Hojun-Cho/ww
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

lib/bytes+test: port split family from Hare

Browse Source
This commit is contained in:
Hojun-Cho
2026-05-19 13:50:54 +09:00
parent d5e8d699d1
commit 7e1b681701
5 changed files with 706 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

131
lib/bytes/bytes.ww
View File

@@ -351,3 +351,134 @@ export fn next_token(s: *tokenizer) ([]u8 | done) = {
export fn remaining_tokens(s: *tokenizer) []u8 = {
return s.in;
};
// rt_ensure is the runtime slice-growth helper invoked by the
// `append(s, v)` builtin. We bind it directly because the builtin's
// expansion stores only 8 bytes of the new element (cgen emits a
// single MOVQ), losing the .len/.cap fields of a []u8 element (24B).
// Mirrors the same workaround in lib/shlex.shlex (appendstr, 16B) and
// lib/getopt.getopt (appendoption, 24B); collapses in one go when the
// append builtin learns to store the full element width.
@symbol("rt_ensure") fn rtensure(s: *void, membsz: u64) void;
// appendslice — grow `*slice` by one and store `item` (24B). Mirror
// of [[shlex.appendstr]] / [[getopt.appendoption]]. Bypasses the
// `append` builtin's first-8B-only-store gap for a slice-element.
fn appendslice(slice: *[][]u8, item: []u8) void = {
let newlen: i32 = slice.len + 1;
slice.len = newlen;
rtensure(slice: *void, 24u64);
let dst: *[]u8 = &slice.ptr[newlen - 1];
dst.ptr = item.ptr;
dst.len = item.len;
dst.cap = item.cap;
};
// splitn — split `in` on any byte in `delim`, returning up to `n`
// tokens via forward iteration. The trailing slot (when more than
// `n - 1` tokens exist) holds the unconsumed remainder.
//
// The caller frees the returned slice via
// `os.free(r.ptr: *void, (r.cap: u64) * 24u64)`. Element bytes are
// borrowed from `in`.
//
// Hare's `([][]u8 | nomem)` collapses to `[][]u8` here: ww os.alloc
// has no recoverable failure path. Same precedent as
// shlex.split / getopt.tryparse.
//
// ref/hare/bytes/tokenize.ha:156.
export fn splitn(in: []u8, delim: []u8, n: i32) [][]u8 = {
os.assert(delim.len > 0,
"bytes.splitn must not be called with an empty delimiter");
let toks: [][]u8;
toks.ptr = nil: *[]u8;
toks.len = 0;
toks.cap = 0;
let tok: tokenizer = tokenize(in, delim...);
let i: i32 = 0;
for (i < n - 1) {
match (next_token(&tok)) {
case let s: []u8 => { appendslice(&toks, s); };
case done => { return toks; };
};
i += 1;
};
match (peek_token(&tok)) {
case done => void;
case let pk: []u8 => {
let r: []u8 = remaining_tokens(&tok);
appendslice(&toks, r);
};
};
return toks;
};
// rsplitn — reverse-direction counterpart to [[splitn]]: tokens are
// collected from the end of `in`. The trailing slot holds the
// unconsumed prefix (everything before the n-th-from-last delim hit).
//
// When the input has fewer than n tokens, the `done` short-circuit
// returns toks UN-reversed (in last-token-first order). Mirrors Hare
// at ref/hare/bytes/tokenize.ha:196-199 where the in-place reverse
// step is gated behind the n-1 loop running to completion. Only the
// "loop ran to completion AND peek saw a remainder" path applies the
// reverse; both early-exit paths skip it.
//
// ref/hare/bytes/tokenize.ha:186.
export fn rsplitn(in: []u8, delim: []u8, n: i32) [][]u8 = {
os.assert(delim.len > 0,
"bytes.rsplitn called with empty delimiter");
let toks: [][]u8;
toks.ptr = nil: *[]u8;
toks.len = 0;
toks.cap = 0;
let tok: tokenizer = rtokenize(in, delim...);
let i: i32 = 0;
for (i < n - 1) {
match (next_token(&tok)) {
case let s: []u8 => { appendslice(&toks, s); };
case done => { return toks; };
};
i += 1;
};
match (peek_token(&tok)) {
case done => void;
case let pk: []u8 => {
let r: []u8 = remaining_tokens(&tok);
appendslice(&toks, r);
};
};
// In-place reverse so callers see argv-order, matching Hare
// (ref/hare/bytes/tokenize.ha:207). Element copy is field-wise
// through `*[]u8` because `toks[i] = toks[j]` (full 24B slice
// store) lands in the multi-word-store gap noted at
// cmd/w6c/cgen.c:6515-6523.
let a: i32 = 0;
let b: i32 = toks.len - 1;
for (a < b) {
let pa: *[]u8 = &toks.ptr[a];
let pb: *[]u8 = &toks.ptr[b];
let tp: *u8 = pa.ptr;
let tl: i32 = pa.len;
let tc: i32 = pa.cap;
pa.ptr = pb.ptr;
pa.len = pb.len;
pa.cap = pb.cap;
pb.ptr = tp;
pb.len = tl;
pb.cap = tc;
a += 1;
b -= 1;
};
return toks;
};
// split — full split of `in` on `delim` (no token cap). Mirrors
// `splitn(in, delim, types::SIZE_MAX)`. ww uses `types.I32_MAX`
// because the index type is i32 (lib/CLAUDE.md).
//
// ref/hare/bytes/tokenize.ha:225.
export fn split(in: []u8, delim: []u8) [][]u8 = {
return splitn(in, delim, types.I32_MAX);
};

182
lib/bytes/bytestest.ww
View File

@@ -467,6 +467,185 @@ fn expect_done(t: *bytes.tokenizer) void = {
if (!bytes.equal(r2, e_12[0:2])) { fail(); };
};
// ---- splitn / rsplitn / split -----------------------------------------
// ref/hare/bytes/tokenize.ha:330 (@test fn split). Hare's table mixes
// strings; ww spells the vectors as byte arrays explicitly.
//
// Element reads go through `&toks.ptr[i]: *[]u8` rather than `toks[i]`
// — the full 24B slice-element copy lands in the multi-word-store gap
// noted at cmd/w6c/cgen.c:6515-6523, so a value-load drops .len/.cap.
// Pointer-then-fields lifts 8B at a time, which the cgen routes correctly.
fn expect_tok(toks: [][]u8, i: i32, want: []u8) void = {
if (i >= toks.len) { fail(); };
let p: *[]u8 = &toks.ptr[i];
if (p.len != want.len) { fail(); };
let j: i32 = 0;
for (j < want.len) {
if (p.ptr[j] != want[j]) { fail(); };
j += 1;
};
};
@test fn splitn_cases() void = {
// "Hello, my name is Drew" ─ space-delimited, n=4 yields the 4-th
// token as the unconsumed remainder. Hare pins this exact shape
// at ref/hare/bytes/tokenize.ha:340.
signalled = 1750;
let a: [22]u8;
a[0] = 72u8; a[1] = 101u8; a[2] = 108u8; a[3] = 108u8; a[4] = 111u8;
a[5] = 44u8; a[6] = 32u8; a[7] = 109u8; a[8] = 121u8; a[9] = 32u8;
a[10] = 110u8; a[11] = 97u8; a[12] = 109u8; a[13] = 101u8; a[14] = 32u8;
a[15] = 105u8; a[16] = 115u8; a[17] = 32u8;
a[18] = 68u8; a[19] = 114u8; a[20] = 101u8; a[21] = 119u8;
let sp: [1]u8; sp[0] = 32u8;
let t1: [][]u8 = bytes.splitn(a[0:22], sp[0:1], 4);
if (t1.len != 4) { fail(); };
expect_tok(t1, 0, a[0:6]);
expect_tok(t1, 1, a[7:9]);
expect_tok(t1, 2, a[10:14]);
expect_tok(t1, 3, a[15:22]);
os.free(t1.ptr: *void, (t1.cap: u64) * 24u64);
// n > token count — final slot is "" if input ends in delim,
// otherwise the last token. Here three tokens, n=10 → 3 entries.
signalled = 1751;
let b: [5]u8; b[0] = 1u8; b[1] = 0u8; b[2] = 2u8; b[3] = 0u8; b[4] = 3u8;
let zd: [1]u8; zd[0] = 0u8;
let t2: [][]u8 = bytes.splitn(b[0:5], zd[0:1], 10);
if (t2.len != 3) { fail(); };
expect_tok(t2, 0, b[0:1]);
expect_tok(t2, 1, b[2:3]);
expect_tok(t2, 2, b[4:5]);
os.free(t2.ptr: *void, (t2.cap: u64) * 24u64);
// n == 1 — single slot holding the whole input as remainder.
signalled = 1752;
let t3: [][]u8 = bytes.splitn(b[0:5], zd[0:1], 1);
if (t3.len != 1) { fail(); };
expect_tok(t3, 0, b[0:5]);
os.free(t3.ptr: *void, (t3.cap: u64) * 24u64);
// delim absent — single slot holding input unchanged.
signalled = 1753;
let c: [3]u8; c[0] = 1u8; c[1] = 2u8; c[2] = 3u8;
let t4: [][]u8 = bytes.splitn(c[0:3], zd[0:1], 5);
if (t4.len != 1) { fail(); };
expect_tok(t4, 0, c[0:3]);
os.free(t4.ptr: *void, (t4.cap: u64) * 24u64);
// empty input — empty result.
signalled = 1754;
let z: [1]u8;
let t5: [][]u8 = bytes.splitn(z[0:0], zd[0:1], 5);
if (t5.len != 0) { fail(); };
if (t5.cap > 0) {
os.free(t5.ptr: *void, (t5.cap: u64) * 24u64);
};
// Multi-byte delimiter set — both 0 and 42 split.
signalled = 1755;
let d: [8]u8;
d[0] = 1u8; d[1] = 2u8; d[2] = 0u8; d[3] = 3u8;
d[4] = 4u8; d[5] = 42u8; d[6] = 5u8; d[7] = 6u8;
let dd: [2]u8; dd[0] = 0u8; dd[1] = 42u8;
let t6: [][]u8 = bytes.splitn(d[0:8], dd[0:2], 100);
if (t6.len != 3) { fail(); };
expect_tok(t6, 0, d[0:2]);
expect_tok(t6, 1, d[3:5]);
expect_tok(t6, 2, d[6:8]);
os.free(t6.ptr: *void, (t6.cap: u64) * 24u64);
};
@test fn rsplitn_cases() void = {
// "Hello, my name is Drew" ─ rsplitn n=4 buckets the first three
// tokens from the *end*; the remainder ("Hello, my") is index 0.
// Hare pins this at ref/hare/bytes/tokenize.ha:379.
signalled = 1760;
let a: [22]u8;
a[0] = 72u8; a[1] = 101u8; a[2] = 108u8; a[3] = 108u8; a[4] = 111u8;
a[5] = 44u8; a[6] = 32u8; a[7] = 109u8; a[8] = 121u8; a[9] = 32u8;
a[10] = 110u8; a[11] = 97u8; a[12] = 109u8; a[13] = 101u8; a[14] = 32u8;
a[15] = 105u8; a[16] = 115u8; a[17] = 32u8;
a[18] = 68u8; a[19] = 114u8; a[20] = 101u8; a[21] = 119u8;
let sp: [1]u8; sp[0] = 32u8;
let t1: [][]u8 = bytes.rsplitn(a[0:22], sp[0:1], 4);
if (t1.len != 4) { fail(); };
expect_tok(t1, 0, a[0:9]);
expect_tok(t1, 1, a[10:14]);
expect_tok(t1, 2, a[15:17]);
expect_tok(t1, 3, a[18:22]);
os.free(t1.ptr: *void, (t1.cap: u64) * 24u64);
// n > token count — done short-circuit returns the toks in
// reverse-iteration order (last token first). Mirrors Hare's
// behavior at ref/hare/bytes/tokenize.ha:196-199 where the
// reverse-step is gated behind the n-1 loop completion.
signalled = 1761;
let b: [5]u8; b[0] = 1u8; b[1] = 0u8; b[2] = 2u8; b[3] = 0u8; b[4] = 3u8;
let zd: [1]u8; zd[0] = 0u8;
let t2: [][]u8 = bytes.rsplitn(b[0:5], zd[0:1], 10);
if (t2.len != 3) { fail(); };
expect_tok(t2, 0, b[4:5]);
expect_tok(t2, 1, b[2:3]);
expect_tok(t2, 2, b[0:1]);
os.free(t2.ptr: *void, (t2.cap: u64) * 24u64);
// n == 1 — single slot holding the whole input as remainder.
signalled = 1762;
let t3: [][]u8 = bytes.rsplitn(b[0:5], zd[0:1], 1);
if (t3.len != 1) { fail(); };
expect_tok(t3, 0, b[0:5]);
os.free(t3.ptr: *void, (t3.cap: u64) * 24u64);
// delim absent — done short-circuits immediately at iter 0;
// toks ends up holding only the eventual remainder if n=1
// (the loop never runs and peek picks up the full input).
// With n>1 the loop's first next_token sees done (no delim
// match anywhere) and returns toks={} per Hare's early-exit.
signalled = 1763;
let c: [3]u8; c[0] = 1u8; c[1] = 2u8; c[2] = 3u8;
let t4: [][]u8 = bytes.rsplitn(c[0:3], zd[0:1], 5);
if (t4.len != 1) { fail(); };
expect_tok(t4, 0, c[0:3]);
os.free(t4.ptr: *void, (t4.cap: u64) * 24u64);
};
@test fn split_cases() void = {
// Full split — every delim hit is a boundary; no per-call cap.
// Hare pins five tokens for the canonical input at
// ref/hare/bytes/tokenize.ha:347.
signalled = 1770;
let a: [22]u8;
a[0] = 72u8; a[1] = 101u8; a[2] = 108u8; a[3] = 108u8; a[4] = 111u8;
a[5] = 44u8; a[6] = 32u8; a[7] = 109u8; a[8] = 121u8; a[9] = 32u8;
a[10] = 110u8; a[11] = 97u8; a[12] = 109u8; a[13] = 101u8; a[14] = 32u8;
a[15] = 105u8; a[16] = 115u8; a[17] = 32u8;
a[18] = 68u8; a[19] = 114u8; a[20] = 101u8; a[21] = 119u8;
let sp: [1]u8; sp[0] = 32u8;
let t1: [][]u8 = bytes.split(a[0:22], sp[0:1]);
if (t1.len != 5) { fail(); };
expect_tok(t1, 0, a[0:6]);
expect_tok(t1, 1, a[7:9]);
expect_tok(t1, 2, a[10:14]);
expect_tok(t1, 3, a[15:17]);
expect_tok(t1, 4, a[18:22]);
os.free(t1.ptr: *void, (t1.cap: u64) * 24u64);
// Leading + trailing empty tokens — five entries, three of which
// are []. Splits across all positions exactly the way Hare does.
signalled = 1771;
let b: [5]u8; b[0] = 0u8; b[1] = 1u8; b[2] = 2u8; b[3] = 3u8; b[4] = 0u8;
let zd: [1]u8; zd[0] = 0u8;
let z: [1]u8;
let t2: [][]u8 = bytes.split(b[0:5], zd[0:1]);
if (t2.len != 3) { fail(); };
expect_tok(t2, 0, z[0:0]);
expect_tok(t2, 1, b[1:4]);
expect_tok(t2, 2, z[0:0]);
os.free(t2.ptr: *void, (t2.cap: u64) * 24u64);
};
export fn main() i32 = {
signalled = 1; equal_cases();
signalled = 2; index_byte_cases();
@@ -480,5 +659,8 @@ export fn main() i32 = {
signalled = 10; rtokenize_cases();
signalled = 11; peek_token_cases();
signalled = 12; remaining_tokens_cases();
signalled = 13; splitn_cases();
signalled = 14; rsplitn_cases();
signalled = 15; split_cases();
return 0;
};