diff --git a/lib/strings/strings.ww b/lib/strings/strings.ww index 334bd4e5..dc35e7f2 100644 --- a/lib/strings/strings.ww +++ b/lib/strings/strings.ww @@ -31,6 +31,7 @@ package strings; import bytes; import encoding.utf8; import os; +import types; // toutf8 — borrowed []u8 view of `s`. ref/hare/strings/utf8.ha:29. // `cap` equals `len`; the slice does not own a separate allocation. @@ -606,3 +607,120 @@ export fn remaining_tokens(s: *tokenizer) str = { let b: *bytes.tokenizer = s: *bytes.tokenizer; return fromutf8_unsafe(bytes.remaining_tokens(b)); }; + +// rt_ensure is the runtime slice-growth helper invoked by the +// `append(s, v)` builtin. Direct bind for the same reason as +// lib/shlex.shlex (appendstr, 16B): the builtin's expansion stores +// only 8B of the new element, losing the `.len` half of a `str`. +@symbol("rt_ensure") fn rtensure(s: *void, membsz: u64) void; + +// appendstr — grow `*slice` by one and store `item` (16B). Mirror of +// lib/shlex.shlex appendstr. Collapses when the append builtin learns +// to store the full element width. +fn appendstr(slice: *[]str, item: str) void = { + let newlen: i32 = slice.len + 1; + slice.len = newlen; + rtensure(slice: *void, 16u64); + let dst: *str = &slice.ptr[newlen - 1]; + dst.ptr = item.ptr; + dst.len = item.len; +}; + +// splitn — split `in` on any byte in `delim`, returning up to `n` +// tokens via forward iteration. The trailing slot (when more than +// `n - 1` tokens exist) holds the unconsumed remainder. Strings +// within the result are borrowed from `in`. +// +// The caller frees the returned slice via +// `os.free(r.ptr: *void, (r.cap: u64) * 16u64)`. +// +// Hare's `([]str | nomem)` collapses to `[]str` here: ww os.alloc +// has no recoverable failure path. Same precedent as +// shlex.split / bytes.splitn. +// +// ref/hare/strings/tokenize.ha:172. +export fn splitn(in: str, delim: str, n: i32) []str = { + let toks: []str; + toks.ptr = nil: *str; + toks.len = 0; + toks.cap = 0; + let tok: tokenizer = tokenize(in, delim); + let i: i32 = 0; + for (i < n - 1) { + match (next_token(&tok)) { + case let s: str => { appendstr(&toks, s); }; + case bytes.done => { return toks; }; + }; + i += 1; + }; + match (peek_token(&tok)) { + case bytes.done => void; + case let pk: str => { + let r: str = remaining_tokens(&tok); + appendstr(&toks, r); + }; + }; + return toks; +}; + +// rsplitn — reverse-direction counterpart to [[splitn]]: tokens are +// collected from the end of `in`. The trailing slot holds the +// unconsumed prefix (everything before the n-th-from-last delim hit). +// +// When the input has fewer than n tokens, the `done` short-circuit +// returns toks UN-reversed (in last-token-first order). Mirrors Hare +// at ref/hare/strings/tokenize.ha:219-224 where the in-place reverse +// step is gated behind the n-1 loop running to completion. +// +// ref/hare/strings/tokenize.ha:200. +export fn rsplitn(in: str, delim: str, n: i32) []str = { + let toks: []str; + toks.ptr = nil: *str; + toks.len = 0; + toks.cap = 0; + let tok: tokenizer = rtokenize(in, delim); + let i: i32 = 0; + for (i < n - 1) { + match (next_token(&tok)) { + case let s: str => { appendstr(&toks, s); }; + case bytes.done => { return toks; }; + }; + i += 1; + }; + match (peek_token(&tok)) { + case bytes.done => void; + case let pk: str => { + let r: str = remaining_tokens(&tok); + appendstr(&toks, r); + }; + }; + + // In-place reverse so callers see argv-order, matching Hare + // (ref/hare/strings/tokenize.ha:220). Element copy is field-wise + // through `*str` because `toks[i] = toks[j]` (full 16B str store) + // lands in the multi-word-store gap noted at cmd/w6c/cgen.c:6515. + let a: i32 = 0; + let b: i32 = toks.len - 1; + for (a < b) { + let pa: *str = &toks.ptr[a]; + let pb: *str = &toks.ptr[b]; + let tp: *u8 = pa.ptr; + let tl: i32 = pa.len; + pa.ptr = pb.ptr; + pa.len = pb.len; + pb.ptr = tp; + pb.len = tl; + a += 1; + b -= 1; + }; + return toks; +}; + +// split — full split of `in` on `delim` (no token cap). Mirrors +// `splitn(in, delim, types::SIZE_MAX)`. ww uses `types.I32_MAX` +// because the index type is i32 (lib/CLAUDE.md). +// +// ref/hare/strings/tokenize.ha:242. +export fn split(in: str, delim: str) []str = { + return splitn(in, delim, types.I32_MAX); +}; diff --git a/lib/strings/stringstest.ww b/lib/strings/stringstest.ww index 1fa936be..25e5bd1b 100644 --- a/lib/strings/stringstest.ww +++ b/lib/strings/stringstest.ww @@ -1064,6 +1064,146 @@ fn expect_str_done(t: *strings.tokenizer) void = { if (!streq(strings.remaining_tokens(&t2), "a b c")) { fail(); }; }; +// ---- splitn / rsplitn / split ---------------------------------------- +// ref/hare/strings/tokenize.ha:245 @test fn split. Hare's vectors +// mirrored here directly; element reads go through `&toks.ptr[i]: *str` +// rather than `toks[i]` so the 16B str element copy stays out of the +// multi-word-store gap noted at cmd/w6c/cgen.c:6515. + +fn expect_str(toks: []str, i: i32, want: str) void = { + if (i >= toks.len) { fail(); }; + let p: *str = &toks.ptr[i]; + if (p.len != want.len) { fail(); }; + let j: i32 = 0; + for (j < want.len) { + if (p.ptr[j] != want[j]) { fail(); }; + j += 1; + }; +}; + +@test fn splitn_cases() void = { + // ref/hare/strings/tokenize.ha:247 — n=4 buckets the trailing + // "is Drew" as the remainder slot. + signalled = 1740; + let t1: []str = strings.splitn("Hello, my name is Drew", " ", 4); + if (t1.len != 4) { fail(); }; + expect_str(t1, 0, "Hello,"); + expect_str(t1, 1, "my"); + expect_str(t1, 2, "name"); + expect_str(t1, 3, "is Drew"); + os.free(t1.ptr: *void, (t1.cap: u64) * 16u64); + + // ref/hare/strings/tokenize.ha:263 — n > tokens leaves a single + // slot holding the unchanged input (delim not found). + signalled = 1741; + let t2: []str = strings.splitn("one", "=", 2); + if (t2.len != 1) { fail(); }; + expect_str(t2, 0, "one"); + os.free(t2.ptr: *void, (t2.cap: u64) * 16u64); + + // n == 1 — single slot holding the whole input as remainder. + signalled = 1742; + let t3: []str = strings.splitn("a b c", " ", 1); + if (t3.len != 1) { fail(); }; + expect_str(t3, 0, "a b c"); + os.free(t3.ptr: *void, (t3.cap: u64) * 16u64); + + // Empty input — empty result. + signalled = 1743; + let t4: []str = strings.splitn("", " ", 5); + if (t4.len != 0) { fail(); }; + if (t4.cap > 0) { + os.free(t4.ptr: *void, (t4.cap: u64) * 16u64); + }; + + // Multi-byte delim set (byte-set semantics per + // ref/hare/strings/tokenize.ha:35) — split on ',' OR ':' OR ';'. + signalled = 1744; + let t5: []str = strings.splitn("hello;world,foo:bar", ",:;", 10); + if (t5.len != 4) { fail(); }; + expect_str(t5, 0, "hello"); + expect_str(t5, 1, "world"); + expect_str(t5, 2, "foo"); + expect_str(t5, 3, "bar"); + os.free(t5.ptr: *void, (t5.cap: u64) * 16u64); +}; + +@test fn rsplitn_cases() void = { + // ref/hare/strings/tokenize.ha:271 — reverse n=4 with the + // "Hello, my" prefix as the remainder slot at index 0. + signalled = 1750; + let t1: []str = strings.rsplitn("Hello, my name is Drew", " ", 4); + if (t1.len != 4) { fail(); }; + expect_str(t1, 0, "Hello, my"); + expect_str(t1, 1, "name"); + expect_str(t1, 2, "is"); + expect_str(t1, 3, "Drew"); + os.free(t1.ptr: *void, (t1.cap: u64) * 16u64); + + // n > token count — done short-circuit returns toks UN-reversed + // (last-token-first order). Mirrors bytes.rsplitn (Hare's + // ref/hare/strings/tokenize.ha:219-224 reverse step is gated + // behind the n-1 loop completion). + signalled = 1751; + let t2: []str = strings.rsplitn("a b c", " ", 10); + if (t2.len != 3) { fail(); }; + expect_str(t2, 0, "c"); + expect_str(t2, 1, "b"); + expect_str(t2, 2, "a"); + os.free(t2.ptr: *void, (t2.cap: u64) * 16u64); + + // n == 1 — single slot holding the whole input as remainder. + signalled = 1752; + let t3: []str = strings.rsplitn("a b c", " ", 1); + if (t3.len != 1) { fail(); }; + expect_str(t3, 0, "a b c"); + os.free(t3.ptr: *void, (t3.cap: u64) * 16u64); + + // delim absent — first next_token yields the entire input as the + // sole token; second iter sees done and short-circuits with the + // 1-elem toks un-reversed (single element, reverse is a no-op). + signalled = 1753; + let t4: []str = strings.rsplitn("abc", "=", 5); + if (t4.len != 1) { fail(); }; + expect_str(t4, 0, "abc"); + os.free(t4.ptr: *void, (t4.cap: u64) * 16u64); +}; + +@test fn split_cases() void = { + // ref/hare/strings/tokenize.ha:255 — full split, every delim hit + // is a boundary. + signalled = 1760; + let t1: []str = strings.split("Hello, my name is Drew", " "); + if (t1.len != 5) { fail(); }; + expect_str(t1, 0, "Hello,"); + expect_str(t1, 1, "my"); + expect_str(t1, 2, "name"); + expect_str(t1, 3, "is"); + expect_str(t1, 4, "Drew"); + os.free(t1.ptr: *void, (t1.cap: u64) * 16u64); + + // Leading + trailing delim — empty tokens at ends. + signalled = 1761; + let t2: []str = strings.split(" a b ", " "); + if (t2.len != 4) { fail(); }; + expect_str(t2, 0, ""); + expect_str(t2, 1, "a"); + expect_str(t2, 2, "b"); + expect_str(t2, 3, ""); + os.free(t2.ptr: *void, (t2.cap: u64) * 16u64); + + // Multi-byte delim set, byte-set semantics matching Hare's + // strings::split example at ref/hare/strings/tokenize.ha:235. + signalled = 1762; + let t3: []str = strings.split("hello;world,foo:bar", ",:;"); + if (t3.len != 4) { fail(); }; + expect_str(t3, 0, "hello"); + expect_str(t3, 1, "world"); + expect_str(t3, 2, "foo"); + expect_str(t3, 3, "bar"); + os.free(t3.ptr: *void, (t3.cap: u64) * 16u64); +}; + export fn main() i32 = { signalled = 1; dup_cases(); signalled = 2; concat_cases(); @@ -1099,5 +1239,8 @@ export fn main() i32 = { signalled = 32; rtokenize_cases(); signalled = 33; peek_token_cases(); signalled = 34; remaining_tokens_cases(); + signalled = 35; splitn_cases(); + signalled = 36; rsplitn_cases(); + signalled = 37; split_cases(); return 0; }; diff --git a/selfhost/cmd/w6c/main.combined.ww b/selfhost/cmd/w6c/main.combined.ww index a75b32b6..ceca605c 100644 --- a/selfhost/cmd/w6c/main.combined.ww +++ b/selfhost/cmd/w6c/main.combined.ww @@ -1828,6 +1828,7 @@ package strings; import bytes; import encoding.utf8; import os; +import types; // toutf8 — borrowed []u8 view of `s`. ref/hare/strings/utf8.ha:29. // `cap` equals `len`; the slice does not own a separate allocation. @@ -2404,6 +2405,123 @@ export fn remaining_tokens(s: *tokenizer) str = { return fromutf8_unsafe(bytes.remaining_tokens(b)); }; +// rt_ensure is the runtime slice-growth helper invoked by the +// `append(s, v)` builtin. Direct bind for the same reason as +// lib/shlex.shlex (appendstr, 16B): the builtin's expansion stores +// only 8B of the new element, losing the `.len` half of a `str`. +@symbol("rt_ensure") fn rtensure(s: *void, membsz: u64) void; + +// appendstr — grow `*slice` by one and store `item` (16B). Mirror of +// lib/shlex.shlex appendstr. Collapses when the append builtin learns +// to store the full element width. +fn appendstr(slice: *[]str, item: str) void = { + let newlen: i32 = slice.len + 1; + slice.len = newlen; + rtensure(slice: *void, 16u64); + let dst: *str = &slice.ptr[newlen - 1]; + dst.ptr = item.ptr; + dst.len = item.len; +}; + +// splitn — split `in` on any byte in `delim`, returning up to `n` +// tokens via forward iteration. The trailing slot (when more than +// `n - 1` tokens exist) holds the unconsumed remainder. Strings +// within the result are borrowed from `in`. +// +// The caller frees the returned slice via +// `os.free(r.ptr: *void, (r.cap: u64) * 16u64)`. +// +// Hare's `([]str | nomem)` collapses to `[]str` here: ww os.alloc +// has no recoverable failure path. Same precedent as +// shlex.split / bytes.splitn. +// +// ref/hare/strings/tokenize.ha:172. +export fn splitn(in: str, delim: str, n: i32) []str = { + let toks: []str; + toks.ptr = nil: *str; + toks.len = 0; + toks.cap = 0; + let tok: tokenizer = tokenize(in, delim); + let i: i32 = 0; + for (i < n - 1) { + match (next_token(&tok)) { + case let s: str => { appendstr(&toks, s); }; + case bytes.done => { return toks; }; + }; + i += 1; + }; + match (peek_token(&tok)) { + case bytes.done => void; + case let pk: str => { + let r: str = remaining_tokens(&tok); + appendstr(&toks, r); + }; + }; + return toks; +}; + +// rsplitn — reverse-direction counterpart to [[splitn]]: tokens are +// collected from the end of `in`. The trailing slot holds the +// unconsumed prefix (everything before the n-th-from-last delim hit). +// +// When the input has fewer than n tokens, the `done` short-circuit +// returns toks UN-reversed (in last-token-first order). Mirrors Hare +// at ref/hare/strings/tokenize.ha:219-224 where the in-place reverse +// step is gated behind the n-1 loop running to completion. +// +// ref/hare/strings/tokenize.ha:200. +export fn rsplitn(in: str, delim: str, n: i32) []str = { + let toks: []str; + toks.ptr = nil: *str; + toks.len = 0; + toks.cap = 0; + let tok: tokenizer = rtokenize(in, delim); + let i: i32 = 0; + for (i < n - 1) { + match (next_token(&tok)) { + case let s: str => { appendstr(&toks, s); }; + case bytes.done => { return toks; }; + }; + i += 1; + }; + match (peek_token(&tok)) { + case bytes.done => void; + case let pk: str => { + let r: str = remaining_tokens(&tok); + appendstr(&toks, r); + }; + }; + + // In-place reverse so callers see argv-order, matching Hare + // (ref/hare/strings/tokenize.ha:220). Element copy is field-wise + // through `*str` because `toks[i] = toks[j]` (full 16B str store) + // lands in the multi-word-store gap noted at cmd/w6c/cgen.c:6515. + let a: i32 = 0; + let b: i32 = toks.len - 1; + for (a < b) { + let pa: *str = &toks.ptr[a]; + let pb: *str = &toks.ptr[b]; + let tp: *u8 = pa.ptr; + let tl: i32 = pa.len; + pa.ptr = pb.ptr; + pa.len = pb.len; + pb.ptr = tp; + pb.len = tl; + a += 1; + b -= 1; + }; + return toks; +}; + +// split — full split of `in` on `delim` (no token cap). Mirrors +// `splitn(in, delim, types::SIZE_MAX)`. ww uses `types.I32_MAX` +// because the index type is i32 (lib/CLAUDE.md). +// +// ref/hare/strings/tokenize.ha:242. +export fn split(in: str, delim: str) []str = { + return splitn(in, delim, types.I32_MAX); +}; + // strconv — number↔string conversions. // // Mirrors Hare's strconv:: surface. The *tos functions return a diff --git a/selfhost/cmd/wwdump/main.combined.ww b/selfhost/cmd/wwdump/main.combined.ww index 4396dc37..6ead08e5 100644 --- a/selfhost/cmd/wwdump/main.combined.ww +++ b/selfhost/cmd/wwdump/main.combined.ww @@ -1828,6 +1828,7 @@ package strings; import bytes; import encoding.utf8; import os; +import types; // toutf8 — borrowed []u8 view of `s`. ref/hare/strings/utf8.ha:29. // `cap` equals `len`; the slice does not own a separate allocation. @@ -2404,6 +2405,123 @@ export fn remaining_tokens(s: *tokenizer) str = { return fromutf8_unsafe(bytes.remaining_tokens(b)); }; +// rt_ensure is the runtime slice-growth helper invoked by the +// `append(s, v)` builtin. Direct bind for the same reason as +// lib/shlex.shlex (appendstr, 16B): the builtin's expansion stores +// only 8B of the new element, losing the `.len` half of a `str`. +@symbol("rt_ensure") fn rtensure(s: *void, membsz: u64) void; + +// appendstr — grow `*slice` by one and store `item` (16B). Mirror of +// lib/shlex.shlex appendstr. Collapses when the append builtin learns +// to store the full element width. +fn appendstr(slice: *[]str, item: str) void = { + let newlen: i32 = slice.len + 1; + slice.len = newlen; + rtensure(slice: *void, 16u64); + let dst: *str = &slice.ptr[newlen - 1]; + dst.ptr = item.ptr; + dst.len = item.len; +}; + +// splitn — split `in` on any byte in `delim`, returning up to `n` +// tokens via forward iteration. The trailing slot (when more than +// `n - 1` tokens exist) holds the unconsumed remainder. Strings +// within the result are borrowed from `in`. +// +// The caller frees the returned slice via +// `os.free(r.ptr: *void, (r.cap: u64) * 16u64)`. +// +// Hare's `([]str | nomem)` collapses to `[]str` here: ww os.alloc +// has no recoverable failure path. Same precedent as +// shlex.split / bytes.splitn. +// +// ref/hare/strings/tokenize.ha:172. +export fn splitn(in: str, delim: str, n: i32) []str = { + let toks: []str; + toks.ptr = nil: *str; + toks.len = 0; + toks.cap = 0; + let tok: tokenizer = tokenize(in, delim); + let i: i32 = 0; + for (i < n - 1) { + match (next_token(&tok)) { + case let s: str => { appendstr(&toks, s); }; + case bytes.done => { return toks; }; + }; + i += 1; + }; + match (peek_token(&tok)) { + case bytes.done => void; + case let pk: str => { + let r: str = remaining_tokens(&tok); + appendstr(&toks, r); + }; + }; + return toks; +}; + +// rsplitn — reverse-direction counterpart to [[splitn]]: tokens are +// collected from the end of `in`. The trailing slot holds the +// unconsumed prefix (everything before the n-th-from-last delim hit). +// +// When the input has fewer than n tokens, the `done` short-circuit +// returns toks UN-reversed (in last-token-first order). Mirrors Hare +// at ref/hare/strings/tokenize.ha:219-224 where the in-place reverse +// step is gated behind the n-1 loop running to completion. +// +// ref/hare/strings/tokenize.ha:200. +export fn rsplitn(in: str, delim: str, n: i32) []str = { + let toks: []str; + toks.ptr = nil: *str; + toks.len = 0; + toks.cap = 0; + let tok: tokenizer = rtokenize(in, delim); + let i: i32 = 0; + for (i < n - 1) { + match (next_token(&tok)) { + case let s: str => { appendstr(&toks, s); }; + case bytes.done => { return toks; }; + }; + i += 1; + }; + match (peek_token(&tok)) { + case bytes.done => void; + case let pk: str => { + let r: str = remaining_tokens(&tok); + appendstr(&toks, r); + }; + }; + + // In-place reverse so callers see argv-order, matching Hare + // (ref/hare/strings/tokenize.ha:220). Element copy is field-wise + // through `*str` because `toks[i] = toks[j]` (full 16B str store) + // lands in the multi-word-store gap noted at cmd/w6c/cgen.c:6515. + let a: i32 = 0; + let b: i32 = toks.len - 1; + for (a < b) { + let pa: *str = &toks.ptr[a]; + let pb: *str = &toks.ptr[b]; + let tp: *u8 = pa.ptr; + let tl: i32 = pa.len; + pa.ptr = pb.ptr; + pa.len = pb.len; + pb.ptr = tp; + pb.len = tl; + a += 1; + b -= 1; + }; + return toks; +}; + +// split — full split of `in` on `delim` (no token cap). Mirrors +// `splitn(in, delim, types::SIZE_MAX)`. ww uses `types.I32_MAX` +// because the index type is i32 (lib/CLAUDE.md). +// +// ref/hare/strings/tokenize.ha:242. +export fn split(in: str, delim: str) []str = { + return splitn(in, delim, types.I32_MAX); +}; + // strconv — number↔string conversions. // // Mirrors Hare's strconv:: surface. The *tos functions return a diff --git a/selfhost/test/smoke.combined.ww b/selfhost/test/smoke.combined.ww index a036f883..dcd607a2 100644 --- a/selfhost/test/smoke.combined.ww +++ b/selfhost/test/smoke.combined.ww @@ -1719,6 +1719,7 @@ package strings; import bytes; import encoding.utf8; import os; +import types; // toutf8 — borrowed []u8 view of `s`. ref/hare/strings/utf8.ha:29. // `cap` equals `len`; the slice does not own a separate allocation. @@ -2295,6 +2296,123 @@ export fn remaining_tokens(s: *tokenizer) str = { return fromutf8_unsafe(bytes.remaining_tokens(b)); }; +// rt_ensure is the runtime slice-growth helper invoked by the +// `append(s, v)` builtin. Direct bind for the same reason as +// lib/shlex.shlex (appendstr, 16B): the builtin's expansion stores +// only 8B of the new element, losing the `.len` half of a `str`. +@symbol("rt_ensure") fn rtensure(s: *void, membsz: u64) void; + +// appendstr — grow `*slice` by one and store `item` (16B). Mirror of +// lib/shlex.shlex appendstr. Collapses when the append builtin learns +// to store the full element width. +fn appendstr(slice: *[]str, item: str) void = { + let newlen: i32 = slice.len + 1; + slice.len = newlen; + rtensure(slice: *void, 16u64); + let dst: *str = &slice.ptr[newlen - 1]; + dst.ptr = item.ptr; + dst.len = item.len; +}; + +// splitn — split `in` on any byte in `delim`, returning up to `n` +// tokens via forward iteration. The trailing slot (when more than +// `n - 1` tokens exist) holds the unconsumed remainder. Strings +// within the result are borrowed from `in`. +// +// The caller frees the returned slice via +// `os.free(r.ptr: *void, (r.cap: u64) * 16u64)`. +// +// Hare's `([]str | nomem)` collapses to `[]str` here: ww os.alloc +// has no recoverable failure path. Same precedent as +// shlex.split / bytes.splitn. +// +// ref/hare/strings/tokenize.ha:172. +export fn splitn(in: str, delim: str, n: i32) []str = { + let toks: []str; + toks.ptr = nil: *str; + toks.len = 0; + toks.cap = 0; + let tok: tokenizer = tokenize(in, delim); + let i: i32 = 0; + for (i < n - 1) { + match (next_token(&tok)) { + case let s: str => { appendstr(&toks, s); }; + case bytes.done => { return toks; }; + }; + i += 1; + }; + match (peek_token(&tok)) { + case bytes.done => void; + case let pk: str => { + let r: str = remaining_tokens(&tok); + appendstr(&toks, r); + }; + }; + return toks; +}; + +// rsplitn — reverse-direction counterpart to [[splitn]]: tokens are +// collected from the end of `in`. The trailing slot holds the +// unconsumed prefix (everything before the n-th-from-last delim hit). +// +// When the input has fewer than n tokens, the `done` short-circuit +// returns toks UN-reversed (in last-token-first order). Mirrors Hare +// at ref/hare/strings/tokenize.ha:219-224 where the in-place reverse +// step is gated behind the n-1 loop running to completion. +// +// ref/hare/strings/tokenize.ha:200. +export fn rsplitn(in: str, delim: str, n: i32) []str = { + let toks: []str; + toks.ptr = nil: *str; + toks.len = 0; + toks.cap = 0; + let tok: tokenizer = rtokenize(in, delim); + let i: i32 = 0; + for (i < n - 1) { + match (next_token(&tok)) { + case let s: str => { appendstr(&toks, s); }; + case bytes.done => { return toks; }; + }; + i += 1; + }; + match (peek_token(&tok)) { + case bytes.done => void; + case let pk: str => { + let r: str = remaining_tokens(&tok); + appendstr(&toks, r); + }; + }; + + // In-place reverse so callers see argv-order, matching Hare + // (ref/hare/strings/tokenize.ha:220). Element copy is field-wise + // through `*str` because `toks[i] = toks[j]` (full 16B str store) + // lands in the multi-word-store gap noted at cmd/w6c/cgen.c:6515. + let a: i32 = 0; + let b: i32 = toks.len - 1; + for (a < b) { + let pa: *str = &toks.ptr[a]; + let pb: *str = &toks.ptr[b]; + let tp: *u8 = pa.ptr; + let tl: i32 = pa.len; + pa.ptr = pb.ptr; + pa.len = pb.len; + pb.ptr = tp; + pb.len = tl; + a += 1; + b -= 1; + }; + return toks; +}; + +// split — full split of `in` on `delim` (no token cap). Mirrors +// `splitn(in, delim, types::SIZE_MAX)`. ww uses `types.I32_MAX` +// because the index type is i32 (lib/CLAUDE.md). +// +// ref/hare/strings/tokenize.ha:242. +export fn split(in: str, delim: str) []str = { + return splitn(in, delim, types.I32_MAX); +}; + // strconv — number↔string conversions. // // Mirrors Hare's strconv:: surface. The *tos functions return a