From 7e1b68170155c3249ed43fc5c4147485a9e85f41 Mon Sep 17 00:00:00 2001 From: Hojun-Cho Date: Tue, 19 May 2026 13:50:54 +0900 Subject: [PATCH] lib/bytes+test: port split family from Hare --- lib/bytes/bytes.ww | 131 +++++++++++++++++++ lib/bytes/bytestest.ww | 182 +++++++++++++++++++++++++++ selfhost/cmd/w6c/main.combined.ww | 131 +++++++++++++++++++ selfhost/cmd/wwdump/main.combined.ww | 131 +++++++++++++++++++ selfhost/test/smoke.combined.ww | 131 +++++++++++++++++++ 5 files changed, 706 insertions(+) diff --git a/lib/bytes/bytes.ww b/lib/bytes/bytes.ww index e821036f..03097fa1 100644 --- a/lib/bytes/bytes.ww +++ b/lib/bytes/bytes.ww @@ -351,3 +351,134 @@ export fn next_token(s: *tokenizer) ([]u8 | done) = { export fn remaining_tokens(s: *tokenizer) []u8 = { return s.in; }; + +// rt_ensure is the runtime slice-growth helper invoked by the +// `append(s, v)` builtin. We bind it directly because the builtin's +// expansion stores only 8 bytes of the new element (cgen emits a +// single MOVQ), losing the .len/.cap fields of a []u8 element (24B). +// Mirrors the same workaround in lib/shlex.shlex (appendstr, 16B) and +// lib/getopt.getopt (appendoption, 24B); collapses in one go when the +// append builtin learns to store the full element width. +@symbol("rt_ensure") fn rtensure(s: *void, membsz: u64) void; + +// appendslice — grow `*slice` by one and store `item` (24B). Mirror +// of [[shlex.appendstr]] / [[getopt.appendoption]]. Bypasses the +// `append` builtin's first-8B-only-store gap for a slice-element. +fn appendslice(slice: *[][]u8, item: []u8) void = { + let newlen: i32 = slice.len + 1; + slice.len = newlen; + rtensure(slice: *void, 24u64); + let dst: *[]u8 = &slice.ptr[newlen - 1]; + dst.ptr = item.ptr; + dst.len = item.len; + dst.cap = item.cap; +}; + +// splitn — split `in` on any byte in `delim`, returning up to `n` +// tokens via forward iteration. The trailing slot (when more than +// `n - 1` tokens exist) holds the unconsumed remainder. +// +// The caller frees the returned slice via +// `os.free(r.ptr: *void, (r.cap: u64) * 24u64)`. Element bytes are +// borrowed from `in`. +// +// Hare's `([][]u8 | nomem)` collapses to `[][]u8` here: ww os.alloc +// has no recoverable failure path. Same precedent as +// shlex.split / getopt.tryparse. +// +// ref/hare/bytes/tokenize.ha:156. +export fn splitn(in: []u8, delim: []u8, n: i32) [][]u8 = { + os.assert(delim.len > 0, + "bytes.splitn must not be called with an empty delimiter"); + let toks: [][]u8; + toks.ptr = nil: *[]u8; + toks.len = 0; + toks.cap = 0; + let tok: tokenizer = tokenize(in, delim...); + let i: i32 = 0; + for (i < n - 1) { + match (next_token(&tok)) { + case let s: []u8 => { appendslice(&toks, s); }; + case done => { return toks; }; + }; + i += 1; + }; + match (peek_token(&tok)) { + case done => void; + case let pk: []u8 => { + let r: []u8 = remaining_tokens(&tok); + appendslice(&toks, r); + }; + }; + return toks; +}; + +// rsplitn — reverse-direction counterpart to [[splitn]]: tokens are +// collected from the end of `in`. The trailing slot holds the +// unconsumed prefix (everything before the n-th-from-last delim hit). +// +// When the input has fewer than n tokens, the `done` short-circuit +// returns toks UN-reversed (in last-token-first order). Mirrors Hare +// at ref/hare/bytes/tokenize.ha:196-199 where the in-place reverse +// step is gated behind the n-1 loop running to completion. Only the +// "loop ran to completion AND peek saw a remainder" path applies the +// reverse; both early-exit paths skip it. +// +// ref/hare/bytes/tokenize.ha:186. +export fn rsplitn(in: []u8, delim: []u8, n: i32) [][]u8 = { + os.assert(delim.len > 0, + "bytes.rsplitn called with empty delimiter"); + let toks: [][]u8; + toks.ptr = nil: *[]u8; + toks.len = 0; + toks.cap = 0; + let tok: tokenizer = rtokenize(in, delim...); + let i: i32 = 0; + for (i < n - 1) { + match (next_token(&tok)) { + case let s: []u8 => { appendslice(&toks, s); }; + case done => { return toks; }; + }; + i += 1; + }; + match (peek_token(&tok)) { + case done => void; + case let pk: []u8 => { + let r: []u8 = remaining_tokens(&tok); + appendslice(&toks, r); + }; + }; + + // In-place reverse so callers see argv-order, matching Hare + // (ref/hare/bytes/tokenize.ha:207). Element copy is field-wise + // through `*[]u8` because `toks[i] = toks[j]` (full 24B slice + // store) lands in the multi-word-store gap noted at + // cmd/w6c/cgen.c:6515-6523. + let a: i32 = 0; + let b: i32 = toks.len - 1; + for (a < b) { + let pa: *[]u8 = &toks.ptr[a]; + let pb: *[]u8 = &toks.ptr[b]; + let tp: *u8 = pa.ptr; + let tl: i32 = pa.len; + let tc: i32 = pa.cap; + pa.ptr = pb.ptr; + pa.len = pb.len; + pa.cap = pb.cap; + pb.ptr = tp; + pb.len = tl; + pb.cap = tc; + a += 1; + b -= 1; + }; + return toks; +}; + +// split — full split of `in` on `delim` (no token cap). Mirrors +// `splitn(in, delim, types::SIZE_MAX)`. ww uses `types.I32_MAX` +// because the index type is i32 (lib/CLAUDE.md). +// +// ref/hare/bytes/tokenize.ha:225. +export fn split(in: []u8, delim: []u8) [][]u8 = { + return splitn(in, delim, types.I32_MAX); +}; diff --git a/lib/bytes/bytestest.ww b/lib/bytes/bytestest.ww index 20436a03..9a0288df 100644 --- a/lib/bytes/bytestest.ww +++ b/lib/bytes/bytestest.ww @@ -467,6 +467,185 @@ fn expect_done(t: *bytes.tokenizer) void = { if (!bytes.equal(r2, e_12[0:2])) { fail(); }; }; +// ---- splitn / rsplitn / split ----------------------------------------- +// ref/hare/bytes/tokenize.ha:330 (@test fn split). Hare's table mixes +// strings; ww spells the vectors as byte arrays explicitly. +// +// Element reads go through `&toks.ptr[i]: *[]u8` rather than `toks[i]` +// — the full 24B slice-element copy lands in the multi-word-store gap +// noted at cmd/w6c/cgen.c:6515-6523, so a value-load drops .len/.cap. +// Pointer-then-fields lifts 8B at a time, which the cgen routes correctly. + +fn expect_tok(toks: [][]u8, i: i32, want: []u8) void = { + if (i >= toks.len) { fail(); }; + let p: *[]u8 = &toks.ptr[i]; + if (p.len != want.len) { fail(); }; + let j: i32 = 0; + for (j < want.len) { + if (p.ptr[j] != want[j]) { fail(); }; + j += 1; + }; +}; + +@test fn splitn_cases() void = { + // "Hello, my name is Drew" ─ space-delimited, n=4 yields the 4-th + // token as the unconsumed remainder. Hare pins this exact shape + // at ref/hare/bytes/tokenize.ha:340. + signalled = 1750; + let a: [22]u8; + a[0] = 72u8; a[1] = 101u8; a[2] = 108u8; a[3] = 108u8; a[4] = 111u8; + a[5] = 44u8; a[6] = 32u8; a[7] = 109u8; a[8] = 121u8; a[9] = 32u8; + a[10] = 110u8; a[11] = 97u8; a[12] = 109u8; a[13] = 101u8; a[14] = 32u8; + a[15] = 105u8; a[16] = 115u8; a[17] = 32u8; + a[18] = 68u8; a[19] = 114u8; a[20] = 101u8; a[21] = 119u8; + let sp: [1]u8; sp[0] = 32u8; + let t1: [][]u8 = bytes.splitn(a[0:22], sp[0:1], 4); + if (t1.len != 4) { fail(); }; + expect_tok(t1, 0, a[0:6]); + expect_tok(t1, 1, a[7:9]); + expect_tok(t1, 2, a[10:14]); + expect_tok(t1, 3, a[15:22]); + os.free(t1.ptr: *void, (t1.cap: u64) * 24u64); + + // n > token count — final slot is "" if input ends in delim, + // otherwise the last token. Here three tokens, n=10 → 3 entries. + signalled = 1751; + let b: [5]u8; b[0] = 1u8; b[1] = 0u8; b[2] = 2u8; b[3] = 0u8; b[4] = 3u8; + let zd: [1]u8; zd[0] = 0u8; + let t2: [][]u8 = bytes.splitn(b[0:5], zd[0:1], 10); + if (t2.len != 3) { fail(); }; + expect_tok(t2, 0, b[0:1]); + expect_tok(t2, 1, b[2:3]); + expect_tok(t2, 2, b[4:5]); + os.free(t2.ptr: *void, (t2.cap: u64) * 24u64); + + // n == 1 — single slot holding the whole input as remainder. + signalled = 1752; + let t3: [][]u8 = bytes.splitn(b[0:5], zd[0:1], 1); + if (t3.len != 1) { fail(); }; + expect_tok(t3, 0, b[0:5]); + os.free(t3.ptr: *void, (t3.cap: u64) * 24u64); + + // delim absent — single slot holding input unchanged. + signalled = 1753; + let c: [3]u8; c[0] = 1u8; c[1] = 2u8; c[2] = 3u8; + let t4: [][]u8 = bytes.splitn(c[0:3], zd[0:1], 5); + if (t4.len != 1) { fail(); }; + expect_tok(t4, 0, c[0:3]); + os.free(t4.ptr: *void, (t4.cap: u64) * 24u64); + + // empty input — empty result. + signalled = 1754; + let z: [1]u8; + let t5: [][]u8 = bytes.splitn(z[0:0], zd[0:1], 5); + if (t5.len != 0) { fail(); }; + if (t5.cap > 0) { + os.free(t5.ptr: *void, (t5.cap: u64) * 24u64); + }; + + // Multi-byte delimiter set — both 0 and 42 split. + signalled = 1755; + let d: [8]u8; + d[0] = 1u8; d[1] = 2u8; d[2] = 0u8; d[3] = 3u8; + d[4] = 4u8; d[5] = 42u8; d[6] = 5u8; d[7] = 6u8; + let dd: [2]u8; dd[0] = 0u8; dd[1] = 42u8; + let t6: [][]u8 = bytes.splitn(d[0:8], dd[0:2], 100); + if (t6.len != 3) { fail(); }; + expect_tok(t6, 0, d[0:2]); + expect_tok(t6, 1, d[3:5]); + expect_tok(t6, 2, d[6:8]); + os.free(t6.ptr: *void, (t6.cap: u64) * 24u64); +}; + +@test fn rsplitn_cases() void = { + // "Hello, my name is Drew" ─ rsplitn n=4 buckets the first three + // tokens from the *end*; the remainder ("Hello, my") is index 0. + // Hare pins this at ref/hare/bytes/tokenize.ha:379. + signalled = 1760; + let a: [22]u8; + a[0] = 72u8; a[1] = 101u8; a[2] = 108u8; a[3] = 108u8; a[4] = 111u8; + a[5] = 44u8; a[6] = 32u8; a[7] = 109u8; a[8] = 121u8; a[9] = 32u8; + a[10] = 110u8; a[11] = 97u8; a[12] = 109u8; a[13] = 101u8; a[14] = 32u8; + a[15] = 105u8; a[16] = 115u8; a[17] = 32u8; + a[18] = 68u8; a[19] = 114u8; a[20] = 101u8; a[21] = 119u8; + let sp: [1]u8; sp[0] = 32u8; + let t1: [][]u8 = bytes.rsplitn(a[0:22], sp[0:1], 4); + if (t1.len != 4) { fail(); }; + expect_tok(t1, 0, a[0:9]); + expect_tok(t1, 1, a[10:14]); + expect_tok(t1, 2, a[15:17]); + expect_tok(t1, 3, a[18:22]); + os.free(t1.ptr: *void, (t1.cap: u64) * 24u64); + + // n > token count — done short-circuit returns the toks in + // reverse-iteration order (last token first). Mirrors Hare's + // behavior at ref/hare/bytes/tokenize.ha:196-199 where the + // reverse-step is gated behind the n-1 loop completion. + signalled = 1761; + let b: [5]u8; b[0] = 1u8; b[1] = 0u8; b[2] = 2u8; b[3] = 0u8; b[4] = 3u8; + let zd: [1]u8; zd[0] = 0u8; + let t2: [][]u8 = bytes.rsplitn(b[0:5], zd[0:1], 10); + if (t2.len != 3) { fail(); }; + expect_tok(t2, 0, b[4:5]); + expect_tok(t2, 1, b[2:3]); + expect_tok(t2, 2, b[0:1]); + os.free(t2.ptr: *void, (t2.cap: u64) * 24u64); + + // n == 1 — single slot holding the whole input as remainder. + signalled = 1762; + let t3: [][]u8 = bytes.rsplitn(b[0:5], zd[0:1], 1); + if (t3.len != 1) { fail(); }; + expect_tok(t3, 0, b[0:5]); + os.free(t3.ptr: *void, (t3.cap: u64) * 24u64); + + // delim absent — done short-circuits immediately at iter 0; + // toks ends up holding only the eventual remainder if n=1 + // (the loop never runs and peek picks up the full input). + // With n>1 the loop's first next_token sees done (no delim + // match anywhere) and returns toks={} per Hare's early-exit. + signalled = 1763; + let c: [3]u8; c[0] = 1u8; c[1] = 2u8; c[2] = 3u8; + let t4: [][]u8 = bytes.rsplitn(c[0:3], zd[0:1], 5); + if (t4.len != 1) { fail(); }; + expect_tok(t4, 0, c[0:3]); + os.free(t4.ptr: *void, (t4.cap: u64) * 24u64); +}; + +@test fn split_cases() void = { + // Full split — every delim hit is a boundary; no per-call cap. + // Hare pins five tokens for the canonical input at + // ref/hare/bytes/tokenize.ha:347. + signalled = 1770; + let a: [22]u8; + a[0] = 72u8; a[1] = 101u8; a[2] = 108u8; a[3] = 108u8; a[4] = 111u8; + a[5] = 44u8; a[6] = 32u8; a[7] = 109u8; a[8] = 121u8; a[9] = 32u8; + a[10] = 110u8; a[11] = 97u8; a[12] = 109u8; a[13] = 101u8; a[14] = 32u8; + a[15] = 105u8; a[16] = 115u8; a[17] = 32u8; + a[18] = 68u8; a[19] = 114u8; a[20] = 101u8; a[21] = 119u8; + let sp: [1]u8; sp[0] = 32u8; + let t1: [][]u8 = bytes.split(a[0:22], sp[0:1]); + if (t1.len != 5) { fail(); }; + expect_tok(t1, 0, a[0:6]); + expect_tok(t1, 1, a[7:9]); + expect_tok(t1, 2, a[10:14]); + expect_tok(t1, 3, a[15:17]); + expect_tok(t1, 4, a[18:22]); + os.free(t1.ptr: *void, (t1.cap: u64) * 24u64); + + // Leading + trailing empty tokens — five entries, three of which + // are []. Splits across all positions exactly the way Hare does. + signalled = 1771; + let b: [5]u8; b[0] = 0u8; b[1] = 1u8; b[2] = 2u8; b[3] = 3u8; b[4] = 0u8; + let zd: [1]u8; zd[0] = 0u8; + let z: [1]u8; + let t2: [][]u8 = bytes.split(b[0:5], zd[0:1]); + if (t2.len != 3) { fail(); }; + expect_tok(t2, 0, z[0:0]); + expect_tok(t2, 1, b[1:4]); + expect_tok(t2, 2, z[0:0]); + os.free(t2.ptr: *void, (t2.cap: u64) * 24u64); +}; + export fn main() i32 = { signalled = 1; equal_cases(); signalled = 2; index_byte_cases(); @@ -480,5 +659,8 @@ export fn main() i32 = { signalled = 10; rtokenize_cases(); signalled = 11; peek_token_cases(); signalled = 12; remaining_tokens_cases(); + signalled = 13; splitn_cases(); + signalled = 14; rsplitn_cases(); + signalled = 15; split_cases(); return 0; }; diff --git a/selfhost/cmd/w6c/main.combined.ww b/selfhost/cmd/w6c/main.combined.ww index 6e0f4c07..670cfb91 100644 --- a/selfhost/cmd/w6c/main.combined.ww +++ b/selfhost/cmd/w6c/main.combined.ww @@ -1233,6 +1233,137 @@ export fn remaining_tokens(s: *tokenizer) []u8 = { return s.in; }; +// rt_ensure is the runtime slice-growth helper invoked by the +// `append(s, v)` builtin. We bind it directly because the builtin's +// expansion stores only 8 bytes of the new element (cgen emits a +// single MOVQ), losing the .len/.cap fields of a []u8 element (24B). +// Mirrors the same workaround in lib/shlex.shlex (appendstr, 16B) and +// lib/getopt.getopt (appendoption, 24B); collapses in one go when the +// append builtin learns to store the full element width. +@symbol("rt_ensure") fn rtensure(s: *void, membsz: u64) void; + +// appendslice — grow `*slice` by one and store `item` (24B). Mirror +// of [[shlex.appendstr]] / [[getopt.appendoption]]. Bypasses the +// `append` builtin's first-8B-only-store gap for a slice-element. +fn appendslice(slice: *[][]u8, item: []u8) void = { + let newlen: i32 = slice.len + 1; + slice.len = newlen; + rtensure(slice: *void, 24u64); + let dst: *[]u8 = &slice.ptr[newlen - 1]; + dst.ptr = item.ptr; + dst.len = item.len; + dst.cap = item.cap; +}; + +// splitn — split `in` on any byte in `delim`, returning up to `n` +// tokens via forward iteration. The trailing slot (when more than +// `n - 1` tokens exist) holds the unconsumed remainder. +// +// The caller frees the returned slice via +// `os.free(r.ptr: *void, (r.cap: u64) * 24u64)`. Element bytes are +// borrowed from `in`. +// +// Hare's `([][]u8 | nomem)` collapses to `[][]u8` here: ww os.alloc +// has no recoverable failure path. Same precedent as +// shlex.split / getopt.tryparse. +// +// ref/hare/bytes/tokenize.ha:156. +export fn splitn(in: []u8, delim: []u8, n: i32) [][]u8 = { + os.assert(delim.len > 0, + "bytes.splitn must not be called with an empty delimiter"); + let toks: [][]u8; + toks.ptr = nil: *[]u8; + toks.len = 0; + toks.cap = 0; + let tok: tokenizer = tokenize(in, delim...); + let i: i32 = 0; + for (i < n - 1) { + match (next_token(&tok)) { + case let s: []u8 => { appendslice(&toks, s); }; + case done => { return toks; }; + }; + i += 1; + }; + match (peek_token(&tok)) { + case done => void; + case let pk: []u8 => { + let r: []u8 = remaining_tokens(&tok); + appendslice(&toks, r); + }; + }; + return toks; +}; + +// rsplitn — reverse-direction counterpart to [[splitn]]: tokens are +// collected from the end of `in`. The trailing slot holds the +// unconsumed prefix (everything before the n-th-from-last delim hit). +// +// When the input has fewer than n tokens, the `done` short-circuit +// returns toks UN-reversed (in last-token-first order). Mirrors Hare +// at ref/hare/bytes/tokenize.ha:196-199 where the in-place reverse +// step is gated behind the n-1 loop running to completion. Only the +// "loop ran to completion AND peek saw a remainder" path applies the +// reverse; both early-exit paths skip it. +// +// ref/hare/bytes/tokenize.ha:186. +export fn rsplitn(in: []u8, delim: []u8, n: i32) [][]u8 = { + os.assert(delim.len > 0, + "bytes.rsplitn called with empty delimiter"); + let toks: [][]u8; + toks.ptr = nil: *[]u8; + toks.len = 0; + toks.cap = 0; + let tok: tokenizer = rtokenize(in, delim...); + let i: i32 = 0; + for (i < n - 1) { + match (next_token(&tok)) { + case let s: []u8 => { appendslice(&toks, s); }; + case done => { return toks; }; + }; + i += 1; + }; + match (peek_token(&tok)) { + case done => void; + case let pk: []u8 => { + let r: []u8 = remaining_tokens(&tok); + appendslice(&toks, r); + }; + }; + + // In-place reverse so callers see argv-order, matching Hare + // (ref/hare/bytes/tokenize.ha:207). Element copy is field-wise + // through `*[]u8` because `toks[i] = toks[j]` (full 24B slice + // store) lands in the multi-word-store gap noted at + // cmd/w6c/cgen.c:6515-6523. + let a: i32 = 0; + let b: i32 = toks.len - 1; + for (a < b) { + let pa: *[]u8 = &toks.ptr[a]; + let pb: *[]u8 = &toks.ptr[b]; + let tp: *u8 = pa.ptr; + let tl: i32 = pa.len; + let tc: i32 = pa.cap; + pa.ptr = pb.ptr; + pa.len = pb.len; + pa.cap = pb.cap; + pb.ptr = tp; + pb.len = tl; + pb.cap = tc; + a += 1; + b -= 1; + }; + return toks; +}; + +// split — full split of `in` on `delim` (no token cap). Mirrors +// `splitn(in, delim, types::SIZE_MAX)`. ww uses `types.I32_MAX` +// because the index type is i32 (lib/CLAUDE.md). +// +// ref/hare/bytes/tokenize.ha:225. +export fn split(in: []u8, delim: []u8) [][]u8 = { + return splitn(in, delim, types.I32_MAX); +}; + // encoding/utf8 — UTF-8 encode/decode. Hare port; see // ref/hare/encoding/utf8/{types,rune,encode,decode,decodetable}.ha. // diff --git a/selfhost/cmd/wwdump/main.combined.ww b/selfhost/cmd/wwdump/main.combined.ww index d11ddf43..d4acb44a 100644 --- a/selfhost/cmd/wwdump/main.combined.ww +++ b/selfhost/cmd/wwdump/main.combined.ww @@ -1233,6 +1233,137 @@ export fn remaining_tokens(s: *tokenizer) []u8 = { return s.in; }; +// rt_ensure is the runtime slice-growth helper invoked by the +// `append(s, v)` builtin. We bind it directly because the builtin's +// expansion stores only 8 bytes of the new element (cgen emits a +// single MOVQ), losing the .len/.cap fields of a []u8 element (24B). +// Mirrors the same workaround in lib/shlex.shlex (appendstr, 16B) and +// lib/getopt.getopt (appendoption, 24B); collapses in one go when the +// append builtin learns to store the full element width. +@symbol("rt_ensure") fn rtensure(s: *void, membsz: u64) void; + +// appendslice — grow `*slice` by one and store `item` (24B). Mirror +// of [[shlex.appendstr]] / [[getopt.appendoption]]. Bypasses the +// `append` builtin's first-8B-only-store gap for a slice-element. +fn appendslice(slice: *[][]u8, item: []u8) void = { + let newlen: i32 = slice.len + 1; + slice.len = newlen; + rtensure(slice: *void, 24u64); + let dst: *[]u8 = &slice.ptr[newlen - 1]; + dst.ptr = item.ptr; + dst.len = item.len; + dst.cap = item.cap; +}; + +// splitn — split `in` on any byte in `delim`, returning up to `n` +// tokens via forward iteration. The trailing slot (when more than +// `n - 1` tokens exist) holds the unconsumed remainder. +// +// The caller frees the returned slice via +// `os.free(r.ptr: *void, (r.cap: u64) * 24u64)`. Element bytes are +// borrowed from `in`. +// +// Hare's `([][]u8 | nomem)` collapses to `[][]u8` here: ww os.alloc +// has no recoverable failure path. Same precedent as +// shlex.split / getopt.tryparse. +// +// ref/hare/bytes/tokenize.ha:156. +export fn splitn(in: []u8, delim: []u8, n: i32) [][]u8 = { + os.assert(delim.len > 0, + "bytes.splitn must not be called with an empty delimiter"); + let toks: [][]u8; + toks.ptr = nil: *[]u8; + toks.len = 0; + toks.cap = 0; + let tok: tokenizer = tokenize(in, delim...); + let i: i32 = 0; + for (i < n - 1) { + match (next_token(&tok)) { + case let s: []u8 => { appendslice(&toks, s); }; + case done => { return toks; }; + }; + i += 1; + }; + match (peek_token(&tok)) { + case done => void; + case let pk: []u8 => { + let r: []u8 = remaining_tokens(&tok); + appendslice(&toks, r); + }; + }; + return toks; +}; + +// rsplitn — reverse-direction counterpart to [[splitn]]: tokens are +// collected from the end of `in`. The trailing slot holds the +// unconsumed prefix (everything before the n-th-from-last delim hit). +// +// When the input has fewer than n tokens, the `done` short-circuit +// returns toks UN-reversed (in last-token-first order). Mirrors Hare +// at ref/hare/bytes/tokenize.ha:196-199 where the in-place reverse +// step is gated behind the n-1 loop running to completion. Only the +// "loop ran to completion AND peek saw a remainder" path applies the +// reverse; both early-exit paths skip it. +// +// ref/hare/bytes/tokenize.ha:186. +export fn rsplitn(in: []u8, delim: []u8, n: i32) [][]u8 = { + os.assert(delim.len > 0, + "bytes.rsplitn called with empty delimiter"); + let toks: [][]u8; + toks.ptr = nil: *[]u8; + toks.len = 0; + toks.cap = 0; + let tok: tokenizer = rtokenize(in, delim...); + let i: i32 = 0; + for (i < n - 1) { + match (next_token(&tok)) { + case let s: []u8 => { appendslice(&toks, s); }; + case done => { return toks; }; + }; + i += 1; + }; + match (peek_token(&tok)) { + case done => void; + case let pk: []u8 => { + let r: []u8 = remaining_tokens(&tok); + appendslice(&toks, r); + }; + }; + + // In-place reverse so callers see argv-order, matching Hare + // (ref/hare/bytes/tokenize.ha:207). Element copy is field-wise + // through `*[]u8` because `toks[i] = toks[j]` (full 24B slice + // store) lands in the multi-word-store gap noted at + // cmd/w6c/cgen.c:6515-6523. + let a: i32 = 0; + let b: i32 = toks.len - 1; + for (a < b) { + let pa: *[]u8 = &toks.ptr[a]; + let pb: *[]u8 = &toks.ptr[b]; + let tp: *u8 = pa.ptr; + let tl: i32 = pa.len; + let tc: i32 = pa.cap; + pa.ptr = pb.ptr; + pa.len = pb.len; + pa.cap = pb.cap; + pb.ptr = tp; + pb.len = tl; + pb.cap = tc; + a += 1; + b -= 1; + }; + return toks; +}; + +// split — full split of `in` on `delim` (no token cap). Mirrors +// `splitn(in, delim, types::SIZE_MAX)`. ww uses `types.I32_MAX` +// because the index type is i32 (lib/CLAUDE.md). +// +// ref/hare/bytes/tokenize.ha:225. +export fn split(in: []u8, delim: []u8) [][]u8 = { + return splitn(in, delim, types.I32_MAX); +}; + // encoding/utf8 — UTF-8 encode/decode. Hare port; see // ref/hare/encoding/utf8/{types,rune,encode,decode,decodetable}.ha. // diff --git a/selfhost/test/smoke.combined.ww b/selfhost/test/smoke.combined.ww index d27bcc8e..f2719ca8 100644 --- a/selfhost/test/smoke.combined.ww +++ b/selfhost/test/smoke.combined.ww @@ -1124,6 +1124,137 @@ export fn remaining_tokens(s: *tokenizer) []u8 = { return s.in; }; +// rt_ensure is the runtime slice-growth helper invoked by the +// `append(s, v)` builtin. We bind it directly because the builtin's +// expansion stores only 8 bytes of the new element (cgen emits a +// single MOVQ), losing the .len/.cap fields of a []u8 element (24B). +// Mirrors the same workaround in lib/shlex.shlex (appendstr, 16B) and +// lib/getopt.getopt (appendoption, 24B); collapses in one go when the +// append builtin learns to store the full element width. +@symbol("rt_ensure") fn rtensure(s: *void, membsz: u64) void; + +// appendslice — grow `*slice` by one and store `item` (24B). Mirror +// of [[shlex.appendstr]] / [[getopt.appendoption]]. Bypasses the +// `append` builtin's first-8B-only-store gap for a slice-element. +fn appendslice(slice: *[][]u8, item: []u8) void = { + let newlen: i32 = slice.len + 1; + slice.len = newlen; + rtensure(slice: *void, 24u64); + let dst: *[]u8 = &slice.ptr[newlen - 1]; + dst.ptr = item.ptr; + dst.len = item.len; + dst.cap = item.cap; +}; + +// splitn — split `in` on any byte in `delim`, returning up to `n` +// tokens via forward iteration. The trailing slot (when more than +// `n - 1` tokens exist) holds the unconsumed remainder. +// +// The caller frees the returned slice via +// `os.free(r.ptr: *void, (r.cap: u64) * 24u64)`. Element bytes are +// borrowed from `in`. +// +// Hare's `([][]u8 | nomem)` collapses to `[][]u8` here: ww os.alloc +// has no recoverable failure path. Same precedent as +// shlex.split / getopt.tryparse. +// +// ref/hare/bytes/tokenize.ha:156. +export fn splitn(in: []u8, delim: []u8, n: i32) [][]u8 = { + os.assert(delim.len > 0, + "bytes.splitn must not be called with an empty delimiter"); + let toks: [][]u8; + toks.ptr = nil: *[]u8; + toks.len = 0; + toks.cap = 0; + let tok: tokenizer = tokenize(in, delim...); + let i: i32 = 0; + for (i < n - 1) { + match (next_token(&tok)) { + case let s: []u8 => { appendslice(&toks, s); }; + case done => { return toks; }; + }; + i += 1; + }; + match (peek_token(&tok)) { + case done => void; + case let pk: []u8 => { + let r: []u8 = remaining_tokens(&tok); + appendslice(&toks, r); + }; + }; + return toks; +}; + +// rsplitn — reverse-direction counterpart to [[splitn]]: tokens are +// collected from the end of `in`. The trailing slot holds the +// unconsumed prefix (everything before the n-th-from-last delim hit). +// +// When the input has fewer than n tokens, the `done` short-circuit +// returns toks UN-reversed (in last-token-first order). Mirrors Hare +// at ref/hare/bytes/tokenize.ha:196-199 where the in-place reverse +// step is gated behind the n-1 loop running to completion. Only the +// "loop ran to completion AND peek saw a remainder" path applies the +// reverse; both early-exit paths skip it. +// +// ref/hare/bytes/tokenize.ha:186. +export fn rsplitn(in: []u8, delim: []u8, n: i32) [][]u8 = { + os.assert(delim.len > 0, + "bytes.rsplitn called with empty delimiter"); + let toks: [][]u8; + toks.ptr = nil: *[]u8; + toks.len = 0; + toks.cap = 0; + let tok: tokenizer = rtokenize(in, delim...); + let i: i32 = 0; + for (i < n - 1) { + match (next_token(&tok)) { + case let s: []u8 => { appendslice(&toks, s); }; + case done => { return toks; }; + }; + i += 1; + }; + match (peek_token(&tok)) { + case done => void; + case let pk: []u8 => { + let r: []u8 = remaining_tokens(&tok); + appendslice(&toks, r); + }; + }; + + // In-place reverse so callers see argv-order, matching Hare + // (ref/hare/bytes/tokenize.ha:207). Element copy is field-wise + // through `*[]u8` because `toks[i] = toks[j]` (full 24B slice + // store) lands in the multi-word-store gap noted at + // cmd/w6c/cgen.c:6515-6523. + let a: i32 = 0; + let b: i32 = toks.len - 1; + for (a < b) { + let pa: *[]u8 = &toks.ptr[a]; + let pb: *[]u8 = &toks.ptr[b]; + let tp: *u8 = pa.ptr; + let tl: i32 = pa.len; + let tc: i32 = pa.cap; + pa.ptr = pb.ptr; + pa.len = pb.len; + pa.cap = pb.cap; + pb.ptr = tp; + pb.len = tl; + pb.cap = tc; + a += 1; + b -= 1; + }; + return toks; +}; + +// split — full split of `in` on `delim` (no token cap). Mirrors +// `splitn(in, delim, types::SIZE_MAX)`. ww uses `types.I32_MAX` +// because the index type is i32 (lib/CLAUDE.md). +// +// ref/hare/bytes/tokenize.ha:225. +export fn split(in: []u8, delim: []u8) [][]u8 = { + return splitn(in, delim, types.I32_MAX); +}; + // encoding/utf8 — UTF-8 encode/decode. Hare port; see // ref/hare/encoding/utf8/{types,rune,encode,decode,decodetable}.ha. //