lib/bytes+test: port split family from Hare
This commit is contained in:
@@ -351,3 +351,134 @@ export fn next_token(s: *tokenizer) ([]u8 | done) = {
|
|||||||
export fn remaining_tokens(s: *tokenizer) []u8 = {
|
export fn remaining_tokens(s: *tokenizer) []u8 = {
|
||||||
return s.in;
|
return s.in;
|
||||||
};
|
};
|
||||||
|
|
||||||
|
// rt_ensure is the runtime slice-growth helper invoked by the
|
||||||
|
// `append(s, v)` builtin. We bind it directly because the builtin's
|
||||||
|
// expansion stores only 8 bytes of the new element (cgen emits a
|
||||||
|
// single MOVQ), losing the .len/.cap fields of a []u8 element (24B).
|
||||||
|
// Mirrors the same workaround in lib/shlex.shlex (appendstr, 16B) and
|
||||||
|
// lib/getopt.getopt (appendoption, 24B); collapses in one go when the
|
||||||
|
// append builtin learns to store the full element width.
|
||||||
|
@symbol("rt_ensure") fn rtensure(s: *void, membsz: u64) void;
|
||||||
|
|
||||||
|
// appendslice — grow `*slice` by one and store `item` (24B). Mirror
|
||||||
|
// of [[shlex.appendstr]] / [[getopt.appendoption]]. Bypasses the
|
||||||
|
// `append` builtin's first-8B-only-store gap for a slice-element.
|
||||||
|
fn appendslice(slice: *[][]u8, item: []u8) void = {
|
||||||
|
let newlen: i32 = slice.len + 1;
|
||||||
|
slice.len = newlen;
|
||||||
|
rtensure(slice: *void, 24u64);
|
||||||
|
let dst: *[]u8 = &slice.ptr[newlen - 1];
|
||||||
|
dst.ptr = item.ptr;
|
||||||
|
dst.len = item.len;
|
||||||
|
dst.cap = item.cap;
|
||||||
|
};
|
||||||
|
|
||||||
|
// splitn — split `in` on any byte in `delim`, returning up to `n`
|
||||||
|
// tokens via forward iteration. The trailing slot (when more than
|
||||||
|
// `n - 1` tokens exist) holds the unconsumed remainder.
|
||||||
|
//
|
||||||
|
// The caller frees the returned slice via
|
||||||
|
// `os.free(r.ptr: *void, (r.cap: u64) * 24u64)`. Element bytes are
|
||||||
|
// borrowed from `in`.
|
||||||
|
//
|
||||||
|
// Hare's `([][]u8 | nomem)` collapses to `[][]u8` here: ww os.alloc
|
||||||
|
// has no recoverable failure path. Same precedent as
|
||||||
|
// shlex.split / getopt.tryparse.
|
||||||
|
//
|
||||||
|
// ref/hare/bytes/tokenize.ha:156.
|
||||||
|
export fn splitn(in: []u8, delim: []u8, n: i32) [][]u8 = {
|
||||||
|
os.assert(delim.len > 0,
|
||||||
|
"bytes.splitn must not be called with an empty delimiter");
|
||||||
|
let toks: [][]u8;
|
||||||
|
toks.ptr = nil: *[]u8;
|
||||||
|
toks.len = 0;
|
||||||
|
toks.cap = 0;
|
||||||
|
let tok: tokenizer = tokenize(in, delim...);
|
||||||
|
let i: i32 = 0;
|
||||||
|
for (i < n - 1) {
|
||||||
|
match (next_token(&tok)) {
|
||||||
|
case let s: []u8 => { appendslice(&toks, s); };
|
||||||
|
case done => { return toks; };
|
||||||
|
};
|
||||||
|
i += 1;
|
||||||
|
};
|
||||||
|
match (peek_token(&tok)) {
|
||||||
|
case done => void;
|
||||||
|
case let pk: []u8 => {
|
||||||
|
let r: []u8 = remaining_tokens(&tok);
|
||||||
|
appendslice(&toks, r);
|
||||||
|
};
|
||||||
|
};
|
||||||
|
return toks;
|
||||||
|
};
|
||||||
|
|
||||||
|
// rsplitn — reverse-direction counterpart to [[splitn]]: tokens are
|
||||||
|
// collected from the end of `in`. The trailing slot holds the
|
||||||
|
// unconsumed prefix (everything before the n-th-from-last delim hit).
|
||||||
|
//
|
||||||
|
// When the input has fewer than n tokens, the `done` short-circuit
|
||||||
|
// returns toks UN-reversed (in last-token-first order). Mirrors Hare
|
||||||
|
// at ref/hare/bytes/tokenize.ha:196-199 where the in-place reverse
|
||||||
|
// step is gated behind the n-1 loop running to completion. Only the
|
||||||
|
// "loop ran to completion AND peek saw a remainder" path applies the
|
||||||
|
// reverse; both early-exit paths skip it.
|
||||||
|
//
|
||||||
|
// ref/hare/bytes/tokenize.ha:186.
|
||||||
|
export fn rsplitn(in: []u8, delim: []u8, n: i32) [][]u8 = {
|
||||||
|
os.assert(delim.len > 0,
|
||||||
|
"bytes.rsplitn called with empty delimiter");
|
||||||
|
let toks: [][]u8;
|
||||||
|
toks.ptr = nil: *[]u8;
|
||||||
|
toks.len = 0;
|
||||||
|
toks.cap = 0;
|
||||||
|
let tok: tokenizer = rtokenize(in, delim...);
|
||||||
|
let i: i32 = 0;
|
||||||
|
for (i < n - 1) {
|
||||||
|
match (next_token(&tok)) {
|
||||||
|
case let s: []u8 => { appendslice(&toks, s); };
|
||||||
|
case done => { return toks; };
|
||||||
|
};
|
||||||
|
i += 1;
|
||||||
|
};
|
||||||
|
match (peek_token(&tok)) {
|
||||||
|
case done => void;
|
||||||
|
case let pk: []u8 => {
|
||||||
|
let r: []u8 = remaining_tokens(&tok);
|
||||||
|
appendslice(&toks, r);
|
||||||
|
};
|
||||||
|
};
|
||||||
|
|
||||||
|
// In-place reverse so callers see argv-order, matching Hare
|
||||||
|
// (ref/hare/bytes/tokenize.ha:207). Element copy is field-wise
|
||||||
|
// through `*[]u8` because `toks[i] = toks[j]` (full 24B slice
|
||||||
|
// store) lands in the multi-word-store gap noted at
|
||||||
|
// cmd/w6c/cgen.c:6515-6523.
|
||||||
|
let a: i32 = 0;
|
||||||
|
let b: i32 = toks.len - 1;
|
||||||
|
for (a < b) {
|
||||||
|
let pa: *[]u8 = &toks.ptr[a];
|
||||||
|
let pb: *[]u8 = &toks.ptr[b];
|
||||||
|
let tp: *u8 = pa.ptr;
|
||||||
|
let tl: i32 = pa.len;
|
||||||
|
let tc: i32 = pa.cap;
|
||||||
|
pa.ptr = pb.ptr;
|
||||||
|
pa.len = pb.len;
|
||||||
|
pa.cap = pb.cap;
|
||||||
|
pb.ptr = tp;
|
||||||
|
pb.len = tl;
|
||||||
|
pb.cap = tc;
|
||||||
|
a += 1;
|
||||||
|
b -= 1;
|
||||||
|
};
|
||||||
|
return toks;
|
||||||
|
};
|
||||||
|
|
||||||
|
// split — full split of `in` on `delim` (no token cap). Mirrors
|
||||||
|
// `splitn(in, delim, types::SIZE_MAX)`. ww uses `types.I32_MAX`
|
||||||
|
// because the index type is i32 (lib/CLAUDE.md).
|
||||||
|
//
|
||||||
|
// ref/hare/bytes/tokenize.ha:225.
|
||||||
|
export fn split(in: []u8, delim: []u8) [][]u8 = {
|
||||||
|
return splitn(in, delim, types.I32_MAX);
|
||||||
|
};
|
||||||
|
|||||||
@@ -467,6 +467,185 @@ fn expect_done(t: *bytes.tokenizer) void = {
|
|||||||
if (!bytes.equal(r2, e_12[0:2])) { fail(); };
|
if (!bytes.equal(r2, e_12[0:2])) { fail(); };
|
||||||
};
|
};
|
||||||
|
|
||||||
|
// ---- splitn / rsplitn / split -----------------------------------------
|
||||||
|
// ref/hare/bytes/tokenize.ha:330 (@test fn split). Hare's table mixes
|
||||||
|
// strings; ww spells the vectors as byte arrays explicitly.
|
||||||
|
//
|
||||||
|
// Element reads go through `&toks.ptr[i]: *[]u8` rather than `toks[i]`
|
||||||
|
// — the full 24B slice-element copy lands in the multi-word-store gap
|
||||||
|
// noted at cmd/w6c/cgen.c:6515-6523, so a value-load drops .len/.cap.
|
||||||
|
// Pointer-then-fields lifts 8B at a time, which the cgen routes correctly.
|
||||||
|
|
||||||
|
fn expect_tok(toks: [][]u8, i: i32, want: []u8) void = {
|
||||||
|
if (i >= toks.len) { fail(); };
|
||||||
|
let p: *[]u8 = &toks.ptr[i];
|
||||||
|
if (p.len != want.len) { fail(); };
|
||||||
|
let j: i32 = 0;
|
||||||
|
for (j < want.len) {
|
||||||
|
if (p.ptr[j] != want[j]) { fail(); };
|
||||||
|
j += 1;
|
||||||
|
};
|
||||||
|
};
|
||||||
|
|
||||||
|
@test fn splitn_cases() void = {
|
||||||
|
// "Hello, my name is Drew" ─ space-delimited, n=4 yields the 4-th
|
||||||
|
// token as the unconsumed remainder. Hare pins this exact shape
|
||||||
|
// at ref/hare/bytes/tokenize.ha:340.
|
||||||
|
signalled = 1750;
|
||||||
|
let a: [22]u8;
|
||||||
|
a[0] = 72u8; a[1] = 101u8; a[2] = 108u8; a[3] = 108u8; a[4] = 111u8;
|
||||||
|
a[5] = 44u8; a[6] = 32u8; a[7] = 109u8; a[8] = 121u8; a[9] = 32u8;
|
||||||
|
a[10] = 110u8; a[11] = 97u8; a[12] = 109u8; a[13] = 101u8; a[14] = 32u8;
|
||||||
|
a[15] = 105u8; a[16] = 115u8; a[17] = 32u8;
|
||||||
|
a[18] = 68u8; a[19] = 114u8; a[20] = 101u8; a[21] = 119u8;
|
||||||
|
let sp: [1]u8; sp[0] = 32u8;
|
||||||
|
let t1: [][]u8 = bytes.splitn(a[0:22], sp[0:1], 4);
|
||||||
|
if (t1.len != 4) { fail(); };
|
||||||
|
expect_tok(t1, 0, a[0:6]);
|
||||||
|
expect_tok(t1, 1, a[7:9]);
|
||||||
|
expect_tok(t1, 2, a[10:14]);
|
||||||
|
expect_tok(t1, 3, a[15:22]);
|
||||||
|
os.free(t1.ptr: *void, (t1.cap: u64) * 24u64);
|
||||||
|
|
||||||
|
// n > token count — final slot is "" if input ends in delim,
|
||||||
|
// otherwise the last token. Here three tokens, n=10 → 3 entries.
|
||||||
|
signalled = 1751;
|
||||||
|
let b: [5]u8; b[0] = 1u8; b[1] = 0u8; b[2] = 2u8; b[3] = 0u8; b[4] = 3u8;
|
||||||
|
let zd: [1]u8; zd[0] = 0u8;
|
||||||
|
let t2: [][]u8 = bytes.splitn(b[0:5], zd[0:1], 10);
|
||||||
|
if (t2.len != 3) { fail(); };
|
||||||
|
expect_tok(t2, 0, b[0:1]);
|
||||||
|
expect_tok(t2, 1, b[2:3]);
|
||||||
|
expect_tok(t2, 2, b[4:5]);
|
||||||
|
os.free(t2.ptr: *void, (t2.cap: u64) * 24u64);
|
||||||
|
|
||||||
|
// n == 1 — single slot holding the whole input as remainder.
|
||||||
|
signalled = 1752;
|
||||||
|
let t3: [][]u8 = bytes.splitn(b[0:5], zd[0:1], 1);
|
||||||
|
if (t3.len != 1) { fail(); };
|
||||||
|
expect_tok(t3, 0, b[0:5]);
|
||||||
|
os.free(t3.ptr: *void, (t3.cap: u64) * 24u64);
|
||||||
|
|
||||||
|
// delim absent — single slot holding input unchanged.
|
||||||
|
signalled = 1753;
|
||||||
|
let c: [3]u8; c[0] = 1u8; c[1] = 2u8; c[2] = 3u8;
|
||||||
|
let t4: [][]u8 = bytes.splitn(c[0:3], zd[0:1], 5);
|
||||||
|
if (t4.len != 1) { fail(); };
|
||||||
|
expect_tok(t4, 0, c[0:3]);
|
||||||
|
os.free(t4.ptr: *void, (t4.cap: u64) * 24u64);
|
||||||
|
|
||||||
|
// empty input — empty result.
|
||||||
|
signalled = 1754;
|
||||||
|
let z: [1]u8;
|
||||||
|
let t5: [][]u8 = bytes.splitn(z[0:0], zd[0:1], 5);
|
||||||
|
if (t5.len != 0) { fail(); };
|
||||||
|
if (t5.cap > 0) {
|
||||||
|
os.free(t5.ptr: *void, (t5.cap: u64) * 24u64);
|
||||||
|
};
|
||||||
|
|
||||||
|
// Multi-byte delimiter set — both 0 and 42 split.
|
||||||
|
signalled = 1755;
|
||||||
|
let d: [8]u8;
|
||||||
|
d[0] = 1u8; d[1] = 2u8; d[2] = 0u8; d[3] = 3u8;
|
||||||
|
d[4] = 4u8; d[5] = 42u8; d[6] = 5u8; d[7] = 6u8;
|
||||||
|
let dd: [2]u8; dd[0] = 0u8; dd[1] = 42u8;
|
||||||
|
let t6: [][]u8 = bytes.splitn(d[0:8], dd[0:2], 100);
|
||||||
|
if (t6.len != 3) { fail(); };
|
||||||
|
expect_tok(t6, 0, d[0:2]);
|
||||||
|
expect_tok(t6, 1, d[3:5]);
|
||||||
|
expect_tok(t6, 2, d[6:8]);
|
||||||
|
os.free(t6.ptr: *void, (t6.cap: u64) * 24u64);
|
||||||
|
};
|
||||||
|
|
||||||
|
@test fn rsplitn_cases() void = {
|
||||||
|
// "Hello, my name is Drew" ─ rsplitn n=4 buckets the first three
|
||||||
|
// tokens from the *end*; the remainder ("Hello, my") is index 0.
|
||||||
|
// Hare pins this at ref/hare/bytes/tokenize.ha:379.
|
||||||
|
signalled = 1760;
|
||||||
|
let a: [22]u8;
|
||||||
|
a[0] = 72u8; a[1] = 101u8; a[2] = 108u8; a[3] = 108u8; a[4] = 111u8;
|
||||||
|
a[5] = 44u8; a[6] = 32u8; a[7] = 109u8; a[8] = 121u8; a[9] = 32u8;
|
||||||
|
a[10] = 110u8; a[11] = 97u8; a[12] = 109u8; a[13] = 101u8; a[14] = 32u8;
|
||||||
|
a[15] = 105u8; a[16] = 115u8; a[17] = 32u8;
|
||||||
|
a[18] = 68u8; a[19] = 114u8; a[20] = 101u8; a[21] = 119u8;
|
||||||
|
let sp: [1]u8; sp[0] = 32u8;
|
||||||
|
let t1: [][]u8 = bytes.rsplitn(a[0:22], sp[0:1], 4);
|
||||||
|
if (t1.len != 4) { fail(); };
|
||||||
|
expect_tok(t1, 0, a[0:9]);
|
||||||
|
expect_tok(t1, 1, a[10:14]);
|
||||||
|
expect_tok(t1, 2, a[15:17]);
|
||||||
|
expect_tok(t1, 3, a[18:22]);
|
||||||
|
os.free(t1.ptr: *void, (t1.cap: u64) * 24u64);
|
||||||
|
|
||||||
|
// n > token count — done short-circuit returns the toks in
|
||||||
|
// reverse-iteration order (last token first). Mirrors Hare's
|
||||||
|
// behavior at ref/hare/bytes/tokenize.ha:196-199 where the
|
||||||
|
// reverse-step is gated behind the n-1 loop completion.
|
||||||
|
signalled = 1761;
|
||||||
|
let b: [5]u8; b[0] = 1u8; b[1] = 0u8; b[2] = 2u8; b[3] = 0u8; b[4] = 3u8;
|
||||||
|
let zd: [1]u8; zd[0] = 0u8;
|
||||||
|
let t2: [][]u8 = bytes.rsplitn(b[0:5], zd[0:1], 10);
|
||||||
|
if (t2.len != 3) { fail(); };
|
||||||
|
expect_tok(t2, 0, b[4:5]);
|
||||||
|
expect_tok(t2, 1, b[2:3]);
|
||||||
|
expect_tok(t2, 2, b[0:1]);
|
||||||
|
os.free(t2.ptr: *void, (t2.cap: u64) * 24u64);
|
||||||
|
|
||||||
|
// n == 1 — single slot holding the whole input as remainder.
|
||||||
|
signalled = 1762;
|
||||||
|
let t3: [][]u8 = bytes.rsplitn(b[0:5], zd[0:1], 1);
|
||||||
|
if (t3.len != 1) { fail(); };
|
||||||
|
expect_tok(t3, 0, b[0:5]);
|
||||||
|
os.free(t3.ptr: *void, (t3.cap: u64) * 24u64);
|
||||||
|
|
||||||
|
// delim absent — done short-circuits immediately at iter 0;
|
||||||
|
// toks ends up holding only the eventual remainder if n=1
|
||||||
|
// (the loop never runs and peek picks up the full input).
|
||||||
|
// With n>1 the loop's first next_token sees done (no delim
|
||||||
|
// match anywhere) and returns toks={} per Hare's early-exit.
|
||||||
|
signalled = 1763;
|
||||||
|
let c: [3]u8; c[0] = 1u8; c[1] = 2u8; c[2] = 3u8;
|
||||||
|
let t4: [][]u8 = bytes.rsplitn(c[0:3], zd[0:1], 5);
|
||||||
|
if (t4.len != 1) { fail(); };
|
||||||
|
expect_tok(t4, 0, c[0:3]);
|
||||||
|
os.free(t4.ptr: *void, (t4.cap: u64) * 24u64);
|
||||||
|
};
|
||||||
|
|
||||||
|
@test fn split_cases() void = {
|
||||||
|
// Full split — every delim hit is a boundary; no per-call cap.
|
||||||
|
// Hare pins five tokens for the canonical input at
|
||||||
|
// ref/hare/bytes/tokenize.ha:347.
|
||||||
|
signalled = 1770;
|
||||||
|
let a: [22]u8;
|
||||||
|
a[0] = 72u8; a[1] = 101u8; a[2] = 108u8; a[3] = 108u8; a[4] = 111u8;
|
||||||
|
a[5] = 44u8; a[6] = 32u8; a[7] = 109u8; a[8] = 121u8; a[9] = 32u8;
|
||||||
|
a[10] = 110u8; a[11] = 97u8; a[12] = 109u8; a[13] = 101u8; a[14] = 32u8;
|
||||||
|
a[15] = 105u8; a[16] = 115u8; a[17] = 32u8;
|
||||||
|
a[18] = 68u8; a[19] = 114u8; a[20] = 101u8; a[21] = 119u8;
|
||||||
|
let sp: [1]u8; sp[0] = 32u8;
|
||||||
|
let t1: [][]u8 = bytes.split(a[0:22], sp[0:1]);
|
||||||
|
if (t1.len != 5) { fail(); };
|
||||||
|
expect_tok(t1, 0, a[0:6]);
|
||||||
|
expect_tok(t1, 1, a[7:9]);
|
||||||
|
expect_tok(t1, 2, a[10:14]);
|
||||||
|
expect_tok(t1, 3, a[15:17]);
|
||||||
|
expect_tok(t1, 4, a[18:22]);
|
||||||
|
os.free(t1.ptr: *void, (t1.cap: u64) * 24u64);
|
||||||
|
|
||||||
|
// Leading + trailing empty tokens — five entries, three of which
|
||||||
|
// are []. Splits across all positions exactly the way Hare does.
|
||||||
|
signalled = 1771;
|
||||||
|
let b: [5]u8; b[0] = 0u8; b[1] = 1u8; b[2] = 2u8; b[3] = 3u8; b[4] = 0u8;
|
||||||
|
let zd: [1]u8; zd[0] = 0u8;
|
||||||
|
let z: [1]u8;
|
||||||
|
let t2: [][]u8 = bytes.split(b[0:5], zd[0:1]);
|
||||||
|
if (t2.len != 3) { fail(); };
|
||||||
|
expect_tok(t2, 0, z[0:0]);
|
||||||
|
expect_tok(t2, 1, b[1:4]);
|
||||||
|
expect_tok(t2, 2, z[0:0]);
|
||||||
|
os.free(t2.ptr: *void, (t2.cap: u64) * 24u64);
|
||||||
|
};
|
||||||
|
|
||||||
export fn main() i32 = {
|
export fn main() i32 = {
|
||||||
signalled = 1; equal_cases();
|
signalled = 1; equal_cases();
|
||||||
signalled = 2; index_byte_cases();
|
signalled = 2; index_byte_cases();
|
||||||
@@ -480,5 +659,8 @@ export fn main() i32 = {
|
|||||||
signalled = 10; rtokenize_cases();
|
signalled = 10; rtokenize_cases();
|
||||||
signalled = 11; peek_token_cases();
|
signalled = 11; peek_token_cases();
|
||||||
signalled = 12; remaining_tokens_cases();
|
signalled = 12; remaining_tokens_cases();
|
||||||
|
signalled = 13; splitn_cases();
|
||||||
|
signalled = 14; rsplitn_cases();
|
||||||
|
signalled = 15; split_cases();
|
||||||
return 0;
|
return 0;
|
||||||
};
|
};
|
||||||
|
|||||||
@@ -1233,6 +1233,137 @@ export fn remaining_tokens(s: *tokenizer) []u8 = {
|
|||||||
return s.in;
|
return s.in;
|
||||||
};
|
};
|
||||||
|
|
||||||
|
// rt_ensure is the runtime slice-growth helper invoked by the
|
||||||
|
// `append(s, v)` builtin. We bind it directly because the builtin's
|
||||||
|
// expansion stores only 8 bytes of the new element (cgen emits a
|
||||||
|
// single MOVQ), losing the .len/.cap fields of a []u8 element (24B).
|
||||||
|
// Mirrors the same workaround in lib/shlex.shlex (appendstr, 16B) and
|
||||||
|
// lib/getopt.getopt (appendoption, 24B); collapses in one go when the
|
||||||
|
// append builtin learns to store the full element width.
|
||||||
|
@symbol("rt_ensure") fn rtensure(s: *void, membsz: u64) void;
|
||||||
|
|
||||||
|
// appendslice — grow `*slice` by one and store `item` (24B). Mirror
|
||||||
|
// of [[shlex.appendstr]] / [[getopt.appendoption]]. Bypasses the
|
||||||
|
// `append` builtin's first-8B-only-store gap for a slice-element.
|
||||||
|
fn appendslice(slice: *[][]u8, item: []u8) void = {
|
||||||
|
let newlen: i32 = slice.len + 1;
|
||||||
|
slice.len = newlen;
|
||||||
|
rtensure(slice: *void, 24u64);
|
||||||
|
let dst: *[]u8 = &slice.ptr[newlen - 1];
|
||||||
|
dst.ptr = item.ptr;
|
||||||
|
dst.len = item.len;
|
||||||
|
dst.cap = item.cap;
|
||||||
|
};
|
||||||
|
|
||||||
|
// splitn — split `in` on any byte in `delim`, returning up to `n`
|
||||||
|
// tokens via forward iteration. The trailing slot (when more than
|
||||||
|
// `n - 1` tokens exist) holds the unconsumed remainder.
|
||||||
|
//
|
||||||
|
// The caller frees the returned slice via
|
||||||
|
// `os.free(r.ptr: *void, (r.cap: u64) * 24u64)`. Element bytes are
|
||||||
|
// borrowed from `in`.
|
||||||
|
//
|
||||||
|
// Hare's `([][]u8 | nomem)` collapses to `[][]u8` here: ww os.alloc
|
||||||
|
// has no recoverable failure path. Same precedent as
|
||||||
|
// shlex.split / getopt.tryparse.
|
||||||
|
//
|
||||||
|
// ref/hare/bytes/tokenize.ha:156.
|
||||||
|
export fn splitn(in: []u8, delim: []u8, n: i32) [][]u8 = {
|
||||||
|
os.assert(delim.len > 0,
|
||||||
|
"bytes.splitn must not be called with an empty delimiter");
|
||||||
|
let toks: [][]u8;
|
||||||
|
toks.ptr = nil: *[]u8;
|
||||||
|
toks.len = 0;
|
||||||
|
toks.cap = 0;
|
||||||
|
let tok: tokenizer = tokenize(in, delim...);
|
||||||
|
let i: i32 = 0;
|
||||||
|
for (i < n - 1) {
|
||||||
|
match (next_token(&tok)) {
|
||||||
|
case let s: []u8 => { appendslice(&toks, s); };
|
||||||
|
case done => { return toks; };
|
||||||
|
};
|
||||||
|
i += 1;
|
||||||
|
};
|
||||||
|
match (peek_token(&tok)) {
|
||||||
|
case done => void;
|
||||||
|
case let pk: []u8 => {
|
||||||
|
let r: []u8 = remaining_tokens(&tok);
|
||||||
|
appendslice(&toks, r);
|
||||||
|
};
|
||||||
|
};
|
||||||
|
return toks;
|
||||||
|
};
|
||||||
|
|
||||||
|
// rsplitn — reverse-direction counterpart to [[splitn]]: tokens are
|
||||||
|
// collected from the end of `in`. The trailing slot holds the
|
||||||
|
// unconsumed prefix (everything before the n-th-from-last delim hit).
|
||||||
|
//
|
||||||
|
// When the input has fewer than n tokens, the `done` short-circuit
|
||||||
|
// returns toks UN-reversed (in last-token-first order). Mirrors Hare
|
||||||
|
// at ref/hare/bytes/tokenize.ha:196-199 where the in-place reverse
|
||||||
|
// step is gated behind the n-1 loop running to completion. Only the
|
||||||
|
// "loop ran to completion AND peek saw a remainder" path applies the
|
||||||
|
// reverse; both early-exit paths skip it.
|
||||||
|
//
|
||||||
|
// ref/hare/bytes/tokenize.ha:186.
|
||||||
|
export fn rsplitn(in: []u8, delim: []u8, n: i32) [][]u8 = {
|
||||||
|
os.assert(delim.len > 0,
|
||||||
|
"bytes.rsplitn called with empty delimiter");
|
||||||
|
let toks: [][]u8;
|
||||||
|
toks.ptr = nil: *[]u8;
|
||||||
|
toks.len = 0;
|
||||||
|
toks.cap = 0;
|
||||||
|
let tok: tokenizer = rtokenize(in, delim...);
|
||||||
|
let i: i32 = 0;
|
||||||
|
for (i < n - 1) {
|
||||||
|
match (next_token(&tok)) {
|
||||||
|
case let s: []u8 => { appendslice(&toks, s); };
|
||||||
|
case done => { return toks; };
|
||||||
|
};
|
||||||
|
i += 1;
|
||||||
|
};
|
||||||
|
match (peek_token(&tok)) {
|
||||||
|
case done => void;
|
||||||
|
case let pk: []u8 => {
|
||||||
|
let r: []u8 = remaining_tokens(&tok);
|
||||||
|
appendslice(&toks, r);
|
||||||
|
};
|
||||||
|
};
|
||||||
|
|
||||||
|
// In-place reverse so callers see argv-order, matching Hare
|
||||||
|
// (ref/hare/bytes/tokenize.ha:207). Element copy is field-wise
|
||||||
|
// through `*[]u8` because `toks[i] = toks[j]` (full 24B slice
|
||||||
|
// store) lands in the multi-word-store gap noted at
|
||||||
|
// cmd/w6c/cgen.c:6515-6523.
|
||||||
|
let a: i32 = 0;
|
||||||
|
let b: i32 = toks.len - 1;
|
||||||
|
for (a < b) {
|
||||||
|
let pa: *[]u8 = &toks.ptr[a];
|
||||||
|
let pb: *[]u8 = &toks.ptr[b];
|
||||||
|
let tp: *u8 = pa.ptr;
|
||||||
|
let tl: i32 = pa.len;
|
||||||
|
let tc: i32 = pa.cap;
|
||||||
|
pa.ptr = pb.ptr;
|
||||||
|
pa.len = pb.len;
|
||||||
|
pa.cap = pb.cap;
|
||||||
|
pb.ptr = tp;
|
||||||
|
pb.len = tl;
|
||||||
|
pb.cap = tc;
|
||||||
|
a += 1;
|
||||||
|
b -= 1;
|
||||||
|
};
|
||||||
|
return toks;
|
||||||
|
};
|
||||||
|
|
||||||
|
// split — full split of `in` on `delim` (no token cap). Mirrors
|
||||||
|
// `splitn(in, delim, types::SIZE_MAX)`. ww uses `types.I32_MAX`
|
||||||
|
// because the index type is i32 (lib/CLAUDE.md).
|
||||||
|
//
|
||||||
|
// ref/hare/bytes/tokenize.ha:225.
|
||||||
|
export fn split(in: []u8, delim: []u8) [][]u8 = {
|
||||||
|
return splitn(in, delim, types.I32_MAX);
|
||||||
|
};
|
||||||
|
|
||||||
// encoding/utf8 — UTF-8 encode/decode. Hare port; see
|
// encoding/utf8 — UTF-8 encode/decode. Hare port; see
|
||||||
// ref/hare/encoding/utf8/{types,rune,encode,decode,decodetable}.ha.
|
// ref/hare/encoding/utf8/{types,rune,encode,decode,decodetable}.ha.
|
||||||
//
|
//
|
||||||
|
|||||||
@@ -1233,6 +1233,137 @@ export fn remaining_tokens(s: *tokenizer) []u8 = {
|
|||||||
return s.in;
|
return s.in;
|
||||||
};
|
};
|
||||||
|
|
||||||
|
// rt_ensure is the runtime slice-growth helper invoked by the
|
||||||
|
// `append(s, v)` builtin. We bind it directly because the builtin's
|
||||||
|
// expansion stores only 8 bytes of the new element (cgen emits a
|
||||||
|
// single MOVQ), losing the .len/.cap fields of a []u8 element (24B).
|
||||||
|
// Mirrors the same workaround in lib/shlex.shlex (appendstr, 16B) and
|
||||||
|
// lib/getopt.getopt (appendoption, 24B); collapses in one go when the
|
||||||
|
// append builtin learns to store the full element width.
|
||||||
|
@symbol("rt_ensure") fn rtensure(s: *void, membsz: u64) void;
|
||||||
|
|
||||||
|
// appendslice — grow `*slice` by one and store `item` (24B). Mirror
|
||||||
|
// of [[shlex.appendstr]] / [[getopt.appendoption]]. Bypasses the
|
||||||
|
// `append` builtin's first-8B-only-store gap for a slice-element.
|
||||||
|
fn appendslice(slice: *[][]u8, item: []u8) void = {
|
||||||
|
let newlen: i32 = slice.len + 1;
|
||||||
|
slice.len = newlen;
|
||||||
|
rtensure(slice: *void, 24u64);
|
||||||
|
let dst: *[]u8 = &slice.ptr[newlen - 1];
|
||||||
|
dst.ptr = item.ptr;
|
||||||
|
dst.len = item.len;
|
||||||
|
dst.cap = item.cap;
|
||||||
|
};
|
||||||
|
|
||||||
|
// splitn — split `in` on any byte in `delim`, returning up to `n`
|
||||||
|
// tokens via forward iteration. The trailing slot (when more than
|
||||||
|
// `n - 1` tokens exist) holds the unconsumed remainder.
|
||||||
|
//
|
||||||
|
// The caller frees the returned slice via
|
||||||
|
// `os.free(r.ptr: *void, (r.cap: u64) * 24u64)`. Element bytes are
|
||||||
|
// borrowed from `in`.
|
||||||
|
//
|
||||||
|
// Hare's `([][]u8 | nomem)` collapses to `[][]u8` here: ww os.alloc
|
||||||
|
// has no recoverable failure path. Same precedent as
|
||||||
|
// shlex.split / getopt.tryparse.
|
||||||
|
//
|
||||||
|
// ref/hare/bytes/tokenize.ha:156.
|
||||||
|
export fn splitn(in: []u8, delim: []u8, n: i32) [][]u8 = {
|
||||||
|
os.assert(delim.len > 0,
|
||||||
|
"bytes.splitn must not be called with an empty delimiter");
|
||||||
|
let toks: [][]u8;
|
||||||
|
toks.ptr = nil: *[]u8;
|
||||||
|
toks.len = 0;
|
||||||
|
toks.cap = 0;
|
||||||
|
let tok: tokenizer = tokenize(in, delim...);
|
||||||
|
let i: i32 = 0;
|
||||||
|
for (i < n - 1) {
|
||||||
|
match (next_token(&tok)) {
|
||||||
|
case let s: []u8 => { appendslice(&toks, s); };
|
||||||
|
case done => { return toks; };
|
||||||
|
};
|
||||||
|
i += 1;
|
||||||
|
};
|
||||||
|
match (peek_token(&tok)) {
|
||||||
|
case done => void;
|
||||||
|
case let pk: []u8 => {
|
||||||
|
let r: []u8 = remaining_tokens(&tok);
|
||||||
|
appendslice(&toks, r);
|
||||||
|
};
|
||||||
|
};
|
||||||
|
return toks;
|
||||||
|
};
|
||||||
|
|
||||||
|
// rsplitn — reverse-direction counterpart to [[splitn]]: tokens are
|
||||||
|
// collected from the end of `in`. The trailing slot holds the
|
||||||
|
// unconsumed prefix (everything before the n-th-from-last delim hit).
|
||||||
|
//
|
||||||
|
// When the input has fewer than n tokens, the `done` short-circuit
|
||||||
|
// returns toks UN-reversed (in last-token-first order). Mirrors Hare
|
||||||
|
// at ref/hare/bytes/tokenize.ha:196-199 where the in-place reverse
|
||||||
|
// step is gated behind the n-1 loop running to completion. Only the
|
||||||
|
// "loop ran to completion AND peek saw a remainder" path applies the
|
||||||
|
// reverse; both early-exit paths skip it.
|
||||||
|
//
|
||||||
|
// ref/hare/bytes/tokenize.ha:186.
|
||||||
|
export fn rsplitn(in: []u8, delim: []u8, n: i32) [][]u8 = {
|
||||||
|
os.assert(delim.len > 0,
|
||||||
|
"bytes.rsplitn called with empty delimiter");
|
||||||
|
let toks: [][]u8;
|
||||||
|
toks.ptr = nil: *[]u8;
|
||||||
|
toks.len = 0;
|
||||||
|
toks.cap = 0;
|
||||||
|
let tok: tokenizer = rtokenize(in, delim...);
|
||||||
|
let i: i32 = 0;
|
||||||
|
for (i < n - 1) {
|
||||||
|
match (next_token(&tok)) {
|
||||||
|
case let s: []u8 => { appendslice(&toks, s); };
|
||||||
|
case done => { return toks; };
|
||||||
|
};
|
||||||
|
i += 1;
|
||||||
|
};
|
||||||
|
match (peek_token(&tok)) {
|
||||||
|
case done => void;
|
||||||
|
case let pk: []u8 => {
|
||||||
|
let r: []u8 = remaining_tokens(&tok);
|
||||||
|
appendslice(&toks, r);
|
||||||
|
};
|
||||||
|
};
|
||||||
|
|
||||||
|
// In-place reverse so callers see argv-order, matching Hare
|
||||||
|
// (ref/hare/bytes/tokenize.ha:207). Element copy is field-wise
|
||||||
|
// through `*[]u8` because `toks[i] = toks[j]` (full 24B slice
|
||||||
|
// store) lands in the multi-word-store gap noted at
|
||||||
|
// cmd/w6c/cgen.c:6515-6523.
|
||||||
|
let a: i32 = 0;
|
||||||
|
let b: i32 = toks.len - 1;
|
||||||
|
for (a < b) {
|
||||||
|
let pa: *[]u8 = &toks.ptr[a];
|
||||||
|
let pb: *[]u8 = &toks.ptr[b];
|
||||||
|
let tp: *u8 = pa.ptr;
|
||||||
|
let tl: i32 = pa.len;
|
||||||
|
let tc: i32 = pa.cap;
|
||||||
|
pa.ptr = pb.ptr;
|
||||||
|
pa.len = pb.len;
|
||||||
|
pa.cap = pb.cap;
|
||||||
|
pb.ptr = tp;
|
||||||
|
pb.len = tl;
|
||||||
|
pb.cap = tc;
|
||||||
|
a += 1;
|
||||||
|
b -= 1;
|
||||||
|
};
|
||||||
|
return toks;
|
||||||
|
};
|
||||||
|
|
||||||
|
// split — full split of `in` on `delim` (no token cap). Mirrors
|
||||||
|
// `splitn(in, delim, types::SIZE_MAX)`. ww uses `types.I32_MAX`
|
||||||
|
// because the index type is i32 (lib/CLAUDE.md).
|
||||||
|
//
|
||||||
|
// ref/hare/bytes/tokenize.ha:225.
|
||||||
|
export fn split(in: []u8, delim: []u8) [][]u8 = {
|
||||||
|
return splitn(in, delim, types.I32_MAX);
|
||||||
|
};
|
||||||
|
|
||||||
// encoding/utf8 — UTF-8 encode/decode. Hare port; see
|
// encoding/utf8 — UTF-8 encode/decode. Hare port; see
|
||||||
// ref/hare/encoding/utf8/{types,rune,encode,decode,decodetable}.ha.
|
// ref/hare/encoding/utf8/{types,rune,encode,decode,decodetable}.ha.
|
||||||
//
|
//
|
||||||
|
|||||||
@@ -1124,6 +1124,137 @@ export fn remaining_tokens(s: *tokenizer) []u8 = {
|
|||||||
return s.in;
|
return s.in;
|
||||||
};
|
};
|
||||||
|
|
||||||
|
// rt_ensure is the runtime slice-growth helper invoked by the
|
||||||
|
// `append(s, v)` builtin. We bind it directly because the builtin's
|
||||||
|
// expansion stores only 8 bytes of the new element (cgen emits a
|
||||||
|
// single MOVQ), losing the .len/.cap fields of a []u8 element (24B).
|
||||||
|
// Mirrors the same workaround in lib/shlex.shlex (appendstr, 16B) and
|
||||||
|
// lib/getopt.getopt (appendoption, 24B); collapses in one go when the
|
||||||
|
// append builtin learns to store the full element width.
|
||||||
|
@symbol("rt_ensure") fn rtensure(s: *void, membsz: u64) void;
|
||||||
|
|
||||||
|
// appendslice — grow `*slice` by one and store `item` (24B). Mirror
|
||||||
|
// of [[shlex.appendstr]] / [[getopt.appendoption]]. Bypasses the
|
||||||
|
// `append` builtin's first-8B-only-store gap for a slice-element.
|
||||||
|
fn appendslice(slice: *[][]u8, item: []u8) void = {
|
||||||
|
let newlen: i32 = slice.len + 1;
|
||||||
|
slice.len = newlen;
|
||||||
|
rtensure(slice: *void, 24u64);
|
||||||
|
let dst: *[]u8 = &slice.ptr[newlen - 1];
|
||||||
|
dst.ptr = item.ptr;
|
||||||
|
dst.len = item.len;
|
||||||
|
dst.cap = item.cap;
|
||||||
|
};
|
||||||
|
|
||||||
|
// splitn — split `in` on any byte in `delim`, returning up to `n`
|
||||||
|
// tokens via forward iteration. The trailing slot (when more than
|
||||||
|
// `n - 1` tokens exist) holds the unconsumed remainder.
|
||||||
|
//
|
||||||
|
// The caller frees the returned slice via
|
||||||
|
// `os.free(r.ptr: *void, (r.cap: u64) * 24u64)`. Element bytes are
|
||||||
|
// borrowed from `in`.
|
||||||
|
//
|
||||||
|
// Hare's `([][]u8 | nomem)` collapses to `[][]u8` here: ww os.alloc
|
||||||
|
// has no recoverable failure path. Same precedent as
|
||||||
|
// shlex.split / getopt.tryparse.
|
||||||
|
//
|
||||||
|
// ref/hare/bytes/tokenize.ha:156.
|
||||||
|
export fn splitn(in: []u8, delim: []u8, n: i32) [][]u8 = {
|
||||||
|
os.assert(delim.len > 0,
|
||||||
|
"bytes.splitn must not be called with an empty delimiter");
|
||||||
|
let toks: [][]u8;
|
||||||
|
toks.ptr = nil: *[]u8;
|
||||||
|
toks.len = 0;
|
||||||
|
toks.cap = 0;
|
||||||
|
let tok: tokenizer = tokenize(in, delim...);
|
||||||
|
let i: i32 = 0;
|
||||||
|
for (i < n - 1) {
|
||||||
|
match (next_token(&tok)) {
|
||||||
|
case let s: []u8 => { appendslice(&toks, s); };
|
||||||
|
case done => { return toks; };
|
||||||
|
};
|
||||||
|
i += 1;
|
||||||
|
};
|
||||||
|
match (peek_token(&tok)) {
|
||||||
|
case done => void;
|
||||||
|
case let pk: []u8 => {
|
||||||
|
let r: []u8 = remaining_tokens(&tok);
|
||||||
|
appendslice(&toks, r);
|
||||||
|
};
|
||||||
|
};
|
||||||
|
return toks;
|
||||||
|
};
|
||||||
|
|
||||||
|
// rsplitn — reverse-direction counterpart to [[splitn]]: tokens are
|
||||||
|
// collected from the end of `in`. The trailing slot holds the
|
||||||
|
// unconsumed prefix (everything before the n-th-from-last delim hit).
|
||||||
|
//
|
||||||
|
// When the input has fewer than n tokens, the `done` short-circuit
|
||||||
|
// returns toks UN-reversed (in last-token-first order). Mirrors Hare
|
||||||
|
// at ref/hare/bytes/tokenize.ha:196-199 where the in-place reverse
|
||||||
|
// step is gated behind the n-1 loop running to completion. Only the
|
||||||
|
// "loop ran to completion AND peek saw a remainder" path applies the
|
||||||
|
// reverse; both early-exit paths skip it.
|
||||||
|
//
|
||||||
|
// ref/hare/bytes/tokenize.ha:186.
|
||||||
|
export fn rsplitn(in: []u8, delim: []u8, n: i32) [][]u8 = {
|
||||||
|
os.assert(delim.len > 0,
|
||||||
|
"bytes.rsplitn called with empty delimiter");
|
||||||
|
let toks: [][]u8;
|
||||||
|
toks.ptr = nil: *[]u8;
|
||||||
|
toks.len = 0;
|
||||||
|
toks.cap = 0;
|
||||||
|
let tok: tokenizer = rtokenize(in, delim...);
|
||||||
|
let i: i32 = 0;
|
||||||
|
for (i < n - 1) {
|
||||||
|
match (next_token(&tok)) {
|
||||||
|
case let s: []u8 => { appendslice(&toks, s); };
|
||||||
|
case done => { return toks; };
|
||||||
|
};
|
||||||
|
i += 1;
|
||||||
|
};
|
||||||
|
match (peek_token(&tok)) {
|
||||||
|
case done => void;
|
||||||
|
case let pk: []u8 => {
|
||||||
|
let r: []u8 = remaining_tokens(&tok);
|
||||||
|
appendslice(&toks, r);
|
||||||
|
};
|
||||||
|
};
|
||||||
|
|
||||||
|
// In-place reverse so callers see argv-order, matching Hare
|
||||||
|
// (ref/hare/bytes/tokenize.ha:207). Element copy is field-wise
|
||||||
|
// through `*[]u8` because `toks[i] = toks[j]` (full 24B slice
|
||||||
|
// store) lands in the multi-word-store gap noted at
|
||||||
|
// cmd/w6c/cgen.c:6515-6523.
|
||||||
|
let a: i32 = 0;
|
||||||
|
let b: i32 = toks.len - 1;
|
||||||
|
for (a < b) {
|
||||||
|
let pa: *[]u8 = &toks.ptr[a];
|
||||||
|
let pb: *[]u8 = &toks.ptr[b];
|
||||||
|
let tp: *u8 = pa.ptr;
|
||||||
|
let tl: i32 = pa.len;
|
||||||
|
let tc: i32 = pa.cap;
|
||||||
|
pa.ptr = pb.ptr;
|
||||||
|
pa.len = pb.len;
|
||||||
|
pa.cap = pb.cap;
|
||||||
|
pb.ptr = tp;
|
||||||
|
pb.len = tl;
|
||||||
|
pb.cap = tc;
|
||||||
|
a += 1;
|
||||||
|
b -= 1;
|
||||||
|
};
|
||||||
|
return toks;
|
||||||
|
};
|
||||||
|
|
||||||
|
// split — full split of `in` on `delim` (no token cap). Mirrors
|
||||||
|
// `splitn(in, delim, types::SIZE_MAX)`. ww uses `types.I32_MAX`
|
||||||
|
// because the index type is i32 (lib/CLAUDE.md).
|
||||||
|
//
|
||||||
|
// ref/hare/bytes/tokenize.ha:225.
|
||||||
|
export fn split(in: []u8, delim: []u8) [][]u8 = {
|
||||||
|
return splitn(in, delim, types.I32_MAX);
|
||||||
|
};
|
||||||
|
|
||||||
// encoding/utf8 — UTF-8 encode/decode. Hare port; see
|
// encoding/utf8 — UTF-8 encode/decode. Hare port; see
|
||||||
// ref/hare/encoding/utf8/{types,rune,encode,decode,decodetable}.ha.
|
// ref/hare/encoding/utf8/{types,rune,encode,decode,decodetable}.ha.
|
||||||
//
|
//
|
||||||
|
|||||||
Reference in New Issue
Block a user