lib/bytes+test: port split family from Hare
This commit is contained in:
@@ -351,3 +351,134 @@ export fn next_token(s: *tokenizer) ([]u8 | done) = {
|
||||
export fn remaining_tokens(s: *tokenizer) []u8 = {
|
||||
return s.in;
|
||||
};
|
||||
|
||||
// rt_ensure is the runtime slice-growth helper invoked by the
|
||||
// `append(s, v)` builtin. We bind it directly because the builtin's
|
||||
// expansion stores only 8 bytes of the new element (cgen emits a
|
||||
// single MOVQ), losing the .len/.cap fields of a []u8 element (24B).
|
||||
// Mirrors the same workaround in lib/shlex.shlex (appendstr, 16B) and
|
||||
// lib/getopt.getopt (appendoption, 24B); collapses in one go when the
|
||||
// append builtin learns to store the full element width.
|
||||
@symbol("rt_ensure") fn rtensure(s: *void, membsz: u64) void;
|
||||
|
||||
// appendslice — grow `*slice` by one and store `item` (24B). Mirror
|
||||
// of [[shlex.appendstr]] / [[getopt.appendoption]]. Bypasses the
|
||||
// `append` builtin's first-8B-only-store gap for a slice-element.
|
||||
fn appendslice(slice: *[][]u8, item: []u8) void = {
|
||||
let newlen: i32 = slice.len + 1;
|
||||
slice.len = newlen;
|
||||
rtensure(slice: *void, 24u64);
|
||||
let dst: *[]u8 = &slice.ptr[newlen - 1];
|
||||
dst.ptr = item.ptr;
|
||||
dst.len = item.len;
|
||||
dst.cap = item.cap;
|
||||
};
|
||||
|
||||
// splitn — split `in` on any byte in `delim`, returning up to `n`
|
||||
// tokens via forward iteration. The trailing slot (when more than
|
||||
// `n - 1` tokens exist) holds the unconsumed remainder.
|
||||
//
|
||||
// The caller frees the returned slice via
|
||||
// `os.free(r.ptr: *void, (r.cap: u64) * 24u64)`. Element bytes are
|
||||
// borrowed from `in`.
|
||||
//
|
||||
// Hare's `([][]u8 | nomem)` collapses to `[][]u8` here: ww os.alloc
|
||||
// has no recoverable failure path. Same precedent as
|
||||
// shlex.split / getopt.tryparse.
|
||||
//
|
||||
// ref/hare/bytes/tokenize.ha:156.
|
||||
export fn splitn(in: []u8, delim: []u8, n: i32) [][]u8 = {
|
||||
os.assert(delim.len > 0,
|
||||
"bytes.splitn must not be called with an empty delimiter");
|
||||
let toks: [][]u8;
|
||||
toks.ptr = nil: *[]u8;
|
||||
toks.len = 0;
|
||||
toks.cap = 0;
|
||||
let tok: tokenizer = tokenize(in, delim...);
|
||||
let i: i32 = 0;
|
||||
for (i < n - 1) {
|
||||
match (next_token(&tok)) {
|
||||
case let s: []u8 => { appendslice(&toks, s); };
|
||||
case done => { return toks; };
|
||||
};
|
||||
i += 1;
|
||||
};
|
||||
match (peek_token(&tok)) {
|
||||
case done => void;
|
||||
case let pk: []u8 => {
|
||||
let r: []u8 = remaining_tokens(&tok);
|
||||
appendslice(&toks, r);
|
||||
};
|
||||
};
|
||||
return toks;
|
||||
};
|
||||
|
||||
// rsplitn — reverse-direction counterpart to [[splitn]]: tokens are
|
||||
// collected from the end of `in`. The trailing slot holds the
|
||||
// unconsumed prefix (everything before the n-th-from-last delim hit).
|
||||
//
|
||||
// When the input has fewer than n tokens, the `done` short-circuit
|
||||
// returns toks UN-reversed (in last-token-first order). Mirrors Hare
|
||||
// at ref/hare/bytes/tokenize.ha:196-199 where the in-place reverse
|
||||
// step is gated behind the n-1 loop running to completion. Only the
|
||||
// "loop ran to completion AND peek saw a remainder" path applies the
|
||||
// reverse; both early-exit paths skip it.
|
||||
//
|
||||
// ref/hare/bytes/tokenize.ha:186.
|
||||
export fn rsplitn(in: []u8, delim: []u8, n: i32) [][]u8 = {
|
||||
os.assert(delim.len > 0,
|
||||
"bytes.rsplitn called with empty delimiter");
|
||||
let toks: [][]u8;
|
||||
toks.ptr = nil: *[]u8;
|
||||
toks.len = 0;
|
||||
toks.cap = 0;
|
||||
let tok: tokenizer = rtokenize(in, delim...);
|
||||
let i: i32 = 0;
|
||||
for (i < n - 1) {
|
||||
match (next_token(&tok)) {
|
||||
case let s: []u8 => { appendslice(&toks, s); };
|
||||
case done => { return toks; };
|
||||
};
|
||||
i += 1;
|
||||
};
|
||||
match (peek_token(&tok)) {
|
||||
case done => void;
|
||||
case let pk: []u8 => {
|
||||
let r: []u8 = remaining_tokens(&tok);
|
||||
appendslice(&toks, r);
|
||||
};
|
||||
};
|
||||
|
||||
// In-place reverse so callers see argv-order, matching Hare
|
||||
// (ref/hare/bytes/tokenize.ha:207). Element copy is field-wise
|
||||
// through `*[]u8` because `toks[i] = toks[j]` (full 24B slice
|
||||
// store) lands in the multi-word-store gap noted at
|
||||
// cmd/w6c/cgen.c:6515-6523.
|
||||
let a: i32 = 0;
|
||||
let b: i32 = toks.len - 1;
|
||||
for (a < b) {
|
||||
let pa: *[]u8 = &toks.ptr[a];
|
||||
let pb: *[]u8 = &toks.ptr[b];
|
||||
let tp: *u8 = pa.ptr;
|
||||
let tl: i32 = pa.len;
|
||||
let tc: i32 = pa.cap;
|
||||
pa.ptr = pb.ptr;
|
||||
pa.len = pb.len;
|
||||
pa.cap = pb.cap;
|
||||
pb.ptr = tp;
|
||||
pb.len = tl;
|
||||
pb.cap = tc;
|
||||
a += 1;
|
||||
b -= 1;
|
||||
};
|
||||
return toks;
|
||||
};
|
||||
|
||||
// split — full split of `in` on `delim` (no token cap). Mirrors
|
||||
// `splitn(in, delim, types::SIZE_MAX)`. ww uses `types.I32_MAX`
|
||||
// because the index type is i32 (lib/CLAUDE.md).
|
||||
//
|
||||
// ref/hare/bytes/tokenize.ha:225.
|
||||
export fn split(in: []u8, delim: []u8) [][]u8 = {
|
||||
return splitn(in, delim, types.I32_MAX);
|
||||
};
|
||||
|
||||
@@ -467,6 +467,185 @@ fn expect_done(t: *bytes.tokenizer) void = {
|
||||
if (!bytes.equal(r2, e_12[0:2])) { fail(); };
|
||||
};
|
||||
|
||||
// ---- splitn / rsplitn / split -----------------------------------------
|
||||
// ref/hare/bytes/tokenize.ha:330 (@test fn split). Hare's table mixes
|
||||
// strings; ww spells the vectors as byte arrays explicitly.
|
||||
//
|
||||
// Element reads go through `&toks.ptr[i]: *[]u8` rather than `toks[i]`
|
||||
// — the full 24B slice-element copy lands in the multi-word-store gap
|
||||
// noted at cmd/w6c/cgen.c:6515-6523, so a value-load drops .len/.cap.
|
||||
// Pointer-then-fields lifts 8B at a time, which the cgen routes correctly.
|
||||
|
||||
fn expect_tok(toks: [][]u8, i: i32, want: []u8) void = {
|
||||
if (i >= toks.len) { fail(); };
|
||||
let p: *[]u8 = &toks.ptr[i];
|
||||
if (p.len != want.len) { fail(); };
|
||||
let j: i32 = 0;
|
||||
for (j < want.len) {
|
||||
if (p.ptr[j] != want[j]) { fail(); };
|
||||
j += 1;
|
||||
};
|
||||
};
|
||||
|
||||
@test fn splitn_cases() void = {
|
||||
// "Hello, my name is Drew" ─ space-delimited, n=4 yields the 4-th
|
||||
// token as the unconsumed remainder. Hare pins this exact shape
|
||||
// at ref/hare/bytes/tokenize.ha:340.
|
||||
signalled = 1750;
|
||||
let a: [22]u8;
|
||||
a[0] = 72u8; a[1] = 101u8; a[2] = 108u8; a[3] = 108u8; a[4] = 111u8;
|
||||
a[5] = 44u8; a[6] = 32u8; a[7] = 109u8; a[8] = 121u8; a[9] = 32u8;
|
||||
a[10] = 110u8; a[11] = 97u8; a[12] = 109u8; a[13] = 101u8; a[14] = 32u8;
|
||||
a[15] = 105u8; a[16] = 115u8; a[17] = 32u8;
|
||||
a[18] = 68u8; a[19] = 114u8; a[20] = 101u8; a[21] = 119u8;
|
||||
let sp: [1]u8; sp[0] = 32u8;
|
||||
let t1: [][]u8 = bytes.splitn(a[0:22], sp[0:1], 4);
|
||||
if (t1.len != 4) { fail(); };
|
||||
expect_tok(t1, 0, a[0:6]);
|
||||
expect_tok(t1, 1, a[7:9]);
|
||||
expect_tok(t1, 2, a[10:14]);
|
||||
expect_tok(t1, 3, a[15:22]);
|
||||
os.free(t1.ptr: *void, (t1.cap: u64) * 24u64);
|
||||
|
||||
// n > token count — final slot is "" if input ends in delim,
|
||||
// otherwise the last token. Here three tokens, n=10 → 3 entries.
|
||||
signalled = 1751;
|
||||
let b: [5]u8; b[0] = 1u8; b[1] = 0u8; b[2] = 2u8; b[3] = 0u8; b[4] = 3u8;
|
||||
let zd: [1]u8; zd[0] = 0u8;
|
||||
let t2: [][]u8 = bytes.splitn(b[0:5], zd[0:1], 10);
|
||||
if (t2.len != 3) { fail(); };
|
||||
expect_tok(t2, 0, b[0:1]);
|
||||
expect_tok(t2, 1, b[2:3]);
|
||||
expect_tok(t2, 2, b[4:5]);
|
||||
os.free(t2.ptr: *void, (t2.cap: u64) * 24u64);
|
||||
|
||||
// n == 1 — single slot holding the whole input as remainder.
|
||||
signalled = 1752;
|
||||
let t3: [][]u8 = bytes.splitn(b[0:5], zd[0:1], 1);
|
||||
if (t3.len != 1) { fail(); };
|
||||
expect_tok(t3, 0, b[0:5]);
|
||||
os.free(t3.ptr: *void, (t3.cap: u64) * 24u64);
|
||||
|
||||
// delim absent — single slot holding input unchanged.
|
||||
signalled = 1753;
|
||||
let c: [3]u8; c[0] = 1u8; c[1] = 2u8; c[2] = 3u8;
|
||||
let t4: [][]u8 = bytes.splitn(c[0:3], zd[0:1], 5);
|
||||
if (t4.len != 1) { fail(); };
|
||||
expect_tok(t4, 0, c[0:3]);
|
||||
os.free(t4.ptr: *void, (t4.cap: u64) * 24u64);
|
||||
|
||||
// empty input — empty result.
|
||||
signalled = 1754;
|
||||
let z: [1]u8;
|
||||
let t5: [][]u8 = bytes.splitn(z[0:0], zd[0:1], 5);
|
||||
if (t5.len != 0) { fail(); };
|
||||
if (t5.cap > 0) {
|
||||
os.free(t5.ptr: *void, (t5.cap: u64) * 24u64);
|
||||
};
|
||||
|
||||
// Multi-byte delimiter set — both 0 and 42 split.
|
||||
signalled = 1755;
|
||||
let d: [8]u8;
|
||||
d[0] = 1u8; d[1] = 2u8; d[2] = 0u8; d[3] = 3u8;
|
||||
d[4] = 4u8; d[5] = 42u8; d[6] = 5u8; d[7] = 6u8;
|
||||
let dd: [2]u8; dd[0] = 0u8; dd[1] = 42u8;
|
||||
let t6: [][]u8 = bytes.splitn(d[0:8], dd[0:2], 100);
|
||||
if (t6.len != 3) { fail(); };
|
||||
expect_tok(t6, 0, d[0:2]);
|
||||
expect_tok(t6, 1, d[3:5]);
|
||||
expect_tok(t6, 2, d[6:8]);
|
||||
os.free(t6.ptr: *void, (t6.cap: u64) * 24u64);
|
||||
};
|
||||
|
||||
@test fn rsplitn_cases() void = {
|
||||
// "Hello, my name is Drew" ─ rsplitn n=4 buckets the first three
|
||||
// tokens from the *end*; the remainder ("Hello, my") is index 0.
|
||||
// Hare pins this at ref/hare/bytes/tokenize.ha:379.
|
||||
signalled = 1760;
|
||||
let a: [22]u8;
|
||||
a[0] = 72u8; a[1] = 101u8; a[2] = 108u8; a[3] = 108u8; a[4] = 111u8;
|
||||
a[5] = 44u8; a[6] = 32u8; a[7] = 109u8; a[8] = 121u8; a[9] = 32u8;
|
||||
a[10] = 110u8; a[11] = 97u8; a[12] = 109u8; a[13] = 101u8; a[14] = 32u8;
|
||||
a[15] = 105u8; a[16] = 115u8; a[17] = 32u8;
|
||||
a[18] = 68u8; a[19] = 114u8; a[20] = 101u8; a[21] = 119u8;
|
||||
let sp: [1]u8; sp[0] = 32u8;
|
||||
let t1: [][]u8 = bytes.rsplitn(a[0:22], sp[0:1], 4);
|
||||
if (t1.len != 4) { fail(); };
|
||||
expect_tok(t1, 0, a[0:9]);
|
||||
expect_tok(t1, 1, a[10:14]);
|
||||
expect_tok(t1, 2, a[15:17]);
|
||||
expect_tok(t1, 3, a[18:22]);
|
||||
os.free(t1.ptr: *void, (t1.cap: u64) * 24u64);
|
||||
|
||||
// n > token count — done short-circuit returns the toks in
|
||||
// reverse-iteration order (last token first). Mirrors Hare's
|
||||
// behavior at ref/hare/bytes/tokenize.ha:196-199 where the
|
||||
// reverse-step is gated behind the n-1 loop completion.
|
||||
signalled = 1761;
|
||||
let b: [5]u8; b[0] = 1u8; b[1] = 0u8; b[2] = 2u8; b[3] = 0u8; b[4] = 3u8;
|
||||
let zd: [1]u8; zd[0] = 0u8;
|
||||
let t2: [][]u8 = bytes.rsplitn(b[0:5], zd[0:1], 10);
|
||||
if (t2.len != 3) { fail(); };
|
||||
expect_tok(t2, 0, b[4:5]);
|
||||
expect_tok(t2, 1, b[2:3]);
|
||||
expect_tok(t2, 2, b[0:1]);
|
||||
os.free(t2.ptr: *void, (t2.cap: u64) * 24u64);
|
||||
|
||||
// n == 1 — single slot holding the whole input as remainder.
|
||||
signalled = 1762;
|
||||
let t3: [][]u8 = bytes.rsplitn(b[0:5], zd[0:1], 1);
|
||||
if (t3.len != 1) { fail(); };
|
||||
expect_tok(t3, 0, b[0:5]);
|
||||
os.free(t3.ptr: *void, (t3.cap: u64) * 24u64);
|
||||
|
||||
// delim absent — done short-circuits immediately at iter 0;
|
||||
// toks ends up holding only the eventual remainder if n=1
|
||||
// (the loop never runs and peek picks up the full input).
|
||||
// With n>1 the loop's first next_token sees done (no delim
|
||||
// match anywhere) and returns toks={} per Hare's early-exit.
|
||||
signalled = 1763;
|
||||
let c: [3]u8; c[0] = 1u8; c[1] = 2u8; c[2] = 3u8;
|
||||
let t4: [][]u8 = bytes.rsplitn(c[0:3], zd[0:1], 5);
|
||||
if (t4.len != 1) { fail(); };
|
||||
expect_tok(t4, 0, c[0:3]);
|
||||
os.free(t4.ptr: *void, (t4.cap: u64) * 24u64);
|
||||
};
|
||||
|
||||
@test fn split_cases() void = {
|
||||
// Full split — every delim hit is a boundary; no per-call cap.
|
||||
// Hare pins five tokens for the canonical input at
|
||||
// ref/hare/bytes/tokenize.ha:347.
|
||||
signalled = 1770;
|
||||
let a: [22]u8;
|
||||
a[0] = 72u8; a[1] = 101u8; a[2] = 108u8; a[3] = 108u8; a[4] = 111u8;
|
||||
a[5] = 44u8; a[6] = 32u8; a[7] = 109u8; a[8] = 121u8; a[9] = 32u8;
|
||||
a[10] = 110u8; a[11] = 97u8; a[12] = 109u8; a[13] = 101u8; a[14] = 32u8;
|
||||
a[15] = 105u8; a[16] = 115u8; a[17] = 32u8;
|
||||
a[18] = 68u8; a[19] = 114u8; a[20] = 101u8; a[21] = 119u8;
|
||||
let sp: [1]u8; sp[0] = 32u8;
|
||||
let t1: [][]u8 = bytes.split(a[0:22], sp[0:1]);
|
||||
if (t1.len != 5) { fail(); };
|
||||
expect_tok(t1, 0, a[0:6]);
|
||||
expect_tok(t1, 1, a[7:9]);
|
||||
expect_tok(t1, 2, a[10:14]);
|
||||
expect_tok(t1, 3, a[15:17]);
|
||||
expect_tok(t1, 4, a[18:22]);
|
||||
os.free(t1.ptr: *void, (t1.cap: u64) * 24u64);
|
||||
|
||||
// Leading + trailing empty tokens — five entries, three of which
|
||||
// are []. Splits across all positions exactly the way Hare does.
|
||||
signalled = 1771;
|
||||
let b: [5]u8; b[0] = 0u8; b[1] = 1u8; b[2] = 2u8; b[3] = 3u8; b[4] = 0u8;
|
||||
let zd: [1]u8; zd[0] = 0u8;
|
||||
let z: [1]u8;
|
||||
let t2: [][]u8 = bytes.split(b[0:5], zd[0:1]);
|
||||
if (t2.len != 3) { fail(); };
|
||||
expect_tok(t2, 0, z[0:0]);
|
||||
expect_tok(t2, 1, b[1:4]);
|
||||
expect_tok(t2, 2, z[0:0]);
|
||||
os.free(t2.ptr: *void, (t2.cap: u64) * 24u64);
|
||||
};
|
||||
|
||||
export fn main() i32 = {
|
||||
signalled = 1; equal_cases();
|
||||
signalled = 2; index_byte_cases();
|
||||
@@ -480,5 +659,8 @@ export fn main() i32 = {
|
||||
signalled = 10; rtokenize_cases();
|
||||
signalled = 11; peek_token_cases();
|
||||
signalled = 12; remaining_tokens_cases();
|
||||
signalled = 13; splitn_cases();
|
||||
signalled = 14; rsplitn_cases();
|
||||
signalled = 15; split_cases();
|
||||
return 0;
|
||||
};
|
||||
|
||||
@@ -1233,6 +1233,137 @@ export fn remaining_tokens(s: *tokenizer) []u8 = {
|
||||
return s.in;
|
||||
};
|
||||
|
||||
// rt_ensure is the runtime slice-growth helper invoked by the
|
||||
// `append(s, v)` builtin. We bind it directly because the builtin's
|
||||
// expansion stores only 8 bytes of the new element (cgen emits a
|
||||
// single MOVQ), losing the .len/.cap fields of a []u8 element (24B).
|
||||
// Mirrors the same workaround in lib/shlex.shlex (appendstr, 16B) and
|
||||
// lib/getopt.getopt (appendoption, 24B); collapses in one go when the
|
||||
// append builtin learns to store the full element width.
|
||||
@symbol("rt_ensure") fn rtensure(s: *void, membsz: u64) void;
|
||||
|
||||
// appendslice — grow `*slice` by one and store `item` (24B). Mirror
|
||||
// of [[shlex.appendstr]] / [[getopt.appendoption]]. Bypasses the
|
||||
// `append` builtin's first-8B-only-store gap for a slice-element.
|
||||
fn appendslice(slice: *[][]u8, item: []u8) void = {
|
||||
let newlen: i32 = slice.len + 1;
|
||||
slice.len = newlen;
|
||||
rtensure(slice: *void, 24u64);
|
||||
let dst: *[]u8 = &slice.ptr[newlen - 1];
|
||||
dst.ptr = item.ptr;
|
||||
dst.len = item.len;
|
||||
dst.cap = item.cap;
|
||||
};
|
||||
|
||||
// splitn — split `in` on any byte in `delim`, returning up to `n`
|
||||
// tokens via forward iteration. The trailing slot (when more than
|
||||
// `n - 1` tokens exist) holds the unconsumed remainder.
|
||||
//
|
||||
// The caller frees the returned slice via
|
||||
// `os.free(r.ptr: *void, (r.cap: u64) * 24u64)`. Element bytes are
|
||||
// borrowed from `in`.
|
||||
//
|
||||
// Hare's `([][]u8 | nomem)` collapses to `[][]u8` here: ww os.alloc
|
||||
// has no recoverable failure path. Same precedent as
|
||||
// shlex.split / getopt.tryparse.
|
||||
//
|
||||
// ref/hare/bytes/tokenize.ha:156.
|
||||
export fn splitn(in: []u8, delim: []u8, n: i32) [][]u8 = {
|
||||
os.assert(delim.len > 0,
|
||||
"bytes.splitn must not be called with an empty delimiter");
|
||||
let toks: [][]u8;
|
||||
toks.ptr = nil: *[]u8;
|
||||
toks.len = 0;
|
||||
toks.cap = 0;
|
||||
let tok: tokenizer = tokenize(in, delim...);
|
||||
let i: i32 = 0;
|
||||
for (i < n - 1) {
|
||||
match (next_token(&tok)) {
|
||||
case let s: []u8 => { appendslice(&toks, s); };
|
||||
case done => { return toks; };
|
||||
};
|
||||
i += 1;
|
||||
};
|
||||
match (peek_token(&tok)) {
|
||||
case done => void;
|
||||
case let pk: []u8 => {
|
||||
let r: []u8 = remaining_tokens(&tok);
|
||||
appendslice(&toks, r);
|
||||
};
|
||||
};
|
||||
return toks;
|
||||
};
|
||||
|
||||
// rsplitn — reverse-direction counterpart to [[splitn]]: tokens are
|
||||
// collected from the end of `in`. The trailing slot holds the
|
||||
// unconsumed prefix (everything before the n-th-from-last delim hit).
|
||||
//
|
||||
// When the input has fewer than n tokens, the `done` short-circuit
|
||||
// returns toks UN-reversed (in last-token-first order). Mirrors Hare
|
||||
// at ref/hare/bytes/tokenize.ha:196-199 where the in-place reverse
|
||||
// step is gated behind the n-1 loop running to completion. Only the
|
||||
// "loop ran to completion AND peek saw a remainder" path applies the
|
||||
// reverse; both early-exit paths skip it.
|
||||
//
|
||||
// ref/hare/bytes/tokenize.ha:186.
|
||||
export fn rsplitn(in: []u8, delim: []u8, n: i32) [][]u8 = {
|
||||
os.assert(delim.len > 0,
|
||||
"bytes.rsplitn called with empty delimiter");
|
||||
let toks: [][]u8;
|
||||
toks.ptr = nil: *[]u8;
|
||||
toks.len = 0;
|
||||
toks.cap = 0;
|
||||
let tok: tokenizer = rtokenize(in, delim...);
|
||||
let i: i32 = 0;
|
||||
for (i < n - 1) {
|
||||
match (next_token(&tok)) {
|
||||
case let s: []u8 => { appendslice(&toks, s); };
|
||||
case done => { return toks; };
|
||||
};
|
||||
i += 1;
|
||||
};
|
||||
match (peek_token(&tok)) {
|
||||
case done => void;
|
||||
case let pk: []u8 => {
|
||||
let r: []u8 = remaining_tokens(&tok);
|
||||
appendslice(&toks, r);
|
||||
};
|
||||
};
|
||||
|
||||
// In-place reverse so callers see argv-order, matching Hare
|
||||
// (ref/hare/bytes/tokenize.ha:207). Element copy is field-wise
|
||||
// through `*[]u8` because `toks[i] = toks[j]` (full 24B slice
|
||||
// store) lands in the multi-word-store gap noted at
|
||||
// cmd/w6c/cgen.c:6515-6523.
|
||||
let a: i32 = 0;
|
||||
let b: i32 = toks.len - 1;
|
||||
for (a < b) {
|
||||
let pa: *[]u8 = &toks.ptr[a];
|
||||
let pb: *[]u8 = &toks.ptr[b];
|
||||
let tp: *u8 = pa.ptr;
|
||||
let tl: i32 = pa.len;
|
||||
let tc: i32 = pa.cap;
|
||||
pa.ptr = pb.ptr;
|
||||
pa.len = pb.len;
|
||||
pa.cap = pb.cap;
|
||||
pb.ptr = tp;
|
||||
pb.len = tl;
|
||||
pb.cap = tc;
|
||||
a += 1;
|
||||
b -= 1;
|
||||
};
|
||||
return toks;
|
||||
};
|
||||
|
||||
// split — full split of `in` on `delim` (no token cap). Mirrors
|
||||
// `splitn(in, delim, types::SIZE_MAX)`. ww uses `types.I32_MAX`
|
||||
// because the index type is i32 (lib/CLAUDE.md).
|
||||
//
|
||||
// ref/hare/bytes/tokenize.ha:225.
|
||||
export fn split(in: []u8, delim: []u8) [][]u8 = {
|
||||
return splitn(in, delim, types.I32_MAX);
|
||||
};
|
||||
|
||||
// encoding/utf8 — UTF-8 encode/decode. Hare port; see
|
||||
// ref/hare/encoding/utf8/{types,rune,encode,decode,decodetable}.ha.
|
||||
//
|
||||
|
||||
@@ -1233,6 +1233,137 @@ export fn remaining_tokens(s: *tokenizer) []u8 = {
|
||||
return s.in;
|
||||
};
|
||||
|
||||
// rt_ensure is the runtime slice-growth helper invoked by the
|
||||
// `append(s, v)` builtin. We bind it directly because the builtin's
|
||||
// expansion stores only 8 bytes of the new element (cgen emits a
|
||||
// single MOVQ), losing the .len/.cap fields of a []u8 element (24B).
|
||||
// Mirrors the same workaround in lib/shlex.shlex (appendstr, 16B) and
|
||||
// lib/getopt.getopt (appendoption, 24B); collapses in one go when the
|
||||
// append builtin learns to store the full element width.
|
||||
@symbol("rt_ensure") fn rtensure(s: *void, membsz: u64) void;
|
||||
|
||||
// appendslice — grow `*slice` by one and store `item` (24B). Mirror
|
||||
// of [[shlex.appendstr]] / [[getopt.appendoption]]. Bypasses the
|
||||
// `append` builtin's first-8B-only-store gap for a slice-element.
|
||||
fn appendslice(slice: *[][]u8, item: []u8) void = {
|
||||
let newlen: i32 = slice.len + 1;
|
||||
slice.len = newlen;
|
||||
rtensure(slice: *void, 24u64);
|
||||
let dst: *[]u8 = &slice.ptr[newlen - 1];
|
||||
dst.ptr = item.ptr;
|
||||
dst.len = item.len;
|
||||
dst.cap = item.cap;
|
||||
};
|
||||
|
||||
// splitn — split `in` on any byte in `delim`, returning up to `n`
|
||||
// tokens via forward iteration. The trailing slot (when more than
|
||||
// `n - 1` tokens exist) holds the unconsumed remainder.
|
||||
//
|
||||
// The caller frees the returned slice via
|
||||
// `os.free(r.ptr: *void, (r.cap: u64) * 24u64)`. Element bytes are
|
||||
// borrowed from `in`.
|
||||
//
|
||||
// Hare's `([][]u8 | nomem)` collapses to `[][]u8` here: ww os.alloc
|
||||
// has no recoverable failure path. Same precedent as
|
||||
// shlex.split / getopt.tryparse.
|
||||
//
|
||||
// ref/hare/bytes/tokenize.ha:156.
|
||||
export fn splitn(in: []u8, delim: []u8, n: i32) [][]u8 = {
|
||||
os.assert(delim.len > 0,
|
||||
"bytes.splitn must not be called with an empty delimiter");
|
||||
let toks: [][]u8;
|
||||
toks.ptr = nil: *[]u8;
|
||||
toks.len = 0;
|
||||
toks.cap = 0;
|
||||
let tok: tokenizer = tokenize(in, delim...);
|
||||
let i: i32 = 0;
|
||||
for (i < n - 1) {
|
||||
match (next_token(&tok)) {
|
||||
case let s: []u8 => { appendslice(&toks, s); };
|
||||
case done => { return toks; };
|
||||
};
|
||||
i += 1;
|
||||
};
|
||||
match (peek_token(&tok)) {
|
||||
case done => void;
|
||||
case let pk: []u8 => {
|
||||
let r: []u8 = remaining_tokens(&tok);
|
||||
appendslice(&toks, r);
|
||||
};
|
||||
};
|
||||
return toks;
|
||||
};
|
||||
|
||||
// rsplitn — reverse-direction counterpart to [[splitn]]: tokens are
|
||||
// collected from the end of `in`. The trailing slot holds the
|
||||
// unconsumed prefix (everything before the n-th-from-last delim hit).
|
||||
//
|
||||
// When the input has fewer than n tokens, the `done` short-circuit
|
||||
// returns toks UN-reversed (in last-token-first order). Mirrors Hare
|
||||
// at ref/hare/bytes/tokenize.ha:196-199 where the in-place reverse
|
||||
// step is gated behind the n-1 loop running to completion. Only the
|
||||
// "loop ran to completion AND peek saw a remainder" path applies the
|
||||
// reverse; both early-exit paths skip it.
|
||||
//
|
||||
// ref/hare/bytes/tokenize.ha:186.
|
||||
export fn rsplitn(in: []u8, delim: []u8, n: i32) [][]u8 = {
|
||||
os.assert(delim.len > 0,
|
||||
"bytes.rsplitn called with empty delimiter");
|
||||
let toks: [][]u8;
|
||||
toks.ptr = nil: *[]u8;
|
||||
toks.len = 0;
|
||||
toks.cap = 0;
|
||||
let tok: tokenizer = rtokenize(in, delim...);
|
||||
let i: i32 = 0;
|
||||
for (i < n - 1) {
|
||||
match (next_token(&tok)) {
|
||||
case let s: []u8 => { appendslice(&toks, s); };
|
||||
case done => { return toks; };
|
||||
};
|
||||
i += 1;
|
||||
};
|
||||
match (peek_token(&tok)) {
|
||||
case done => void;
|
||||
case let pk: []u8 => {
|
||||
let r: []u8 = remaining_tokens(&tok);
|
||||
appendslice(&toks, r);
|
||||
};
|
||||
};
|
||||
|
||||
// In-place reverse so callers see argv-order, matching Hare
|
||||
// (ref/hare/bytes/tokenize.ha:207). Element copy is field-wise
|
||||
// through `*[]u8` because `toks[i] = toks[j]` (full 24B slice
|
||||
// store) lands in the multi-word-store gap noted at
|
||||
// cmd/w6c/cgen.c:6515-6523.
|
||||
let a: i32 = 0;
|
||||
let b: i32 = toks.len - 1;
|
||||
for (a < b) {
|
||||
let pa: *[]u8 = &toks.ptr[a];
|
||||
let pb: *[]u8 = &toks.ptr[b];
|
||||
let tp: *u8 = pa.ptr;
|
||||
let tl: i32 = pa.len;
|
||||
let tc: i32 = pa.cap;
|
||||
pa.ptr = pb.ptr;
|
||||
pa.len = pb.len;
|
||||
pa.cap = pb.cap;
|
||||
pb.ptr = tp;
|
||||
pb.len = tl;
|
||||
pb.cap = tc;
|
||||
a += 1;
|
||||
b -= 1;
|
||||
};
|
||||
return toks;
|
||||
};
|
||||
|
||||
// split — full split of `in` on `delim` (no token cap). Mirrors
|
||||
// `splitn(in, delim, types::SIZE_MAX)`. ww uses `types.I32_MAX`
|
||||
// because the index type is i32 (lib/CLAUDE.md).
|
||||
//
|
||||
// ref/hare/bytes/tokenize.ha:225.
|
||||
export fn split(in: []u8, delim: []u8) [][]u8 = {
|
||||
return splitn(in, delim, types.I32_MAX);
|
||||
};
|
||||
|
||||
// encoding/utf8 — UTF-8 encode/decode. Hare port; see
|
||||
// ref/hare/encoding/utf8/{types,rune,encode,decode,decodetable}.ha.
|
||||
//
|
||||
|
||||
@@ -1124,6 +1124,137 @@ export fn remaining_tokens(s: *tokenizer) []u8 = {
|
||||
return s.in;
|
||||
};
|
||||
|
||||
// rt_ensure is the runtime slice-growth helper invoked by the
|
||||
// `append(s, v)` builtin. We bind it directly because the builtin's
|
||||
// expansion stores only 8 bytes of the new element (cgen emits a
|
||||
// single MOVQ), losing the .len/.cap fields of a []u8 element (24B).
|
||||
// Mirrors the same workaround in lib/shlex.shlex (appendstr, 16B) and
|
||||
// lib/getopt.getopt (appendoption, 24B); collapses in one go when the
|
||||
// append builtin learns to store the full element width.
|
||||
@symbol("rt_ensure") fn rtensure(s: *void, membsz: u64) void;
|
||||
|
||||
// appendslice — grow `*slice` by one and store `item` (24B). Mirror
|
||||
// of [[shlex.appendstr]] / [[getopt.appendoption]]. Bypasses the
|
||||
// `append` builtin's first-8B-only-store gap for a slice-element.
|
||||
fn appendslice(slice: *[][]u8, item: []u8) void = {
|
||||
let newlen: i32 = slice.len + 1;
|
||||
slice.len = newlen;
|
||||
rtensure(slice: *void, 24u64);
|
||||
let dst: *[]u8 = &slice.ptr[newlen - 1];
|
||||
dst.ptr = item.ptr;
|
||||
dst.len = item.len;
|
||||
dst.cap = item.cap;
|
||||
};
|
||||
|
||||
// splitn — split `in` on any byte in `delim`, returning up to `n`
|
||||
// tokens via forward iteration. The trailing slot (when more than
|
||||
// `n - 1` tokens exist) holds the unconsumed remainder.
|
||||
//
|
||||
// The caller frees the returned slice via
|
||||
// `os.free(r.ptr: *void, (r.cap: u64) * 24u64)`. Element bytes are
|
||||
// borrowed from `in`.
|
||||
//
|
||||
// Hare's `([][]u8 | nomem)` collapses to `[][]u8` here: ww os.alloc
|
||||
// has no recoverable failure path. Same precedent as
|
||||
// shlex.split / getopt.tryparse.
|
||||
//
|
||||
// ref/hare/bytes/tokenize.ha:156.
|
||||
export fn splitn(in: []u8, delim: []u8, n: i32) [][]u8 = {
|
||||
os.assert(delim.len > 0,
|
||||
"bytes.splitn must not be called with an empty delimiter");
|
||||
let toks: [][]u8;
|
||||
toks.ptr = nil: *[]u8;
|
||||
toks.len = 0;
|
||||
toks.cap = 0;
|
||||
let tok: tokenizer = tokenize(in, delim...);
|
||||
let i: i32 = 0;
|
||||
for (i < n - 1) {
|
||||
match (next_token(&tok)) {
|
||||
case let s: []u8 => { appendslice(&toks, s); };
|
||||
case done => { return toks; };
|
||||
};
|
||||
i += 1;
|
||||
};
|
||||
match (peek_token(&tok)) {
|
||||
case done => void;
|
||||
case let pk: []u8 => {
|
||||
let r: []u8 = remaining_tokens(&tok);
|
||||
appendslice(&toks, r);
|
||||
};
|
||||
};
|
||||
return toks;
|
||||
};
|
||||
|
||||
// rsplitn — reverse-direction counterpart to [[splitn]]: tokens are
|
||||
// collected from the end of `in`. The trailing slot holds the
|
||||
// unconsumed prefix (everything before the n-th-from-last delim hit).
|
||||
//
|
||||
// When the input has fewer than n tokens, the `done` short-circuit
|
||||
// returns toks UN-reversed (in last-token-first order). Mirrors Hare
|
||||
// at ref/hare/bytes/tokenize.ha:196-199 where the in-place reverse
|
||||
// step is gated behind the n-1 loop running to completion. Only the
|
||||
// "loop ran to completion AND peek saw a remainder" path applies the
|
||||
// reverse; both early-exit paths skip it.
|
||||
//
|
||||
// ref/hare/bytes/tokenize.ha:186.
|
||||
export fn rsplitn(in: []u8, delim: []u8, n: i32) [][]u8 = {
|
||||
os.assert(delim.len > 0,
|
||||
"bytes.rsplitn called with empty delimiter");
|
||||
let toks: [][]u8;
|
||||
toks.ptr = nil: *[]u8;
|
||||
toks.len = 0;
|
||||
toks.cap = 0;
|
||||
let tok: tokenizer = rtokenize(in, delim...);
|
||||
let i: i32 = 0;
|
||||
for (i < n - 1) {
|
||||
match (next_token(&tok)) {
|
||||
case let s: []u8 => { appendslice(&toks, s); };
|
||||
case done => { return toks; };
|
||||
};
|
||||
i += 1;
|
||||
};
|
||||
match (peek_token(&tok)) {
|
||||
case done => void;
|
||||
case let pk: []u8 => {
|
||||
let r: []u8 = remaining_tokens(&tok);
|
||||
appendslice(&toks, r);
|
||||
};
|
||||
};
|
||||
|
||||
// In-place reverse so callers see argv-order, matching Hare
|
||||
// (ref/hare/bytes/tokenize.ha:207). Element copy is field-wise
|
||||
// through `*[]u8` because `toks[i] = toks[j]` (full 24B slice
|
||||
// store) lands in the multi-word-store gap noted at
|
||||
// cmd/w6c/cgen.c:6515-6523.
|
||||
let a: i32 = 0;
|
||||
let b: i32 = toks.len - 1;
|
||||
for (a < b) {
|
||||
let pa: *[]u8 = &toks.ptr[a];
|
||||
let pb: *[]u8 = &toks.ptr[b];
|
||||
let tp: *u8 = pa.ptr;
|
||||
let tl: i32 = pa.len;
|
||||
let tc: i32 = pa.cap;
|
||||
pa.ptr = pb.ptr;
|
||||
pa.len = pb.len;
|
||||
pa.cap = pb.cap;
|
||||
pb.ptr = tp;
|
||||
pb.len = tl;
|
||||
pb.cap = tc;
|
||||
a += 1;
|
||||
b -= 1;
|
||||
};
|
||||
return toks;
|
||||
};
|
||||
|
||||
// split — full split of `in` on `delim` (no token cap). Mirrors
|
||||
// `splitn(in, delim, types::SIZE_MAX)`. ww uses `types.I32_MAX`
|
||||
// because the index type is i32 (lib/CLAUDE.md).
|
||||
//
|
||||
// ref/hare/bytes/tokenize.ha:225.
|
||||
export fn split(in: []u8, delim: []u8) [][]u8 = {
|
||||
return splitn(in, delim, types.I32_MAX);
|
||||
};
|
||||
|
||||
// encoding/utf8 — UTF-8 encode/decode. Hare port; see
|
||||
// ref/hare/encoding/utf8/{types,rune,encode,decode,decodetable}.ha.
|
||||
//
|
||||
|
||||
Reference in New Issue
Block a user