lib/bytes+test: port split family from Hare
This commit is contained in:
@@ -1233,6 +1233,137 @@ export fn remaining_tokens(s: *tokenizer) []u8 = {
|
||||
return s.in;
|
||||
};
|
||||
|
||||
// rt_ensure is the runtime slice-growth helper invoked by the
|
||||
// `append(s, v)` builtin. We bind it directly because the builtin's
|
||||
// expansion stores only 8 bytes of the new element (cgen emits a
|
||||
// single MOVQ), losing the .len/.cap fields of a []u8 element (24B).
|
||||
// Mirrors the same workaround in lib/shlex.shlex (appendstr, 16B) and
|
||||
// lib/getopt.getopt (appendoption, 24B); collapses in one go when the
|
||||
// append builtin learns to store the full element width.
|
||||
@symbol("rt_ensure") fn rtensure(s: *void, membsz: u64) void;
|
||||
|
||||
// appendslice — grow `*slice` by one and store `item` (24B). Mirror
|
||||
// of [[shlex.appendstr]] / [[getopt.appendoption]]. Bypasses the
|
||||
// `append` builtin's first-8B-only-store gap for a slice-element.
|
||||
fn appendslice(slice: *[][]u8, item: []u8) void = {
|
||||
let newlen: i32 = slice.len + 1;
|
||||
slice.len = newlen;
|
||||
rtensure(slice: *void, 24u64);
|
||||
let dst: *[]u8 = &slice.ptr[newlen - 1];
|
||||
dst.ptr = item.ptr;
|
||||
dst.len = item.len;
|
||||
dst.cap = item.cap;
|
||||
};
|
||||
|
||||
// splitn — split `in` on any byte in `delim`, returning up to `n`
|
||||
// tokens via forward iteration. The trailing slot (when more than
|
||||
// `n - 1` tokens exist) holds the unconsumed remainder.
|
||||
//
|
||||
// The caller frees the returned slice via
|
||||
// `os.free(r.ptr: *void, (r.cap: u64) * 24u64)`. Element bytes are
|
||||
// borrowed from `in`.
|
||||
//
|
||||
// Hare's `([][]u8 | nomem)` collapses to `[][]u8` here: ww os.alloc
|
||||
// has no recoverable failure path. Same precedent as
|
||||
// shlex.split / getopt.tryparse.
|
||||
//
|
||||
// ref/hare/bytes/tokenize.ha:156.
|
||||
export fn splitn(in: []u8, delim: []u8, n: i32) [][]u8 = {
|
||||
os.assert(delim.len > 0,
|
||||
"bytes.splitn must not be called with an empty delimiter");
|
||||
let toks: [][]u8;
|
||||
toks.ptr = nil: *[]u8;
|
||||
toks.len = 0;
|
||||
toks.cap = 0;
|
||||
let tok: tokenizer = tokenize(in, delim...);
|
||||
let i: i32 = 0;
|
||||
for (i < n - 1) {
|
||||
match (next_token(&tok)) {
|
||||
case let s: []u8 => { appendslice(&toks, s); };
|
||||
case done => { return toks; };
|
||||
};
|
||||
i += 1;
|
||||
};
|
||||
match (peek_token(&tok)) {
|
||||
case done => void;
|
||||
case let pk: []u8 => {
|
||||
let r: []u8 = remaining_tokens(&tok);
|
||||
appendslice(&toks, r);
|
||||
};
|
||||
};
|
||||
return toks;
|
||||
};
|
||||
|
||||
// rsplitn — reverse-direction counterpart to [[splitn]]: tokens are
|
||||
// collected from the end of `in`. The trailing slot holds the
|
||||
// unconsumed prefix (everything before the n-th-from-last delim hit).
|
||||
//
|
||||
// When the input has fewer than n tokens, the `done` short-circuit
|
||||
// returns toks UN-reversed (in last-token-first order). Mirrors Hare
|
||||
// at ref/hare/bytes/tokenize.ha:196-199 where the in-place reverse
|
||||
// step is gated behind the n-1 loop running to completion. Only the
|
||||
// "loop ran to completion AND peek saw a remainder" path applies the
|
||||
// reverse; both early-exit paths skip it.
|
||||
//
|
||||
// ref/hare/bytes/tokenize.ha:186.
|
||||
export fn rsplitn(in: []u8, delim: []u8, n: i32) [][]u8 = {
|
||||
os.assert(delim.len > 0,
|
||||
"bytes.rsplitn called with empty delimiter");
|
||||
let toks: [][]u8;
|
||||
toks.ptr = nil: *[]u8;
|
||||
toks.len = 0;
|
||||
toks.cap = 0;
|
||||
let tok: tokenizer = rtokenize(in, delim...);
|
||||
let i: i32 = 0;
|
||||
for (i < n - 1) {
|
||||
match (next_token(&tok)) {
|
||||
case let s: []u8 => { appendslice(&toks, s); };
|
||||
case done => { return toks; };
|
||||
};
|
||||
i += 1;
|
||||
};
|
||||
match (peek_token(&tok)) {
|
||||
case done => void;
|
||||
case let pk: []u8 => {
|
||||
let r: []u8 = remaining_tokens(&tok);
|
||||
appendslice(&toks, r);
|
||||
};
|
||||
};
|
||||
|
||||
// In-place reverse so callers see argv-order, matching Hare
|
||||
// (ref/hare/bytes/tokenize.ha:207). Element copy is field-wise
|
||||
// through `*[]u8` because `toks[i] = toks[j]` (full 24B slice
|
||||
// store) lands in the multi-word-store gap noted at
|
||||
// cmd/w6c/cgen.c:6515-6523.
|
||||
let a: i32 = 0;
|
||||
let b: i32 = toks.len - 1;
|
||||
for (a < b) {
|
||||
let pa: *[]u8 = &toks.ptr[a];
|
||||
let pb: *[]u8 = &toks.ptr[b];
|
||||
let tp: *u8 = pa.ptr;
|
||||
let tl: i32 = pa.len;
|
||||
let tc: i32 = pa.cap;
|
||||
pa.ptr = pb.ptr;
|
||||
pa.len = pb.len;
|
||||
pa.cap = pb.cap;
|
||||
pb.ptr = tp;
|
||||
pb.len = tl;
|
||||
pb.cap = tc;
|
||||
a += 1;
|
||||
b -= 1;
|
||||
};
|
||||
return toks;
|
||||
};
|
||||
|
||||
// split — full split of `in` on `delim` (no token cap). Mirrors
|
||||
// `splitn(in, delim, types::SIZE_MAX)`. ww uses `types.I32_MAX`
|
||||
// because the index type is i32 (lib/CLAUDE.md).
|
||||
//
|
||||
// ref/hare/bytes/tokenize.ha:225.
|
||||
export fn split(in: []u8, delim: []u8) [][]u8 = {
|
||||
return splitn(in, delim, types.I32_MAX);
|
||||
};
|
||||
|
||||
// encoding/utf8 — UTF-8 encode/decode. Hare port; see
|
||||
// ref/hare/encoding/utf8/{types,rune,encode,decode,decodetable}.ha.
|
||||
//
|
||||
|
||||
@@ -1233,6 +1233,137 @@ export fn remaining_tokens(s: *tokenizer) []u8 = {
|
||||
return s.in;
|
||||
};
|
||||
|
||||
// rt_ensure is the runtime slice-growth helper invoked by the
|
||||
// `append(s, v)` builtin. We bind it directly because the builtin's
|
||||
// expansion stores only 8 bytes of the new element (cgen emits a
|
||||
// single MOVQ), losing the .len/.cap fields of a []u8 element (24B).
|
||||
// Mirrors the same workaround in lib/shlex.shlex (appendstr, 16B) and
|
||||
// lib/getopt.getopt (appendoption, 24B); collapses in one go when the
|
||||
// append builtin learns to store the full element width.
|
||||
@symbol("rt_ensure") fn rtensure(s: *void, membsz: u64) void;
|
||||
|
||||
// appendslice — grow `*slice` by one and store `item` (24B). Mirror
|
||||
// of [[shlex.appendstr]] / [[getopt.appendoption]]. Bypasses the
|
||||
// `append` builtin's first-8B-only-store gap for a slice-element.
|
||||
fn appendslice(slice: *[][]u8, item: []u8) void = {
|
||||
let newlen: i32 = slice.len + 1;
|
||||
slice.len = newlen;
|
||||
rtensure(slice: *void, 24u64);
|
||||
let dst: *[]u8 = &slice.ptr[newlen - 1];
|
||||
dst.ptr = item.ptr;
|
||||
dst.len = item.len;
|
||||
dst.cap = item.cap;
|
||||
};
|
||||
|
||||
// splitn — split `in` on any byte in `delim`, returning up to `n`
|
||||
// tokens via forward iteration. The trailing slot (when more than
|
||||
// `n - 1` tokens exist) holds the unconsumed remainder.
|
||||
//
|
||||
// The caller frees the returned slice via
|
||||
// `os.free(r.ptr: *void, (r.cap: u64) * 24u64)`. Element bytes are
|
||||
// borrowed from `in`.
|
||||
//
|
||||
// Hare's `([][]u8 | nomem)` collapses to `[][]u8` here: ww os.alloc
|
||||
// has no recoverable failure path. Same precedent as
|
||||
// shlex.split / getopt.tryparse.
|
||||
//
|
||||
// ref/hare/bytes/tokenize.ha:156.
|
||||
export fn splitn(in: []u8, delim: []u8, n: i32) [][]u8 = {
|
||||
os.assert(delim.len > 0,
|
||||
"bytes.splitn must not be called with an empty delimiter");
|
||||
let toks: [][]u8;
|
||||
toks.ptr = nil: *[]u8;
|
||||
toks.len = 0;
|
||||
toks.cap = 0;
|
||||
let tok: tokenizer = tokenize(in, delim...);
|
||||
let i: i32 = 0;
|
||||
for (i < n - 1) {
|
||||
match (next_token(&tok)) {
|
||||
case let s: []u8 => { appendslice(&toks, s); };
|
||||
case done => { return toks; };
|
||||
};
|
||||
i += 1;
|
||||
};
|
||||
match (peek_token(&tok)) {
|
||||
case done => void;
|
||||
case let pk: []u8 => {
|
||||
let r: []u8 = remaining_tokens(&tok);
|
||||
appendslice(&toks, r);
|
||||
};
|
||||
};
|
||||
return toks;
|
||||
};
|
||||
|
||||
// rsplitn — reverse-direction counterpart to [[splitn]]: tokens are
|
||||
// collected from the end of `in`. The trailing slot holds the
|
||||
// unconsumed prefix (everything before the n-th-from-last delim hit).
|
||||
//
|
||||
// When the input has fewer than n tokens, the `done` short-circuit
|
||||
// returns toks UN-reversed (in last-token-first order). Mirrors Hare
|
||||
// at ref/hare/bytes/tokenize.ha:196-199 where the in-place reverse
|
||||
// step is gated behind the n-1 loop running to completion. Only the
|
||||
// "loop ran to completion AND peek saw a remainder" path applies the
|
||||
// reverse; both early-exit paths skip it.
|
||||
//
|
||||
// ref/hare/bytes/tokenize.ha:186.
|
||||
export fn rsplitn(in: []u8, delim: []u8, n: i32) [][]u8 = {
|
||||
os.assert(delim.len > 0,
|
||||
"bytes.rsplitn called with empty delimiter");
|
||||
let toks: [][]u8;
|
||||
toks.ptr = nil: *[]u8;
|
||||
toks.len = 0;
|
||||
toks.cap = 0;
|
||||
let tok: tokenizer = rtokenize(in, delim...);
|
||||
let i: i32 = 0;
|
||||
for (i < n - 1) {
|
||||
match (next_token(&tok)) {
|
||||
case let s: []u8 => { appendslice(&toks, s); };
|
||||
case done => { return toks; };
|
||||
};
|
||||
i += 1;
|
||||
};
|
||||
match (peek_token(&tok)) {
|
||||
case done => void;
|
||||
case let pk: []u8 => {
|
||||
let r: []u8 = remaining_tokens(&tok);
|
||||
appendslice(&toks, r);
|
||||
};
|
||||
};
|
||||
|
||||
// In-place reverse so callers see argv-order, matching Hare
|
||||
// (ref/hare/bytes/tokenize.ha:207). Element copy is field-wise
|
||||
// through `*[]u8` because `toks[i] = toks[j]` (full 24B slice
|
||||
// store) lands in the multi-word-store gap noted at
|
||||
// cmd/w6c/cgen.c:6515-6523.
|
||||
let a: i32 = 0;
|
||||
let b: i32 = toks.len - 1;
|
||||
for (a < b) {
|
||||
let pa: *[]u8 = &toks.ptr[a];
|
||||
let pb: *[]u8 = &toks.ptr[b];
|
||||
let tp: *u8 = pa.ptr;
|
||||
let tl: i32 = pa.len;
|
||||
let tc: i32 = pa.cap;
|
||||
pa.ptr = pb.ptr;
|
||||
pa.len = pb.len;
|
||||
pa.cap = pb.cap;
|
||||
pb.ptr = tp;
|
||||
pb.len = tl;
|
||||
pb.cap = tc;
|
||||
a += 1;
|
||||
b -= 1;
|
||||
};
|
||||
return toks;
|
||||
};
|
||||
|
||||
// split — full split of `in` on `delim` (no token cap). Mirrors
|
||||
// `splitn(in, delim, types::SIZE_MAX)`. ww uses `types.I32_MAX`
|
||||
// because the index type is i32 (lib/CLAUDE.md).
|
||||
//
|
||||
// ref/hare/bytes/tokenize.ha:225.
|
||||
export fn split(in: []u8, delim: []u8) [][]u8 = {
|
||||
return splitn(in, delim, types.I32_MAX);
|
||||
};
|
||||
|
||||
// encoding/utf8 — UTF-8 encode/decode. Hare port; see
|
||||
// ref/hare/encoding/utf8/{types,rune,encode,decode,decodetable}.ha.
|
||||
//
|
||||
|
||||
Reference in New Issue
Block a user