diff --git a/lib/ww/lex/lex.ww b/lib/ww/lex/lex.ww index 04e164a5..641f7c9d 100644 --- a/lib/ww/lex/lex.ww +++ b/lib/ww/lex/lex.ww @@ -18,6 +18,7 @@ package lex; import os; import ascii; import mem; +import strings; // isidstart / isidpart — identifier classification. Lexer-local // because the "alpha or '_' / alnum or '_'" set isn't part of Hare's @@ -447,7 +448,10 @@ fn lexnum(l: *lex, start: *pos, out: *tok) void = { }; let n: u64 = l.lpos - begin; - out.text = astrndup(l.a, l.src + begin, n); + let view: str; + view.ptr = l.src + begin; + view.len = n: i32; + out.text = strings.dup(view); if (isfloat) { out.kind = tkind.TK_FLOAT; @@ -526,7 +530,10 @@ fn lexnum(l: *lex, start: *pos, out: *tok) void = { }; }; if (isok) { - out.tsuffix = astrndup(l.a, p, sl); + let view: str; + view.ptr = p; + view.len = sl: i32; + out.tsuffix = strings.dup(view); } else { l.lpos = sb; }; @@ -551,7 +558,10 @@ fn lexident(l: *lex, start: *pos, out: *tok) void = { if (n == 1u64) { if (p[0] == 95u8) { out.kind = tkind.TK_UNDER; - out.text = astrndup(l.a, p, n); + let view: str; + view.ptr = p; + view.len = n: i32; + out.text = strings.dup(view); return; }; }; @@ -561,7 +571,10 @@ fn lexident(l: *lex, start: *pos, out: *tok) void = { } else { out.kind = tkind.TK_IDENT; }; - out.text = astrndup(l.a, p, n); + let view: str; + view.ptr = p; + view.len = n: i32; + out.text = strings.dup(view); }; fn lexstr(l: *lex, start: *pos, out: *tok) void = { @@ -576,7 +589,10 @@ fn lexstr(l: *lex, start: *pos, out: *tok) void = { out.file = start.file; out.line = start.line; out.col = start.col; - out.text = astrndup(l.a, "".ptr, 0u64); + let view: str; + view.ptr = "".ptr; + view.len = 0; + out.text = strings.dup(view); return; }; if (c == 34) { lget(l); break; }; @@ -621,7 +637,10 @@ fn lexrune(l: *lex, start: *pos, out: *tok) void = { out.file = start.file; out.line = start.line; out.col = start.col; - out.text = astrndup(l.a, "".ptr, 0u64); + let view: str; + view.ptr = "".ptr; + view.len = 0; + out.text = strings.dup(view); return; }; let ch: i32 = 0; @@ -637,7 +656,10 @@ fn lexrune(l: *lex, start: *pos, out: *tok) void = { out.file = start.file; out.line = start.line; out.col = start.col; - out.text = astrndup(l.a, "".ptr, 0u64); + let view: str; + view.ptr = "".ptr; + view.len = 0; + out.text = strings.dup(view); return; }; lget(l); @@ -791,5 +813,8 @@ export fn lexnext(l: *lex, out: *tok) void = { setposfrom(out, &start); let one: [1]u8; one[0] = c: u8; - out.text = astrndup(l.a, one.ptr, 1u64); + let view: str; + view.ptr = one.ptr; + view.len = 1; + out.text = strings.dup(view); }; diff --git a/selfhost/cmd/w6a/main.combined.ww b/selfhost/cmd/w6a/main.combined.ww index 045d7844..9be84fba 100644 --- a/selfhost/cmd/w6a/main.combined.ww +++ b/selfhost/cmd/w6a/main.combined.ww @@ -1331,6 +1331,7 @@ fn perr(a: *asm_, msg: str) void = { }; // dupstr — copy n bytes from p into a fresh heap str. +// astrndup until #11 (w6a types shadow) is fixed. fn dupstr(a: *arena, p: *u8, n: u64) str = { return astrndup(a, p, n); }; @@ -3120,6 +3121,7 @@ export fn main(argc: i32, argv: **u8) i32 = { let ar: *arena = newarena(); let s: asm_; let nlen: u64 = cstrlen(src); + // astrndup until #11 (w6a types shadow) is fixed. let fname: str = astrndup(ar, src, nlen); init(&s, ar, fname, buf, blen); diff --git a/selfhost/cmd/w6a/main.ww b/selfhost/cmd/w6a/main.ww index 39bc275d..35b6951b 100644 --- a/selfhost/cmd/w6a/main.ww +++ b/selfhost/cmd/w6a/main.ww @@ -114,6 +114,7 @@ export fn main(argc: i32, argv: **u8) i32 = { let ar: *arena = newarena(); let s: asm_; let nlen: u64 = cstrlen(src); + // astrndup until #11 (w6a types shadow) is fixed. let fname: str = astrndup(ar, src, nlen); init(&s, ar, fname, buf, blen); diff --git a/selfhost/cmd/w6a/parse.ww b/selfhost/cmd/w6a/parse.ww index 1c5915cb..1b1d05ca 100644 --- a/selfhost/cmd/w6a/parse.ww +++ b/selfhost/cmd/w6a/parse.ww @@ -179,6 +179,7 @@ fn perr(a: *asm_, msg: str) void = { }; // dupstr — copy n bytes from p into a fresh heap str. +// astrndup until #11 (w6a types shadow) is fixed. fn dupstr(a: *arena, p: *u8, n: u64) str = { return astrndup(a, p, n); }; diff --git a/selfhost/cmd/w6c/main.combined.ww b/selfhost/cmd/w6c/main.combined.ww index b079f03c..67494bd4 100644 --- a/selfhost/cmd/w6c/main.combined.ww +++ b/selfhost/cmd/w6c/main.combined.ww @@ -3730,6 +3730,7 @@ package lex; import os; import ascii; import mem; +import strings; // isidstart / isidpart — identifier classification. Lexer-local // because the "alpha or '_' / alnum or '_'" set isn't part of Hare's @@ -4159,7 +4160,10 @@ fn lexnum(l: *lex, start: *pos, out: *tok) void = { }; let n: u64 = l.lpos - begin; - out.text = astrndup(l.a, l.src + begin, n); + let view: str; + view.ptr = l.src + begin; + view.len = n: i32; + out.text = strings.dup(view); if (isfloat) { out.kind = tkind.TK_FLOAT; @@ -4238,7 +4242,10 @@ fn lexnum(l: *lex, start: *pos, out: *tok) void = { }; }; if (isok) { - out.tsuffix = astrndup(l.a, p, sl); + let view: str; + view.ptr = p; + view.len = sl: i32; + out.tsuffix = strings.dup(view); } else { l.lpos = sb; }; @@ -4263,7 +4270,10 @@ fn lexident(l: *lex, start: *pos, out: *tok) void = { if (n == 1u64) { if (p[0] == 95u8) { out.kind = tkind.TK_UNDER; - out.text = astrndup(l.a, p, n); + let view: str; + view.ptr = p; + view.len = n: i32; + out.text = strings.dup(view); return; }; }; @@ -4273,7 +4283,10 @@ fn lexident(l: *lex, start: *pos, out: *tok) void = { } else { out.kind = tkind.TK_IDENT; }; - out.text = astrndup(l.a, p, n); + let view: str; + view.ptr = p; + view.len = n: i32; + out.text = strings.dup(view); }; fn lexstr(l: *lex, start: *pos, out: *tok) void = { @@ -4288,7 +4301,10 @@ fn lexstr(l: *lex, start: *pos, out: *tok) void = { out.file = start.file; out.line = start.line; out.col = start.col; - out.text = astrndup(l.a, "".ptr, 0u64); + let view: str; + view.ptr = "".ptr; + view.len = 0; + out.text = strings.dup(view); return; }; if (c == 34) { lget(l); break; }; @@ -4333,7 +4349,10 @@ fn lexrune(l: *lex, start: *pos, out: *tok) void = { out.file = start.file; out.line = start.line; out.col = start.col; - out.text = astrndup(l.a, "".ptr, 0u64); + let view: str; + view.ptr = "".ptr; + view.len = 0; + out.text = strings.dup(view); return; }; let ch: i32 = 0; @@ -4349,7 +4368,10 @@ fn lexrune(l: *lex, start: *pos, out: *tok) void = { out.file = start.file; out.line = start.line; out.col = start.col; - out.text = astrndup(l.a, "".ptr, 0u64); + let view: str; + view.ptr = "".ptr; + view.len = 0; + out.text = strings.dup(view); return; }; lget(l); @@ -4503,7 +4525,10 @@ export fn lexnext(l: *lex, out: *tok) void = { setposfrom(out, &start); let one: [1]u8; one[0] = c: u8; - out.text = astrndup(l.a, one.ptr, 1u64); + let view: str; + view.ptr = one.ptr; + view.len = 1; + out.text = strings.dup(view); }; // lib/ww/ast.ww — port of cmd/wcc/ast.c (Node defs + printer). @@ -23367,6 +23392,7 @@ package main; import os; import rt; import mem; +import strings; import tok; import lex; import ast; @@ -23488,7 +23514,10 @@ export fn main(argc: i32, argv: **u8) i32 = { let ar: *arena = newarena(); let nlen: u64 = cstrlen(src); - let fname: str = astrndup(ar, src, nlen); + let view: str; + view.ptr = src; + view.len = nlen: i32; + let fname: str = strings.dup(view); let l: lex; lexinit(&l, ar, fname, buf, blen); diff --git a/selfhost/cmd/w6c/main.ww b/selfhost/cmd/w6c/main.ww index c3b2e330..39af3bad 100644 --- a/selfhost/cmd/w6c/main.ww +++ b/selfhost/cmd/w6c/main.ww @@ -15,6 +15,7 @@ package main; import os; import rt; import mem; +import strings; import tok; import lex; import ast; @@ -136,7 +137,10 @@ export fn main(argc: i32, argv: **u8) i32 = { let ar: *arena = newarena(); let nlen: u64 = cstrlen(src); - let fname: str = astrndup(ar, src, nlen); + let view: str; + view.ptr = src; + view.len = nlen: i32; + let fname: str = strings.dup(view); let l: lex; lexinit(&l, ar, fname, buf, blen); diff --git a/selfhost/cmd/w6l/dyn.ww b/selfhost/cmd/w6l/dyn.ww index 15e47f6a..33f8169d 100644 --- a/selfhost/cmd/w6l/dyn.ww +++ b/selfhost/cmd/w6l/dyn.ww @@ -13,6 +13,7 @@ package w6l; import os; import rt; import mem; +import strings; import sym; def ET_DYN_SO: u16 = 3u16; @@ -108,7 +109,10 @@ fn dcstrlen(p: *u8) u64 = { fn dcstrtostr(a: *arena, p: *u8) str = { let n: u64 = dcstrlen(p); - return astrndup(a, p, n); + let view: str; + view.ptr = p; + view.len = n: i32; + return strings.dup(view); }; // basename: scan for last '/' and return pointer past it. diff --git a/selfhost/cmd/w6l/main.combined.ww b/selfhost/cmd/w6l/main.combined.ww index 7ef734a7..66c82c7d 100644 --- a/selfhost/cmd/w6l/main.combined.ww +++ b/selfhost/cmd/w6l/main.combined.ww @@ -972,6 +972,1937 @@ export fn lookup(l: *lnk, name: str) *lsym = { return nil; }; +// types — integer limits. Mirrors Hare's types::limits (I8_MAX, …) +// platform-fixed for amd64. Numeric helpers live in lib/math, matching +// Hare's split between types::limits and math::. + +package types; + +def I8_MAX: i8 = 127; +def I16_MAX: i16 = 32767; +def I32_MAX: i32 = 2147483647; +def I64_MAX: i64 = 9223372036854775807; + +def I8_MIN: i8 = -128; +def I16_MIN: i16 = -32768; +def I32_MIN: i32 = -2147483648; +def I64_MIN: i64 = -9223372036854775808; + +def U8_MAX: u8 = 255; +def U16_MAX: u16 = 65535; +def U32_MAX: u32 = 4294967295; +def U64_MAX: u64 = 18446744073709551615; + +// bytes — slice operations over []u8. Mirrors Hare's bytes module +// (ref/hare/bytes/) for the in-tree subset: search/equality/prefix +// helpers used by lib/encoding, lib/bufio, lib/memio. +// +// Documented divergences from Hare: +// - index_slice / rindex_slice use naive O(n·m); Hare specialises +// 2/3/4-byte needles and falls back to two_way (Crochemore-Perrin) +// for longer (ref/hare/bytes/index.ha:61, ref/hare/bytes/two_way.ha). +// Correctness equivalent. +// - peek_token dispatches index/rindex by branching on `reverse` +// rather than a function-pointer `ifunc` (ref/hare/bytes/tokenize.ha:97). +// ww has no fn pointers in scope yet — same pattern as lib/strings +// `move`. Outwardly identical. +// - tokenize / rtokenize zero the `delim` field on the constructed +// tokenizer when `in` is empty, rather than mutating the variadic +// param before the struct write (ref/hare/bytes/tokenize.ha:26-28). +// Semantically identical; the variadic param is borrowed and +// captured-by-value into the struct, so mutating either side +// yields the same observable state. + +package bytes; + +import os; +import types; + +// done — iteration sentinel returned by next_token / peek_token at +// end-of-input. ref/hare/bytes/tokenize.ha uses the built-in `done` +// token; ww spells it per-package the same way lib/encoding/utf8 does +// (utf8.ww:36). Plain `void` (not `!void`): continuation signal. +export type done = void; + +// tokenizer — cursor over an input slice. Layout mirrors +// ref/hare/bytes/tokenize.ha:6-10. `p` is the cached peek-position; +// I64_MAX (forward) / I64_MIN (reverse) are the unprimed sentinels. +// p < 0 also identifies a reverse-direction iterator. +export type tokenizer = struct { + in: []u8, + delim: []u8, + p: i64, +}; + +// equal — true iff `a` and `b` have the same length and contents. +// ref/hare/bytes/equal.ha:9. +export fn equal(a: []u8, b: []u8) bool = { + if (a.len != b.len) { return false; }; + let i: i32 = 0; + for (i < a.len) { + if (a[i] != b[i]) { return false; }; + i += 1; + }; + return true; +}; + +// index — first offset of `needle` in `s`. u8 needle scans for the +// byte; []u8 needle scans for the substring. void if absent. +// ref/hare/bytes/index.ha:6. +export fn index(s: []u8, needle: (u8 | []u8)) (i32 | void) = { + match (needle) { + case let c: u8 => { + let i: i32 = 0; + for (i < s.len) { + if (s[i] == c) { return i; }; + i += 1; + }; + return; + }; + case let sub: []u8 => { + if (sub.len == 0) { return 0; }; + if (sub.len > s.len) { return; }; + let last: i32 = s.len - sub.len; + let i: i32 = 0; + for (i <= last) { + let j: i32 = 0; + let ok: bool = true; + for (j < sub.len) { + if (s[i + j] != sub[j]) { ok = false; j = sub.len; } + else { j += 1; }; + }; + if (ok) { return i; }; + i += 1; + }; + return; + }; + }; + return; +}; + +// rindex — last offset of `needle` in `s`. Empty []u8 needle returns +// s.len (ref/hare/bytes/index.ha:103 — Hare's loop yields r-0 at i=0). +// ref/hare/bytes/index.ha:86. +export fn rindex(s: []u8, needle: (u8 | []u8)) (i32 | void) = { + match (needle) { + case let c: u8 => { + let i: i32 = s.len - 1; + for (i >= 0) { + if (s[i] == c) { return i; }; + i -= 1; + }; + return; + }; + case let sub: []u8 => { + if (sub.len == 0) { return s.len; }; + if (sub.len > s.len) { return; }; + let i: i32 = s.len - sub.len; + for (i >= 0) { + let j: i32 = 0; + let ok: bool = true; + for (j < sub.len) { + if (s[i + j] != sub[j]) { ok = false; j = sub.len; } + else { j += 1; }; + }; + if (ok) { return i; }; + i -= 1; + }; + return; + }; + }; + return; +}; + +// contains — true iff any of `needles` (byte or sub-slice) appears in `s`. +// ref/hare/bytes/contains.ha:6. +export fn contains(s: []u8, needles: (u8 | []u8)...) bool = { + let i: i32 = 0; + for (i < needles.len) { + match (needles[i]) { + case let b: u8 => { + match (index(s, b)) { + case let bo: i32 => return true; + case void => void; + }; + }; + case let n: []u8 => { + match (index(s, n)) { + case let bo: i32 => return true; + case void => void; + }; + }; + }; + i += 1; + }; + return false; +}; + +// ltrim — borrowed view of `in` with leading bytes in `trim` stripped. +// `trim` must be non-empty. ref/hare/bytes/trim.ha:7. +export fn ltrim(in: []u8, trim: u8...) []u8 = { + os.assert(trim.len > 0, "bytes.ltrim called with empty trim set"); + let i: i32 = 0; + for (i < in.len && contains(trim, in[i])) { i += 1; }; + let r: []u8; + r.ptr = in.ptr + (i: u64); + r.len = in.len - i; + r.cap = r.len; + return r; +}; + +// rtrim — borrowed view of `in` with trailing bytes in `trim` stripped. +// `trim` must be non-empty. ref/hare/bytes/trim.ha:17. Hare's loop uses +// `size` underflow at i==0 to terminate; ww indices are signed i32, so +// the equivalent termination is spelled `i >= 0` explicitly. +export fn rtrim(in: []u8, trim: u8...) []u8 = { + os.assert(trim.len > 0, "bytes.rtrim called with empty trim set"); + let i: i32 = in.len - 1; + for (i >= 0 && contains(trim, in[i])) { i -= 1; }; + let r: []u8; + r.ptr = in.ptr; + r.len = i + 1; + r.cap = r.len; + return r; +}; + +// trim — borrowed view of `in` with both ends in `trim` stripped. +// ref/hare/bytes/trim.ha:27. +export fn trim(in: []u8, trim: u8...) []u8 = { + return ltrim(rtrim(in, trim...), trim...); +}; + +// hasprefix — true iff `s` starts with `pre`. +// ref/hare/bytes/contains.ha:21. +export fn hasprefix(s: []u8, pre: []u8) bool = { + if (pre.len > s.len) { return false; }; + let i: i32 = 0; + for (i < pre.len) { + if (s[i] != pre[i]) { return false; }; + i += 1; + }; + return true; +}; + +// hassuffix — true iff `s` ends with `suf`. +// ref/hare/bytes/contains.ha:35. +export fn hassuffix(s: []u8, suf: []u8) bool = { + if (suf.len > s.len) { return false; }; + let off: i32 = s.len - suf.len; + let i: i32 = 0; + for (i < suf.len) { + if (s[off + i] != suf[i]) { return false; }; + i += 1; + }; + return true; +}; + +// reverse — in-place reverse of `s`. ref/hare/bytes/reverse.ha:5. +export fn reverse(s: []u8) void = { + let i: i32 = 0; + let j: i32 = s.len - 1; + for (i < j) { + let t: u8 = s[i]; + s[i] = s[j]; + s[j] = t; + i += 1; + j -= 1; + }; +}; + +// zero — set every byte of `s` to 0. ref/hare/bytes/zero.ha:5. +export fn zero(s: []u8) void = { + let i: i32 = 0; + for (i < s.len) { + s[i] = 0u8; + i += 1; + }; +}; + +// tokenize — iterator yielding tokens from `in` separated by any byte +// in `delim`. Leading / trailing / adjacent delims yield empty tokens. +// `delim` is borrowed; caller keeps it valid for the tokenizer's +// lifetime. ref/hare/bytes/tokenize.ha:22. +export fn tokenize(in: []u8, delim: u8...) tokenizer = { + os.assert(delim.len > 0, "bytes.tokenize called with empty slice"); + os.assert((in.len: i64) < types.I64_MAX, + "bytes.tokenize: input length exceeds I64_MAX"); + let t: tokenizer; + t.in = in; + t.delim = delim; + if (in.len == 0) { + t.delim.len = 0; + t.delim.cap = 0; + }; + t.p = types.I64_MAX; + return t; +}; + +// rtokenize — reverse-direction tokenize. First next_token yields the +// last token, last next_token yields the first. ref/hare/bytes/tokenize.ha:40. +export fn rtokenize(in: []u8, delim: u8...) tokenizer = { + os.assert(delim.len > 0, "bytes.rtokenize called with empty slice"); + os.assert((in.len: i64) < types.I64_MAX, + "bytes.rtokenize: input length exceeds I64_MAX"); + let t: tokenizer; + t.in = in; + t.delim = delim; + if (in.len == 0) { + t.delim.len = 0; + t.delim.cap = 0; + }; + t.p = types.I64_MIN; + return t; +}; + +// peek_token — next token without advancing the cursor. Returns done +// once `s.delim` has been zeroed by a prior past-end next_token. +// ref/hare/bytes/tokenize.ha:91. +export fn peek_token(s: *tokenizer) ([]u8 | done) = { + if (s.delim.len == 0) { + let d: done; return d; + }; + + let reverse: bool = s.p < 0i64; + let known: bool = false; + if (reverse) { + if (s.p != types.I64_MIN) { known = true; }; + } else { + if (s.p != types.I64_MAX) { known = true; }; + }; + if (!known) { + let i: i64 = types.I64_MAX; + if (reverse) { i = types.I64_MIN; }; + let dlen: i64 = 0i64; + let slen: i64 = s.in.len: i64; + + let k: i32 = 0; + for (k < s.delim.len) { + let d: u8 = s.delim[k]; + let ix_found: bool = false; + let ix_val: i32 = 0; + if (reverse) { + match (rindex(s.in, d)) { + case let v: i32 => { ix_found = true; ix_val = v; }; + case void => void; + }; + } else { + match (index(s.in, d)) { + case let v: i32 => { ix_found = true; ix_val = v; }; + case void => void; + }; + }; + if (ix_found) { + if (!reverse) { + if ((ix_val: i64) < i) { i = ix_val: i64; dlen = 1i64; }; + } else { + if ((ix_val: i64) > i) { i = ix_val: i64; dlen = 1i64; }; + }; + } else { + if (!reverse) { + if (slen < i) { i = slen; }; + } else { + if (0i64 > i) { i = 0i64; }; + }; + }; + k += 1; + }; + + if (reverse) { + if (i == slen) { + s.p = -(slen + 1i64); + } else { + s.p = i + dlen - slen - 1i64; + }; + } else { + s.p = i; + }; + }; + + let r: []u8; + if (reverse) { + let start: i32 = (s.in.len: i64 + s.p + 1i64): i32; + r.ptr = s.in.ptr + (start: u64); + r.len = s.in.len - start; + r.cap = r.len; + } else { + let end: i32 = s.p: i32; + r.ptr = s.in.ptr; + r.len = end; + r.cap = end; + }; + return r; +}; + +// next_token — current token, then advance past it and the delim. +// Once the input is exhausted, returns done and zeros `s.delim` so +// subsequent peeks short-circuit. ref/hare/bytes/tokenize.ha:59. +export fn next_token(s: *tokenizer) ([]u8 | done) = { + let b: []u8; + match (peek_token(s)) { + case let v: []u8 => { b = v; }; + case done => { let d: done; return d; }; + }; + + let slen: i64 = s.in.len: i64; + let reverse: bool = s.p < 0i64; + if (reverse) { + if (slen + s.p + 1i64 == 0i64) { + s.delim.len = 0; + s.delim.cap = 0; + s.in.len = 0; + s.in.cap = 0; + } else { + let end: i32 = (slen + s.p + 1i64 - 1i64): i32; + s.in.len = end; + s.in.cap = end; + }; + s.p = types.I64_MIN; + } else { + if (s.p == slen) { + s.delim.len = 0; + s.delim.cap = 0; + s.in.len = 0; + s.in.cap = 0; + } else { + let adv: u64 = (s.p: u64) + 1u64; + let adv_i32: i32 = (s.p: i32) + 1; + s.in.ptr = s.in.ptr + adv; + s.in.len = s.in.len - adv_i32; + s.in.cap = s.in.cap - adv_i32; + }; + s.p = types.I64_MAX; + }; + return b; +}; + +// remaining_tokens — the unconsumed portion of `s.in`. Read-only view. +// ref/hare/bytes/tokenize.ha:145. +export fn remaining_tokens(s: *tokenizer) []u8 = { + return s.in; +}; + +// rt_ensure is the runtime slice-growth helper invoked by the +// `append(s, v)` builtin. We bind it directly because the builtin's +// expansion stores only 8 bytes of the new element (cgen emits a +// single MOVQ), losing the .len/.cap fields of a []u8 element (24B). +// Mirrors the same workaround in lib/shlex.shlex (appendstr, 16B) and +// lib/getopt.getopt (appendoption, 24B); collapses in one go when the +// append builtin learns to store the full element width. +@symbol("rt_ensure") fn rtensure(s: *void, membsz: u64) void; + +// appendslice — grow `*slice` by one and store `item` (24B). Mirror +// of [[shlex.appendstr]] / [[getopt.appendoption]]. Bypasses the +// `append` builtin's first-8B-only-store gap for a slice-element. +fn appendslice(slice: *[][]u8, item: []u8) void = { + let newlen: i32 = slice.len + 1; + slice.len = newlen; + rtensure(slice: *void, 24u64); + let dst: *[]u8 = &slice.ptr[newlen - 1]; + dst.ptr = item.ptr; + dst.len = item.len; + dst.cap = item.cap; +}; + +// splitn — split `in` on any byte in `delim`, returning up to `n` +// tokens via forward iteration. The trailing slot (when more than +// `n - 1` tokens exist) holds the unconsumed remainder. +// +// The caller frees the returned slice via +// `os.free(r.ptr: *void, (r.cap: u64) * 24u64)`. Element bytes are +// borrowed from `in`. +// +// Hare's `([][]u8 | nomem)` collapses to `[][]u8` here: ww os.alloc +// has no recoverable failure path. Same precedent as +// shlex.split / getopt.tryparse. +// +// ref/hare/bytes/tokenize.ha:156. +export fn splitn(in: []u8, delim: []u8, n: i32) [][]u8 = { + os.assert(delim.len > 0, + "bytes.splitn must not be called with an empty delimiter"); + let toks: [][]u8; + toks.ptr = nil: *[]u8; + toks.len = 0; + toks.cap = 0; + let tok: tokenizer = tokenize(in, delim...); + let i: i32 = 0; + for (i < n - 1) { + match (next_token(&tok)) { + case let s: []u8 => { appendslice(&toks, s); }; + case done => { return toks; }; + }; + i += 1; + }; + match (peek_token(&tok)) { + case done => void; + case let pk: []u8 => { + let r: []u8 = remaining_tokens(&tok); + appendslice(&toks, r); + }; + }; + return toks; +}; + +// rsplitn — reverse-direction counterpart to [[splitn]]: tokens are +// collected from the end of `in`. The trailing slot holds the +// unconsumed prefix (everything before the n-th-from-last delim hit). +// +// When the input has fewer than n tokens, the `done` short-circuit +// returns toks UN-reversed (in last-token-first order). Mirrors Hare +// at ref/hare/bytes/tokenize.ha:196-199 where the in-place reverse +// step is gated behind the n-1 loop running to completion. Only the +// "loop ran to completion AND peek saw a remainder" path applies the +// reverse; both early-exit paths skip it. +// +// ref/hare/bytes/tokenize.ha:186. +export fn rsplitn(in: []u8, delim: []u8, n: i32) [][]u8 = { + os.assert(delim.len > 0, + "bytes.rsplitn called with empty delimiter"); + let toks: [][]u8; + toks.ptr = nil: *[]u8; + toks.len = 0; + toks.cap = 0; + let tok: tokenizer = rtokenize(in, delim...); + let i: i32 = 0; + for (i < n - 1) { + match (next_token(&tok)) { + case let s: []u8 => { appendslice(&toks, s); }; + case done => { return toks; }; + }; + i += 1; + }; + match (peek_token(&tok)) { + case done => void; + case let pk: []u8 => { + let r: []u8 = remaining_tokens(&tok); + appendslice(&toks, r); + }; + }; + + // In-place reverse so callers see argv-order, matching Hare + // (ref/hare/bytes/tokenize.ha:207). Element copy is field-wise + // through `*[]u8` because `toks[i] = toks[j]` (full 24B slice + // store) lands in the multi-word-store gap noted at + // cmd/w6c/cgen.c:6515-6523. + let a: i32 = 0; + let b: i32 = toks.len - 1; + for (a < b) { + let pa: *[]u8 = &toks.ptr[a]; + let pb: *[]u8 = &toks.ptr[b]; + let tp: *u8 = pa.ptr; + let tl: i32 = pa.len; + let tc: i32 = pa.cap; + pa.ptr = pb.ptr; + pa.len = pb.len; + pa.cap = pb.cap; + pb.ptr = tp; + pb.len = tl; + pb.cap = tc; + a += 1; + b -= 1; + }; + return toks; +}; + +// split — full split of `in` on `delim` (no token cap). Mirrors +// `splitn(in, delim, types::SIZE_MAX)`. ww uses `types.I32_MAX` +// because the index type is i32 (lib/CLAUDE.md). +// +// ref/hare/bytes/tokenize.ha:225. +export fn split(in: []u8, delim: []u8) [][]u8 = { + return splitn(in, delim, types.I32_MAX); +}; + +// encoding/utf8 — UTF-8 encode/decode. Hare port; see +// ref/hare/encoding/utf8/{types,rune,encode,decode,decodetable}.ha. +// +// The decoder is Hoehrmann's branchless DFA, originally published +// at . Hare's +// ref/hare/encoding/utf8/decodetable.ha:4 restructures Hoehrmann's +// flat table to 2D `[8][256]i8`; we flatten back to 1D `[2048]i8` +// because ww cgen does not yet ship 2D arrays (task #20). +// +// Surface deviation from ref/hare/encoding/utf8: +// +// - `encoderune` takes a caller-supplied `out: []u8` and returns +// the byte count. Hare returns a slice into a `static let buf`; +// the caller-buffer form mirrors lib/encoding/hex.encode and +// skips the static-buffer/slice-return pair. +// +// Deferred (no in-tree caller, follow-up tasks): `appendrune`, +// `strencode`, `strdecode`. Hare's string-iteration surface +// (`strings::iterator`/`strings::next` — ref/hare/strings/iter.ha) +// lives under lib/strings, not here. + +// ref/hare/encoding/utf8/types.ha:6 — incomplete trailing sequence. +// Plain `void` (not `!void`): a truncated tail is a control-flow +// signal, not an error caller can ignore. +package utf8; + +export type more = void; + +// ref/hare/encoding/utf8/types.ha:9 — invalid UTF-8 sequence. +export type invalid = !void; + +// ref/hare/encoding/utf8/types.ha:12 — fixed message; `invalid` carries +// no payload, so the rendering is constant. +export fn strerror(err: invalid) str = { + return "Invalid UTF-8"; +}; + +// `done` is not a built-in singleton in ww (Hare ships it as part of +// the type system). Plain `void` (not `!void`): end-of-input is a +// continuation signal, not an error. lib/io spells its EOF the same +// way (lib/io/io.ww:8-11). +export type done = void; + +// ref/hare/encoding/utf8/decodetable.ha:4 — Hoehrmann's UTF-8 DFA, +// flat 1D `[2048]i8`. Layout: dfa[state*256 + byte] gives the next +// state (>0), the accept transition (0 — emit rune), or invalid (-1). +// Values match ref/hare/encoding/utf8/decodetable.ha verbatim. +let dfa: [2048]i8 = [ + // state 0 — initial byte: ASCII accepts (0), continuation/illegal + // byte rejects (-1), legal multibyte start emits a state. + 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, + 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, + 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, + 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, + 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, + 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, + 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, + 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, + -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, + -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, + -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, + -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, + -1i8, -1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, + 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, + 3i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 4i8, 2i8, 2i8, + 5i8, 6i8, 6i8, 6i8, 7i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, + + // state 1 — expecting one continuation byte (0x80..0xBF). + -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, + -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, + -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, + -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, + -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, + -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, + -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, + -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, + 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, + 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, + 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, + 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, + -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, + -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, + -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, + -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, + + // state 2 — expecting one continuation byte (full 0x80..0xBF range). + -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, + -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, + -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, + -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, + -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, + -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, + -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, + -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, + 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, + 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, + 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, + 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, + -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, + -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, + -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, + -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, + + // state 3 — first byte was 0xE0; continuation byte must be 0xA0..0xBF + // (rejects overlong 3-byte encodings). + -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, + -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, + -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, + -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, + -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, + -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, + -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, + -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, + -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, + -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, + 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, + 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, + -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, + -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, + -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, + -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, + + // state 4 — first byte was 0xED; continuation byte must be 0x80..0x9F + // (rejects UTF-16 surrogate codepoints U+D800..U+DFFF). + -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, + -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, + -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, + -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, + -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, + -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, + -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, + -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, + 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, + 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, + -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, + -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, + -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, + -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, + -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, + -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, + + // state 5 — first byte was 0xF0; continuation byte must be 0x90..0xBF + // (rejects overlong 4-byte encodings). + -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, + -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, + -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, + -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, + -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, + -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, + -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, + -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, + -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, + 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, + 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, + 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, + -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, + -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, + -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, + -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, + + // state 6 — middle continuation byte of a 4-byte sequence (0x80..0xBF). + -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, + -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, + -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, + -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, + -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, + -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, + -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, + -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, + 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, + 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, + 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, + 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, + -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, + -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, + -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, + -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, + + // state 7 — first byte was 0xF4; continuation byte must be 0x80..0x8F + // (rejects codepoints above U+10FFFF). + -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, + -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, + -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, + -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, + -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, + -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, + -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, + -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, + 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, + -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, + -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, + -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, + -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, + -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, + -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, + -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, +]; + +// ref/hare/encoding/utf8/decode.ha:17 — payload-bit masks. Hare's +// [2][8]u8 flattened to 1D [16]u8; row 0 (offsets 0..7) is the +// continuation-byte mask (always 0x3F), row 1 (offsets 8..15) is the +// initial-byte payload mask indexed by the transition class. +let masks: [16]u8 = [ + 0x3fu8, 0x3fu8, 0x3fu8, 0x3fu8, 0x3fu8, 0x3fu8, 0x3fu8, 0x3fu8, + 0x7fu8, 0x1fu8, 0x0fu8, 0x0fu8, 0x0fu8, 0x07u8, 0x07u8, 0x07u8, +]; + +// ref/hare/encoding/utf8/decode.ha:6 — incremental decoder state. +export type decoder = struct { + offs: i32, + src: []u8, +}; + +// ref/hare/encoding/utf8/decode.ha:12. +export fn decode(src: []u8) decoder = { + let d: decoder; + d.src = src; + d.offs = 0; + return d; +}; + +// ref/hare/encoding/utf8/decode.ha:27. Returns the next rune from a +// decoder, `done` at end-of-input, `more` on truncated trailing +// sequence, `invalid` on malformed input (overlong, surrogate, +// out-of-range, bad continuation). +// +// Algorithm is verbatim Hoehrmann (see file header). One structural +// rewrite: Hare encodes the "initial vs continuation byte" decision +// as the branchless `(state - 1): uint >> 31`, which assumes a 32-bit +// uint. ww's uint is 64-bit (cmd/wcc/type.c:58), so the shift answer +// would be 0x1_ffff_ffff rather than 1. We spell the same predicate +// with an explicit conditional. +export fn next(d: *decoder) (rune | done | more | invalid) = { + if (d.offs == d.src.len) { + let dn: done; return dn; + }; + let nx: i32 = 0; + let state: i32 = 0; + let r: u32 = 0u32; + for (d.offs < d.src.len) { + let b: u8 = d.src[d.offs]; + let bi: i32 = b: i32; + let row: i32 = state * 256 + bi; + let cell: i8 = dfa[row]; + nx = cell: i32; + let mi: i32 = 0; + if (state == 0) { mi = 1; }; + let m: u8 = masks[mi * 8 + (nx & 7)]; + r = (r << 6u32) | ((b & m): u32); + if (nx <= 0) { + d.offs += 1; + if (nx == 0) { return r: rune; }; + let e: invalid; return e; + }; + state = nx; + d.offs += 1; + }; + let mr: more; return mr; +}; + +// ref/hare/encoding/utf8/decode.ha:207. Strict whole-input check. +// The hot path: tight DFA loop, no rune assembly. Bails the moment +// the table returns -1 so malformed inputs don't pay for the rest +// of the buffer. +export fn validate(src: []u8) (void | invalid) = { + let state: i32 = 0; + let i: i32 = 0; + for (i < src.len) { + if (state < 0) { break; }; + let bi: i32 = src[i]: i32; + let cell: i8 = dfa[state * 256 + bi]; + state = cell: i32; + i += 1; + }; + if (state == 0) { return; }; + let e: invalid; return e; +}; + +// ref/hare/encoding/utf8/rune.ha:5. Encoded byte length of `r` as +// UTF-8. Callers in ww use this to size the buffer they hand to +// [[encoderune]]; values >0x10FFFF or negative are not legal Unicode +// codepoints and Hare aborts on them in `encoderune` itself, so we +// keep `runesz` infallible (matches Hare). +export fn runesz(r: rune) i32 = { + let ch: u32 = r: u32; + if (ch < 128u32) { return 1; }; + if (ch < 2048u32) { return 2; }; + if (ch < 65536u32) { return 3; }; + return 4; +}; + +// ref/hare/encoding/utf8/rune.ha:15. Expected byte length of the +// codepoint that starts with `c`, or `invalid` if `c` cannot start +// a legal UTF-8 sequence. Constants written in decimal because ww +// doesn't accept Hare's `0b1000_0000` binary syntax: 0x80=128, +// 0xC2=194, 0xE0=224, 0xF0=240, 0xF8=248. +export fn utf8sz(c: u8) (i32 | invalid) = { + if (c < 128u8) { return 1; }; + if (c < 194u8) { let e: invalid; return e; }; + if (c >= 248u8) { let e: invalid; return e; }; + if (c < 224u8) { return 2; }; + if (c < 240u8) { return 3; }; + return 4; +}; + +// ref/hare/encoding/utf8/encode.ha:7. Encode `r` into `out` (caller- +// supplied; must hold at least [[runesz]](r) bytes) and return the +// byte count. ABORT if `r` is a UTF-16 surrogate or above U+10FFFF — +// same precondition Hare asserts at ref/hare/encoding/utf8/encode.ha:9. +// +// Surface deviation: Hare returns `[]u8` (slice into a static buf). +// ww uses the caller-buffer form (matches lib/encoding/hex.encode); +// caller can reuse a [4]u8 stack scratch across encodes. +export fn encoderune(out: []u8, r: rune) i32 = { + let ch: u32 = r: u32; + if (ch >= 0xD800u32) { + if (ch <= 0xDFFFu32) { + abort("utf8.encoderune: surrogate codepoint"); + }; + }; + if (ch > 0x10FFFFu32) { + abort("utf8.encoderune: codepoint > U+10FFFF"); + }; + + let n: i32 = 0; + let first: u8 = 0u8; + if (ch < 0x80u32) { + first = 0u8; n = 1; + } else if (ch < 0x800u32) { + first = 0xC0u8; n = 2; + } else if (ch < 0x10000u32) { + first = 0xE0u8; n = 3; + } else { + first = 0xF0u8; n = 4; + }; + + let v: u32 = ch; + let i: i32 = n - 1; + for (i > 0) { + out[i] = ((v: u8) & 0x3Fu8) | 0x80u8; + v = v >> 6u32; + i -= 1; + }; + out[0] = (v: u8) | first; + return n; +}; + +// ref/hare/encoding/utf8/decode.ha:52. Walks back from `d.offs` to a +// byte that could start a codepoint (state-0 dfa cell != -1), re-decodes +// forward from there, and confirms the forward decode lands back at the +// original offset. Returns `done` at start-of-input; `invalid` if no +// initial byte appears within 4 steps (no legal UTF-8 codepoint exceeds +// 4 bytes), if the forward decode returns `more`/`invalid`, or if it +// lands at a different offset than expected. Returns `more` when the +// walk reaches byte 0 without finding any initial byte. +// +// Hare's `for (d.offs < len(d.src); d.offs -= 1)` relies on size_t +// wrap-around to exit when offs underflows past 0; ww's offs is i32, +// so we spell the same exit as `d.offs >= 0`. Hare's `defer d.offs = t` +// is inlined in each match arm — ww has no defer. +export fn prev(d: *decoder) (rune | done | more | invalid) = { + if (d.offs == 0) { + let dn: done; return dn; + }; + let n: i32 = d.offs; + d.offs -= 1; + for (d.offs >= 0) { + let b: u8 = d.src[d.offs]; + let bi: i32 = b: i32; + let cell: i8 = dfa[bi]; + if (cell: i32 != -1) { + let t: i32 = d.offs; + match (next(d)) { + case let r: rune => { + let landed: i32 = d.offs; + d.offs = t; + if (landed != n) { + let e: invalid; return e; + }; + return r; + }; + case let dn: done => { + d.offs = t; + let e: invalid; return e; + }; + case let m: more => { + d.offs = t; + let e: invalid; return e; + }; + case let e: invalid => { + d.offs = t; + let e2: invalid; return e2; + }; + }; + }; + if (n - d.offs == 4) { + let e: invalid; return e; + }; + d.offs -= 1; + }; + let mr: more; return mr; +}; + +// ref/hare/encoding/utf8/decode.ha:74. Borrowed view of the bytes from +// the decoder's current position to the end of its source. +export fn remaining(d: *decoder) []u8 = { + let r: []u8; + r.ptr = d.src.ptr + (d.offs: u64); + r.len = d.src.len - d.offs; + r.cap = d.src.len - d.offs; + return r; +}; + +// ref/hare/encoding/utf8/decode.ha:80. Borrowed view of the bytes +// between two decoders' positions. Precondition (Hare asserts both): +// the decoders share the same source, and `begin.offs <= end.offs`. +export fn slice(begin: *decoder, end: *decoder) []u8 = { + if (begin.src.ptr != end.src.ptr) { + abort("utf8.slice: decoders from different sources"); + }; + if (begin.offs > end.offs) { + abort("utf8.slice: begin past end"); + }; + let r: []u8; + r.ptr = begin.src.ptr + (begin.offs: u64); + r.len = end.offs - begin.offs; + r.cap = end.offs - begin.offs; + return r; +}; + +// ref/hare/encoding/utf8/decode.ha:203. Byte position of the decoder +// in its source. +export fn position(d: *decoder) i32 = { + return d.offs; +}; + + +// strings — operations over str ({ptr,len}). Hare port; see +// ref/hare/strings/. +// +// Documented divergences from Hare: +// +// - `byteindex` / `rbyteindex` rune arms encode via +// `utf8.encoderune`; the legacy impls scanned for `r: u8` (an +// undocumented ASCII-only restriction that silently dropped +// to the wrong byte for U+80..U+7FF and higher). +// - `dup(s: str) str` — Hare returns `(str | nomem)`. ww's +// `os.alloc` aborts on OOM (no `nomem` type), so we return plain +// `str`. Empty input returns `{nil, 0}`; Hare returns the static +// empty string — same observable result. +// - `iterator` is flattened (`offs`, `src`, `reverse` fields). +// Hare uses anonymous-embedded `utf8::decoder` +// (ref/hare/strings/iter.ha:6-9); ww has no anonymous-embed +// syntax, so `next`/`prev`/`slice` copy `offs`/`src` into a +// local `utf8.decoder` for the call (and `next`/`prev` write +// `offs` back). +// - Hare's private `move()` helper dispatches on a `forward: bool` +// using a function-pointer `let fun = if (forward) &utf8::next +// else &utf8::prev`. ww has no fn-pointers in scope yet, so the +// dispatch is a branch on `forward` selecting the call site. + +package strings; + +import bytes; +import encoding.utf8; +import os; +import rt; +import types; + +// toutf8 — borrowed []u8 view of `s`. ref/hare/strings/utf8.ha:29. +// `cap` equals `len`; the slice does not own a separate allocation. +export fn toutf8(s: str) []u8 = { + let r: []u8; + r.ptr = s.ptr; + r.len = s.len; + r.cap = s.len; + return r; +}; + +// frombytes — borrowed str view of `in`. Pure reinterpret per +// CLAUDE.md rule 9 carve-out; ref/hare/strings/utf8.ha:10. +export fn frombytes(in: []u8) str = { + let r: str; + r.ptr = in.ptr; + r.len = in.len; + return r; +}; + +// compare — three-way bytewise codepoint-order comparison. Return is +// a sign (neg/zero/pos), not an index, so it tracks Hare's `int` +// rather than the str-index i32 (#8). ref/hare/strings/compare.ha:12. +export fn compare(a: str, b: str) int = { + let n: i32 = a.len; + if (b.len < n) { n = b.len; }; + let i: i32 = 0; + for (i < n) { + if (a[i] != b[i]) { return (a[i]: int) - (b[i]: int); }; + i += 1; + }; + return (a.len: int) - (b.len: int); +}; + +// dup — allocate a fresh copy of `s`. Caller releases with +// `os.free(r.ptr, r.len: u64)`. ref/hare/strings/dup.ha:7. +export fn dup(s: str) str = { + let r: str; + r.ptr = nil; + r.len = 0; + if (s.len == 0) { return r; }; + let buf: []u8 = alloc([], s.len: u64)!; + let i: i32 = 0; + for (i < s.len) { buf[i] = s[i]; i += 1; }; + buf.len = s.len; + return frombytes(buf); +}; + +// dupall — fresh `[]str` whose elements are independent copies of +// `s`'s elements. Caller releases via [[freeall]]. +// ref/hare/strings/dup.ha:26 (#6). +// +// Hare gates the per-element dup behind `?` and rolls back via +// `defer if (!ok) freeall(newsl)`. ww has no `defer if`; more +// importantly, ww's [[dup]] is still unchecked (returns plain `str`, +// aborts via os.alloc on OOM — see top-of-file divergence note), +// so the only nomem propagation point is the initial slice alloc. +// With no inner failure path, the rollback is structurally a no-op +// and is omitted; it returns once dup graduates to `(str | nomem)` +// (#46). The pre-allocated slice has `cap == s.len`, so appendstr's +// rt_ensure call never reaches the grow branch. +// +// Empty input bypasses the alloc: rt_malloc(0) is an mmap of 0 bytes +// which returns -EINVAL, and the alloc-slice `?` shortcut routes +// that through nomem — Hare's heap allocator hands back a sentinel +// instead (#47). Return `{nil, 0, 0}` directly so callers get the +// Hare-observable shape (len==0, freeall is a no-op via cap==0). +export fn dupall(s: []str) ([]str | nomem) = { + if (s.len == 0) { + let r: []str; + r.ptr = nil: *str; + r.len = 0; + r.cap = 0; + return r; + }; + let newsl: []str = alloc([], s.len)?; + let i: i32 = 0; + for (i < s.len) { + appendstr(&newsl, dup(s[i])); + i += 1; + }; + return newsl; +}; + +// freeall — release each element + the slice header. The natural +// disposer for any `[]str` of dup'd elements (e.g. shlex.split). +// ref/hare/strings/dup.ha:38. +// +// Empty elements (`{nil, 0}` from a zero-length dup) are skipped: +// os.free on a nil pointer at len 0 tickles the rt_free guard. The +// slice header itself is freed at `cap * size(str)` — the literal +// would drift under #1's str-layout bump, so route through the +// typ.ww SSoT. A never-grown slice (cap == 0) skips the header free. +export fn freeall(s: []str) void = { + let i: i32 = 0; + for (i < s.len) { + if (s[i].len > 0) { + os.free(s[i].ptr: *void, s[i].len: u64); + }; + i += 1; + }; + if (s.cap > 0) { + os.free(s.ptr: *void, (s.cap: u64) * size(str): u64); + }; +}; + +// concat — fresh allocation containing each element of `strs` in +// order. Caller releases with `os.free(r.ptr, r.len: u64)`. +// ref/hare/strings/concat.ha:5. Hare's `nomem` return is dropped: +// `os.alloc` aborts on OOM. +export fn concat(strs: str...) str = { + let total: i32 = 0; + let i: i32 = 0; + for (i < strs.len) { total += strs[i].len; i += 1; }; + let r: str; + r.ptr = nil; + r.len = 0; + if (total == 0) { return r; }; + let buf: []u8 = alloc([], total: u64)!; + let off: i32 = 0; + i = 0; + for (i < strs.len) { + let j: i32 = 0; + for (j < strs[i].len) { + buf[off + j] = strs[i][j]; + j += 1; + }; + off += strs[i].len; + i += 1; + }; + buf.len = total; + return frombytes(buf); +}; + +// join — fresh allocation with `delim` placed between each element of +// `strs`. Caller releases with `os.free(r.ptr, r.len: u64)`. +// ref/hare/strings/concat.ha:46. Hare's `nomem` return is dropped: +// `os.alloc` aborts on OOM. +export fn join(delim: str, strs: str...) str = { + let total: i32 = 0; + let i: i32 = 0; + for (i < strs.len) { + total += strs[i].len; + if (i + 1 < strs.len) { total += delim.len; }; + i += 1; + }; + let r: str; + r.ptr = nil; + r.len = 0; + if (total == 0) { return r; }; + let buf: []u8 = alloc([], total: u64)!; + let off: i32 = 0; + i = 0; + for (i < strs.len) { + let j: i32 = 0; + for (j < strs[i].len) { + buf[off + j] = strs[i][j]; + j += 1; + }; + off += strs[i].len; + if (i + 1 < strs.len) { + j = 0; + for (j < delim.len) { + buf[off + j] = delim[j]; + j += 1; + }; + off += delim.len; + }; + i += 1; + }; + buf.len = total; + return frombytes(buf); +}; + +// utf8bytelenbounded — walk `it` forward `end` runes and return the +// resulting byte offset. ref/hare/strings/sub.ha:10. Aborts on +// short input per Hare's contract for the rune-wise [[sub]]. +fn utf8bytelenbounded(it: *iterator, end: i32) i32 = { + let i: i32 = 0; + for (i < end) { + match (next(it)) { + case let r: rune => void; + case utf8.done => abort("strings.sub: index exceeds string length"); + }; + i += 1; + }; + return it.offs; +}; + +// sub — borrowed substring [start, end) where start/end are rune +// indices. ref/hare/strings/sub.ha:30. Hare's 2-arg `sub(s, start)` +// defaulting end=END is omitted: ww has no default-parameter syntax +// (filed as #37). Byte-indexed counterpart: [[bytesub]]. +export fn sub(s: str, start: i32, end: i32) str = { + os.assert(start <= end, "strings.sub: start is higher than end"); + let it: iterator = iter(s); + let starti: i32 = utf8bytelenbounded(&it, start); + let endi: i32 = utf8bytelenbounded(&it, end - start); + let r: str; + r.ptr = s.ptr + (starti: u64); + r.len = endi - starti; + return r; +}; + +// bytesub — borrowed substring [start, end) where start/end are byte +// offsets. ref/hare/strings/sub.ha:59 (#7). Returns `utf8.invalid` if +// either endpoint lands on a continuation byte (would split a +// codepoint); the equivalent Hare predicate is `s[i] & 0xc0 == 0x80` +// at ref/hare/strings/sub.ha:72-73. +export fn bytesub(s: str, start: i32, end: i32) (str | utf8.invalid) = { + os.assert(start <= end, "strings.bytesub: start is higher than end"); + os.assert(end <= s.len, "strings.bytesub: end exceeds string length"); + if (start < s.len) { + if ((s[start] & 0xC0u8) == 0x80u8) { + let e: utf8.invalid; return e; + }; + }; + if (end < s.len) { + if ((s[end] & 0xC0u8) == 0x80u8) { + let e: utf8.invalid; return e; + }; + }; + let r: str; + r.ptr = s.ptr + (start: u64); + r.len = end - start; + return r; +}; + +// runebytes — encode `r` into caller's `scratch` (must hold 4 bytes) +// and return the borrowed slice trimmed to the encoded length. Hare +// inlines the same shape at ref/hare/strings/index.ha:132. +fn runebytes(scratch: []u8, r: rune) []u8 = { + let n: i32 = utf8.encoderune(scratch, r); + let s: []u8; + s.ptr = scratch.ptr; + s.len = n; + s.cap = n; + return s; +}; + +// hasprefix — true iff `in` begins with `prefix`. +// ref/hare/strings/suffix.ha:8. +export fn hasprefix(in: str, prefix: (str | rune)) bool = { + let scratch: [4]u8; + let p: []u8 = match (prefix) { + case let s: str => yield toutf8(s); + case let r: rune => yield runebytes(scratch[0:4], r); + }; + return bytes.hasprefix(toutf8(in), p); +}; + +// hassuffix — true iff `in` ends with `suff`. +// ref/hare/strings/suffix.ha:26. +export fn hassuffix(in: str, suff: (str | rune)) bool = { + let scratch: [4]u8; + let s: []u8 = match (suff) { + case let v: str => yield toutf8(v); + case let r: rune => yield runebytes(scratch[0:4], r); + }; + return bytes.hassuffix(toutf8(in), s); +}; + +// byteindex — byte-wise offset of `needle` in `haystack`, or void if +// absent. ref/hare/strings/index.ha:127. Rune arm encodes via +// utf8.encoderune (Hare passes the encoded slice straight to +// bytes::index). +export fn byteindex(haystack: str, needle: (str | rune)) (i32 | void) = { + let scratch: [4]u8; + let n: []u8 = match (needle) { + case let s: str => yield toutf8(s); + case let r: rune => yield runebytes(scratch[0:4], r); + }; + return bytes.index(toutf8(haystack), n); +}; + +// rbyteindex — byte-wise offset of the last `needle` in `haystack`. +// ref/hare/strings/index.ha:138. +export fn rbyteindex(haystack: str, needle: (str | rune)) (i32 | void) = { + let scratch: [4]u8; + let n: []u8 = match (needle) { + case let s: str => yield toutf8(s); + case let r: rune => yield runebytes(scratch[0:4], r); + }; + return bytes.rindex(toutf8(haystack), n); +}; + +// indexstring — str-arm of [[index]]. Dual-rune-iterator walk: at each +// candidate rune index `i`, compare `haystack` from that position +// against `needle` rune-by-rune until needle is exhausted (match) or +// a mismatch / haystack-exhaustion breaks the inner loop. Mirrors +// ref/hare/strings/index.ha:59 (#10). Hare copies `rest_iter = s_iter` +// directly via struct assignment; ww re-seats `rest_iter` field-wise +// because the let-init struct-copy form diverges between cstage and +// wwstage on this iterator type (993_ww_ww + 995_self_rebuild fail, +// filed as #41) and rule #10 (CLAUDE.md) forbids stage asymmetry. +fn indexstring(haystack: str, needle: str) (i32 | void) = { + let s_iter: iterator = iter(haystack); + let i: i32 = 0; + for (true) { + let rest_iter: iterator; + rest_iter.src = s_iter.src; + rest_iter.offs = s_iter.offs; + rest_iter.reverse = s_iter.reverse; + let needle_iter: iterator = iter(needle); + let matched: bool = false; + for (true) { + let rest_done: bool = false; + let rest_r: rune; + match (next(&rest_iter)) { + case let r: rune => rest_r = r; + case utf8.done => rest_done = true; + }; + let needle_done: bool = false; + let needle_r: rune; + match (next(&needle_iter)) { + case let r: rune => needle_r = r; + case utf8.done => needle_done = true; + }; + if (rest_done && !needle_done) { break; }; + if (needle_done) { matched = true; break; }; + if (rest_r != needle_r) { break; }; + }; + if (matched) { return i; }; + match (next(&s_iter)) { + case let r: rune => i += 1; + case utf8.done => return; + }; + }; + return; +}; + +// index — rune-wise offset of `needle`'s first occurrence in +// `haystack`, or void if absent. ref/hare/strings/index.ha:10. The +// str-arm delegates to [[indexstring]] (dual-iterator rune-by-rune +// walk per Hare's `index_string`, #10); the rune-arm mirrors Hare's +// `index_rune` (ref/hare/strings/index.ha:31). +export fn index(haystack: str, needle: (str | rune)) (i32 | void) = { + match (needle) { + case let s: str => return indexstring(haystack, s); + case let r: rune => { + let it: iterator = iter(haystack); + let i: i32 = 0; + for (true) { + match (next(&it)) { + case let n: rune => { + if (n == r) { return i; }; + i += 1; + }; + case utf8.done => return; + }; + }; + }; + }; + return; +}; + +// rindex — rune-wise offset of `needle`'s last occurrence in +// `haystack`, or void if absent. ref/hare/strings/index.ha:22. The +// str-arm reuses `rbyteindex`; the rune-arm walks forward tracking +// the most recent matching rune index (Hare's `rindex_rune` with +// `riter` returns a byte-offset value for multibyte strings, which +// disagrees with the rune-wise docstring; we keep the docstring's +// contract). +export fn rindex(haystack: str, needle: (str | rune)) (i32 | void) = { + match (needle) { + case let s: str => { + match (rbyteindex(haystack, s)) { + case void => return; + case let bo: i32 => { + let it: iterator = iter(haystack); + let i: i32 = 0; + for (position(&it) < bo) { + match (next(&it)) { + case let r: rune => i += 1; + case utf8.done => break; + }; + }; + return i; + }; + }; + }; + case let r: rune => { + let it: iterator = iter(haystack); + let i: i32 = 0; + let last: i32 = -1; + for (true) { + match (next(&it)) { + case let n: rune => { + if (n == r) { last = i; }; + i += 1; + }; + case utf8.done => break; + }; + }; + if (last < 0) { return; }; + return last; + }; + }; + return; +}; + +// contains — true iff any of `needles` occurs in `haystack`. +// ref/hare/strings/contains.ha:9. +export fn contains(haystack: str, needles: (str | rune)...) bool = { + let i: i32 = 0; + for (i < needles.len) { + match (needles[i]) { + case let s: str => { + match (byteindex(haystack, s)) { + case let bo: i32 => return true; + case void => void; + }; + }; + case let r: rune => { + match (byteindex(haystack, r)) { + case let bo: i32 => return true; + case void => void; + }; + }; + }; + i += 1; + }; + return false; +}; + +// trimprefix — `s` with `prefix` stripped from the front, or `s` +// unchanged if it doesn't start with `prefix`. Borrowed view. +// ref/hare/strings/trim.ha:60. +export fn trimprefix(input: str, prefix: str) str = { + if (!hasprefix(input, prefix)) { return input; }; + let r: str; + r.ptr = input.ptr + (prefix.len: u64); + r.len = input.len - prefix.len; + return r; +}; + +// trimsuffix — symmetric. ref/hare/strings/trim.ha:69. +export fn trimsuffix(input: str, suffix: str) str = { + if (!hassuffix(input, suffix)) { return input; }; + let r: str; + r.ptr = input.ptr; + r.len = input.len - suffix.len; + return r; +}; + +// whitespace — ASCII whitespace set used by the 0-arg ltrim/rtrim/trim +// branches (#9). ref/hare/strings/trim.ha:6. +let whitespace: [4]u8 = [0x20u8, 0x0Au8, 0x09u8, 0x0Du8]; + +// ltrim — strip leading runes that occur in `trim`. Borrowed view. +// 0-arg strips ASCII whitespace via [[bytes.ltrim]] (#9). +// ref/hare/strings/trim.ha:11. The spread expression is inlined +// because `let ws: []u8 = whitespace[0:4]` produces a slice whose +// ptr doesn't track the module-level array storage (filed as #40); +// `b.flush = flushdefault[0:1]` in lib/bufio is the same shape via +// the working field-assign path. +export fn ltrim(input: str, trim: rune...) str = { + if (trim.len == 0) { + return frombytes(bytes.ltrim(toutf8(input), whitespace[0:4]...)); + }; + let it: iterator = iter(input); + for (true) { + match (next(&it)) { + case let r: rune => { + let j: i32 = 0; + let found: bool = false; + for (j < trim.len) { + if (r == trim[j]) { found = true; j = trim.len; } + else { j += 1; }; + }; + if (!found) { + match (prev(&it)) { + case let r2: rune => void; + case utf8.done => void; + }; + break; + }; + }; + case utf8.done => break; + }; + }; + return iterstr(&it); +}; + +// rtrim — strip trailing runes that occur in `trim`. Borrowed view. +// 0-arg strips ASCII whitespace via [[bytes.rtrim]] (#9). Spread is +// inlined to dodge #40 — see [[ltrim]]. +// ref/hare/strings/trim.ha:32. +export fn rtrim(input: str, trim: rune...) str = { + if (trim.len == 0) { + return frombytes(bytes.rtrim(toutf8(input), whitespace[0:4]...)); + }; + let it: iterator = riter(input); + for (true) { + match (next(&it)) { + case let r: rune => { + let j: i32 = 0; + let found: bool = false; + for (j < trim.len) { + if (r == trim[j]) { found = true; j = trim.len; } + else { j += 1; }; + }; + if (!found) { + match (prev(&it)) { + case let r2: rune => void; + case utf8.done => void; + }; + break; + }; + }; + case utf8.done => break; + }; + }; + return iterstr(&it); +}; + +// trim — strip from both ends. ref/hare/strings/trim.ha:54. +export fn trim(input: str, trim: rune...) str = { + return ltrim(rtrim(input, trim...), trim...); +}; + +// iterator — UTF-8 rune cursor over a `str`. Layout flattens Hare's +// anonymous-embedded `utf8::decoder` (ref/hare/strings/iter.ha:6-9) to +// explicit fields. `reverse` selects walk direction: forward iterators +// (`iter`) advance through utf8.next; reverse iterators (`riter`) advance +// through utf8.prev. May be copied to save state. +export type iterator = struct { + offs: i32, + src: []u8, + reverse: bool, +}; + +// iter — initialize a forward iterator at the start of `src`. +// ref/hare/strings/iter.ha:24. +export fn iter(src: str) iterator = { + let r: iterator; + r.src = toutf8(src); + r.offs = 0; + r.reverse = false; + return r; +}; + +// riter — initialize a reverse iterator at the end of `src`. `next` +// on a reverse iterator walks back through the string. +// ref/hare/strings/iter.ha:32. +export fn riter(src: str) iterator = { + let r: iterator; + r.src = toutf8(src); + r.offs = src.len; + r.reverse = true; + return r; +}; + +// move — private dispatch shared by next/prev. `forward` selects +// utf8.next vs utf8.prev. Aborts on more/invalid per Hare's +// ref/hare/strings/iter.ha:51-58 ("Invalid UTF-8 string (this should +// not happen)"). Hare picks the utf8 function via a fn-pointer; ww +// branches on `forward` at each call site instead. +fn move(forward: bool, it: *iterator) (rune | utf8.done) = { + let d: utf8.decoder; + d.src = it.src; + d.offs = it.offs; + if (forward) { + match (utf8.next(&d)) { + case let r: rune => { it.offs = d.offs; return r; }; + case let dn: utf8.done => return dn; + case let m: utf8.more => abort("strings.move: invalid UTF-8"); + case let e: utf8.invalid => abort("strings.move: invalid UTF-8"); + }; + } else { + match (utf8.prev(&d)) { + case let r: rune => { it.offs = d.offs; return r; }; + case let dn: utf8.done => return dn; + case let m: utf8.more => abort("strings.move: invalid UTF-8"); + case let e: utf8.invalid => abort("strings.move: invalid UTF-8"); + }; + }; +}; + +// next — advance the iterator one rune. Forward iterators step +// through utf8.next; reverse iterators (riter) step backward through +// utf8.prev. Returns utf8.done at end-of-walk. ref/hare/strings/iter.ha:45. +export fn next(it: *iterator) (rune | utf8.done) = { + return move(!it.reverse, it); +}; + +// prev — step back one rune. Dual to next: on a forward iterator +// this walks utf8.prev; on a reverse iterator (riter) it walks +// utf8.next. ref/hare/strings/iter.ha:49. +export fn prev(it: *iterator) (rune | utf8.done) = { + return move(it.reverse, it); +}; + +// iterstr — borrowed view of the bytes remaining in the iterator's +// walk direction. Forward iter: bytes from offs to end; reverse iter: +// bytes from start to offs. ref/hare/strings/iter.ha:63. +export fn iterstr(it: *iterator) str = { + let r: []u8; + if (it.reverse) { + r = it.src[0:it.offs]; + } else { + r = it.src[it.offs:it.src.len]; + }; + return frombytes(r); +}; + +// slice — borrowed substring between two iterator positions. +// ref/hare/strings/iter.ha:75. Hare passes `*iterator` directly where +// `*utf8::decoder` is expected via anonymous-embed coercion; ww has +// no anonymous embed, so we reconstruct a local utf8.decoder for each +// endpoint and forward — same pattern as `move` above. +export fn slice(begin: *iterator, end: *iterator) str = { + let b: utf8.decoder; + b.src = begin.src; + b.offs = begin.offs; + let e: utf8.decoder; + e.src = end.src; + e.offs = end.offs; + return frombytes(utf8.slice(&b, &e)); +}; + +// position — byte-wise offset of the iterator in its source. +// ref/hare/strings/iter.ha:82. +export fn position(it: *iterator) i32 = { + return it.offs; +}; + +// tokenizer — re-export of bytes.tokenizer. ref/hare/strings/tokenize.ha:7. +// First cross-module type alias in tree; needs #22's transitive +// alias-chain unwrap (cstage type_chase_named + wwstage +// structlookupchain) to walk struct fields through the chain. +export type tokenizer = bytes.tokenizer; + +// tokenize — yield substrings of `s` split on any byte in `delim`. +// Leading / trailing / adjacent delims yield empty tokens. `s` and +// `delim` are borrowed; caller keeps them live for the tokenizer's +// lifetime. ref/hare/strings/tokenize.ha:32. ASCII-only delim +// asserted per Hare lines 35-37: a multibyte rune in delim would +// split on a single continuation byte and yield invalid UTF-8. +export fn tokenize(s: str, delim: str) tokenizer = { + let d: []u8 = toutf8(delim); + let i: i32 = 0; + for (i < d.len) { + os.assert((d[i] & 0x80u8) == 0u8, + "strings.tokenize cannot tokenize on non-ASCII delimiters"); + i += 1; + }; + return bytes.tokenize(toutf8(s), d...); +}; + +// rtokenize — reverse-direction counterpart to [[tokenize]]. First +// next_token yields the last token, last yields the first. +// ref/hare/strings/tokenize.ha:44. +export fn rtokenize(s: str, delim: str) tokenizer = { + let d: []u8 = toutf8(delim); + let i: i32 = 0; + for (i < d.len) { + os.assert((d[i] & 0x80u8) == 0u8, + "strings.rtokenize cannot tokenize on non-ASCII delimiters"); + i += 1; + }; + return bytes.rtokenize(toutf8(s), d...); +}; + +// next_token — current token, advancing the cursor. +// ref/hare/strings/tokenize.ha:62. +export fn next_token(s: *tokenizer) (str | bytes.done) = { + let b: *bytes.tokenizer = s: *bytes.tokenizer; + match (bytes.next_token(b)) { + case let v: []u8 => return frombytes(v); + case bytes.done => { let d: bytes.done; return d; }; + }; +}; + +// peek_token — current token without advancing. +// ref/hare/strings/tokenize.ha:71. +export fn peek_token(s: *tokenizer) (str | bytes.done) = { + let b: *bytes.tokenizer = s: *bytes.tokenizer; + match (bytes.peek_token(b)) { + case let v: []u8 => return frombytes(v); + case bytes.done => { let d: bytes.done; return d; }; + }; +}; + +// remaining_tokens — unconsumed portion of the input ahead of the +// cursor. ref/hare/strings/tokenize.ha:79. +export fn remaining_tokens(s: *tokenizer) str = { + let b: *bytes.tokenizer = s: *bytes.tokenizer; + return frombytes(bytes.remaining_tokens(b)); +}; + +// rt_ensure is the runtime slice-growth helper invoked by the +// `append(s, v)` builtin. Direct bind for the same reason as +// lib/shlex.shlex (appendstr, 16B): the builtin's expansion stores +// only 8B of the new element, losing the `.len` half of a `str`. +@symbol("rt_ensure") fn rtensure(s: *void, membsz: u64) void; + +// appendstr — grow `*slice` by one and store `item` (16B). Mirror of +// lib/shlex.shlex appendstr. Collapses when the append builtin learns +// to store the full element width. +fn appendstr(slice: *[]str, item: str) void = { + let newlen: i32 = slice.len + 1; + slice.len = newlen; + rtensure(slice: *void, size(str): u64); + let dst: *str = &slice.ptr[newlen - 1]; + dst.ptr = item.ptr; + dst.len = item.len; +}; + +// splitn — split `in` on any byte in `delim`, returning up to `n` +// tokens via forward iteration. The trailing slot (when more than +// `n - 1` tokens exist) holds the unconsumed remainder. Strings +// within the result are borrowed from `in`. +// +// The caller frees the returned slice via +// `os.free(r.ptr: *void, (r.cap: u64) * size(str): u64)`. +// +// Hare's `([]str | nomem)` collapses to `[]str` here: ww os.alloc +// has no recoverable failure path. Same precedent as +// shlex.split / bytes.splitn. +// +// ref/hare/strings/tokenize.ha:172. +export fn splitn(in: str, delim: str, n: i32) []str = { + let toks: []str; + toks.ptr = nil: *str; + toks.len = 0; + toks.cap = 0; + let tok: tokenizer = tokenize(in, delim); + let i: i32 = 0; + for (i < n - 1) { + match (next_token(&tok)) { + case let s: str => { appendstr(&toks, s); }; + case bytes.done => { return toks; }; + }; + i += 1; + }; + match (peek_token(&tok)) { + case bytes.done => void; + case let pk: str => { + let r: str = remaining_tokens(&tok); + appendstr(&toks, r); + }; + }; + return toks; +}; + +// rsplitn — reverse-direction counterpart to [[splitn]]: tokens are +// collected from the end of `in`. The trailing slot holds the +// unconsumed prefix (everything before the n-th-from-last delim hit). +// +// When the input has fewer than n tokens, the `done` short-circuit +// returns toks UN-reversed (in last-token-first order). Mirrors Hare +// at ref/hare/strings/tokenize.ha:219-224 where the in-place reverse +// step is gated behind the n-1 loop running to completion. +// +// ref/hare/strings/tokenize.ha:200. +export fn rsplitn(in: str, delim: str, n: i32) []str = { + let toks: []str; + toks.ptr = nil: *str; + toks.len = 0; + toks.cap = 0; + let tok: tokenizer = rtokenize(in, delim); + let i: i32 = 0; + for (i < n - 1) { + match (next_token(&tok)) { + case let s: str => { appendstr(&toks, s); }; + case bytes.done => { return toks; }; + }; + i += 1; + }; + match (peek_token(&tok)) { + case bytes.done => void; + case let pk: str => { + let r: str = remaining_tokens(&tok); + appendstr(&toks, r); + }; + }; + + // In-place reverse so callers see argv-order, matching Hare + // (ref/hare/strings/tokenize.ha:220). Element copy is field-wise + // through `*str` because `toks[i] = toks[j]` (full 16B str store) + // lands in the multi-word-store gap noted at cmd/w6c/cgen.c:6515. + let a: i32 = 0; + let b: i32 = toks.len - 1; + for (a < b) { + let pa: *str = &toks.ptr[a]; + let pb: *str = &toks.ptr[b]; + let tp: *u8 = pa.ptr; + let tl: i32 = pa.len; + pa.ptr = pb.ptr; + pa.len = pb.len; + pb.ptr = tp; + pb.len = tl; + a += 1; + b -= 1; + }; + return toks; +}; + +// split — full split of `in` on `delim` (no token cap). Mirrors +// `splitn(in, delim, types::SIZE_MAX)`. ww uses `types.I32_MAX` +// because the index type is i32 (lib/CLAUDE.md). +// +// ref/hare/strings/tokenize.ha:242. +export fn split(in: str, delim: str) []str = { + return splitn(in, delim, types.I32_MAX); +}; + +// lpad — left-pad `s` with `p` rune until the result reaches `maxlen` +// bytes. Length comparison is BYTES, mirroring Hare's `len(s) >= maxlen` +// at ref/hare/strings/pad.ha:9. A multibyte `p` whose encoded width +// doesn't divide `maxlen - s.len` evenly leaves a trailing pad byte +// pair sliced mid-codepoint at byte `maxlen-1`, exactly as Hare's +// `res[..maxlen]` does (ref/hare/strings/pad.ha:20). When +// `(maxlen - s.len) * pad.len >= maxlen` (multibyte pad overflows the +// budget), `s` is entirely sliced off — same as Hare. Caller releases +// with `os.free(r.ptr, r.len: u64)`. Hare's `nomem` return is dropped: +// `os.alloc` aborts on OOM. Buf size == r.len keeps the free-contract +// shape of [[dup]] / [[concat]] / [[join]]; Hare's `alloc([], maxlen)!` +// over-allocs via append then slices, but Hare's slice-free recovers +// the true capacity from the heap allocator (rt/ensure.ha:24), which +// ww's munmap-based `os.free` cannot do. +export fn lpad(s: str, p: rune, maxlen: i32) str = { + if (s.len >= maxlen) { return dup(s); }; + let scratch: [4]u8; + let pad: []u8 = runebytes(scratch[0:4], p); + let buf: []u8 = alloc([], maxlen: u64)!; + let padwrite: i32 = (maxlen - s.len) * pad.len; + if (padwrite > maxlen) { padwrite = maxlen; }; + let off: i32 = 0; + for (off < padwrite) { + buf[off] = pad.ptr[off % pad.len]; + off += 1; + }; + let k: i32 = 0; + let srem: i32 = maxlen - off; + if (srem > s.len) { srem = s.len; }; + for (k < srem) { + buf[off + k] = s[k]; + k += 1; + }; + buf.len = maxlen; + return frombytes(buf); +}; + +// replace — fresh allocation of `s` with every non-overlapping +// occurrence of `needle` replaced by `target`. Caller releases with +// `os.free(r.ptr, r.len: u64)`. ref/hare/strings/replace.ha:8 (#4). +// +// Hare delegates to [[multireplace]] with a single pair; ww has no +// `(str, str)` variadic shape today (#39), so this is a standalone +// two-pass implementation: pass 1 counts matches to size the result, +// pass 2 copies chunks and `target` into a single fresh buffer. +// Single nomem path (the `alloc([], total)?`) preserves Hare's +// signature without a per-write `append(...)?` (ww's append builtin +// aborts on OOM, #11). Empty `needle` would hasprefix-match every +// position with a zero stride — same infinite loop Hare exhibits at +// ref/hare/strings/replace.ha:31; not gated. +export fn replace(s: str, needle: str, target: str) (str | nomem) = { + let sb: []u8 = toutf8(s); + let nb: []u8 = toutf8(needle); + let tb: []u8 = toutf8(target); + let count: i32 = 0; + let i: i32 = 0; + for (i < sb.len) { + if (bytes.hasprefix(sb[i:sb.len], nb)) { + count += 1; + i += nb.len; + } else { + i += 1; + }; + }; + let total: i32 = sb.len + count * (tb.len - nb.len); + if (total == 0) { + let r: str; + r.ptr = nil; + r.len = 0; + return r; + }; + let res: []u8 = alloc([], total)?; + let off: i32 = 0; + i = 0; + for (i < sb.len) { + if (bytes.hasprefix(sb[i:sb.len], nb)) { + let j: i32 = 0; + for (j < tb.len) { + res.ptr[off + j] = tb.ptr[j]; + j += 1; + }; + off += tb.len; + i += nb.len; + } else { + res.ptr[off] = sb.ptr[i]; + off += 1; + i += 1; + }; + }; + res.len = total; + return frombytes(res); +}; + +// rpad — right-pad `s` with `p` rune until the result reaches `maxlen` +// bytes. Symmetric with [[lpad]]. ref/hare/strings/pad.ha:39. +export fn rpad(s: str, p: rune, maxlen: i32) str = { + if (s.len >= maxlen) { return dup(s); }; + let scratch: [4]u8; + let pad: []u8 = runebytes(scratch[0:4], p); + let buf: []u8 = alloc([], maxlen: u64)!; + let k: i32 = 0; + for (k < s.len) { + buf[k] = s[k]; + k += 1; + }; + let padwrite: i32 = maxlen - s.len; + let i: i32 = 0; + for (i < padwrite) { + buf[s.len + i] = pad.ptr[i % pad.len]; + i += 1; + }; + buf.len = maxlen; + return frombytes(buf); +}; + // selfhost/cmd/w6l/obj.ww — port of cmd/w6l/obj.c. // // Loads relocatable ELF64 .o files emitted by w6a, appends .text to @@ -987,6 +2918,7 @@ package w6l; import os; import rt; import mem; +import strings; import sym; def ET_REL: i32 = 1; @@ -1156,7 +3088,10 @@ fn cstreq(p: *u8, lit: str) bool = { // Build a ww str from a NUL-terminated *u8 (for passing to intern). fn cstrtostr(a: *arena, p: *u8) str = { let n: u64 = cstrlen(p); - return astrndup(a, p, n); + let view: str; + view.ptr = p; + view.len = n: i32; + return strings.dup(view); }; // ---- archive (SysV ar) types and helpers ------------------------------- @@ -1601,6 +3536,7 @@ package w6l; import os; import rt; import mem; +import strings; import sym; def ET_DYN_SO: u16 = 3u16; @@ -1696,7 +3632,10 @@ fn dcstrlen(p: *u8) u64 = { fn dcstrtostr(a: *arena, p: *u8) str = { let n: u64 = dcstrlen(p); - return astrndup(a, p, n); + let view: str; + view.ptr = p; + view.len = n: i32; + return strings.dup(view); }; // basename: scan for last '/' and return pointer past it. diff --git a/selfhost/cmd/w6l/obj.ww b/selfhost/cmd/w6l/obj.ww index c559ee69..0e82724f 100644 --- a/selfhost/cmd/w6l/obj.ww +++ b/selfhost/cmd/w6l/obj.ww @@ -13,6 +13,7 @@ package w6l; import os; import rt; import mem; +import strings; import sym; def ET_REL: i32 = 1; @@ -182,7 +183,10 @@ fn cstreq(p: *u8, lit: str) bool = { // Build a ww str from a NUL-terminated *u8 (for passing to intern). fn cstrtostr(a: *arena, p: *u8) str = { let n: u64 = cstrlen(p); - return astrndup(a, p, n); + let view: str; + view.ptr = p; + view.len = n: i32; + return strings.dup(view); }; // ---- archive (SysV ar) types and helpers ------------------------------- diff --git a/selfhost/cmd/ww/main.combined.ww b/selfhost/cmd/ww/main.combined.ww index 96242139..ea696d66 100644 --- a/selfhost/cmd/ww/main.combined.ww +++ b/selfhost/cmd/ww/main.combined.ww @@ -860,6 +860,1937 @@ export fn freearena(a: *arena) void = { }; }; +// types — integer limits. Mirrors Hare's types::limits (I8_MAX, …) +// platform-fixed for amd64. Numeric helpers live in lib/math, matching +// Hare's split between types::limits and math::. + +package types; + +def I8_MAX: i8 = 127; +def I16_MAX: i16 = 32767; +def I32_MAX: i32 = 2147483647; +def I64_MAX: i64 = 9223372036854775807; + +def I8_MIN: i8 = -128; +def I16_MIN: i16 = -32768; +def I32_MIN: i32 = -2147483648; +def I64_MIN: i64 = -9223372036854775808; + +def U8_MAX: u8 = 255; +def U16_MAX: u16 = 65535; +def U32_MAX: u32 = 4294967295; +def U64_MAX: u64 = 18446744073709551615; + +// bytes — slice operations over []u8. Mirrors Hare's bytes module +// (ref/hare/bytes/) for the in-tree subset: search/equality/prefix +// helpers used by lib/encoding, lib/bufio, lib/memio. +// +// Documented divergences from Hare: +// - index_slice / rindex_slice use naive O(n·m); Hare specialises +// 2/3/4-byte needles and falls back to two_way (Crochemore-Perrin) +// for longer (ref/hare/bytes/index.ha:61, ref/hare/bytes/two_way.ha). +// Correctness equivalent. +// - peek_token dispatches index/rindex by branching on `reverse` +// rather than a function-pointer `ifunc` (ref/hare/bytes/tokenize.ha:97). +// ww has no fn pointers in scope yet — same pattern as lib/strings +// `move`. Outwardly identical. +// - tokenize / rtokenize zero the `delim` field on the constructed +// tokenizer when `in` is empty, rather than mutating the variadic +// param before the struct write (ref/hare/bytes/tokenize.ha:26-28). +// Semantically identical; the variadic param is borrowed and +// captured-by-value into the struct, so mutating either side +// yields the same observable state. + +package bytes; + +import os; +import types; + +// done — iteration sentinel returned by next_token / peek_token at +// end-of-input. ref/hare/bytes/tokenize.ha uses the built-in `done` +// token; ww spells it per-package the same way lib/encoding/utf8 does +// (utf8.ww:36). Plain `void` (not `!void`): continuation signal. +export type done = void; + +// tokenizer — cursor over an input slice. Layout mirrors +// ref/hare/bytes/tokenize.ha:6-10. `p` is the cached peek-position; +// I64_MAX (forward) / I64_MIN (reverse) are the unprimed sentinels. +// p < 0 also identifies a reverse-direction iterator. +export type tokenizer = struct { + in: []u8, + delim: []u8, + p: i64, +}; + +// equal — true iff `a` and `b` have the same length and contents. +// ref/hare/bytes/equal.ha:9. +export fn equal(a: []u8, b: []u8) bool = { + if (a.len != b.len) { return false; }; + let i: i32 = 0; + for (i < a.len) { + if (a[i] != b[i]) { return false; }; + i += 1; + }; + return true; +}; + +// index — first offset of `needle` in `s`. u8 needle scans for the +// byte; []u8 needle scans for the substring. void if absent. +// ref/hare/bytes/index.ha:6. +export fn index(s: []u8, needle: (u8 | []u8)) (i32 | void) = { + match (needle) { + case let c: u8 => { + let i: i32 = 0; + for (i < s.len) { + if (s[i] == c) { return i; }; + i += 1; + }; + return; + }; + case let sub: []u8 => { + if (sub.len == 0) { return 0; }; + if (sub.len > s.len) { return; }; + let last: i32 = s.len - sub.len; + let i: i32 = 0; + for (i <= last) { + let j: i32 = 0; + let ok: bool = true; + for (j < sub.len) { + if (s[i + j] != sub[j]) { ok = false; j = sub.len; } + else { j += 1; }; + }; + if (ok) { return i; }; + i += 1; + }; + return; + }; + }; + return; +}; + +// rindex — last offset of `needle` in `s`. Empty []u8 needle returns +// s.len (ref/hare/bytes/index.ha:103 — Hare's loop yields r-0 at i=0). +// ref/hare/bytes/index.ha:86. +export fn rindex(s: []u8, needle: (u8 | []u8)) (i32 | void) = { + match (needle) { + case let c: u8 => { + let i: i32 = s.len - 1; + for (i >= 0) { + if (s[i] == c) { return i; }; + i -= 1; + }; + return; + }; + case let sub: []u8 => { + if (sub.len == 0) { return s.len; }; + if (sub.len > s.len) { return; }; + let i: i32 = s.len - sub.len; + for (i >= 0) { + let j: i32 = 0; + let ok: bool = true; + for (j < sub.len) { + if (s[i + j] != sub[j]) { ok = false; j = sub.len; } + else { j += 1; }; + }; + if (ok) { return i; }; + i -= 1; + }; + return; + }; + }; + return; +}; + +// contains — true iff any of `needles` (byte or sub-slice) appears in `s`. +// ref/hare/bytes/contains.ha:6. +export fn contains(s: []u8, needles: (u8 | []u8)...) bool = { + let i: i32 = 0; + for (i < needles.len) { + match (needles[i]) { + case let b: u8 => { + match (index(s, b)) { + case let bo: i32 => return true; + case void => void; + }; + }; + case let n: []u8 => { + match (index(s, n)) { + case let bo: i32 => return true; + case void => void; + }; + }; + }; + i += 1; + }; + return false; +}; + +// ltrim — borrowed view of `in` with leading bytes in `trim` stripped. +// `trim` must be non-empty. ref/hare/bytes/trim.ha:7. +export fn ltrim(in: []u8, trim: u8...) []u8 = { + os.assert(trim.len > 0, "bytes.ltrim called with empty trim set"); + let i: i32 = 0; + for (i < in.len && contains(trim, in[i])) { i += 1; }; + let r: []u8; + r.ptr = in.ptr + (i: u64); + r.len = in.len - i; + r.cap = r.len; + return r; +}; + +// rtrim — borrowed view of `in` with trailing bytes in `trim` stripped. +// `trim` must be non-empty. ref/hare/bytes/trim.ha:17. Hare's loop uses +// `size` underflow at i==0 to terminate; ww indices are signed i32, so +// the equivalent termination is spelled `i >= 0` explicitly. +export fn rtrim(in: []u8, trim: u8...) []u8 = { + os.assert(trim.len > 0, "bytes.rtrim called with empty trim set"); + let i: i32 = in.len - 1; + for (i >= 0 && contains(trim, in[i])) { i -= 1; }; + let r: []u8; + r.ptr = in.ptr; + r.len = i + 1; + r.cap = r.len; + return r; +}; + +// trim — borrowed view of `in` with both ends in `trim` stripped. +// ref/hare/bytes/trim.ha:27. +export fn trim(in: []u8, trim: u8...) []u8 = { + return ltrim(rtrim(in, trim...), trim...); +}; + +// hasprefix — true iff `s` starts with `pre`. +// ref/hare/bytes/contains.ha:21. +export fn hasprefix(s: []u8, pre: []u8) bool = { + if (pre.len > s.len) { return false; }; + let i: i32 = 0; + for (i < pre.len) { + if (s[i] != pre[i]) { return false; }; + i += 1; + }; + return true; +}; + +// hassuffix — true iff `s` ends with `suf`. +// ref/hare/bytes/contains.ha:35. +export fn hassuffix(s: []u8, suf: []u8) bool = { + if (suf.len > s.len) { return false; }; + let off: i32 = s.len - suf.len; + let i: i32 = 0; + for (i < suf.len) { + if (s[off + i] != suf[i]) { return false; }; + i += 1; + }; + return true; +}; + +// reverse — in-place reverse of `s`. ref/hare/bytes/reverse.ha:5. +export fn reverse(s: []u8) void = { + let i: i32 = 0; + let j: i32 = s.len - 1; + for (i < j) { + let t: u8 = s[i]; + s[i] = s[j]; + s[j] = t; + i += 1; + j -= 1; + }; +}; + +// zero — set every byte of `s` to 0. ref/hare/bytes/zero.ha:5. +export fn zero(s: []u8) void = { + let i: i32 = 0; + for (i < s.len) { + s[i] = 0u8; + i += 1; + }; +}; + +// tokenize — iterator yielding tokens from `in` separated by any byte +// in `delim`. Leading / trailing / adjacent delims yield empty tokens. +// `delim` is borrowed; caller keeps it valid for the tokenizer's +// lifetime. ref/hare/bytes/tokenize.ha:22. +export fn tokenize(in: []u8, delim: u8...) tokenizer = { + os.assert(delim.len > 0, "bytes.tokenize called with empty slice"); + os.assert((in.len: i64) < types.I64_MAX, + "bytes.tokenize: input length exceeds I64_MAX"); + let t: tokenizer; + t.in = in; + t.delim = delim; + if (in.len == 0) { + t.delim.len = 0; + t.delim.cap = 0; + }; + t.p = types.I64_MAX; + return t; +}; + +// rtokenize — reverse-direction tokenize. First next_token yields the +// last token, last next_token yields the first. ref/hare/bytes/tokenize.ha:40. +export fn rtokenize(in: []u8, delim: u8...) tokenizer = { + os.assert(delim.len > 0, "bytes.rtokenize called with empty slice"); + os.assert((in.len: i64) < types.I64_MAX, + "bytes.rtokenize: input length exceeds I64_MAX"); + let t: tokenizer; + t.in = in; + t.delim = delim; + if (in.len == 0) { + t.delim.len = 0; + t.delim.cap = 0; + }; + t.p = types.I64_MIN; + return t; +}; + +// peek_token — next token without advancing the cursor. Returns done +// once `s.delim` has been zeroed by a prior past-end next_token. +// ref/hare/bytes/tokenize.ha:91. +export fn peek_token(s: *tokenizer) ([]u8 | done) = { + if (s.delim.len == 0) { + let d: done; return d; + }; + + let reverse: bool = s.p < 0i64; + let known: bool = false; + if (reverse) { + if (s.p != types.I64_MIN) { known = true; }; + } else { + if (s.p != types.I64_MAX) { known = true; }; + }; + if (!known) { + let i: i64 = types.I64_MAX; + if (reverse) { i = types.I64_MIN; }; + let dlen: i64 = 0i64; + let slen: i64 = s.in.len: i64; + + let k: i32 = 0; + for (k < s.delim.len) { + let d: u8 = s.delim[k]; + let ix_found: bool = false; + let ix_val: i32 = 0; + if (reverse) { + match (rindex(s.in, d)) { + case let v: i32 => { ix_found = true; ix_val = v; }; + case void => void; + }; + } else { + match (index(s.in, d)) { + case let v: i32 => { ix_found = true; ix_val = v; }; + case void => void; + }; + }; + if (ix_found) { + if (!reverse) { + if ((ix_val: i64) < i) { i = ix_val: i64; dlen = 1i64; }; + } else { + if ((ix_val: i64) > i) { i = ix_val: i64; dlen = 1i64; }; + }; + } else { + if (!reverse) { + if (slen < i) { i = slen; }; + } else { + if (0i64 > i) { i = 0i64; }; + }; + }; + k += 1; + }; + + if (reverse) { + if (i == slen) { + s.p = -(slen + 1i64); + } else { + s.p = i + dlen - slen - 1i64; + }; + } else { + s.p = i; + }; + }; + + let r: []u8; + if (reverse) { + let start: i32 = (s.in.len: i64 + s.p + 1i64): i32; + r.ptr = s.in.ptr + (start: u64); + r.len = s.in.len - start; + r.cap = r.len; + } else { + let end: i32 = s.p: i32; + r.ptr = s.in.ptr; + r.len = end; + r.cap = end; + }; + return r; +}; + +// next_token — current token, then advance past it and the delim. +// Once the input is exhausted, returns done and zeros `s.delim` so +// subsequent peeks short-circuit. ref/hare/bytes/tokenize.ha:59. +export fn next_token(s: *tokenizer) ([]u8 | done) = { + let b: []u8; + match (peek_token(s)) { + case let v: []u8 => { b = v; }; + case done => { let d: done; return d; }; + }; + + let slen: i64 = s.in.len: i64; + let reverse: bool = s.p < 0i64; + if (reverse) { + if (slen + s.p + 1i64 == 0i64) { + s.delim.len = 0; + s.delim.cap = 0; + s.in.len = 0; + s.in.cap = 0; + } else { + let end: i32 = (slen + s.p + 1i64 - 1i64): i32; + s.in.len = end; + s.in.cap = end; + }; + s.p = types.I64_MIN; + } else { + if (s.p == slen) { + s.delim.len = 0; + s.delim.cap = 0; + s.in.len = 0; + s.in.cap = 0; + } else { + let adv: u64 = (s.p: u64) + 1u64; + let adv_i32: i32 = (s.p: i32) + 1; + s.in.ptr = s.in.ptr + adv; + s.in.len = s.in.len - adv_i32; + s.in.cap = s.in.cap - adv_i32; + }; + s.p = types.I64_MAX; + }; + return b; +}; + +// remaining_tokens — the unconsumed portion of `s.in`. Read-only view. +// ref/hare/bytes/tokenize.ha:145. +export fn remaining_tokens(s: *tokenizer) []u8 = { + return s.in; +}; + +// rt_ensure is the runtime slice-growth helper invoked by the +// `append(s, v)` builtin. We bind it directly because the builtin's +// expansion stores only 8 bytes of the new element (cgen emits a +// single MOVQ), losing the .len/.cap fields of a []u8 element (24B). +// Mirrors the same workaround in lib/shlex.shlex (appendstr, 16B) and +// lib/getopt.getopt (appendoption, 24B); collapses in one go when the +// append builtin learns to store the full element width. +@symbol("rt_ensure") fn rtensure(s: *void, membsz: u64) void; + +// appendslice — grow `*slice` by one and store `item` (24B). Mirror +// of [[shlex.appendstr]] / [[getopt.appendoption]]. Bypasses the +// `append` builtin's first-8B-only-store gap for a slice-element. +fn appendslice(slice: *[][]u8, item: []u8) void = { + let newlen: i32 = slice.len + 1; + slice.len = newlen; + rtensure(slice: *void, 24u64); + let dst: *[]u8 = &slice.ptr[newlen - 1]; + dst.ptr = item.ptr; + dst.len = item.len; + dst.cap = item.cap; +}; + +// splitn — split `in` on any byte in `delim`, returning up to `n` +// tokens via forward iteration. The trailing slot (when more than +// `n - 1` tokens exist) holds the unconsumed remainder. +// +// The caller frees the returned slice via +// `os.free(r.ptr: *void, (r.cap: u64) * 24u64)`. Element bytes are +// borrowed from `in`. +// +// Hare's `([][]u8 | nomem)` collapses to `[][]u8` here: ww os.alloc +// has no recoverable failure path. Same precedent as +// shlex.split / getopt.tryparse. +// +// ref/hare/bytes/tokenize.ha:156. +export fn splitn(in: []u8, delim: []u8, n: i32) [][]u8 = { + os.assert(delim.len > 0, + "bytes.splitn must not be called with an empty delimiter"); + let toks: [][]u8; + toks.ptr = nil: *[]u8; + toks.len = 0; + toks.cap = 0; + let tok: tokenizer = tokenize(in, delim...); + let i: i32 = 0; + for (i < n - 1) { + match (next_token(&tok)) { + case let s: []u8 => { appendslice(&toks, s); }; + case done => { return toks; }; + }; + i += 1; + }; + match (peek_token(&tok)) { + case done => void; + case let pk: []u8 => { + let r: []u8 = remaining_tokens(&tok); + appendslice(&toks, r); + }; + }; + return toks; +}; + +// rsplitn — reverse-direction counterpart to [[splitn]]: tokens are +// collected from the end of `in`. The trailing slot holds the +// unconsumed prefix (everything before the n-th-from-last delim hit). +// +// When the input has fewer than n tokens, the `done` short-circuit +// returns toks UN-reversed (in last-token-first order). Mirrors Hare +// at ref/hare/bytes/tokenize.ha:196-199 where the in-place reverse +// step is gated behind the n-1 loop running to completion. Only the +// "loop ran to completion AND peek saw a remainder" path applies the +// reverse; both early-exit paths skip it. +// +// ref/hare/bytes/tokenize.ha:186. +export fn rsplitn(in: []u8, delim: []u8, n: i32) [][]u8 = { + os.assert(delim.len > 0, + "bytes.rsplitn called with empty delimiter"); + let toks: [][]u8; + toks.ptr = nil: *[]u8; + toks.len = 0; + toks.cap = 0; + let tok: tokenizer = rtokenize(in, delim...); + let i: i32 = 0; + for (i < n - 1) { + match (next_token(&tok)) { + case let s: []u8 => { appendslice(&toks, s); }; + case done => { return toks; }; + }; + i += 1; + }; + match (peek_token(&tok)) { + case done => void; + case let pk: []u8 => { + let r: []u8 = remaining_tokens(&tok); + appendslice(&toks, r); + }; + }; + + // In-place reverse so callers see argv-order, matching Hare + // (ref/hare/bytes/tokenize.ha:207). Element copy is field-wise + // through `*[]u8` because `toks[i] = toks[j]` (full 24B slice + // store) lands in the multi-word-store gap noted at + // cmd/w6c/cgen.c:6515-6523. + let a: i32 = 0; + let b: i32 = toks.len - 1; + for (a < b) { + let pa: *[]u8 = &toks.ptr[a]; + let pb: *[]u8 = &toks.ptr[b]; + let tp: *u8 = pa.ptr; + let tl: i32 = pa.len; + let tc: i32 = pa.cap; + pa.ptr = pb.ptr; + pa.len = pb.len; + pa.cap = pb.cap; + pb.ptr = tp; + pb.len = tl; + pb.cap = tc; + a += 1; + b -= 1; + }; + return toks; +}; + +// split — full split of `in` on `delim` (no token cap). Mirrors +// `splitn(in, delim, types::SIZE_MAX)`. ww uses `types.I32_MAX` +// because the index type is i32 (lib/CLAUDE.md). +// +// ref/hare/bytes/tokenize.ha:225. +export fn split(in: []u8, delim: []u8) [][]u8 = { + return splitn(in, delim, types.I32_MAX); +}; + +// encoding/utf8 — UTF-8 encode/decode. Hare port; see +// ref/hare/encoding/utf8/{types,rune,encode,decode,decodetable}.ha. +// +// The decoder is Hoehrmann's branchless DFA, originally published +// at . Hare's +// ref/hare/encoding/utf8/decodetable.ha:4 restructures Hoehrmann's +// flat table to 2D `[8][256]i8`; we flatten back to 1D `[2048]i8` +// because ww cgen does not yet ship 2D arrays (task #20). +// +// Surface deviation from ref/hare/encoding/utf8: +// +// - `encoderune` takes a caller-supplied `out: []u8` and returns +// the byte count. Hare returns a slice into a `static let buf`; +// the caller-buffer form mirrors lib/encoding/hex.encode and +// skips the static-buffer/slice-return pair. +// +// Deferred (no in-tree caller, follow-up tasks): `appendrune`, +// `strencode`, `strdecode`. Hare's string-iteration surface +// (`strings::iterator`/`strings::next` — ref/hare/strings/iter.ha) +// lives under lib/strings, not here. + +// ref/hare/encoding/utf8/types.ha:6 — incomplete trailing sequence. +// Plain `void` (not `!void`): a truncated tail is a control-flow +// signal, not an error caller can ignore. +package utf8; + +export type more = void; + +// ref/hare/encoding/utf8/types.ha:9 — invalid UTF-8 sequence. +export type invalid = !void; + +// ref/hare/encoding/utf8/types.ha:12 — fixed message; `invalid` carries +// no payload, so the rendering is constant. +export fn strerror(err: invalid) str = { + return "Invalid UTF-8"; +}; + +// `done` is not a built-in singleton in ww (Hare ships it as part of +// the type system). Plain `void` (not `!void`): end-of-input is a +// continuation signal, not an error. lib/io spells its EOF the same +// way (lib/io/io.ww:8-11). +export type done = void; + +// ref/hare/encoding/utf8/decodetable.ha:4 — Hoehrmann's UTF-8 DFA, +// flat 1D `[2048]i8`. Layout: dfa[state*256 + byte] gives the next +// state (>0), the accept transition (0 — emit rune), or invalid (-1). +// Values match ref/hare/encoding/utf8/decodetable.ha verbatim. +let dfa: [2048]i8 = [ + // state 0 — initial byte: ASCII accepts (0), continuation/illegal + // byte rejects (-1), legal multibyte start emits a state. + 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, + 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, + 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, + 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, + 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, + 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, + 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, + 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, + -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, + -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, + -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, + -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, + -1i8, -1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, + 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, + 3i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 4i8, 2i8, 2i8, + 5i8, 6i8, 6i8, 6i8, 7i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, + + // state 1 — expecting one continuation byte (0x80..0xBF). + -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, + -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, + -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, + -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, + -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, + -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, + -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, + -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, + 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, + 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, + 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, + 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, + -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, + -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, + -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, + -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, + + // state 2 — expecting one continuation byte (full 0x80..0xBF range). + -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, + -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, + -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, + -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, + -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, + -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, + -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, + -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, + 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, + 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, + 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, + 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, + -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, + -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, + -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, + -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, + + // state 3 — first byte was 0xE0; continuation byte must be 0xA0..0xBF + // (rejects overlong 3-byte encodings). + -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, + -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, + -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, + -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, + -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, + -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, + -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, + -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, + -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, + -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, + 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, + 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, + -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, + -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, + -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, + -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, + + // state 4 — first byte was 0xED; continuation byte must be 0x80..0x9F + // (rejects UTF-16 surrogate codepoints U+D800..U+DFFF). + -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, + -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, + -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, + -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, + -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, + -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, + -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, + -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, + 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, + 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, + -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, + -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, + -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, + -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, + -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, + -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, + + // state 5 — first byte was 0xF0; continuation byte must be 0x90..0xBF + // (rejects overlong 4-byte encodings). + -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, + -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, + -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, + -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, + -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, + -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, + -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, + -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, + -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, + 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, + 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, + 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, + -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, + -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, + -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, + -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, + + // state 6 — middle continuation byte of a 4-byte sequence (0x80..0xBF). + -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, + -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, + -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, + -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, + -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, + -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, + -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, + -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, + 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, + 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, + 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, + 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, + -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, + -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, + -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, + -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, + + // state 7 — first byte was 0xF4; continuation byte must be 0x80..0x8F + // (rejects codepoints above U+10FFFF). + -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, + -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, + -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, + -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, + -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, + -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, + -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, + -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, + 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, + -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, + -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, + -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, + -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, + -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, + -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, + -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, +]; + +// ref/hare/encoding/utf8/decode.ha:17 — payload-bit masks. Hare's +// [2][8]u8 flattened to 1D [16]u8; row 0 (offsets 0..7) is the +// continuation-byte mask (always 0x3F), row 1 (offsets 8..15) is the +// initial-byte payload mask indexed by the transition class. +let masks: [16]u8 = [ + 0x3fu8, 0x3fu8, 0x3fu8, 0x3fu8, 0x3fu8, 0x3fu8, 0x3fu8, 0x3fu8, + 0x7fu8, 0x1fu8, 0x0fu8, 0x0fu8, 0x0fu8, 0x07u8, 0x07u8, 0x07u8, +]; + +// ref/hare/encoding/utf8/decode.ha:6 — incremental decoder state. +export type decoder = struct { + offs: i32, + src: []u8, +}; + +// ref/hare/encoding/utf8/decode.ha:12. +export fn decode(src: []u8) decoder = { + let d: decoder; + d.src = src; + d.offs = 0; + return d; +}; + +// ref/hare/encoding/utf8/decode.ha:27. Returns the next rune from a +// decoder, `done` at end-of-input, `more` on truncated trailing +// sequence, `invalid` on malformed input (overlong, surrogate, +// out-of-range, bad continuation). +// +// Algorithm is verbatim Hoehrmann (see file header). One structural +// rewrite: Hare encodes the "initial vs continuation byte" decision +// as the branchless `(state - 1): uint >> 31`, which assumes a 32-bit +// uint. ww's uint is 64-bit (cmd/wcc/type.c:58), so the shift answer +// would be 0x1_ffff_ffff rather than 1. We spell the same predicate +// with an explicit conditional. +export fn next(d: *decoder) (rune | done | more | invalid) = { + if (d.offs == d.src.len) { + let dn: done; return dn; + }; + let nx: i32 = 0; + let state: i32 = 0; + let r: u32 = 0u32; + for (d.offs < d.src.len) { + let b: u8 = d.src[d.offs]; + let bi: i32 = b: i32; + let row: i32 = state * 256 + bi; + let cell: i8 = dfa[row]; + nx = cell: i32; + let mi: i32 = 0; + if (state == 0) { mi = 1; }; + let m: u8 = masks[mi * 8 + (nx & 7)]; + r = (r << 6u32) | ((b & m): u32); + if (nx <= 0) { + d.offs += 1; + if (nx == 0) { return r: rune; }; + let e: invalid; return e; + }; + state = nx; + d.offs += 1; + }; + let mr: more; return mr; +}; + +// ref/hare/encoding/utf8/decode.ha:207. Strict whole-input check. +// The hot path: tight DFA loop, no rune assembly. Bails the moment +// the table returns -1 so malformed inputs don't pay for the rest +// of the buffer. +export fn validate(src: []u8) (void | invalid) = { + let state: i32 = 0; + let i: i32 = 0; + for (i < src.len) { + if (state < 0) { break; }; + let bi: i32 = src[i]: i32; + let cell: i8 = dfa[state * 256 + bi]; + state = cell: i32; + i += 1; + }; + if (state == 0) { return; }; + let e: invalid; return e; +}; + +// ref/hare/encoding/utf8/rune.ha:5. Encoded byte length of `r` as +// UTF-8. Callers in ww use this to size the buffer they hand to +// [[encoderune]]; values >0x10FFFF or negative are not legal Unicode +// codepoints and Hare aborts on them in `encoderune` itself, so we +// keep `runesz` infallible (matches Hare). +export fn runesz(r: rune) i32 = { + let ch: u32 = r: u32; + if (ch < 128u32) { return 1; }; + if (ch < 2048u32) { return 2; }; + if (ch < 65536u32) { return 3; }; + return 4; +}; + +// ref/hare/encoding/utf8/rune.ha:15. Expected byte length of the +// codepoint that starts with `c`, or `invalid` if `c` cannot start +// a legal UTF-8 sequence. Constants written in decimal because ww +// doesn't accept Hare's `0b1000_0000` binary syntax: 0x80=128, +// 0xC2=194, 0xE0=224, 0xF0=240, 0xF8=248. +export fn utf8sz(c: u8) (i32 | invalid) = { + if (c < 128u8) { return 1; }; + if (c < 194u8) { let e: invalid; return e; }; + if (c >= 248u8) { let e: invalid; return e; }; + if (c < 224u8) { return 2; }; + if (c < 240u8) { return 3; }; + return 4; +}; + +// ref/hare/encoding/utf8/encode.ha:7. Encode `r` into `out` (caller- +// supplied; must hold at least [[runesz]](r) bytes) and return the +// byte count. ABORT if `r` is a UTF-16 surrogate or above U+10FFFF — +// same precondition Hare asserts at ref/hare/encoding/utf8/encode.ha:9. +// +// Surface deviation: Hare returns `[]u8` (slice into a static buf). +// ww uses the caller-buffer form (matches lib/encoding/hex.encode); +// caller can reuse a [4]u8 stack scratch across encodes. +export fn encoderune(out: []u8, r: rune) i32 = { + let ch: u32 = r: u32; + if (ch >= 0xD800u32) { + if (ch <= 0xDFFFu32) { + abort("utf8.encoderune: surrogate codepoint"); + }; + }; + if (ch > 0x10FFFFu32) { + abort("utf8.encoderune: codepoint > U+10FFFF"); + }; + + let n: i32 = 0; + let first: u8 = 0u8; + if (ch < 0x80u32) { + first = 0u8; n = 1; + } else if (ch < 0x800u32) { + first = 0xC0u8; n = 2; + } else if (ch < 0x10000u32) { + first = 0xE0u8; n = 3; + } else { + first = 0xF0u8; n = 4; + }; + + let v: u32 = ch; + let i: i32 = n - 1; + for (i > 0) { + out[i] = ((v: u8) & 0x3Fu8) | 0x80u8; + v = v >> 6u32; + i -= 1; + }; + out[0] = (v: u8) | first; + return n; +}; + +// ref/hare/encoding/utf8/decode.ha:52. Walks back from `d.offs` to a +// byte that could start a codepoint (state-0 dfa cell != -1), re-decodes +// forward from there, and confirms the forward decode lands back at the +// original offset. Returns `done` at start-of-input; `invalid` if no +// initial byte appears within 4 steps (no legal UTF-8 codepoint exceeds +// 4 bytes), if the forward decode returns `more`/`invalid`, or if it +// lands at a different offset than expected. Returns `more` when the +// walk reaches byte 0 without finding any initial byte. +// +// Hare's `for (d.offs < len(d.src); d.offs -= 1)` relies on size_t +// wrap-around to exit when offs underflows past 0; ww's offs is i32, +// so we spell the same exit as `d.offs >= 0`. Hare's `defer d.offs = t` +// is inlined in each match arm — ww has no defer. +export fn prev(d: *decoder) (rune | done | more | invalid) = { + if (d.offs == 0) { + let dn: done; return dn; + }; + let n: i32 = d.offs; + d.offs -= 1; + for (d.offs >= 0) { + let b: u8 = d.src[d.offs]; + let bi: i32 = b: i32; + let cell: i8 = dfa[bi]; + if (cell: i32 != -1) { + let t: i32 = d.offs; + match (next(d)) { + case let r: rune => { + let landed: i32 = d.offs; + d.offs = t; + if (landed != n) { + let e: invalid; return e; + }; + return r; + }; + case let dn: done => { + d.offs = t; + let e: invalid; return e; + }; + case let m: more => { + d.offs = t; + let e: invalid; return e; + }; + case let e: invalid => { + d.offs = t; + let e2: invalid; return e2; + }; + }; + }; + if (n - d.offs == 4) { + let e: invalid; return e; + }; + d.offs -= 1; + }; + let mr: more; return mr; +}; + +// ref/hare/encoding/utf8/decode.ha:74. Borrowed view of the bytes from +// the decoder's current position to the end of its source. +export fn remaining(d: *decoder) []u8 = { + let r: []u8; + r.ptr = d.src.ptr + (d.offs: u64); + r.len = d.src.len - d.offs; + r.cap = d.src.len - d.offs; + return r; +}; + +// ref/hare/encoding/utf8/decode.ha:80. Borrowed view of the bytes +// between two decoders' positions. Precondition (Hare asserts both): +// the decoders share the same source, and `begin.offs <= end.offs`. +export fn slice(begin: *decoder, end: *decoder) []u8 = { + if (begin.src.ptr != end.src.ptr) { + abort("utf8.slice: decoders from different sources"); + }; + if (begin.offs > end.offs) { + abort("utf8.slice: begin past end"); + }; + let r: []u8; + r.ptr = begin.src.ptr + (begin.offs: u64); + r.len = end.offs - begin.offs; + r.cap = end.offs - begin.offs; + return r; +}; + +// ref/hare/encoding/utf8/decode.ha:203. Byte position of the decoder +// in its source. +export fn position(d: *decoder) i32 = { + return d.offs; +}; + + +// strings — operations over str ({ptr,len}). Hare port; see +// ref/hare/strings/. +// +// Documented divergences from Hare: +// +// - `byteindex` / `rbyteindex` rune arms encode via +// `utf8.encoderune`; the legacy impls scanned for `r: u8` (an +// undocumented ASCII-only restriction that silently dropped +// to the wrong byte for U+80..U+7FF and higher). +// - `dup(s: str) str` — Hare returns `(str | nomem)`. ww's +// `os.alloc` aborts on OOM (no `nomem` type), so we return plain +// `str`. Empty input returns `{nil, 0}`; Hare returns the static +// empty string — same observable result. +// - `iterator` is flattened (`offs`, `src`, `reverse` fields). +// Hare uses anonymous-embedded `utf8::decoder` +// (ref/hare/strings/iter.ha:6-9); ww has no anonymous-embed +// syntax, so `next`/`prev`/`slice` copy `offs`/`src` into a +// local `utf8.decoder` for the call (and `next`/`prev` write +// `offs` back). +// - Hare's private `move()` helper dispatches on a `forward: bool` +// using a function-pointer `let fun = if (forward) &utf8::next +// else &utf8::prev`. ww has no fn-pointers in scope yet, so the +// dispatch is a branch on `forward` selecting the call site. + +package strings; + +import bytes; +import encoding.utf8; +import os; +import rt; +import types; + +// toutf8 — borrowed []u8 view of `s`. ref/hare/strings/utf8.ha:29. +// `cap` equals `len`; the slice does not own a separate allocation. +export fn toutf8(s: str) []u8 = { + let r: []u8; + r.ptr = s.ptr; + r.len = s.len; + r.cap = s.len; + return r; +}; + +// frombytes — borrowed str view of `in`. Pure reinterpret per +// CLAUDE.md rule 9 carve-out; ref/hare/strings/utf8.ha:10. +export fn frombytes(in: []u8) str = { + let r: str; + r.ptr = in.ptr; + r.len = in.len; + return r; +}; + +// compare — three-way bytewise codepoint-order comparison. Return is +// a sign (neg/zero/pos), not an index, so it tracks Hare's `int` +// rather than the str-index i32 (#8). ref/hare/strings/compare.ha:12. +export fn compare(a: str, b: str) int = { + let n: i32 = a.len; + if (b.len < n) { n = b.len; }; + let i: i32 = 0; + for (i < n) { + if (a[i] != b[i]) { return (a[i]: int) - (b[i]: int); }; + i += 1; + }; + return (a.len: int) - (b.len: int); +}; + +// dup — allocate a fresh copy of `s`. Caller releases with +// `os.free(r.ptr, r.len: u64)`. ref/hare/strings/dup.ha:7. +export fn dup(s: str) str = { + let r: str; + r.ptr = nil; + r.len = 0; + if (s.len == 0) { return r; }; + let buf: []u8 = alloc([], s.len: u64)!; + let i: i32 = 0; + for (i < s.len) { buf[i] = s[i]; i += 1; }; + buf.len = s.len; + return frombytes(buf); +}; + +// dupall — fresh `[]str` whose elements are independent copies of +// `s`'s elements. Caller releases via [[freeall]]. +// ref/hare/strings/dup.ha:26 (#6). +// +// Hare gates the per-element dup behind `?` and rolls back via +// `defer if (!ok) freeall(newsl)`. ww has no `defer if`; more +// importantly, ww's [[dup]] is still unchecked (returns plain `str`, +// aborts via os.alloc on OOM — see top-of-file divergence note), +// so the only nomem propagation point is the initial slice alloc. +// With no inner failure path, the rollback is structurally a no-op +// and is omitted; it returns once dup graduates to `(str | nomem)` +// (#46). The pre-allocated slice has `cap == s.len`, so appendstr's +// rt_ensure call never reaches the grow branch. +// +// Empty input bypasses the alloc: rt_malloc(0) is an mmap of 0 bytes +// which returns -EINVAL, and the alloc-slice `?` shortcut routes +// that through nomem — Hare's heap allocator hands back a sentinel +// instead (#47). Return `{nil, 0, 0}` directly so callers get the +// Hare-observable shape (len==0, freeall is a no-op via cap==0). +export fn dupall(s: []str) ([]str | nomem) = { + if (s.len == 0) { + let r: []str; + r.ptr = nil: *str; + r.len = 0; + r.cap = 0; + return r; + }; + let newsl: []str = alloc([], s.len)?; + let i: i32 = 0; + for (i < s.len) { + appendstr(&newsl, dup(s[i])); + i += 1; + }; + return newsl; +}; + +// freeall — release each element + the slice header. The natural +// disposer for any `[]str` of dup'd elements (e.g. shlex.split). +// ref/hare/strings/dup.ha:38. +// +// Empty elements (`{nil, 0}` from a zero-length dup) are skipped: +// os.free on a nil pointer at len 0 tickles the rt_free guard. The +// slice header itself is freed at `cap * size(str)` — the literal +// would drift under #1's str-layout bump, so route through the +// typ.ww SSoT. A never-grown slice (cap == 0) skips the header free. +export fn freeall(s: []str) void = { + let i: i32 = 0; + for (i < s.len) { + if (s[i].len > 0) { + os.free(s[i].ptr: *void, s[i].len: u64); + }; + i += 1; + }; + if (s.cap > 0) { + os.free(s.ptr: *void, (s.cap: u64) * size(str): u64); + }; +}; + +// concat — fresh allocation containing each element of `strs` in +// order. Caller releases with `os.free(r.ptr, r.len: u64)`. +// ref/hare/strings/concat.ha:5. Hare's `nomem` return is dropped: +// `os.alloc` aborts on OOM. +export fn concat(strs: str...) str = { + let total: i32 = 0; + let i: i32 = 0; + for (i < strs.len) { total += strs[i].len; i += 1; }; + let r: str; + r.ptr = nil; + r.len = 0; + if (total == 0) { return r; }; + let buf: []u8 = alloc([], total: u64)!; + let off: i32 = 0; + i = 0; + for (i < strs.len) { + let j: i32 = 0; + for (j < strs[i].len) { + buf[off + j] = strs[i][j]; + j += 1; + }; + off += strs[i].len; + i += 1; + }; + buf.len = total; + return frombytes(buf); +}; + +// join — fresh allocation with `delim` placed between each element of +// `strs`. Caller releases with `os.free(r.ptr, r.len: u64)`. +// ref/hare/strings/concat.ha:46. Hare's `nomem` return is dropped: +// `os.alloc` aborts on OOM. +export fn join(delim: str, strs: str...) str = { + let total: i32 = 0; + let i: i32 = 0; + for (i < strs.len) { + total += strs[i].len; + if (i + 1 < strs.len) { total += delim.len; }; + i += 1; + }; + let r: str; + r.ptr = nil; + r.len = 0; + if (total == 0) { return r; }; + let buf: []u8 = alloc([], total: u64)!; + let off: i32 = 0; + i = 0; + for (i < strs.len) { + let j: i32 = 0; + for (j < strs[i].len) { + buf[off + j] = strs[i][j]; + j += 1; + }; + off += strs[i].len; + if (i + 1 < strs.len) { + j = 0; + for (j < delim.len) { + buf[off + j] = delim[j]; + j += 1; + }; + off += delim.len; + }; + i += 1; + }; + buf.len = total; + return frombytes(buf); +}; + +// utf8bytelenbounded — walk `it` forward `end` runes and return the +// resulting byte offset. ref/hare/strings/sub.ha:10. Aborts on +// short input per Hare's contract for the rune-wise [[sub]]. +fn utf8bytelenbounded(it: *iterator, end: i32) i32 = { + let i: i32 = 0; + for (i < end) { + match (next(it)) { + case let r: rune => void; + case utf8.done => abort("strings.sub: index exceeds string length"); + }; + i += 1; + }; + return it.offs; +}; + +// sub — borrowed substring [start, end) where start/end are rune +// indices. ref/hare/strings/sub.ha:30. Hare's 2-arg `sub(s, start)` +// defaulting end=END is omitted: ww has no default-parameter syntax +// (filed as #37). Byte-indexed counterpart: [[bytesub]]. +export fn sub(s: str, start: i32, end: i32) str = { + os.assert(start <= end, "strings.sub: start is higher than end"); + let it: iterator = iter(s); + let starti: i32 = utf8bytelenbounded(&it, start); + let endi: i32 = utf8bytelenbounded(&it, end - start); + let r: str; + r.ptr = s.ptr + (starti: u64); + r.len = endi - starti; + return r; +}; + +// bytesub — borrowed substring [start, end) where start/end are byte +// offsets. ref/hare/strings/sub.ha:59 (#7). Returns `utf8.invalid` if +// either endpoint lands on a continuation byte (would split a +// codepoint); the equivalent Hare predicate is `s[i] & 0xc0 == 0x80` +// at ref/hare/strings/sub.ha:72-73. +export fn bytesub(s: str, start: i32, end: i32) (str | utf8.invalid) = { + os.assert(start <= end, "strings.bytesub: start is higher than end"); + os.assert(end <= s.len, "strings.bytesub: end exceeds string length"); + if (start < s.len) { + if ((s[start] & 0xC0u8) == 0x80u8) { + let e: utf8.invalid; return e; + }; + }; + if (end < s.len) { + if ((s[end] & 0xC0u8) == 0x80u8) { + let e: utf8.invalid; return e; + }; + }; + let r: str; + r.ptr = s.ptr + (start: u64); + r.len = end - start; + return r; +}; + +// runebytes — encode `r` into caller's `scratch` (must hold 4 bytes) +// and return the borrowed slice trimmed to the encoded length. Hare +// inlines the same shape at ref/hare/strings/index.ha:132. +fn runebytes(scratch: []u8, r: rune) []u8 = { + let n: i32 = utf8.encoderune(scratch, r); + let s: []u8; + s.ptr = scratch.ptr; + s.len = n; + s.cap = n; + return s; +}; + +// hasprefix — true iff `in` begins with `prefix`. +// ref/hare/strings/suffix.ha:8. +export fn hasprefix(in: str, prefix: (str | rune)) bool = { + let scratch: [4]u8; + let p: []u8 = match (prefix) { + case let s: str => yield toutf8(s); + case let r: rune => yield runebytes(scratch[0:4], r); + }; + return bytes.hasprefix(toutf8(in), p); +}; + +// hassuffix — true iff `in` ends with `suff`. +// ref/hare/strings/suffix.ha:26. +export fn hassuffix(in: str, suff: (str | rune)) bool = { + let scratch: [4]u8; + let s: []u8 = match (suff) { + case let v: str => yield toutf8(v); + case let r: rune => yield runebytes(scratch[0:4], r); + }; + return bytes.hassuffix(toutf8(in), s); +}; + +// byteindex — byte-wise offset of `needle` in `haystack`, or void if +// absent. ref/hare/strings/index.ha:127. Rune arm encodes via +// utf8.encoderune (Hare passes the encoded slice straight to +// bytes::index). +export fn byteindex(haystack: str, needle: (str | rune)) (i32 | void) = { + let scratch: [4]u8; + let n: []u8 = match (needle) { + case let s: str => yield toutf8(s); + case let r: rune => yield runebytes(scratch[0:4], r); + }; + return bytes.index(toutf8(haystack), n); +}; + +// rbyteindex — byte-wise offset of the last `needle` in `haystack`. +// ref/hare/strings/index.ha:138. +export fn rbyteindex(haystack: str, needle: (str | rune)) (i32 | void) = { + let scratch: [4]u8; + let n: []u8 = match (needle) { + case let s: str => yield toutf8(s); + case let r: rune => yield runebytes(scratch[0:4], r); + }; + return bytes.rindex(toutf8(haystack), n); +}; + +// indexstring — str-arm of [[index]]. Dual-rune-iterator walk: at each +// candidate rune index `i`, compare `haystack` from that position +// against `needle` rune-by-rune until needle is exhausted (match) or +// a mismatch / haystack-exhaustion breaks the inner loop. Mirrors +// ref/hare/strings/index.ha:59 (#10). Hare copies `rest_iter = s_iter` +// directly via struct assignment; ww re-seats `rest_iter` field-wise +// because the let-init struct-copy form diverges between cstage and +// wwstage on this iterator type (993_ww_ww + 995_self_rebuild fail, +// filed as #41) and rule #10 (CLAUDE.md) forbids stage asymmetry. +fn indexstring(haystack: str, needle: str) (i32 | void) = { + let s_iter: iterator = iter(haystack); + let i: i32 = 0; + for (true) { + let rest_iter: iterator; + rest_iter.src = s_iter.src; + rest_iter.offs = s_iter.offs; + rest_iter.reverse = s_iter.reverse; + let needle_iter: iterator = iter(needle); + let matched: bool = false; + for (true) { + let rest_done: bool = false; + let rest_r: rune; + match (next(&rest_iter)) { + case let r: rune => rest_r = r; + case utf8.done => rest_done = true; + }; + let needle_done: bool = false; + let needle_r: rune; + match (next(&needle_iter)) { + case let r: rune => needle_r = r; + case utf8.done => needle_done = true; + }; + if (rest_done && !needle_done) { break; }; + if (needle_done) { matched = true; break; }; + if (rest_r != needle_r) { break; }; + }; + if (matched) { return i; }; + match (next(&s_iter)) { + case let r: rune => i += 1; + case utf8.done => return; + }; + }; + return; +}; + +// index — rune-wise offset of `needle`'s first occurrence in +// `haystack`, or void if absent. ref/hare/strings/index.ha:10. The +// str-arm delegates to [[indexstring]] (dual-iterator rune-by-rune +// walk per Hare's `index_string`, #10); the rune-arm mirrors Hare's +// `index_rune` (ref/hare/strings/index.ha:31). +export fn index(haystack: str, needle: (str | rune)) (i32 | void) = { + match (needle) { + case let s: str => return indexstring(haystack, s); + case let r: rune => { + let it: iterator = iter(haystack); + let i: i32 = 0; + for (true) { + match (next(&it)) { + case let n: rune => { + if (n == r) { return i; }; + i += 1; + }; + case utf8.done => return; + }; + }; + }; + }; + return; +}; + +// rindex — rune-wise offset of `needle`'s last occurrence in +// `haystack`, or void if absent. ref/hare/strings/index.ha:22. The +// str-arm reuses `rbyteindex`; the rune-arm walks forward tracking +// the most recent matching rune index (Hare's `rindex_rune` with +// `riter` returns a byte-offset value for multibyte strings, which +// disagrees with the rune-wise docstring; we keep the docstring's +// contract). +export fn rindex(haystack: str, needle: (str | rune)) (i32 | void) = { + match (needle) { + case let s: str => { + match (rbyteindex(haystack, s)) { + case void => return; + case let bo: i32 => { + let it: iterator = iter(haystack); + let i: i32 = 0; + for (position(&it) < bo) { + match (next(&it)) { + case let r: rune => i += 1; + case utf8.done => break; + }; + }; + return i; + }; + }; + }; + case let r: rune => { + let it: iterator = iter(haystack); + let i: i32 = 0; + let last: i32 = -1; + for (true) { + match (next(&it)) { + case let n: rune => { + if (n == r) { last = i; }; + i += 1; + }; + case utf8.done => break; + }; + }; + if (last < 0) { return; }; + return last; + }; + }; + return; +}; + +// contains — true iff any of `needles` occurs in `haystack`. +// ref/hare/strings/contains.ha:9. +export fn contains(haystack: str, needles: (str | rune)...) bool = { + let i: i32 = 0; + for (i < needles.len) { + match (needles[i]) { + case let s: str => { + match (byteindex(haystack, s)) { + case let bo: i32 => return true; + case void => void; + }; + }; + case let r: rune => { + match (byteindex(haystack, r)) { + case let bo: i32 => return true; + case void => void; + }; + }; + }; + i += 1; + }; + return false; +}; + +// trimprefix — `s` with `prefix` stripped from the front, or `s` +// unchanged if it doesn't start with `prefix`. Borrowed view. +// ref/hare/strings/trim.ha:60. +export fn trimprefix(input: str, prefix: str) str = { + if (!hasprefix(input, prefix)) { return input; }; + let r: str; + r.ptr = input.ptr + (prefix.len: u64); + r.len = input.len - prefix.len; + return r; +}; + +// trimsuffix — symmetric. ref/hare/strings/trim.ha:69. +export fn trimsuffix(input: str, suffix: str) str = { + if (!hassuffix(input, suffix)) { return input; }; + let r: str; + r.ptr = input.ptr; + r.len = input.len - suffix.len; + return r; +}; + +// whitespace — ASCII whitespace set used by the 0-arg ltrim/rtrim/trim +// branches (#9). ref/hare/strings/trim.ha:6. +let whitespace: [4]u8 = [0x20u8, 0x0Au8, 0x09u8, 0x0Du8]; + +// ltrim — strip leading runes that occur in `trim`. Borrowed view. +// 0-arg strips ASCII whitespace via [[bytes.ltrim]] (#9). +// ref/hare/strings/trim.ha:11. The spread expression is inlined +// because `let ws: []u8 = whitespace[0:4]` produces a slice whose +// ptr doesn't track the module-level array storage (filed as #40); +// `b.flush = flushdefault[0:1]` in lib/bufio is the same shape via +// the working field-assign path. +export fn ltrim(input: str, trim: rune...) str = { + if (trim.len == 0) { + return frombytes(bytes.ltrim(toutf8(input), whitespace[0:4]...)); + }; + let it: iterator = iter(input); + for (true) { + match (next(&it)) { + case let r: rune => { + let j: i32 = 0; + let found: bool = false; + for (j < trim.len) { + if (r == trim[j]) { found = true; j = trim.len; } + else { j += 1; }; + }; + if (!found) { + match (prev(&it)) { + case let r2: rune => void; + case utf8.done => void; + }; + break; + }; + }; + case utf8.done => break; + }; + }; + return iterstr(&it); +}; + +// rtrim — strip trailing runes that occur in `trim`. Borrowed view. +// 0-arg strips ASCII whitespace via [[bytes.rtrim]] (#9). Spread is +// inlined to dodge #40 — see [[ltrim]]. +// ref/hare/strings/trim.ha:32. +export fn rtrim(input: str, trim: rune...) str = { + if (trim.len == 0) { + return frombytes(bytes.rtrim(toutf8(input), whitespace[0:4]...)); + }; + let it: iterator = riter(input); + for (true) { + match (next(&it)) { + case let r: rune => { + let j: i32 = 0; + let found: bool = false; + for (j < trim.len) { + if (r == trim[j]) { found = true; j = trim.len; } + else { j += 1; }; + }; + if (!found) { + match (prev(&it)) { + case let r2: rune => void; + case utf8.done => void; + }; + break; + }; + }; + case utf8.done => break; + }; + }; + return iterstr(&it); +}; + +// trim — strip from both ends. ref/hare/strings/trim.ha:54. +export fn trim(input: str, trim: rune...) str = { + return ltrim(rtrim(input, trim...), trim...); +}; + +// iterator — UTF-8 rune cursor over a `str`. Layout flattens Hare's +// anonymous-embedded `utf8::decoder` (ref/hare/strings/iter.ha:6-9) to +// explicit fields. `reverse` selects walk direction: forward iterators +// (`iter`) advance through utf8.next; reverse iterators (`riter`) advance +// through utf8.prev. May be copied to save state. +export type iterator = struct { + offs: i32, + src: []u8, + reverse: bool, +}; + +// iter — initialize a forward iterator at the start of `src`. +// ref/hare/strings/iter.ha:24. +export fn iter(src: str) iterator = { + let r: iterator; + r.src = toutf8(src); + r.offs = 0; + r.reverse = false; + return r; +}; + +// riter — initialize a reverse iterator at the end of `src`. `next` +// on a reverse iterator walks back through the string. +// ref/hare/strings/iter.ha:32. +export fn riter(src: str) iterator = { + let r: iterator; + r.src = toutf8(src); + r.offs = src.len; + r.reverse = true; + return r; +}; + +// move — private dispatch shared by next/prev. `forward` selects +// utf8.next vs utf8.prev. Aborts on more/invalid per Hare's +// ref/hare/strings/iter.ha:51-58 ("Invalid UTF-8 string (this should +// not happen)"). Hare picks the utf8 function via a fn-pointer; ww +// branches on `forward` at each call site instead. +fn move(forward: bool, it: *iterator) (rune | utf8.done) = { + let d: utf8.decoder; + d.src = it.src; + d.offs = it.offs; + if (forward) { + match (utf8.next(&d)) { + case let r: rune => { it.offs = d.offs; return r; }; + case let dn: utf8.done => return dn; + case let m: utf8.more => abort("strings.move: invalid UTF-8"); + case let e: utf8.invalid => abort("strings.move: invalid UTF-8"); + }; + } else { + match (utf8.prev(&d)) { + case let r: rune => { it.offs = d.offs; return r; }; + case let dn: utf8.done => return dn; + case let m: utf8.more => abort("strings.move: invalid UTF-8"); + case let e: utf8.invalid => abort("strings.move: invalid UTF-8"); + }; + }; +}; + +// next — advance the iterator one rune. Forward iterators step +// through utf8.next; reverse iterators (riter) step backward through +// utf8.prev. Returns utf8.done at end-of-walk. ref/hare/strings/iter.ha:45. +export fn next(it: *iterator) (rune | utf8.done) = { + return move(!it.reverse, it); +}; + +// prev — step back one rune. Dual to next: on a forward iterator +// this walks utf8.prev; on a reverse iterator (riter) it walks +// utf8.next. ref/hare/strings/iter.ha:49. +export fn prev(it: *iterator) (rune | utf8.done) = { + return move(it.reverse, it); +}; + +// iterstr — borrowed view of the bytes remaining in the iterator's +// walk direction. Forward iter: bytes from offs to end; reverse iter: +// bytes from start to offs. ref/hare/strings/iter.ha:63. +export fn iterstr(it: *iterator) str = { + let r: []u8; + if (it.reverse) { + r = it.src[0:it.offs]; + } else { + r = it.src[it.offs:it.src.len]; + }; + return frombytes(r); +}; + +// slice — borrowed substring between two iterator positions. +// ref/hare/strings/iter.ha:75. Hare passes `*iterator` directly where +// `*utf8::decoder` is expected via anonymous-embed coercion; ww has +// no anonymous embed, so we reconstruct a local utf8.decoder for each +// endpoint and forward — same pattern as `move` above. +export fn slice(begin: *iterator, end: *iterator) str = { + let b: utf8.decoder; + b.src = begin.src; + b.offs = begin.offs; + let e: utf8.decoder; + e.src = end.src; + e.offs = end.offs; + return frombytes(utf8.slice(&b, &e)); +}; + +// position — byte-wise offset of the iterator in its source. +// ref/hare/strings/iter.ha:82. +export fn position(it: *iterator) i32 = { + return it.offs; +}; + +// tokenizer — re-export of bytes.tokenizer. ref/hare/strings/tokenize.ha:7. +// First cross-module type alias in tree; needs #22's transitive +// alias-chain unwrap (cstage type_chase_named + wwstage +// structlookupchain) to walk struct fields through the chain. +export type tokenizer = bytes.tokenizer; + +// tokenize — yield substrings of `s` split on any byte in `delim`. +// Leading / trailing / adjacent delims yield empty tokens. `s` and +// `delim` are borrowed; caller keeps them live for the tokenizer's +// lifetime. ref/hare/strings/tokenize.ha:32. ASCII-only delim +// asserted per Hare lines 35-37: a multibyte rune in delim would +// split on a single continuation byte and yield invalid UTF-8. +export fn tokenize(s: str, delim: str) tokenizer = { + let d: []u8 = toutf8(delim); + let i: i32 = 0; + for (i < d.len) { + os.assert((d[i] & 0x80u8) == 0u8, + "strings.tokenize cannot tokenize on non-ASCII delimiters"); + i += 1; + }; + return bytes.tokenize(toutf8(s), d...); +}; + +// rtokenize — reverse-direction counterpart to [[tokenize]]. First +// next_token yields the last token, last yields the first. +// ref/hare/strings/tokenize.ha:44. +export fn rtokenize(s: str, delim: str) tokenizer = { + let d: []u8 = toutf8(delim); + let i: i32 = 0; + for (i < d.len) { + os.assert((d[i] & 0x80u8) == 0u8, + "strings.rtokenize cannot tokenize on non-ASCII delimiters"); + i += 1; + }; + return bytes.rtokenize(toutf8(s), d...); +}; + +// next_token — current token, advancing the cursor. +// ref/hare/strings/tokenize.ha:62. +export fn next_token(s: *tokenizer) (str | bytes.done) = { + let b: *bytes.tokenizer = s: *bytes.tokenizer; + match (bytes.next_token(b)) { + case let v: []u8 => return frombytes(v); + case bytes.done => { let d: bytes.done; return d; }; + }; +}; + +// peek_token — current token without advancing. +// ref/hare/strings/tokenize.ha:71. +export fn peek_token(s: *tokenizer) (str | bytes.done) = { + let b: *bytes.tokenizer = s: *bytes.tokenizer; + match (bytes.peek_token(b)) { + case let v: []u8 => return frombytes(v); + case bytes.done => { let d: bytes.done; return d; }; + }; +}; + +// remaining_tokens — unconsumed portion of the input ahead of the +// cursor. ref/hare/strings/tokenize.ha:79. +export fn remaining_tokens(s: *tokenizer) str = { + let b: *bytes.tokenizer = s: *bytes.tokenizer; + return frombytes(bytes.remaining_tokens(b)); +}; + +// rt_ensure is the runtime slice-growth helper invoked by the +// `append(s, v)` builtin. Direct bind for the same reason as +// lib/shlex.shlex (appendstr, 16B): the builtin's expansion stores +// only 8B of the new element, losing the `.len` half of a `str`. +@symbol("rt_ensure") fn rtensure(s: *void, membsz: u64) void; + +// appendstr — grow `*slice` by one and store `item` (16B). Mirror of +// lib/shlex.shlex appendstr. Collapses when the append builtin learns +// to store the full element width. +fn appendstr(slice: *[]str, item: str) void = { + let newlen: i32 = slice.len + 1; + slice.len = newlen; + rtensure(slice: *void, size(str): u64); + let dst: *str = &slice.ptr[newlen - 1]; + dst.ptr = item.ptr; + dst.len = item.len; +}; + +// splitn — split `in` on any byte in `delim`, returning up to `n` +// tokens via forward iteration. The trailing slot (when more than +// `n - 1` tokens exist) holds the unconsumed remainder. Strings +// within the result are borrowed from `in`. +// +// The caller frees the returned slice via +// `os.free(r.ptr: *void, (r.cap: u64) * size(str): u64)`. +// +// Hare's `([]str | nomem)` collapses to `[]str` here: ww os.alloc +// has no recoverable failure path. Same precedent as +// shlex.split / bytes.splitn. +// +// ref/hare/strings/tokenize.ha:172. +export fn splitn(in: str, delim: str, n: i32) []str = { + let toks: []str; + toks.ptr = nil: *str; + toks.len = 0; + toks.cap = 0; + let tok: tokenizer = tokenize(in, delim); + let i: i32 = 0; + for (i < n - 1) { + match (next_token(&tok)) { + case let s: str => { appendstr(&toks, s); }; + case bytes.done => { return toks; }; + }; + i += 1; + }; + match (peek_token(&tok)) { + case bytes.done => void; + case let pk: str => { + let r: str = remaining_tokens(&tok); + appendstr(&toks, r); + }; + }; + return toks; +}; + +// rsplitn — reverse-direction counterpart to [[splitn]]: tokens are +// collected from the end of `in`. The trailing slot holds the +// unconsumed prefix (everything before the n-th-from-last delim hit). +// +// When the input has fewer than n tokens, the `done` short-circuit +// returns toks UN-reversed (in last-token-first order). Mirrors Hare +// at ref/hare/strings/tokenize.ha:219-224 where the in-place reverse +// step is gated behind the n-1 loop running to completion. +// +// ref/hare/strings/tokenize.ha:200. +export fn rsplitn(in: str, delim: str, n: i32) []str = { + let toks: []str; + toks.ptr = nil: *str; + toks.len = 0; + toks.cap = 0; + let tok: tokenizer = rtokenize(in, delim); + let i: i32 = 0; + for (i < n - 1) { + match (next_token(&tok)) { + case let s: str => { appendstr(&toks, s); }; + case bytes.done => { return toks; }; + }; + i += 1; + }; + match (peek_token(&tok)) { + case bytes.done => void; + case let pk: str => { + let r: str = remaining_tokens(&tok); + appendstr(&toks, r); + }; + }; + + // In-place reverse so callers see argv-order, matching Hare + // (ref/hare/strings/tokenize.ha:220). Element copy is field-wise + // through `*str` because `toks[i] = toks[j]` (full 16B str store) + // lands in the multi-word-store gap noted at cmd/w6c/cgen.c:6515. + let a: i32 = 0; + let b: i32 = toks.len - 1; + for (a < b) { + let pa: *str = &toks.ptr[a]; + let pb: *str = &toks.ptr[b]; + let tp: *u8 = pa.ptr; + let tl: i32 = pa.len; + pa.ptr = pb.ptr; + pa.len = pb.len; + pb.ptr = tp; + pb.len = tl; + a += 1; + b -= 1; + }; + return toks; +}; + +// split — full split of `in` on `delim` (no token cap). Mirrors +// `splitn(in, delim, types::SIZE_MAX)`. ww uses `types.I32_MAX` +// because the index type is i32 (lib/CLAUDE.md). +// +// ref/hare/strings/tokenize.ha:242. +export fn split(in: str, delim: str) []str = { + return splitn(in, delim, types.I32_MAX); +}; + +// lpad — left-pad `s` with `p` rune until the result reaches `maxlen` +// bytes. Length comparison is BYTES, mirroring Hare's `len(s) >= maxlen` +// at ref/hare/strings/pad.ha:9. A multibyte `p` whose encoded width +// doesn't divide `maxlen - s.len` evenly leaves a trailing pad byte +// pair sliced mid-codepoint at byte `maxlen-1`, exactly as Hare's +// `res[..maxlen]` does (ref/hare/strings/pad.ha:20). When +// `(maxlen - s.len) * pad.len >= maxlen` (multibyte pad overflows the +// budget), `s` is entirely sliced off — same as Hare. Caller releases +// with `os.free(r.ptr, r.len: u64)`. Hare's `nomem` return is dropped: +// `os.alloc` aborts on OOM. Buf size == r.len keeps the free-contract +// shape of [[dup]] / [[concat]] / [[join]]; Hare's `alloc([], maxlen)!` +// over-allocs via append then slices, but Hare's slice-free recovers +// the true capacity from the heap allocator (rt/ensure.ha:24), which +// ww's munmap-based `os.free` cannot do. +export fn lpad(s: str, p: rune, maxlen: i32) str = { + if (s.len >= maxlen) { return dup(s); }; + let scratch: [4]u8; + let pad: []u8 = runebytes(scratch[0:4], p); + let buf: []u8 = alloc([], maxlen: u64)!; + let padwrite: i32 = (maxlen - s.len) * pad.len; + if (padwrite > maxlen) { padwrite = maxlen; }; + let off: i32 = 0; + for (off < padwrite) { + buf[off] = pad.ptr[off % pad.len]; + off += 1; + }; + let k: i32 = 0; + let srem: i32 = maxlen - off; + if (srem > s.len) { srem = s.len; }; + for (k < srem) { + buf[off + k] = s[k]; + k += 1; + }; + buf.len = maxlen; + return frombytes(buf); +}; + +// replace — fresh allocation of `s` with every non-overlapping +// occurrence of `needle` replaced by `target`. Caller releases with +// `os.free(r.ptr, r.len: u64)`. ref/hare/strings/replace.ha:8 (#4). +// +// Hare delegates to [[multireplace]] with a single pair; ww has no +// `(str, str)` variadic shape today (#39), so this is a standalone +// two-pass implementation: pass 1 counts matches to size the result, +// pass 2 copies chunks and `target` into a single fresh buffer. +// Single nomem path (the `alloc([], total)?`) preserves Hare's +// signature without a per-write `append(...)?` (ww's append builtin +// aborts on OOM, #11). Empty `needle` would hasprefix-match every +// position with a zero stride — same infinite loop Hare exhibits at +// ref/hare/strings/replace.ha:31; not gated. +export fn replace(s: str, needle: str, target: str) (str | nomem) = { + let sb: []u8 = toutf8(s); + let nb: []u8 = toutf8(needle); + let tb: []u8 = toutf8(target); + let count: i32 = 0; + let i: i32 = 0; + for (i < sb.len) { + if (bytes.hasprefix(sb[i:sb.len], nb)) { + count += 1; + i += nb.len; + } else { + i += 1; + }; + }; + let total: i32 = sb.len + count * (tb.len - nb.len); + if (total == 0) { + let r: str; + r.ptr = nil; + r.len = 0; + return r; + }; + let res: []u8 = alloc([], total)?; + let off: i32 = 0; + i = 0; + for (i < sb.len) { + if (bytes.hasprefix(sb[i:sb.len], nb)) { + let j: i32 = 0; + for (j < tb.len) { + res.ptr[off + j] = tb.ptr[j]; + j += 1; + }; + off += tb.len; + i += nb.len; + } else { + res.ptr[off] = sb.ptr[i]; + off += 1; + i += 1; + }; + }; + res.len = total; + return frombytes(res); +}; + +// rpad — right-pad `s` with `p` rune until the result reaches `maxlen` +// bytes. Symmetric with [[lpad]]. ref/hare/strings/pad.ha:39. +export fn rpad(s: str, p: rune, maxlen: i32) str = { + if (s.len >= maxlen) { return dup(s); }; + let scratch: [4]u8; + let pad: []u8 = runebytes(scratch[0:4], p); + let buf: []u8 = alloc([], maxlen: u64)!; + let k: i32 = 0; + for (k < s.len) { + buf[k] = s[k]; + k += 1; + }; + let padwrite: i32 = maxlen - s.len; + let i: i32 = 0; + for (i < padwrite) { + buf[s.len + i] = pad.ptr[i % pad.len]; + i += 1; + }; + buf.len = maxlen; + return frombytes(buf); +}; + // selfhost/cmd/ww/main.ww — port of cmd/ww/main.c. // // The user-facing driver. Plan 9 cc(1) / Hare hare(1) analogue: @@ -878,6 +2809,7 @@ package main; import os; import rt; import mem; +import strings; // All path/string scratch buffers go on the runtime page allocator. // One page is plenty for any path we build. @@ -1390,7 +3322,10 @@ fn scanuse(src: *u8, len: u64) (*u8, u64) = { // decls). fn expand(c: *expctx, pathcs: *u8) void = { let plen: u64 = cstrlen(pathcs); - let pathstr: str = astrndup(c.a, pathcs, plen); + let view: str; + view.ptr = pathcs; + view.len = plen: i32; + let pathstr: str = strings.dup(view); if (visitseen(c, pathstr)) { return; }; visitadd(c, pathstr); diff --git a/selfhost/cmd/ww/main.ww b/selfhost/cmd/ww/main.ww index 7c521932..6444b532 100644 --- a/selfhost/cmd/ww/main.ww +++ b/selfhost/cmd/ww/main.ww @@ -16,6 +16,7 @@ package main; import os; import rt; import mem; +import strings; // All path/string scratch buffers go on the runtime page allocator. // One page is plenty for any path we build. @@ -528,7 +529,10 @@ fn scanuse(src: *u8, len: u64) (*u8, u64) = { // decls). fn expand(c: *expctx, pathcs: *u8) void = { let plen: u64 = cstrlen(pathcs); - let pathstr: str = astrndup(c.a, pathcs, plen); + let view: str; + view.ptr = pathcs; + view.len = plen: i32; + let pathstr: str = strings.dup(view); if (visitseen(c, pathstr)) { return; }; visitadd(c, pathstr); diff --git a/selfhost/cmd/wwdump/main.combined.ww b/selfhost/cmd/wwdump/main.combined.ww index 2dd8ed63..d62e0fa9 100644 --- a/selfhost/cmd/wwdump/main.combined.ww +++ b/selfhost/cmd/wwdump/main.combined.ww @@ -3730,6 +3730,7 @@ package lex; import os; import ascii; import mem; +import strings; // isidstart / isidpart — identifier classification. Lexer-local // because the "alpha or '_' / alnum or '_'" set isn't part of Hare's @@ -4159,7 +4160,10 @@ fn lexnum(l: *lex, start: *pos, out: *tok) void = { }; let n: u64 = l.lpos - begin; - out.text = astrndup(l.a, l.src + begin, n); + let view: str; + view.ptr = l.src + begin; + view.len = n: i32; + out.text = strings.dup(view); if (isfloat) { out.kind = tkind.TK_FLOAT; @@ -4238,7 +4242,10 @@ fn lexnum(l: *lex, start: *pos, out: *tok) void = { }; }; if (isok) { - out.tsuffix = astrndup(l.a, p, sl); + let view: str; + view.ptr = p; + view.len = sl: i32; + out.tsuffix = strings.dup(view); } else { l.lpos = sb; }; @@ -4263,7 +4270,10 @@ fn lexident(l: *lex, start: *pos, out: *tok) void = { if (n == 1u64) { if (p[0] == 95u8) { out.kind = tkind.TK_UNDER; - out.text = astrndup(l.a, p, n); + let view: str; + view.ptr = p; + view.len = n: i32; + out.text = strings.dup(view); return; }; }; @@ -4273,7 +4283,10 @@ fn lexident(l: *lex, start: *pos, out: *tok) void = { } else { out.kind = tkind.TK_IDENT; }; - out.text = astrndup(l.a, p, n); + let view: str; + view.ptr = p; + view.len = n: i32; + out.text = strings.dup(view); }; fn lexstr(l: *lex, start: *pos, out: *tok) void = { @@ -4288,7 +4301,10 @@ fn lexstr(l: *lex, start: *pos, out: *tok) void = { out.file = start.file; out.line = start.line; out.col = start.col; - out.text = astrndup(l.a, "".ptr, 0u64); + let view: str; + view.ptr = "".ptr; + view.len = 0; + out.text = strings.dup(view); return; }; if (c == 34) { lget(l); break; }; @@ -4333,7 +4349,10 @@ fn lexrune(l: *lex, start: *pos, out: *tok) void = { out.file = start.file; out.line = start.line; out.col = start.col; - out.text = astrndup(l.a, "".ptr, 0u64); + let view: str; + view.ptr = "".ptr; + view.len = 0; + out.text = strings.dup(view); return; }; let ch: i32 = 0; @@ -4349,7 +4368,10 @@ fn lexrune(l: *lex, start: *pos, out: *tok) void = { out.file = start.file; out.line = start.line; out.col = start.col; - out.text = astrndup(l.a, "".ptr, 0u64); + let view: str; + view.ptr = "".ptr; + view.len = 0; + out.text = strings.dup(view); return; }; lget(l); @@ -4503,7 +4525,10 @@ export fn lexnext(l: *lex, out: *tok) void = { setposfrom(out, &start); let one: [1]u8; one[0] = c: u8; - out.text = astrndup(l.a, one.ptr, 1u64); + let view: str; + view.ptr = one.ptr; + view.len = 1; + out.text = strings.dup(view); }; // lib/ww/ast.ww — port of cmd/wcc/ast.c (Node defs + printer).