Hojun-Cho/ww
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

lib/bytes+test: port tokenize family from Hare (#18)

Browse Source
This commit is contained in:
Hojun-Cho
2026-05-19 11:41:36 +09:00
parent 006df414aa
commit 3a85db0f3f
6 changed files with 1079 additions and 20 deletions
Download Patch File Download Diff File Expand all files Collapse all files

196
lib/bytes/bytes.ww
View File

@@ -7,11 +7,40 @@
// 2/3/4-byte needles and falls back to two_way (Crochemore-Perrin)
// for longer (ref/hare/bytes/index.ha:61, ref/hare/bytes/two_way.ha).
// Correctness equivalent.
// - peek_token dispatches index/rindex by branching on `reverse`
// rather than a function-pointer `ifunc` (ref/hare/bytes/tokenize.ha:97).
// ww has no fn pointers in scope yet — same pattern as lib/strings
// `move`. Outwardly identical.
// - tokenize / rtokenize zero the `delim` field on the constructed
// tokenizer when `in` is empty, rather than mutating the variadic
// param before the struct write (ref/hare/bytes/tokenize.ha:26-28).
// Semantically identical; the variadic param is borrowed and
// captured-by-value into the struct, so mutating either side
// yields the same observable state.
package bytes;
import os;
import types;
// done — iteration sentinel returned by next_token / peek_token at
// end-of-input. ref/hare/bytes/tokenize.ha uses the built-in `done`
// token; ww spells it per-package the same way lib/encoding/utf8 does
// (utf8.ww:36). Plain `void` (not `!void`): continuation signal.
export type done = void;
// tokenizer — cursor over an input slice. Layout mirrors
// ref/hare/bytes/tokenize.ha:6-10. `p` is the cached peek-position;
// I64_MAX (forward) / I64_MIN (reverse) are the unprimed sentinels.
// p < 0 also identifies a reverse-direction iterator.
export type tokenizer = struct {
in: []u8,
delim: []u8,
p: i64,
};
// equal — true iff `a` and `b` have the same length and contents.
// ref/hare/bytes/equal.ha:9.
package bytes;
export fn equal(a: []u8, b: []u8) bool = {
if (a.len != b.len) { return false; };
let i: i32 = 0;
@@ -159,3 +188,166 @@ export fn zero(s: []u8) void = {
i += 1;
};
};
// tokenize — iterator yielding tokens from `in` separated by any byte
// in `delim`. Leading / trailing / adjacent delims yield empty tokens.
// `delim` is borrowed; caller keeps it valid for the tokenizer's
// lifetime. ref/hare/bytes/tokenize.ha:22.
export fn tokenize(in: []u8, delim: u8...) tokenizer = {
os.assert(delim.len > 0, "bytes.tokenize called with empty slice");
os.assert((in.len: i64) < types.I64_MAX,
"bytes.tokenize: input length exceeds I64_MAX");
let t: tokenizer;
t.in = in;
t.delim = delim;
if (in.len == 0) {
t.delim.len = 0;
t.delim.cap = 0;
};
t.p = types.I64_MAX;
return t;
};
// rtokenize — reverse-direction tokenize. First next_token yields the
// last token, last next_token yields the first. ref/hare/bytes/tokenize.ha:40.
export fn rtokenize(in: []u8, delim: u8...) tokenizer = {
os.assert(delim.len > 0, "bytes.rtokenize called with empty slice");
os.assert((in.len: i64) < types.I64_MAX,
"bytes.rtokenize: input length exceeds I64_MAX");
let t: tokenizer;
t.in = in;
t.delim = delim;
if (in.len == 0) {
t.delim.len = 0;
t.delim.cap = 0;
};
t.p = types.I64_MIN;
return t;
};
// peek_token — next token without advancing the cursor. Returns done
// once `s.delim` has been zeroed by a prior past-end next_token.
// ref/hare/bytes/tokenize.ha:91.
export fn peek_token(s: *tokenizer) ([]u8 | done) = {
if (s.delim.len == 0) {
let d: done; return d;
};
let reverse: bool = s.p < 0i64;
let known: bool = false;
if (reverse) {
if (s.p != types.I64_MIN) { known = true; };
} else {
if (s.p != types.I64_MAX) { known = true; };
};
if (!known) {
let i: i64 = types.I64_MAX;
if (reverse) { i = types.I64_MIN; };
let dlen: i64 = 0i64;
let slen: i64 = s.in.len: i64;
let k: i32 = 0;
for (k < s.delim.len) {
let d: u8 = s.delim[k];
let ix_found: bool = false;
let ix_val: i32 = 0;
if (reverse) {
match (rindex(s.in, d)) {
case let v: i32 => { ix_found = true; ix_val = v; };
case void => void;
};
} else {
match (index(s.in, d)) {
case let v: i32 => { ix_found = true; ix_val = v; };
case void => void;
};
};
if (ix_found) {
if (!reverse) {
if ((ix_val: i64) < i) { i = ix_val: i64; dlen = 1i64; };
} else {
if ((ix_val: i64) > i) { i = ix_val: i64; dlen = 1i64; };
};
} else {
if (!reverse) {
if (slen < i) { i = slen; };
} else {
if (0i64 > i) { i = 0i64; };
};
};
k += 1;
};
if (reverse) {
if (i == slen) {
s.p = -(slen + 1i64);
} else {
s.p = i + dlen - slen - 1i64;
};
} else {
s.p = i;
};
};
let r: []u8;
if (reverse) {
let start: i32 = (s.in.len: i64 + s.p + 1i64): i32;
r.ptr = s.in.ptr + (start: u64);
r.len = s.in.len - start;
r.cap = r.len;
} else {
let end: i32 = s.p: i32;
r.ptr = s.in.ptr;
r.len = end;
r.cap = end;
};
return r;
};
// next_token — current token, then advance past it and the delim.
// Once the input is exhausted, returns done and zeros `s.delim` so
// subsequent peeks short-circuit. ref/hare/bytes/tokenize.ha:59.
export fn next_token(s: *tokenizer) ([]u8 | done) = {
let b: []u8;
match (peek_token(s)) {
case let v: []u8 => { b = v; };
case done => { let d: done; return d; };
};
let slen: i64 = s.in.len: i64;
let reverse: bool = s.p < 0i64;
if (reverse) {
if (slen + s.p + 1i64 == 0i64) {
s.delim.len = 0;
s.delim.cap = 0;
s.in.len = 0;
s.in.cap = 0;
} else {
let end: i32 = (slen + s.p + 1i64 - 1i64): i32;
s.in.len = end;
s.in.cap = end;
};
s.p = types.I64_MIN;
} else {
if (s.p == slen) {
s.delim.len = 0;
s.delim.cap = 0;
s.in.len = 0;
s.in.cap = 0;
} else {
let adv: u64 = (s.p: u64) + 1u64;
let adv_i32: i32 = (s.p: i32) + 1;
s.in.ptr = s.in.ptr + adv;
s.in.len = s.in.len - adv_i32;
s.in.cap = s.in.cap - adv_i32;
};
s.p = types.I64_MAX;
};
return b;
};
// remaining_tokens — the unconsumed portion of `s.in`. Read-only view.
// ref/hare/bytes/tokenize.ha:145.
export fn remaining_tokens(s: *tokenizer) []u8 = {
return s.in;
};

225
lib/bytes/bytestest.ww
View File

@@ -246,6 +246,227 @@ fn fail() void = { os.exit(signalled + 10); };
if (!bytes.hassuffix(a4[0:4], s234[0:3])) { fail(); };
};
// ---- tokenize / rtokenize / peek_token / remaining_tokens -----------
// ref/hare/bytes/tokenize.ha:258. Hare's @test fn tokenize / rtokenize
// drives the iterator through an expected-token sequence and asserts
// `equal(p, n)` (peek == next), `equal(n, want)` (next == expected).
// expect_token — table row driver. Advances `t` once, asserts the
// returned token matches `want`. peek invariant: peek must equal next.
fn expect_token(t: *bytes.tokenizer, want: []u8) void = {
match (bytes.peek_token(t)) {
case let p: []u8 => {
if (!bytes.equal(p, want)) { fail(); };
};
case bytes.done => { fail(); };
};
match (bytes.next_token(t)) {
case let n: []u8 => {
if (!bytes.equal(n, want)) { fail(); };
};
case bytes.done => { fail(); };
};
};
// expect_done — table-row driver. peek and next must both be done.
fn expect_done(t: *bytes.tokenizer) void = {
match (bytes.peek_token(t)) {
case let p: []u8 => { fail(); };
case bytes.done => void;
};
match (bytes.next_token(t)) {
case let n: []u8 => { fail(); };
case bytes.done => void;
};
};
@test fn tokenize_cases() void = {
let z: [1]u8;
// simple — [1,2,0,3,4] / [0] -> [1,2],[3,4]
signalled = 1710;
let a: [5]u8; a[0] = 1u8; a[1] = 2u8; a[2] = 0u8; a[3] = 3u8; a[4] = 4u8;
let e_12: [2]u8; e_12[0] = 1u8; e_12[1] = 2u8;
let e_34: [2]u8; e_34[0] = 3u8; e_34[1] = 4u8;
let t: bytes.tokenizer = bytes.tokenize(a[0:5], 0u8);
expect_token(&t, e_12[0:2]);
expect_token(&t, e_34[0:2]);
expect_done(&t);
// multiple delimiters — [1,2,0,3,4,42,5,6] / [0,42] -> [1,2],[3,4],[5,6]
signalled = 1711;
let b: [8]u8;
b[0] = 1u8; b[1] = 2u8; b[2] = 0u8; b[3] = 3u8;
b[4] = 4u8; b[5] = 42u8; b[6] = 5u8; b[7] = 6u8;
let e_56: [2]u8; e_56[0] = 5u8; e_56[1] = 6u8;
let t2: bytes.tokenizer = bytes.tokenize(b[0:8], 0u8, 42u8);
expect_token(&t2, e_12[0:2]);
expect_token(&t2, e_34[0:2]);
expect_token(&t2, e_56[0:2]);
expect_done(&t2);
// empty interior tokens — [1,2,0,0,0,3,4] / [0] -> [1,2],[],[],[3,4]
signalled = 1712;
let c: [7]u8;
c[0] = 1u8; c[1] = 2u8; c[2] = 0u8; c[3] = 0u8;
c[4] = 0u8; c[5] = 3u8; c[6] = 4u8;
let t3: bytes.tokenizer = bytes.tokenize(c[0:7], 0u8);
expect_token(&t3, e_12[0:2]);
expect_token(&t3, z[0:0]);
expect_token(&t3, z[0:0]);
expect_token(&t3, e_34[0:2]);
expect_done(&t3);
// leading + trailing empty — [0,1,2,3,0] / [0] -> [],[1,2,3],[]
signalled = 1713;
let d: [5]u8;
d[0] = 0u8; d[1] = 1u8; d[2] = 2u8; d[3] = 3u8; d[4] = 0u8;
let e_123: [3]u8; e_123[0] = 1u8; e_123[1] = 2u8; e_123[2] = 3u8;
let t4: bytes.tokenizer = bytes.tokenize(d[0:5], 0u8);
expect_token(&t4, z[0:0]);
expect_token(&t4, e_123[0:3]);
expect_token(&t4, z[0:0]);
expect_done(&t4);
// no delim hit — [1,2,3] / [0] -> [1,2,3]
signalled = 1714;
let f: [3]u8; f[0] = 1u8; f[1] = 2u8; f[2] = 3u8;
let t5: bytes.tokenizer = bytes.tokenize(f[0:3], 0u8);
expect_token(&t5, e_123[0:3]);
expect_done(&t5);
// empty input — [] / [0] -> done immediately
signalled = 1715;
let t6: bytes.tokenizer = bytes.tokenize(z[0:0], 0u8);
expect_done(&t6);
};
@test fn rtokenize_cases() void = {
let z: [1]u8;
// simple — [1,2,0,3,4] / [0] -> [3,4],[1,2]
signalled = 1720;
let a: [5]u8; a[0] = 1u8; a[1] = 2u8; a[2] = 0u8; a[3] = 3u8; a[4] = 4u8;
let e_12: [2]u8; e_12[0] = 1u8; e_12[1] = 2u8;
let e_34: [2]u8; e_34[0] = 3u8; e_34[1] = 4u8;
let t: bytes.tokenizer = bytes.rtokenize(a[0:5], 0u8);
expect_token(&t, e_34[0:2]);
expect_token(&t, e_12[0:2]);
expect_done(&t);
// multiple delimiters — [1,2,0,3,4,42,5,6] / [0,42] -> [5,6],[3,4],[1,2]
signalled = 1721;
let b: [8]u8;
b[0] = 1u8; b[1] = 2u8; b[2] = 0u8; b[3] = 3u8;
b[4] = 4u8; b[5] = 42u8; b[6] = 5u8; b[7] = 6u8;
let e_56: [2]u8; e_56[0] = 5u8; e_56[1] = 6u8;
let t2: bytes.tokenizer = bytes.rtokenize(b[0:8], 0u8, 42u8);
expect_token(&t2, e_56[0:2]);
expect_token(&t2, e_34[0:2]);
expect_token(&t2, e_12[0:2]);
expect_done(&t2);
// empty interior tokens — [1,2,0,0,0,3,4] / [0] -> [3,4],[],[],[1,2]
signalled = 1722;
let c: [7]u8;
c[0] = 1u8; c[1] = 2u8; c[2] = 0u8; c[3] = 0u8;
c[4] = 0u8; c[5] = 3u8; c[6] = 4u8;
let t3: bytes.tokenizer = bytes.rtokenize(c[0:7], 0u8);
expect_token(&t3, e_34[0:2]);
expect_token(&t3, z[0:0]);
expect_token(&t3, z[0:0]);
expect_token(&t3, e_12[0:2]);
expect_done(&t3);
// leading + trailing empty — [0,1,2,3,0] / [0] -> [],[1,2,3],[]
signalled = 1723;
let d: [5]u8;
d[0] = 0u8; d[1] = 1u8; d[2] = 2u8; d[3] = 3u8; d[4] = 0u8;
let e_123: [3]u8; e_123[0] = 1u8; e_123[1] = 2u8; e_123[2] = 3u8;
let t4: bytes.tokenizer = bytes.rtokenize(d[0:5], 0u8);
expect_token(&t4, z[0:0]);
expect_token(&t4, e_123[0:3]);
expect_token(&t4, z[0:0]);
expect_done(&t4);
// no delim hit — [1,2,3] / [0] -> [1,2,3]
signalled = 1724;
let f: [3]u8; f[0] = 1u8; f[1] = 2u8; f[2] = 3u8;
let t5: bytes.tokenizer = bytes.rtokenize(f[0:3], 0u8);
expect_token(&t5, e_123[0:3]);
expect_done(&t5);
// empty input — [] / [0] -> done immediately
signalled = 1725;
let t6: bytes.tokenizer = bytes.rtokenize(z[0:0], 0u8);
expect_done(&t6);
};
@test fn peek_token_cases() void = {
// Peeking twice without advancing returns the same token.
signalled = 1730;
let a: [5]u8; a[0] = 1u8; a[1] = 2u8; a[2] = 0u8; a[3] = 3u8; a[4] = 4u8;
let e_12: [2]u8; e_12[0] = 1u8; e_12[1] = 2u8;
let t: bytes.tokenizer = bytes.tokenize(a[0:5], 0u8);
match (bytes.peek_token(&t)) {
case let p: []u8 => { if (!bytes.equal(p, e_12[0:2])) { fail(); }; };
case bytes.done => { fail(); };
};
match (bytes.peek_token(&t)) {
case let p: []u8 => { if (!bytes.equal(p, e_12[0:2])) { fail(); }; };
case bytes.done => { fail(); };
};
// Then advance once — peek-after-next is the second token.
signalled = 1731;
let e_34: [2]u8; e_34[0] = 3u8; e_34[1] = 4u8;
match (bytes.next_token(&t)) {
case let n: []u8 => { if (!bytes.equal(n, e_12[0:2])) { fail(); }; };
case bytes.done => { fail(); };
};
match (bytes.peek_token(&t)) {
case let p: []u8 => { if (!bytes.equal(p, e_34[0:2])) { fail(); }; };
case bytes.done => { fail(); };
};
// Reverse peek symmetry — first peek is last token.
signalled = 1732;
let t2: bytes.tokenizer = bytes.rtokenize(a[0:5], 0u8);
match (bytes.peek_token(&t2)) {
case let p: []u8 => { if (!bytes.equal(p, e_34[0:2])) { fail(); }; };
case bytes.done => { fail(); };
};
match (bytes.peek_token(&t2)) {
case let p: []u8 => { if (!bytes.equal(p, e_34[0:2])) { fail(); }; };
case bytes.done => { fail(); };
};
};
@test fn remaining_tokens_cases() void = {
// After one next_token, remaining is bytes past the consumed delim.
signalled = 1740;
let a: [5]u8; a[0] = 1u8; a[1] = 2u8; a[2] = 0u8; a[3] = 3u8; a[4] = 4u8;
let e_34: [2]u8; e_34[0] = 3u8; e_34[1] = 4u8;
let t: bytes.tokenizer = bytes.tokenize(a[0:5], 0u8);
match (bytes.next_token(&t)) {
case let n: []u8 => void;
case bytes.done => { fail(); };
};
let r: []u8 = bytes.remaining_tokens(&t);
if (!bytes.equal(r, e_34[0:2])) { fail(); };
// Reverse — after one next_token, remaining is bytes before the
// consumed delim. ref/hare/bytes/tokenize.ha:323-327 pins [1,2].
signalled = 1741;
let e_12: [2]u8; e_12[0] = 1u8; e_12[1] = 2u8;
let t2: bytes.tokenizer = bytes.rtokenize(a[0:5], 0u8);
match (bytes.next_token(&t2)) {
case let n: []u8 => void;
case bytes.done => { fail(); };
};
let r2: []u8 = bytes.remaining_tokens(&t2);
if (!bytes.equal(r2, e_12[0:2])) { fail(); };
};
export fn main() i32 = {
signalled = 1; equal_cases();
signalled = 2; index_byte_cases();
@@ -255,5 +476,9 @@ export fn main() i32 = {
signalled = 6; contains_cases();
signalled = 7; hasprefix_cases();
signalled = 8; hassuffix_cases();
signalled = 9; tokenize_cases();
signalled = 10; rtokenize_cases();
signalled = 11; peek_token_cases();
signalled = 12; remaining_tokens_cases();
return 0;
};