Hojun-Cho/ww
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

lib/bytes+test: port tokenize family from Hare (#18)

Browse Source
This commit is contained in:
Hojun-Cho
2026-05-19 11:41:36 +09:00
parent 006df414aa
commit 3a85db0f3f
6 changed files with 1079 additions and 20 deletions
Download Patch File Download Diff File Expand all files Collapse all files

225
lib/bytes/bytestest.ww
View File

@@ -246,6 +246,227 @@ fn fail() void = { os.exit(signalled + 10); };
if (!bytes.hassuffix(a4[0:4], s234[0:3])) { fail(); };
};
// ---- tokenize / rtokenize / peek_token / remaining_tokens -----------
// ref/hare/bytes/tokenize.ha:258. Hare's @test fn tokenize / rtokenize
// drives the iterator through an expected-token sequence and asserts
// `equal(p, n)` (peek == next), `equal(n, want)` (next == expected).
// expect_token — table row driver. Advances `t` once, asserts the
// returned token matches `want`. peek invariant: peek must equal next.
fn expect_token(t: *bytes.tokenizer, want: []u8) void = {
match (bytes.peek_token(t)) {
case let p: []u8 => {
if (!bytes.equal(p, want)) { fail(); };
};
case bytes.done => { fail(); };
};
match (bytes.next_token(t)) {
case let n: []u8 => {
if (!bytes.equal(n, want)) { fail(); };
};
case bytes.done => { fail(); };
};
};
// expect_done — table-row driver. peek and next must both be done.
fn expect_done(t: *bytes.tokenizer) void = {
match (bytes.peek_token(t)) {
case let p: []u8 => { fail(); };
case bytes.done => void;
};
match (bytes.next_token(t)) {
case let n: []u8 => { fail(); };
case bytes.done => void;
};
};
@test fn tokenize_cases() void = {
let z: [1]u8;
// simple — [1,2,0,3,4] / [0] -> [1,2],[3,4]
signalled = 1710;
let a: [5]u8; a[0] = 1u8; a[1] = 2u8; a[2] = 0u8; a[3] = 3u8; a[4] = 4u8;
let e_12: [2]u8; e_12[0] = 1u8; e_12[1] = 2u8;
let e_34: [2]u8; e_34[0] = 3u8; e_34[1] = 4u8;
let t: bytes.tokenizer = bytes.tokenize(a[0:5], 0u8);
expect_token(&t, e_12[0:2]);
expect_token(&t, e_34[0:2]);
expect_done(&t);
// multiple delimiters — [1,2,0,3,4,42,5,6] / [0,42] -> [1,2],[3,4],[5,6]
signalled = 1711;
let b: [8]u8;
b[0] = 1u8; b[1] = 2u8; b[2] = 0u8; b[3] = 3u8;
b[4] = 4u8; b[5] = 42u8; b[6] = 5u8; b[7] = 6u8;
let e_56: [2]u8; e_56[0] = 5u8; e_56[1] = 6u8;
let t2: bytes.tokenizer = bytes.tokenize(b[0:8], 0u8, 42u8);
expect_token(&t2, e_12[0:2]);
expect_token(&t2, e_34[0:2]);
expect_token(&t2, e_56[0:2]);
expect_done(&t2);
// empty interior tokens — [1,2,0,0,0,3,4] / [0] -> [1,2],[],[],[3,4]
signalled = 1712;
let c: [7]u8;
c[0] = 1u8; c[1] = 2u8; c[2] = 0u8; c[3] = 0u8;
c[4] = 0u8; c[5] = 3u8; c[6] = 4u8;
let t3: bytes.tokenizer = bytes.tokenize(c[0:7], 0u8);
expect_token(&t3, e_12[0:2]);
expect_token(&t3, z[0:0]);
expect_token(&t3, z[0:0]);
expect_token(&t3, e_34[0:2]);
expect_done(&t3);
// leading + trailing empty — [0,1,2,3,0] / [0] -> [],[1,2,3],[]
signalled = 1713;
let d: [5]u8;
d[0] = 0u8; d[1] = 1u8; d[2] = 2u8; d[3] = 3u8; d[4] = 0u8;
let e_123: [3]u8; e_123[0] = 1u8; e_123[1] = 2u8; e_123[2] = 3u8;
let t4: bytes.tokenizer = bytes.tokenize(d[0:5], 0u8);
expect_token(&t4, z[0:0]);
expect_token(&t4, e_123[0:3]);
expect_token(&t4, z[0:0]);
expect_done(&t4);
// no delim hit — [1,2,3] / [0] -> [1,2,3]
signalled = 1714;
let f: [3]u8; f[0] = 1u8; f[1] = 2u8; f[2] = 3u8;
let t5: bytes.tokenizer = bytes.tokenize(f[0:3], 0u8);
expect_token(&t5, e_123[0:3]);
expect_done(&t5);
// empty input — [] / [0] -> done immediately
signalled = 1715;
let t6: bytes.tokenizer = bytes.tokenize(z[0:0], 0u8);
expect_done(&t6);
};
@test fn rtokenize_cases() void = {
let z: [1]u8;
// simple — [1,2,0,3,4] / [0] -> [3,4],[1,2]
signalled = 1720;
let a: [5]u8; a[0] = 1u8; a[1] = 2u8; a[2] = 0u8; a[3] = 3u8; a[4] = 4u8;
let e_12: [2]u8; e_12[0] = 1u8; e_12[1] = 2u8;
let e_34: [2]u8; e_34[0] = 3u8; e_34[1] = 4u8;
let t: bytes.tokenizer = bytes.rtokenize(a[0:5], 0u8);
expect_token(&t, e_34[0:2]);
expect_token(&t, e_12[0:2]);
expect_done(&t);
// multiple delimiters — [1,2,0,3,4,42,5,6] / [0,42] -> [5,6],[3,4],[1,2]
signalled = 1721;
let b: [8]u8;
b[0] = 1u8; b[1] = 2u8; b[2] = 0u8; b[3] = 3u8;
b[4] = 4u8; b[5] = 42u8; b[6] = 5u8; b[7] = 6u8;
let e_56: [2]u8; e_56[0] = 5u8; e_56[1] = 6u8;
let t2: bytes.tokenizer = bytes.rtokenize(b[0:8], 0u8, 42u8);
expect_token(&t2, e_56[0:2]);
expect_token(&t2, e_34[0:2]);
expect_token(&t2, e_12[0:2]);
expect_done(&t2);
// empty interior tokens — [1,2,0,0,0,3,4] / [0] -> [3,4],[],[],[1,2]
signalled = 1722;
let c: [7]u8;
c[0] = 1u8; c[1] = 2u8; c[2] = 0u8; c[3] = 0u8;
c[4] = 0u8; c[5] = 3u8; c[6] = 4u8;
let t3: bytes.tokenizer = bytes.rtokenize(c[0:7], 0u8);
expect_token(&t3, e_34[0:2]);
expect_token(&t3, z[0:0]);
expect_token(&t3, z[0:0]);
expect_token(&t3, e_12[0:2]);
expect_done(&t3);
// leading + trailing empty — [0,1,2,3,0] / [0] -> [],[1,2,3],[]
signalled = 1723;
let d: [5]u8;
d[0] = 0u8; d[1] = 1u8; d[2] = 2u8; d[3] = 3u8; d[4] = 0u8;
let e_123: [3]u8; e_123[0] = 1u8; e_123[1] = 2u8; e_123[2] = 3u8;
let t4: bytes.tokenizer = bytes.rtokenize(d[0:5], 0u8);
expect_token(&t4, z[0:0]);
expect_token(&t4, e_123[0:3]);
expect_token(&t4, z[0:0]);
expect_done(&t4);
// no delim hit — [1,2,3] / [0] -> [1,2,3]
signalled = 1724;
let f: [3]u8; f[0] = 1u8; f[1] = 2u8; f[2] = 3u8;
let t5: bytes.tokenizer = bytes.rtokenize(f[0:3], 0u8);
expect_token(&t5, e_123[0:3]);
expect_done(&t5);
// empty input — [] / [0] -> done immediately
signalled = 1725;
let t6: bytes.tokenizer = bytes.rtokenize(z[0:0], 0u8);
expect_done(&t6);
};
@test fn peek_token_cases() void = {
// Peeking twice without advancing returns the same token.
signalled = 1730;
let a: [5]u8; a[0] = 1u8; a[1] = 2u8; a[2] = 0u8; a[3] = 3u8; a[4] = 4u8;
let e_12: [2]u8; e_12[0] = 1u8; e_12[1] = 2u8;
let t: bytes.tokenizer = bytes.tokenize(a[0:5], 0u8);
match (bytes.peek_token(&t)) {
case let p: []u8 => { if (!bytes.equal(p, e_12[0:2])) { fail(); }; };
case bytes.done => { fail(); };
};
match (bytes.peek_token(&t)) {
case let p: []u8 => { if (!bytes.equal(p, e_12[0:2])) { fail(); }; };
case bytes.done => { fail(); };
};
// Then advance once — peek-after-next is the second token.
signalled = 1731;
let e_34: [2]u8; e_34[0] = 3u8; e_34[1] = 4u8;
match (bytes.next_token(&t)) {
case let n: []u8 => { if (!bytes.equal(n, e_12[0:2])) { fail(); }; };
case bytes.done => { fail(); };
};
match (bytes.peek_token(&t)) {
case let p: []u8 => { if (!bytes.equal(p, e_34[0:2])) { fail(); }; };
case bytes.done => { fail(); };
};
// Reverse peek symmetry — first peek is last token.
signalled = 1732;
let t2: bytes.tokenizer = bytes.rtokenize(a[0:5], 0u8);
match (bytes.peek_token(&t2)) {
case let p: []u8 => { if (!bytes.equal(p, e_34[0:2])) { fail(); }; };
case bytes.done => { fail(); };
};
match (bytes.peek_token(&t2)) {
case let p: []u8 => { if (!bytes.equal(p, e_34[0:2])) { fail(); }; };
case bytes.done => { fail(); };
};
};
@test fn remaining_tokens_cases() void = {
// After one next_token, remaining is bytes past the consumed delim.
signalled = 1740;
let a: [5]u8; a[0] = 1u8; a[1] = 2u8; a[2] = 0u8; a[3] = 3u8; a[4] = 4u8;
let e_34: [2]u8; e_34[0] = 3u8; e_34[1] = 4u8;
let t: bytes.tokenizer = bytes.tokenize(a[0:5], 0u8);
match (bytes.next_token(&t)) {
case let n: []u8 => void;
case bytes.done => { fail(); };
};
let r: []u8 = bytes.remaining_tokens(&t);
if (!bytes.equal(r, e_34[0:2])) { fail(); };
// Reverse — after one next_token, remaining is bytes before the
// consumed delim. ref/hare/bytes/tokenize.ha:323-327 pins [1,2].
signalled = 1741;
let e_12: [2]u8; e_12[0] = 1u8; e_12[1] = 2u8;
let t2: bytes.tokenizer = bytes.rtokenize(a[0:5], 0u8);
match (bytes.next_token(&t2)) {
case let n: []u8 => void;
case bytes.done => { fail(); };
};
let r2: []u8 = bytes.remaining_tokens(&t2);
if (!bytes.equal(r2, e_12[0:2])) { fail(); };
};
export fn main() i32 = {
signalled = 1; equal_cases();
signalled = 2; index_byte_cases();
@@ -255,5 +476,9 @@ export fn main() i32 = {
signalled = 6; contains_cases();
signalled = 7; hasprefix_cases();
signalled = 8; hassuffix_cases();
signalled = 9; tokenize_cases();
signalled = 10; rtokenize_cases();
signalled = 11; peek_token_cases();
signalled = 12; remaining_tokens_cases();
return 0;
};