// Vectors mirror ref/hare/bytes/tokenize.ha (task #5 @test conversion). package bytes_test; import bytes; import os; // ref/hare/bytes/tokenize.ha:258. Hare's @test fn tokenize / rtokenize // drives the iterator through an expected-token sequence and asserts // `equal(p, n)` (peek == next), `equal(n, want)` (next == expected). fn expect_token(t: *bytes.tokenizer, want: []u8) void = { match (bytes.peektoken(t)) { case let p: []u8 => { assert(!(!bytes.equal(p, want))); }; case bytes.done => abort(); }; match (bytes.nexttoken(t)) { case let n: []u8 => { assert(!(!bytes.equal(n, want))); }; case bytes.done => abort(); }; }; fn expect_done(t: *bytes.tokenizer) void = { match (bytes.peektoken(t)) { case let p: []u8 => abort(); case bytes.done => void; }; match (bytes.nexttoken(t)) { case let n: []u8 => abort(); case bytes.done => void; }; }; @test fn tokenize_cases() void = { let z: [1]u8; // simple — [1,2,0,3,4] / [0] -> [1,2],[3,4] let a: [5]u8; a[0] = 1u8; a[1] = 2u8; a[2] = 0u8; a[3] = 3u8; a[4] = 4u8; let e_12: [2]u8; e_12[0] = 1u8; e_12[1] = 2u8; let e_34: [2]u8; e_34[0] = 3u8; e_34[1] = 4u8; let t: bytes.tokenizer = bytes.tokenize(a[0:5], 0u8); expect_token(&t, e_12[0:2]); expect_token(&t, e_34[0:2]); expect_done(&t); // multiple delimiters — [1,2,0,3,4,42,5,6] / [0,42] -> [1,2],[3,4],[5,6] let b: [8]u8; b[0] = 1u8; b[1] = 2u8; b[2] = 0u8; b[3] = 3u8; b[4] = 4u8; b[5] = 42u8; b[6] = 5u8; b[7] = 6u8; let e_56: [2]u8; e_56[0] = 5u8; e_56[1] = 6u8; let t2: bytes.tokenizer = bytes.tokenize(b[0:8], 0u8, 42u8); expect_token(&t2, e_12[0:2]); expect_token(&t2, e_34[0:2]); expect_token(&t2, e_56[0:2]); expect_done(&t2); // empty interior tokens — [1,2,0,0,0,3,4] / [0] -> [1,2],[],[],[3,4] let c: [7]u8; c[0] = 1u8; c[1] = 2u8; c[2] = 0u8; c[3] = 0u8; c[4] = 0u8; c[5] = 3u8; c[6] = 4u8; let t3: bytes.tokenizer = bytes.tokenize(c[0:7], 0u8); expect_token(&t3, e_12[0:2]); expect_token(&t3, z[0:0]); expect_token(&t3, z[0:0]); expect_token(&t3, e_34[0:2]); expect_done(&t3); // leading + trailing empty — [0,1,2,3,0] / [0] -> [],[1,2,3],[] let d: [5]u8; d[0] = 0u8; d[1] = 1u8; d[2] = 2u8; d[3] = 3u8; d[4] = 0u8; let e_123: [3]u8; e_123[0] = 1u8; e_123[1] = 2u8; e_123[2] = 3u8; let t4: bytes.tokenizer = bytes.tokenize(d[0:5], 0u8); expect_token(&t4, z[0:0]); expect_token(&t4, e_123[0:3]); expect_token(&t4, z[0:0]); expect_done(&t4); // no delim hit — [1,2,3] / [0] -> [1,2,3] let f: [3]u8; f[0] = 1u8; f[1] = 2u8; f[2] = 3u8; let t5: bytes.tokenizer = bytes.tokenize(f[0:3], 0u8); expect_token(&t5, e_123[0:3]); expect_done(&t5); // empty input — [] / [0] -> done immediately let t6: bytes.tokenizer = bytes.tokenize(z[0:0], 0u8); expect_done(&t6); }; @test fn rtokenize_cases() void = { let z: [1]u8; // simple — [1,2,0,3,4] / [0] -> [3,4],[1,2] let a: [5]u8; a[0] = 1u8; a[1] = 2u8; a[2] = 0u8; a[3] = 3u8; a[4] = 4u8; let e_12: [2]u8; e_12[0] = 1u8; e_12[1] = 2u8; let e_34: [2]u8; e_34[0] = 3u8; e_34[1] = 4u8; let t: bytes.tokenizer = bytes.rtokenize(a[0:5], 0u8); expect_token(&t, e_34[0:2]); expect_token(&t, e_12[0:2]); expect_done(&t); // multiple delimiters — [1,2,0,3,4,42,5,6] / [0,42] -> [5,6],[3,4],[1,2] let b: [8]u8; b[0] = 1u8; b[1] = 2u8; b[2] = 0u8; b[3] = 3u8; b[4] = 4u8; b[5] = 42u8; b[6] = 5u8; b[7] = 6u8; let e_56: [2]u8; e_56[0] = 5u8; e_56[1] = 6u8; let t2: bytes.tokenizer = bytes.rtokenize(b[0:8], 0u8, 42u8); expect_token(&t2, e_56[0:2]); expect_token(&t2, e_34[0:2]); expect_token(&t2, e_12[0:2]); expect_done(&t2); // empty interior tokens — [1,2,0,0,0,3,4] / [0] -> [3,4],[],[],[1,2] let c: [7]u8; c[0] = 1u8; c[1] = 2u8; c[2] = 0u8; c[3] = 0u8; c[4] = 0u8; c[5] = 3u8; c[6] = 4u8; let t3: bytes.tokenizer = bytes.rtokenize(c[0:7], 0u8); expect_token(&t3, e_34[0:2]); expect_token(&t3, z[0:0]); expect_token(&t3, z[0:0]); expect_token(&t3, e_12[0:2]); expect_done(&t3); // leading + trailing empty — [0,1,2,3,0] / [0] -> [],[1,2,3],[] let d: [5]u8; d[0] = 0u8; d[1] = 1u8; d[2] = 2u8; d[3] = 3u8; d[4] = 0u8; let e_123: [3]u8; e_123[0] = 1u8; e_123[1] = 2u8; e_123[2] = 3u8; let t4: bytes.tokenizer = bytes.rtokenize(d[0:5], 0u8); expect_token(&t4, z[0:0]); expect_token(&t4, e_123[0:3]); expect_token(&t4, z[0:0]); expect_done(&t4); // no delim hit — [1,2,3] / [0] -> [1,2,3] let f: [3]u8; f[0] = 1u8; f[1] = 2u8; f[2] = 3u8; let t5: bytes.tokenizer = bytes.rtokenize(f[0:3], 0u8); expect_token(&t5, e_123[0:3]); expect_done(&t5); // empty input — [] / [0] -> done immediately let t6: bytes.tokenizer = bytes.rtokenize(z[0:0], 0u8); expect_done(&t6); }; @test fn peektoken_cases() void = { // Peeking twice without advancing returns the same token. let a: [5]u8; a[0] = 1u8; a[1] = 2u8; a[2] = 0u8; a[3] = 3u8; a[4] = 4u8; let e_12: [2]u8; e_12[0] = 1u8; e_12[1] = 2u8; let t: bytes.tokenizer = bytes.tokenize(a[0:5], 0u8); match (bytes.peektoken(&t)) { case let p: []u8 => { assert(!(!bytes.equal(p, e_12[0:2]))); }; case bytes.done => abort(); }; match (bytes.peektoken(&t)) { case let p: []u8 => { assert(!(!bytes.equal(p, e_12[0:2]))); }; case bytes.done => abort(); }; // Then advance once — peek-after-next is the second token. let e_34: [2]u8; e_34[0] = 3u8; e_34[1] = 4u8; match (bytes.nexttoken(&t)) { case let n: []u8 => { assert(!(!bytes.equal(n, e_12[0:2]))); }; case bytes.done => abort(); }; match (bytes.peektoken(&t)) { case let p: []u8 => { assert(!(!bytes.equal(p, e_34[0:2]))); }; case bytes.done => abort(); }; // Reverse peek symmetry — first peek is last token. let t2: bytes.tokenizer = bytes.rtokenize(a[0:5], 0u8); match (bytes.peektoken(&t2)) { case let p: []u8 => { assert(!(!bytes.equal(p, e_34[0:2]))); }; case bytes.done => abort(); }; match (bytes.peektoken(&t2)) { case let p: []u8 => { assert(!(!bytes.equal(p, e_34[0:2]))); }; case bytes.done => abort(); }; }; @test fn remainingtokens_cases() void = { // After one nexttoken, remaining is bytes past the consumed delim. let a: [5]u8; a[0] = 1u8; a[1] = 2u8; a[2] = 0u8; a[3] = 3u8; a[4] = 4u8; let e_34: [2]u8; e_34[0] = 3u8; e_34[1] = 4u8; let t: bytes.tokenizer = bytes.tokenize(a[0:5], 0u8); match (bytes.nexttoken(&t)) { case let n: []u8 => void; case bytes.done => abort(); }; let r: []u8 = bytes.remainingtokens(&t); assert(!(!bytes.equal(r, e_34[0:2]))); // Reverse — after one nexttoken, remaining is bytes before the // consumed delim. ref/hare/bytes/tokenize.ha:323-327 pins [1,2]. let e_12: [2]u8; e_12[0] = 1u8; e_12[1] = 2u8; let t2: bytes.tokenizer = bytes.rtokenize(a[0:5], 0u8); match (bytes.nexttoken(&t2)) { case let n: []u8 => void; case bytes.done => abort(); }; let r2: []u8 = bytes.remainingtokens(&t2); assert(!(!bytes.equal(r2, e_12[0:2]))); }; // ref/hare/bytes/tokenize.ha:330 (@test fn split). Hare's table mixes // strings; ww spells the vectors as byte arrays explicitly. // // Element reads go through `&toks.ptr[i]: *[]u8` rather than `toks[i]` // — the full 24B slice-element copy lands in the multi-word-store gap // noted at cmd/w6c/cgen.c:6515-6523, so a value-load drops .len/.cap. // Pointer-then-fields lifts 8B at a time, which the cgen routes correctly. fn expect_tok(toks: [][]u8, i: i32, want: []u8) void = { assert(!(i >= toks.len)); let p: *[]u8 = &toks.ptr[i]; assert(!(p.len != want.len)); let j: i32 = 0; for (j < want.len) { assert(!(p.ptr[j] != want[j])); j += 1; }; }; @test fn splitn_cases() void = { // "Hello, my name is Drew" ─ space-delimited, n=4 yields the 4-th // token as the unconsumed remainder. Hare pins this exact shape // at ref/hare/bytes/tokenize.ha:340. let a: [22]u8; a[0] = 72u8; a[1] = 101u8; a[2] = 108u8; a[3] = 108u8; a[4] = 111u8; a[5] = 44u8; a[6] = 32u8; a[7] = 109u8; a[8] = 121u8; a[9] = 32u8; a[10] = 110u8; a[11] = 97u8; a[12] = 109u8; a[13] = 101u8; a[14] = 32u8; a[15] = 105u8; a[16] = 115u8; a[17] = 32u8; a[18] = 68u8; a[19] = 114u8; a[20] = 101u8; a[21] = 119u8; let sp: [1]u8; sp[0] = 32u8; let t1: [][]u8 = bytes.splitn(a[0:22], sp[0:1], 4); assert(!(t1.len != 4)); expect_tok(t1, 0, a[0:6]); expect_tok(t1, 1, a[7:9]); expect_tok(t1, 2, a[10:14]); expect_tok(t1, 3, a[15:22]); os.free(t1.ptr: *void, (t1.cap: u64) * 24u64); // n > token count — final slot is "" if input ends in delim, // otherwise the last token. Here three tokens, n=10 → 3 entries. let b: [5]u8; b[0] = 1u8; b[1] = 0u8; b[2] = 2u8; b[3] = 0u8; b[4] = 3u8; let zd: [1]u8; zd[0] = 0u8; let t2: [][]u8 = bytes.splitn(b[0:5], zd[0:1], 10); assert(!(t2.len != 3)); expect_tok(t2, 0, b[0:1]); expect_tok(t2, 1, b[2:3]); expect_tok(t2, 2, b[4:5]); os.free(t2.ptr: *void, (t2.cap: u64) * 24u64); // n == 1 — single slot holding the whole input as remainder. let t3: [][]u8 = bytes.splitn(b[0:5], zd[0:1], 1); assert(!(t3.len != 1)); expect_tok(t3, 0, b[0:5]); os.free(t3.ptr: *void, (t3.cap: u64) * 24u64); // delim absent — single slot holding input unchanged. let c: [3]u8; c[0] = 1u8; c[1] = 2u8; c[2] = 3u8; let t4: [][]u8 = bytes.splitn(c[0:3], zd[0:1], 5); assert(!(t4.len != 1)); expect_tok(t4, 0, c[0:3]); os.free(t4.ptr: *void, (t4.cap: u64) * 24u64); // empty input — empty result. let z: [1]u8; let t5: [][]u8 = bytes.splitn(z[0:0], zd[0:1], 5); assert(!(t5.len != 0)); if (t5.cap > 0) { os.free(t5.ptr: *void, (t5.cap: u64) * 24u64); }; // Multi-byte delimiter set — both 0 and 42 split. let d: [8]u8; d[0] = 1u8; d[1] = 2u8; d[2] = 0u8; d[3] = 3u8; d[4] = 4u8; d[5] = 42u8; d[6] = 5u8; d[7] = 6u8; let dd: [2]u8; dd[0] = 0u8; dd[1] = 42u8; let t6: [][]u8 = bytes.splitn(d[0:8], dd[0:2], 100); assert(!(t6.len != 3)); expect_tok(t6, 0, d[0:2]); expect_tok(t6, 1, d[3:5]); expect_tok(t6, 2, d[6:8]); os.free(t6.ptr: *void, (t6.cap: u64) * 24u64); }; @test fn rsplitn_cases() void = { // "Hello, my name is Drew" ─ rsplitn n=4 buckets the first three // tokens from the *end*; the remainder ("Hello, my") is index 0. // Hare pins this at ref/hare/bytes/tokenize.ha:379. let a: [22]u8; a[0] = 72u8; a[1] = 101u8; a[2] = 108u8; a[3] = 108u8; a[4] = 111u8; a[5] = 44u8; a[6] = 32u8; a[7] = 109u8; a[8] = 121u8; a[9] = 32u8; a[10] = 110u8; a[11] = 97u8; a[12] = 109u8; a[13] = 101u8; a[14] = 32u8; a[15] = 105u8; a[16] = 115u8; a[17] = 32u8; a[18] = 68u8; a[19] = 114u8; a[20] = 101u8; a[21] = 119u8; let sp: [1]u8; sp[0] = 32u8; let t1: [][]u8 = bytes.rsplitn(a[0:22], sp[0:1], 4); assert(!(t1.len != 4)); expect_tok(t1, 0, a[0:9]); expect_tok(t1, 1, a[10:14]); expect_tok(t1, 2, a[15:17]); expect_tok(t1, 3, a[18:22]); os.free(t1.ptr: *void, (t1.cap: u64) * 24u64); // n > token count — done short-circuit returns the toks in // reverse-iteration order (last token first). Mirrors Hare's // behavior at ref/hare/bytes/tokenize.ha:196-199 where the // reverse-step is gated behind the n-1 loop completion. let b: [5]u8; b[0] = 1u8; b[1] = 0u8; b[2] = 2u8; b[3] = 0u8; b[4] = 3u8; let zd: [1]u8; zd[0] = 0u8; let t2: [][]u8 = bytes.rsplitn(b[0:5], zd[0:1], 10); assert(!(t2.len != 3)); expect_tok(t2, 0, b[4:5]); expect_tok(t2, 1, b[2:3]); expect_tok(t2, 2, b[0:1]); os.free(t2.ptr: *void, (t2.cap: u64) * 24u64); // n == 1 — single slot holding the whole input as remainder. let t3: [][]u8 = bytes.rsplitn(b[0:5], zd[0:1], 1); assert(!(t3.len != 1)); expect_tok(t3, 0, b[0:5]); os.free(t3.ptr: *void, (t3.cap: u64) * 24u64); // delim absent — done short-circuits immediately at iter 0; // toks ends up holding only the eventual remainder if n=1 // (the loop never runs and peek picks up the full input). // With n>1 the loop's first nexttoken sees done (no delim // match anywhere) and returns toks={} per Hare's early-exit. let c: [3]u8; c[0] = 1u8; c[1] = 2u8; c[2] = 3u8; let t4: [][]u8 = bytes.rsplitn(c[0:3], zd[0:1], 5); assert(!(t4.len != 1)); expect_tok(t4, 0, c[0:3]); os.free(t4.ptr: *void, (t4.cap: u64) * 24u64); }; @test fn split_cases() void = { // Full split — every delim hit is a boundary; no per-call cap. // Hare pins five tokens for the canonical input at // ref/hare/bytes/tokenize.ha:347. let a: [22]u8; a[0] = 72u8; a[1] = 101u8; a[2] = 108u8; a[3] = 108u8; a[4] = 111u8; a[5] = 44u8; a[6] = 32u8; a[7] = 109u8; a[8] = 121u8; a[9] = 32u8; a[10] = 110u8; a[11] = 97u8; a[12] = 109u8; a[13] = 101u8; a[14] = 32u8; a[15] = 105u8; a[16] = 115u8; a[17] = 32u8; a[18] = 68u8; a[19] = 114u8; a[20] = 101u8; a[21] = 119u8; let sp: [1]u8; sp[0] = 32u8; let t1: [][]u8 = bytes.split(a[0:22], sp[0:1]); assert(!(t1.len != 5)); expect_tok(t1, 0, a[0:6]); expect_tok(t1, 1, a[7:9]); expect_tok(t1, 2, a[10:14]); expect_tok(t1, 3, a[15:17]); expect_tok(t1, 4, a[18:22]); os.free(t1.ptr: *void, (t1.cap: u64) * 24u64); // Leading + trailing empty tokens — five entries, three of which // are []. Splits across all positions exactly the way Hare does. let b: [5]u8; b[0] = 0u8; b[1] = 1u8; b[2] = 2u8; b[3] = 3u8; b[4] = 0u8; let zd: [1]u8; zd[0] = 0u8; let z: [1]u8; let t2: [][]u8 = bytes.split(b[0:5], zd[0:1]); assert(!(t2.len != 3)); expect_tok(t2, 0, z[0:0]); expect_tok(t2, 1, b[1:4]); expect_tok(t2, 2, z[0:0]); os.free(t2.ptr: *void, (t2.cap: u64) * 24u64); }; // ref/hare/bytes/tokenize.ha:429 (@test fn cut). Borrowed views; each // half asserted via equal(). Needle order is ww's (u8 | []u8). Rows use // the destructure form `let (before, after) = cut(...)` to drive the #10 // over-cap tuple-return (sret) end-to-end. @test fn cut_cases() void = { let z: [1]u8; let abc: [3]u8; abc[0] = 97u8; abc[1] = 98u8; abc[2] = 99u8; let ea: [1]u8; ea[0] = 97u8; let ec: [1]u8; ec[0] = 99u8; // ['a','b','c'] along byte 'b' -> ['a'],['c'] let (b0, a0) = bytes.cut(abc[0:3], 98u8); assert(!(!bytes.equal(b0, ea[0:1]))); assert(!(!bytes.equal(a0, ec[0:1]))); // same, single-byte []u8 delim -> identical let nb: [1]u8; nb[0] = 98u8; let (b1, a1) = bytes.cut(abc[0:3], nb[0:1]); assert(!(!bytes.equal(b1, ea[0:1]))); assert(!(!bytes.equal(a1, ec[0:1]))); // ['b','c'] along 'b' -> [],['c'] (empty before) let bc: [2]u8; bc[0] = 98u8; bc[1] = 99u8; let (b2, a2) = bytes.cut(bc[0:2], 98u8); assert(!(!bytes.equal(b2, z[0:0]))); assert(!(!bytes.equal(a2, ec[0:1]))); // ['a','b'] along 'b' -> ['a'],[] (empty after) let ab: [2]u8; ab[0] = 97u8; ab[1] = 98u8; let (b3, a3) = bytes.cut(ab[0:2], 98u8); assert(!(!bytes.equal(b3, ea[0:1]))); assert(!(!bytes.equal(a3, z[0:0]))); // delim absent -> (whole input, []) let (b4, a4) = bytes.cut(abc[0:3], 120u8); assert(!(!bytes.equal(b4, abc[0:3]))); assert(!(!bytes.equal(a4, z[0:0]))); // empty input -> ([],[]) let (b5, a5) = bytes.cut(z[0:0], 120u8); assert(!(!bytes.equal(b5, z[0:0]))); assert(!(!bytes.equal(a5, z[0:0]))); // repeated delim -> FIRST instance: ['a'],['c','b','a'] let abcba: [5]u8; abcba[0] = 97u8; abcba[1] = 98u8; abcba[2] = 99u8; abcba[3] = 98u8; abcba[4] = 97u8; let ecba: [3]u8; ecba[0] = 99u8; ecba[1] = 98u8; ecba[2] = 97u8; let (b6, a6) = bytes.cut(abcba[0:5], 98u8); assert(!(!bytes.equal(b6, ea[0:1]))); assert(!(!bytes.equal(a6, ecba[0:3]))); // 2-byte []u8 delim present — "abc" -> ['a','b'],['c'] let h: [5]u8; h[0] = 97u8; h[1] = 98u8; h[2] = 88u8; h[3] = 89u8; h[4] = 99u8; let xy: [2]u8; xy[0] = 88u8; xy[1] = 89u8; let eab: [2]u8; eab[0] = 97u8; eab[1] = 98u8; let (b7, a7) = bytes.cut(h[0:5], xy[0:2]); assert(!(!bytes.equal(b7, eab[0:2]))); assert(!(!bytes.equal(a7, ec[0:1]))); // 2-byte []u8 delim absent -> (whole, []) let zz: [2]u8; zz[0] = 9u8; zz[1] = 9u8; let (b8, a8) = bytes.cut(h[0:5], zz[0:2]); assert(!(!bytes.equal(b8, h[0:5]))); assert(!(!bytes.equal(a8, z[0:0]))); }; @test fn rcut_cases() void = { let z: [1]u8; let ea: [1]u8; ea[0] = 97u8; // ['a','b','c'] along 'b' -> ['a'],['c'] (single instance == cut) let abc: [3]u8; abc[0] = 97u8; abc[1] = 98u8; abc[2] = 99u8; let ec: [1]u8; ec[0] = 99u8; let (b0, a0) = bytes.rcut(abc[0:3], 98u8); assert(!(!bytes.equal(b0, ea[0:1]))); assert(!(!bytes.equal(a0, ec[0:1]))); // repeated delim -> LAST instance: ['a','b','c'],['a'] let abcba: [5]u8; abcba[0] = 97u8; abcba[1] = 98u8; abcba[2] = 99u8; abcba[3] = 98u8; abcba[4] = 97u8; let eabc: [3]u8; eabc[0] = 97u8; eabc[1] = 98u8; eabc[2] = 99u8; let (b1, a1) = bytes.rcut(abcba[0:5], 98u8); assert(!(!bytes.equal(b1, eabc[0:3]))); assert(!(!bytes.equal(a1, ea[0:1]))); // delim absent -> (whole input, []) let (b2, a2) = bytes.rcut(abc[0:3], 120u8); assert(!(!bytes.equal(b2, abc[0:3]))); assert(!(!bytes.equal(a2, z[0:0]))); // 2-byte []u8 delim present, two instances -> cut at LAST. // "ab" -> ['X','Y','a'],['b'] let h: [6]u8; h[0] = 88u8; h[1] = 89u8; h[2] = 97u8; h[3] = 88u8; h[4] = 89u8; h[5] = 98u8; let xy: [2]u8; xy[0] = 88u8; xy[1] = 89u8; let exya: [3]u8; exya[0] = 88u8; exya[1] = 89u8; exya[2] = 97u8; let eb: [1]u8; eb[0] = 98u8; let (b3, a3) = bytes.rcut(h[0:6], xy[0:2]); assert(!(!bytes.equal(b3, exya[0:3]))); assert(!(!bytes.equal(a3, eb[0:1]))); // 2-byte []u8 delim absent -> (whole, []) let zz: [2]u8; zz[0] = 9u8; zz[1] = 9u8; let (b4, a4) = bytes.rcut(h[0:6], zz[0:2]); assert(!(!bytes.equal(b4, h[0:6]))); assert(!(!bytes.equal(a4, z[0:0]))); };