diff --git a/lib/bytes/bytes.ww b/lib/bytes/bytes.ww index ab3db696..e821036f 100644 --- a/lib/bytes/bytes.ww +++ b/lib/bytes/bytes.ww @@ -7,11 +7,40 @@ // 2/3/4-byte needles and falls back to two_way (Crochemore-Perrin) // for longer (ref/hare/bytes/index.ha:61, ref/hare/bytes/two_way.ha). // Correctness equivalent. +// - peek_token dispatches index/rindex by branching on `reverse` +// rather than a function-pointer `ifunc` (ref/hare/bytes/tokenize.ha:97). +// ww has no fn pointers in scope yet — same pattern as lib/strings +// `move`. Outwardly identical. +// - tokenize / rtokenize zero the `delim` field on the constructed +// tokenizer when `in` is empty, rather than mutating the variadic +// param before the struct write (ref/hare/bytes/tokenize.ha:26-28). +// Semantically identical; the variadic param is borrowed and +// captured-by-value into the struct, so mutating either side +// yields the same observable state. + +package bytes; + +import os; +import types; + +// done — iteration sentinel returned by next_token / peek_token at +// end-of-input. ref/hare/bytes/tokenize.ha uses the built-in `done` +// token; ww spells it per-package the same way lib/encoding/utf8 does +// (utf8.ww:36). Plain `void` (not `!void`): continuation signal. +export type done = void; + +// tokenizer — cursor over an input slice. Layout mirrors +// ref/hare/bytes/tokenize.ha:6-10. `p` is the cached peek-position; +// I64_MAX (forward) / I64_MIN (reverse) are the unprimed sentinels. +// p < 0 also identifies a reverse-direction iterator. +export type tokenizer = struct { + in: []u8, + delim: []u8, + p: i64, +}; // equal — true iff `a` and `b` have the same length and contents. // ref/hare/bytes/equal.ha:9. -package bytes; - export fn equal(a: []u8, b: []u8) bool = { if (a.len != b.len) { return false; }; let i: i32 = 0; @@ -159,3 +188,166 @@ export fn zero(s: []u8) void = { i += 1; }; }; + +// tokenize — iterator yielding tokens from `in` separated by any byte +// in `delim`. Leading / trailing / adjacent delims yield empty tokens. +// `delim` is borrowed; caller keeps it valid for the tokenizer's +// lifetime. ref/hare/bytes/tokenize.ha:22. +export fn tokenize(in: []u8, delim: u8...) tokenizer = { + os.assert(delim.len > 0, "bytes.tokenize called with empty slice"); + os.assert((in.len: i64) < types.I64_MAX, + "bytes.tokenize: input length exceeds I64_MAX"); + let t: tokenizer; + t.in = in; + t.delim = delim; + if (in.len == 0) { + t.delim.len = 0; + t.delim.cap = 0; + }; + t.p = types.I64_MAX; + return t; +}; + +// rtokenize — reverse-direction tokenize. First next_token yields the +// last token, last next_token yields the first. ref/hare/bytes/tokenize.ha:40. +export fn rtokenize(in: []u8, delim: u8...) tokenizer = { + os.assert(delim.len > 0, "bytes.rtokenize called with empty slice"); + os.assert((in.len: i64) < types.I64_MAX, + "bytes.rtokenize: input length exceeds I64_MAX"); + let t: tokenizer; + t.in = in; + t.delim = delim; + if (in.len == 0) { + t.delim.len = 0; + t.delim.cap = 0; + }; + t.p = types.I64_MIN; + return t; +}; + +// peek_token — next token without advancing the cursor. Returns done +// once `s.delim` has been zeroed by a prior past-end next_token. +// ref/hare/bytes/tokenize.ha:91. +export fn peek_token(s: *tokenizer) ([]u8 | done) = { + if (s.delim.len == 0) { + let d: done; return d; + }; + + let reverse: bool = s.p < 0i64; + let known: bool = false; + if (reverse) { + if (s.p != types.I64_MIN) { known = true; }; + } else { + if (s.p != types.I64_MAX) { known = true; }; + }; + if (!known) { + let i: i64 = types.I64_MAX; + if (reverse) { i = types.I64_MIN; }; + let dlen: i64 = 0i64; + let slen: i64 = s.in.len: i64; + + let k: i32 = 0; + for (k < s.delim.len) { + let d: u8 = s.delim[k]; + let ix_found: bool = false; + let ix_val: i32 = 0; + if (reverse) { + match (rindex(s.in, d)) { + case let v: i32 => { ix_found = true; ix_val = v; }; + case void => void; + }; + } else { + match (index(s.in, d)) { + case let v: i32 => { ix_found = true; ix_val = v; }; + case void => void; + }; + }; + if (ix_found) { + if (!reverse) { + if ((ix_val: i64) < i) { i = ix_val: i64; dlen = 1i64; }; + } else { + if ((ix_val: i64) > i) { i = ix_val: i64; dlen = 1i64; }; + }; + } else { + if (!reverse) { + if (slen < i) { i = slen; }; + } else { + if (0i64 > i) { i = 0i64; }; + }; + }; + k += 1; + }; + + if (reverse) { + if (i == slen) { + s.p = -(slen + 1i64); + } else { + s.p = i + dlen - slen - 1i64; + }; + } else { + s.p = i; + }; + }; + + let r: []u8; + if (reverse) { + let start: i32 = (s.in.len: i64 + s.p + 1i64): i32; + r.ptr = s.in.ptr + (start: u64); + r.len = s.in.len - start; + r.cap = r.len; + } else { + let end: i32 = s.p: i32; + r.ptr = s.in.ptr; + r.len = end; + r.cap = end; + }; + return r; +}; + +// next_token — current token, then advance past it and the delim. +// Once the input is exhausted, returns done and zeros `s.delim` so +// subsequent peeks short-circuit. ref/hare/bytes/tokenize.ha:59. +export fn next_token(s: *tokenizer) ([]u8 | done) = { + let b: []u8; + match (peek_token(s)) { + case let v: []u8 => { b = v; }; + case done => { let d: done; return d; }; + }; + + let slen: i64 = s.in.len: i64; + let reverse: bool = s.p < 0i64; + if (reverse) { + if (slen + s.p + 1i64 == 0i64) { + s.delim.len = 0; + s.delim.cap = 0; + s.in.len = 0; + s.in.cap = 0; + } else { + let end: i32 = (slen + s.p + 1i64 - 1i64): i32; + s.in.len = end; + s.in.cap = end; + }; + s.p = types.I64_MIN; + } else { + if (s.p == slen) { + s.delim.len = 0; + s.delim.cap = 0; + s.in.len = 0; + s.in.cap = 0; + } else { + let adv: u64 = (s.p: u64) + 1u64; + let adv_i32: i32 = (s.p: i32) + 1; + s.in.ptr = s.in.ptr + adv; + s.in.len = s.in.len - adv_i32; + s.in.cap = s.in.cap - adv_i32; + }; + s.p = types.I64_MAX; + }; + return b; +}; + +// remaining_tokens — the unconsumed portion of `s.in`. Read-only view. +// ref/hare/bytes/tokenize.ha:145. +export fn remaining_tokens(s: *tokenizer) []u8 = { + return s.in; +}; diff --git a/lib/bytes/bytestest.ww b/lib/bytes/bytestest.ww index 5dcc6350..20436a03 100644 --- a/lib/bytes/bytestest.ww +++ b/lib/bytes/bytestest.ww @@ -246,6 +246,227 @@ fn fail() void = { os.exit(signalled + 10); }; if (!bytes.hassuffix(a4[0:4], s234[0:3])) { fail(); }; }; +// ---- tokenize / rtokenize / peek_token / remaining_tokens ----------- +// ref/hare/bytes/tokenize.ha:258. Hare's @test fn tokenize / rtokenize +// drives the iterator through an expected-token sequence and asserts +// `equal(p, n)` (peek == next), `equal(n, want)` (next == expected). + +// expect_token — table row driver. Advances `t` once, asserts the +// returned token matches `want`. peek invariant: peek must equal next. +fn expect_token(t: *bytes.tokenizer, want: []u8) void = { + match (bytes.peek_token(t)) { + case let p: []u8 => { + if (!bytes.equal(p, want)) { fail(); }; + }; + case bytes.done => { fail(); }; + }; + match (bytes.next_token(t)) { + case let n: []u8 => { + if (!bytes.equal(n, want)) { fail(); }; + }; + case bytes.done => { fail(); }; + }; +}; + +// expect_done — table-row driver. peek and next must both be done. +fn expect_done(t: *bytes.tokenizer) void = { + match (bytes.peek_token(t)) { + case let p: []u8 => { fail(); }; + case bytes.done => void; + }; + match (bytes.next_token(t)) { + case let n: []u8 => { fail(); }; + case bytes.done => void; + }; +}; + +@test fn tokenize_cases() void = { + let z: [1]u8; + + // simple — [1,2,0,3,4] / [0] -> [1,2],[3,4] + signalled = 1710; + let a: [5]u8; a[0] = 1u8; a[1] = 2u8; a[2] = 0u8; a[3] = 3u8; a[4] = 4u8; + let e_12: [2]u8; e_12[0] = 1u8; e_12[1] = 2u8; + let e_34: [2]u8; e_34[0] = 3u8; e_34[1] = 4u8; + let t: bytes.tokenizer = bytes.tokenize(a[0:5], 0u8); + expect_token(&t, e_12[0:2]); + expect_token(&t, e_34[0:2]); + expect_done(&t); + + // multiple delimiters — [1,2,0,3,4,42,5,6] / [0,42] -> [1,2],[3,4],[5,6] + signalled = 1711; + let b: [8]u8; + b[0] = 1u8; b[1] = 2u8; b[2] = 0u8; b[3] = 3u8; + b[4] = 4u8; b[5] = 42u8; b[6] = 5u8; b[7] = 6u8; + let e_56: [2]u8; e_56[0] = 5u8; e_56[1] = 6u8; + let t2: bytes.tokenizer = bytes.tokenize(b[0:8], 0u8, 42u8); + expect_token(&t2, e_12[0:2]); + expect_token(&t2, e_34[0:2]); + expect_token(&t2, e_56[0:2]); + expect_done(&t2); + + // empty interior tokens — [1,2,0,0,0,3,4] / [0] -> [1,2],[],[],[3,4] + signalled = 1712; + let c: [7]u8; + c[0] = 1u8; c[1] = 2u8; c[2] = 0u8; c[3] = 0u8; + c[4] = 0u8; c[5] = 3u8; c[6] = 4u8; + let t3: bytes.tokenizer = bytes.tokenize(c[0:7], 0u8); + expect_token(&t3, e_12[0:2]); + expect_token(&t3, z[0:0]); + expect_token(&t3, z[0:0]); + expect_token(&t3, e_34[0:2]); + expect_done(&t3); + + // leading + trailing empty — [0,1,2,3,0] / [0] -> [],[1,2,3],[] + signalled = 1713; + let d: [5]u8; + d[0] = 0u8; d[1] = 1u8; d[2] = 2u8; d[3] = 3u8; d[4] = 0u8; + let e_123: [3]u8; e_123[0] = 1u8; e_123[1] = 2u8; e_123[2] = 3u8; + let t4: bytes.tokenizer = bytes.tokenize(d[0:5], 0u8); + expect_token(&t4, z[0:0]); + expect_token(&t4, e_123[0:3]); + expect_token(&t4, z[0:0]); + expect_done(&t4); + + // no delim hit — [1,2,3] / [0] -> [1,2,3] + signalled = 1714; + let f: [3]u8; f[0] = 1u8; f[1] = 2u8; f[2] = 3u8; + let t5: bytes.tokenizer = bytes.tokenize(f[0:3], 0u8); + expect_token(&t5, e_123[0:3]); + expect_done(&t5); + + // empty input — [] / [0] -> done immediately + signalled = 1715; + let t6: bytes.tokenizer = bytes.tokenize(z[0:0], 0u8); + expect_done(&t6); +}; + +@test fn rtokenize_cases() void = { + let z: [1]u8; + + // simple — [1,2,0,3,4] / [0] -> [3,4],[1,2] + signalled = 1720; + let a: [5]u8; a[0] = 1u8; a[1] = 2u8; a[2] = 0u8; a[3] = 3u8; a[4] = 4u8; + let e_12: [2]u8; e_12[0] = 1u8; e_12[1] = 2u8; + let e_34: [2]u8; e_34[0] = 3u8; e_34[1] = 4u8; + let t: bytes.tokenizer = bytes.rtokenize(a[0:5], 0u8); + expect_token(&t, e_34[0:2]); + expect_token(&t, e_12[0:2]); + expect_done(&t); + + // multiple delimiters — [1,2,0,3,4,42,5,6] / [0,42] -> [5,6],[3,4],[1,2] + signalled = 1721; + let b: [8]u8; + b[0] = 1u8; b[1] = 2u8; b[2] = 0u8; b[3] = 3u8; + b[4] = 4u8; b[5] = 42u8; b[6] = 5u8; b[7] = 6u8; + let e_56: [2]u8; e_56[0] = 5u8; e_56[1] = 6u8; + let t2: bytes.tokenizer = bytes.rtokenize(b[0:8], 0u8, 42u8); + expect_token(&t2, e_56[0:2]); + expect_token(&t2, e_34[0:2]); + expect_token(&t2, e_12[0:2]); + expect_done(&t2); + + // empty interior tokens — [1,2,0,0,0,3,4] / [0] -> [3,4],[],[],[1,2] + signalled = 1722; + let c: [7]u8; + c[0] = 1u8; c[1] = 2u8; c[2] = 0u8; c[3] = 0u8; + c[4] = 0u8; c[5] = 3u8; c[6] = 4u8; + let t3: bytes.tokenizer = bytes.rtokenize(c[0:7], 0u8); + expect_token(&t3, e_34[0:2]); + expect_token(&t3, z[0:0]); + expect_token(&t3, z[0:0]); + expect_token(&t3, e_12[0:2]); + expect_done(&t3); + + // leading + trailing empty — [0,1,2,3,0] / [0] -> [],[1,2,3],[] + signalled = 1723; + let d: [5]u8; + d[0] = 0u8; d[1] = 1u8; d[2] = 2u8; d[3] = 3u8; d[4] = 0u8; + let e_123: [3]u8; e_123[0] = 1u8; e_123[1] = 2u8; e_123[2] = 3u8; + let t4: bytes.tokenizer = bytes.rtokenize(d[0:5], 0u8); + expect_token(&t4, z[0:0]); + expect_token(&t4, e_123[0:3]); + expect_token(&t4, z[0:0]); + expect_done(&t4); + + // no delim hit — [1,2,3] / [0] -> [1,2,3] + signalled = 1724; + let f: [3]u8; f[0] = 1u8; f[1] = 2u8; f[2] = 3u8; + let t5: bytes.tokenizer = bytes.rtokenize(f[0:3], 0u8); + expect_token(&t5, e_123[0:3]); + expect_done(&t5); + + // empty input — [] / [0] -> done immediately + signalled = 1725; + let t6: bytes.tokenizer = bytes.rtokenize(z[0:0], 0u8); + expect_done(&t6); +}; + +@test fn peek_token_cases() void = { + // Peeking twice without advancing returns the same token. + signalled = 1730; + let a: [5]u8; a[0] = 1u8; a[1] = 2u8; a[2] = 0u8; a[3] = 3u8; a[4] = 4u8; + let e_12: [2]u8; e_12[0] = 1u8; e_12[1] = 2u8; + let t: bytes.tokenizer = bytes.tokenize(a[0:5], 0u8); + match (bytes.peek_token(&t)) { + case let p: []u8 => { if (!bytes.equal(p, e_12[0:2])) { fail(); }; }; + case bytes.done => { fail(); }; + }; + match (bytes.peek_token(&t)) { + case let p: []u8 => { if (!bytes.equal(p, e_12[0:2])) { fail(); }; }; + case bytes.done => { fail(); }; + }; + // Then advance once — peek-after-next is the second token. + signalled = 1731; + let e_34: [2]u8; e_34[0] = 3u8; e_34[1] = 4u8; + match (bytes.next_token(&t)) { + case let n: []u8 => { if (!bytes.equal(n, e_12[0:2])) { fail(); }; }; + case bytes.done => { fail(); }; + }; + match (bytes.peek_token(&t)) { + case let p: []u8 => { if (!bytes.equal(p, e_34[0:2])) { fail(); }; }; + case bytes.done => { fail(); }; + }; + + // Reverse peek symmetry — first peek is last token. + signalled = 1732; + let t2: bytes.tokenizer = bytes.rtokenize(a[0:5], 0u8); + match (bytes.peek_token(&t2)) { + case let p: []u8 => { if (!bytes.equal(p, e_34[0:2])) { fail(); }; }; + case bytes.done => { fail(); }; + }; + match (bytes.peek_token(&t2)) { + case let p: []u8 => { if (!bytes.equal(p, e_34[0:2])) { fail(); }; }; + case bytes.done => { fail(); }; + }; +}; + +@test fn remaining_tokens_cases() void = { + // After one next_token, remaining is bytes past the consumed delim. + signalled = 1740; + let a: [5]u8; a[0] = 1u8; a[1] = 2u8; a[2] = 0u8; a[3] = 3u8; a[4] = 4u8; + let e_34: [2]u8; e_34[0] = 3u8; e_34[1] = 4u8; + let t: bytes.tokenizer = bytes.tokenize(a[0:5], 0u8); + match (bytes.next_token(&t)) { + case let n: []u8 => void; + case bytes.done => { fail(); }; + }; + let r: []u8 = bytes.remaining_tokens(&t); + if (!bytes.equal(r, e_34[0:2])) { fail(); }; + + // Reverse — after one next_token, remaining is bytes before the + // consumed delim. ref/hare/bytes/tokenize.ha:323-327 pins [1,2]. + signalled = 1741; + let e_12: [2]u8; e_12[0] = 1u8; e_12[1] = 2u8; + let t2: bytes.tokenizer = bytes.rtokenize(a[0:5], 0u8); + match (bytes.next_token(&t2)) { + case let n: []u8 => void; + case bytes.done => { fail(); }; + }; + let r2: []u8 = bytes.remaining_tokens(&t2); + if (!bytes.equal(r2, e_12[0:2])) { fail(); }; +}; + export fn main() i32 = { signalled = 1; equal_cases(); signalled = 2; index_byte_cases(); @@ -255,5 +476,9 @@ export fn main() i32 = { signalled = 6; contains_cases(); signalled = 7; hasprefix_cases(); signalled = 8; hassuffix_cases(); + signalled = 9; tokenize_cases(); + signalled = 10; rtokenize_cases(); + signalled = 11; peek_token_cases(); + signalled = 12; remaining_tokens_cases(); return 0; }; diff --git a/selfhost/cmd/w6c/main.combined.ww b/selfhost/cmd/w6c/main.combined.ww index 20b2b58f..abc3382e 100644 --- a/selfhost/cmd/w6c/main.combined.ww +++ b/selfhost/cmd/w6c/main.combined.ww @@ -858,6 +858,27 @@ export fn freearena(a: *arena) void = { }; }; +// types — integer limits. Mirrors Hare's types::limits (I8_MAX, …) +// platform-fixed for amd64. Numeric helpers live in lib/math, matching +// Hare's split between types::limits and math::. + +package types; + +def I8_MAX: i8 = 127; +def I16_MAX: i16 = 32767; +def I32_MAX: i32 = 2147483647; +def I64_MAX: i64 = 9223372036854775807; + +def I8_MIN: i8 = -128; +def I16_MIN: i16 = -32768; +def I32_MIN: i32 = -2147483648; +def I64_MIN: i64 = -9223372036854775808; + +def U8_MAX: u8 = 255; +def U16_MAX: u16 = 65535; +def U32_MAX: u32 = 4294967295; +def U64_MAX: u64 = 18446744073709551615; + // bytes — slice operations over []u8. Mirrors Hare's bytes module // (ref/hare/bytes/) for the in-tree subset: search/equality/prefix // helpers used by lib/encoding, lib/bufio, lib/memio. @@ -867,11 +888,40 @@ export fn freearena(a: *arena) void = { // 2/3/4-byte needles and falls back to two_way (Crochemore-Perrin) // for longer (ref/hare/bytes/index.ha:61, ref/hare/bytes/two_way.ha). // Correctness equivalent. +// - peek_token dispatches index/rindex by branching on `reverse` +// rather than a function-pointer `ifunc` (ref/hare/bytes/tokenize.ha:97). +// ww has no fn pointers in scope yet — same pattern as lib/strings +// `move`. Outwardly identical. +// - tokenize / rtokenize zero the `delim` field on the constructed +// tokenizer when `in` is empty, rather than mutating the variadic +// param before the struct write (ref/hare/bytes/tokenize.ha:26-28). +// Semantically identical; the variadic param is borrowed and +// captured-by-value into the struct, so mutating either side +// yields the same observable state. + +package bytes; + +import os; +import types; + +// done — iteration sentinel returned by next_token / peek_token at +// end-of-input. ref/hare/bytes/tokenize.ha uses the built-in `done` +// token; ww spells it per-package the same way lib/encoding/utf8 does +// (utf8.ww:36). Plain `void` (not `!void`): continuation signal. +export type done = void; + +// tokenizer — cursor over an input slice. Layout mirrors +// ref/hare/bytes/tokenize.ha:6-10. `p` is the cached peek-position; +// I64_MAX (forward) / I64_MIN (reverse) are the unprimed sentinels. +// p < 0 also identifies a reverse-direction iterator. +export type tokenizer = struct { + in: []u8, + delim: []u8, + p: i64, +}; // equal — true iff `a` and `b` have the same length and contents. // ref/hare/bytes/equal.ha:9. -package bytes; - export fn equal(a: []u8, b: []u8) bool = { if (a.len != b.len) { return false; }; let i: i32 = 0; @@ -1020,6 +1070,169 @@ export fn zero(s: []u8) void = { }; }; +// tokenize — iterator yielding tokens from `in` separated by any byte +// in `delim`. Leading / trailing / adjacent delims yield empty tokens. +// `delim` is borrowed; caller keeps it valid for the tokenizer's +// lifetime. ref/hare/bytes/tokenize.ha:22. +export fn tokenize(in: []u8, delim: u8...) tokenizer = { + os.assert(delim.len > 0, "bytes.tokenize called with empty slice"); + os.assert((in.len: i64) < types.I64_MAX, + "bytes.tokenize: input length exceeds I64_MAX"); + let t: tokenizer; + t.in = in; + t.delim = delim; + if (in.len == 0) { + t.delim.len = 0; + t.delim.cap = 0; + }; + t.p = types.I64_MAX; + return t; +}; + +// rtokenize — reverse-direction tokenize. First next_token yields the +// last token, last next_token yields the first. ref/hare/bytes/tokenize.ha:40. +export fn rtokenize(in: []u8, delim: u8...) tokenizer = { + os.assert(delim.len > 0, "bytes.rtokenize called with empty slice"); + os.assert((in.len: i64) < types.I64_MAX, + "bytes.rtokenize: input length exceeds I64_MAX"); + let t: tokenizer; + t.in = in; + t.delim = delim; + if (in.len == 0) { + t.delim.len = 0; + t.delim.cap = 0; + }; + t.p = types.I64_MIN; + return t; +}; + +// peek_token — next token without advancing the cursor. Returns done +// once `s.delim` has been zeroed by a prior past-end next_token. +// ref/hare/bytes/tokenize.ha:91. +export fn peek_token(s: *tokenizer) ([]u8 | done) = { + if (s.delim.len == 0) { + let d: done; return d; + }; + + let reverse: bool = s.p < 0i64; + let known: bool = false; + if (reverse) { + if (s.p != types.I64_MIN) { known = true; }; + } else { + if (s.p != types.I64_MAX) { known = true; }; + }; + if (!known) { + let i: i64 = types.I64_MAX; + if (reverse) { i = types.I64_MIN; }; + let dlen: i64 = 0i64; + let slen: i64 = s.in.len: i64; + + let k: i32 = 0; + for (k < s.delim.len) { + let d: u8 = s.delim[k]; + let ix_found: bool = false; + let ix_val: i32 = 0; + if (reverse) { + match (rindex(s.in, d)) { + case let v: i32 => { ix_found = true; ix_val = v; }; + case void => void; + }; + } else { + match (index(s.in, d)) { + case let v: i32 => { ix_found = true; ix_val = v; }; + case void => void; + }; + }; + if (ix_found) { + if (!reverse) { + if ((ix_val: i64) < i) { i = ix_val: i64; dlen = 1i64; }; + } else { + if ((ix_val: i64) > i) { i = ix_val: i64; dlen = 1i64; }; + }; + } else { + if (!reverse) { + if (slen < i) { i = slen; }; + } else { + if (0i64 > i) { i = 0i64; }; + }; + }; + k += 1; + }; + + if (reverse) { + if (i == slen) { + s.p = -(slen + 1i64); + } else { + s.p = i + dlen - slen - 1i64; + }; + } else { + s.p = i; + }; + }; + + let r: []u8; + if (reverse) { + let start: i32 = (s.in.len: i64 + s.p + 1i64): i32; + r.ptr = s.in.ptr + (start: u64); + r.len = s.in.len - start; + r.cap = r.len; + } else { + let end: i32 = s.p: i32; + r.ptr = s.in.ptr; + r.len = end; + r.cap = end; + }; + return r; +}; + +// next_token — current token, then advance past it and the delim. +// Once the input is exhausted, returns done and zeros `s.delim` so +// subsequent peeks short-circuit. ref/hare/bytes/tokenize.ha:59. +export fn next_token(s: *tokenizer) ([]u8 | done) = { + let b: []u8; + match (peek_token(s)) { + case let v: []u8 => { b = v; }; + case done => { let d: done; return d; }; + }; + + let slen: i64 = s.in.len: i64; + let reverse: bool = s.p < 0i64; + if (reverse) { + if (slen + s.p + 1i64 == 0i64) { + s.delim.len = 0; + s.delim.cap = 0; + s.in.len = 0; + s.in.cap = 0; + } else { + let end: i32 = (slen + s.p + 1i64 - 1i64): i32; + s.in.len = end; + s.in.cap = end; + }; + s.p = types.I64_MIN; + } else { + if (s.p == slen) { + s.delim.len = 0; + s.delim.cap = 0; + s.in.len = 0; + s.in.cap = 0; + } else { + let adv: u64 = (s.p: u64) + 1u64; + let adv_i32: i32 = (s.p: i32) + 1; + s.in.ptr = s.in.ptr + adv; + s.in.len = s.in.len - adv_i32; + s.in.cap = s.in.cap - adv_i32; + }; + s.p = types.I64_MAX; + }; + return b; +}; + +// remaining_tokens — the unconsumed portion of `s.in`. Read-only view. +// ref/hare/bytes/tokenize.ha:145. +export fn remaining_tokens(s: *tokenizer) []u8 = { + return s.in; +}; + // encoding/utf8 — UTF-8 encode/decode. Hare port; see // ref/hare/encoding/utf8/{types,rune,encode,decode,decodetable}.ha. // diff --git a/selfhost/cmd/wwdump/main.combined.ww b/selfhost/cmd/wwdump/main.combined.ww index c4af1ed0..ee68e95c 100644 --- a/selfhost/cmd/wwdump/main.combined.ww +++ b/selfhost/cmd/wwdump/main.combined.ww @@ -858,6 +858,27 @@ export fn freearena(a: *arena) void = { }; }; +// types — integer limits. Mirrors Hare's types::limits (I8_MAX, …) +// platform-fixed for amd64. Numeric helpers live in lib/math, matching +// Hare's split between types::limits and math::. + +package types; + +def I8_MAX: i8 = 127; +def I16_MAX: i16 = 32767; +def I32_MAX: i32 = 2147483647; +def I64_MAX: i64 = 9223372036854775807; + +def I8_MIN: i8 = -128; +def I16_MIN: i16 = -32768; +def I32_MIN: i32 = -2147483648; +def I64_MIN: i64 = -9223372036854775808; + +def U8_MAX: u8 = 255; +def U16_MAX: u16 = 65535; +def U32_MAX: u32 = 4294967295; +def U64_MAX: u64 = 18446744073709551615; + // bytes — slice operations over []u8. Mirrors Hare's bytes module // (ref/hare/bytes/) for the in-tree subset: search/equality/prefix // helpers used by lib/encoding, lib/bufio, lib/memio. @@ -867,11 +888,40 @@ export fn freearena(a: *arena) void = { // 2/3/4-byte needles and falls back to two_way (Crochemore-Perrin) // for longer (ref/hare/bytes/index.ha:61, ref/hare/bytes/two_way.ha). // Correctness equivalent. +// - peek_token dispatches index/rindex by branching on `reverse` +// rather than a function-pointer `ifunc` (ref/hare/bytes/tokenize.ha:97). +// ww has no fn pointers in scope yet — same pattern as lib/strings +// `move`. Outwardly identical. +// - tokenize / rtokenize zero the `delim` field on the constructed +// tokenizer when `in` is empty, rather than mutating the variadic +// param before the struct write (ref/hare/bytes/tokenize.ha:26-28). +// Semantically identical; the variadic param is borrowed and +// captured-by-value into the struct, so mutating either side +// yields the same observable state. + +package bytes; + +import os; +import types; + +// done — iteration sentinel returned by next_token / peek_token at +// end-of-input. ref/hare/bytes/tokenize.ha uses the built-in `done` +// token; ww spells it per-package the same way lib/encoding/utf8 does +// (utf8.ww:36). Plain `void` (not `!void`): continuation signal. +export type done = void; + +// tokenizer — cursor over an input slice. Layout mirrors +// ref/hare/bytes/tokenize.ha:6-10. `p` is the cached peek-position; +// I64_MAX (forward) / I64_MIN (reverse) are the unprimed sentinels. +// p < 0 also identifies a reverse-direction iterator. +export type tokenizer = struct { + in: []u8, + delim: []u8, + p: i64, +}; // equal — true iff `a` and `b` have the same length and contents. // ref/hare/bytes/equal.ha:9. -package bytes; - export fn equal(a: []u8, b: []u8) bool = { if (a.len != b.len) { return false; }; let i: i32 = 0; @@ -1020,6 +1070,169 @@ export fn zero(s: []u8) void = { }; }; +// tokenize — iterator yielding tokens from `in` separated by any byte +// in `delim`. Leading / trailing / adjacent delims yield empty tokens. +// `delim` is borrowed; caller keeps it valid for the tokenizer's +// lifetime. ref/hare/bytes/tokenize.ha:22. +export fn tokenize(in: []u8, delim: u8...) tokenizer = { + os.assert(delim.len > 0, "bytes.tokenize called with empty slice"); + os.assert((in.len: i64) < types.I64_MAX, + "bytes.tokenize: input length exceeds I64_MAX"); + let t: tokenizer; + t.in = in; + t.delim = delim; + if (in.len == 0) { + t.delim.len = 0; + t.delim.cap = 0; + }; + t.p = types.I64_MAX; + return t; +}; + +// rtokenize — reverse-direction tokenize. First next_token yields the +// last token, last next_token yields the first. ref/hare/bytes/tokenize.ha:40. +export fn rtokenize(in: []u8, delim: u8...) tokenizer = { + os.assert(delim.len > 0, "bytes.rtokenize called with empty slice"); + os.assert((in.len: i64) < types.I64_MAX, + "bytes.rtokenize: input length exceeds I64_MAX"); + let t: tokenizer; + t.in = in; + t.delim = delim; + if (in.len == 0) { + t.delim.len = 0; + t.delim.cap = 0; + }; + t.p = types.I64_MIN; + return t; +}; + +// peek_token — next token without advancing the cursor. Returns done +// once `s.delim` has been zeroed by a prior past-end next_token. +// ref/hare/bytes/tokenize.ha:91. +export fn peek_token(s: *tokenizer) ([]u8 | done) = { + if (s.delim.len == 0) { + let d: done; return d; + }; + + let reverse: bool = s.p < 0i64; + let known: bool = false; + if (reverse) { + if (s.p != types.I64_MIN) { known = true; }; + } else { + if (s.p != types.I64_MAX) { known = true; }; + }; + if (!known) { + let i: i64 = types.I64_MAX; + if (reverse) { i = types.I64_MIN; }; + let dlen: i64 = 0i64; + let slen: i64 = s.in.len: i64; + + let k: i32 = 0; + for (k < s.delim.len) { + let d: u8 = s.delim[k]; + let ix_found: bool = false; + let ix_val: i32 = 0; + if (reverse) { + match (rindex(s.in, d)) { + case let v: i32 => { ix_found = true; ix_val = v; }; + case void => void; + }; + } else { + match (index(s.in, d)) { + case let v: i32 => { ix_found = true; ix_val = v; }; + case void => void; + }; + }; + if (ix_found) { + if (!reverse) { + if ((ix_val: i64) < i) { i = ix_val: i64; dlen = 1i64; }; + } else { + if ((ix_val: i64) > i) { i = ix_val: i64; dlen = 1i64; }; + }; + } else { + if (!reverse) { + if (slen < i) { i = slen; }; + } else { + if (0i64 > i) { i = 0i64; }; + }; + }; + k += 1; + }; + + if (reverse) { + if (i == slen) { + s.p = -(slen + 1i64); + } else { + s.p = i + dlen - slen - 1i64; + }; + } else { + s.p = i; + }; + }; + + let r: []u8; + if (reverse) { + let start: i32 = (s.in.len: i64 + s.p + 1i64): i32; + r.ptr = s.in.ptr + (start: u64); + r.len = s.in.len - start; + r.cap = r.len; + } else { + let end: i32 = s.p: i32; + r.ptr = s.in.ptr; + r.len = end; + r.cap = end; + }; + return r; +}; + +// next_token — current token, then advance past it and the delim. +// Once the input is exhausted, returns done and zeros `s.delim` so +// subsequent peeks short-circuit. ref/hare/bytes/tokenize.ha:59. +export fn next_token(s: *tokenizer) ([]u8 | done) = { + let b: []u8; + match (peek_token(s)) { + case let v: []u8 => { b = v; }; + case done => { let d: done; return d; }; + }; + + let slen: i64 = s.in.len: i64; + let reverse: bool = s.p < 0i64; + if (reverse) { + if (slen + s.p + 1i64 == 0i64) { + s.delim.len = 0; + s.delim.cap = 0; + s.in.len = 0; + s.in.cap = 0; + } else { + let end: i32 = (slen + s.p + 1i64 - 1i64): i32; + s.in.len = end; + s.in.cap = end; + }; + s.p = types.I64_MIN; + } else { + if (s.p == slen) { + s.delim.len = 0; + s.delim.cap = 0; + s.in.len = 0; + s.in.cap = 0; + } else { + let adv: u64 = (s.p: u64) + 1u64; + let adv_i32: i32 = (s.p: i32) + 1; + s.in.ptr = s.in.ptr + adv; + s.in.len = s.in.len - adv_i32; + s.in.cap = s.in.cap - adv_i32; + }; + s.p = types.I64_MAX; + }; + return b; +}; + +// remaining_tokens — the unconsumed portion of `s.in`. Read-only view. +// ref/hare/bytes/tokenize.ha:145. +export fn remaining_tokens(s: *tokenizer) []u8 = { + return s.in; +}; + // encoding/utf8 — UTF-8 encode/decode. Hare port; see // ref/hare/encoding/utf8/{types,rune,encode,decode,decodetable}.ha. // diff --git a/selfhost/test/smoke.combined.ww b/selfhost/test/smoke.combined.ww index 7e26bcb9..4f02bf35 100644 --- a/selfhost/test/smoke.combined.ww +++ b/selfhost/test/smoke.combined.ww @@ -749,6 +749,27 @@ export fn exists(path: str) bool = { return r >= 0i64; }; +// types — integer limits. Mirrors Hare's types::limits (I8_MAX, …) +// platform-fixed for amd64. Numeric helpers live in lib/math, matching +// Hare's split between types::limits and math::. + +package types; + +def I8_MAX: i8 = 127; +def I16_MAX: i16 = 32767; +def I32_MAX: i32 = 2147483647; +def I64_MAX: i64 = 9223372036854775807; + +def I8_MIN: i8 = -128; +def I16_MIN: i16 = -32768; +def I32_MIN: i32 = -2147483648; +def I64_MIN: i64 = -9223372036854775808; + +def U8_MAX: u8 = 255; +def U16_MAX: u16 = 65535; +def U32_MAX: u32 = 4294967295; +def U64_MAX: u64 = 18446744073709551615; + // bytes — slice operations over []u8. Mirrors Hare's bytes module // (ref/hare/bytes/) for the in-tree subset: search/equality/prefix // helpers used by lib/encoding, lib/bufio, lib/memio. @@ -758,11 +779,40 @@ export fn exists(path: str) bool = { // 2/3/4-byte needles and falls back to two_way (Crochemore-Perrin) // for longer (ref/hare/bytes/index.ha:61, ref/hare/bytes/two_way.ha). // Correctness equivalent. +// - peek_token dispatches index/rindex by branching on `reverse` +// rather than a function-pointer `ifunc` (ref/hare/bytes/tokenize.ha:97). +// ww has no fn pointers in scope yet — same pattern as lib/strings +// `move`. Outwardly identical. +// - tokenize / rtokenize zero the `delim` field on the constructed +// tokenizer when `in` is empty, rather than mutating the variadic +// param before the struct write (ref/hare/bytes/tokenize.ha:26-28). +// Semantically identical; the variadic param is borrowed and +// captured-by-value into the struct, so mutating either side +// yields the same observable state. + +package bytes; + +import os; +import types; + +// done — iteration sentinel returned by next_token / peek_token at +// end-of-input. ref/hare/bytes/tokenize.ha uses the built-in `done` +// token; ww spells it per-package the same way lib/encoding/utf8 does +// (utf8.ww:36). Plain `void` (not `!void`): continuation signal. +export type done = void; + +// tokenizer — cursor over an input slice. Layout mirrors +// ref/hare/bytes/tokenize.ha:6-10. `p` is the cached peek-position; +// I64_MAX (forward) / I64_MIN (reverse) are the unprimed sentinels. +// p < 0 also identifies a reverse-direction iterator. +export type tokenizer = struct { + in: []u8, + delim: []u8, + p: i64, +}; // equal — true iff `a` and `b` have the same length and contents. // ref/hare/bytes/equal.ha:9. -package bytes; - export fn equal(a: []u8, b: []u8) bool = { if (a.len != b.len) { return false; }; let i: i32 = 0; @@ -911,6 +961,169 @@ export fn zero(s: []u8) void = { }; }; +// tokenize — iterator yielding tokens from `in` separated by any byte +// in `delim`. Leading / trailing / adjacent delims yield empty tokens. +// `delim` is borrowed; caller keeps it valid for the tokenizer's +// lifetime. ref/hare/bytes/tokenize.ha:22. +export fn tokenize(in: []u8, delim: u8...) tokenizer = { + os.assert(delim.len > 0, "bytes.tokenize called with empty slice"); + os.assert((in.len: i64) < types.I64_MAX, + "bytes.tokenize: input length exceeds I64_MAX"); + let t: tokenizer; + t.in = in; + t.delim = delim; + if (in.len == 0) { + t.delim.len = 0; + t.delim.cap = 0; + }; + t.p = types.I64_MAX; + return t; +}; + +// rtokenize — reverse-direction tokenize. First next_token yields the +// last token, last next_token yields the first. ref/hare/bytes/tokenize.ha:40. +export fn rtokenize(in: []u8, delim: u8...) tokenizer = { + os.assert(delim.len > 0, "bytes.rtokenize called with empty slice"); + os.assert((in.len: i64) < types.I64_MAX, + "bytes.rtokenize: input length exceeds I64_MAX"); + let t: tokenizer; + t.in = in; + t.delim = delim; + if (in.len == 0) { + t.delim.len = 0; + t.delim.cap = 0; + }; + t.p = types.I64_MIN; + return t; +}; + +// peek_token — next token without advancing the cursor. Returns done +// once `s.delim` has been zeroed by a prior past-end next_token. +// ref/hare/bytes/tokenize.ha:91. +export fn peek_token(s: *tokenizer) ([]u8 | done) = { + if (s.delim.len == 0) { + let d: done; return d; + }; + + let reverse: bool = s.p < 0i64; + let known: bool = false; + if (reverse) { + if (s.p != types.I64_MIN) { known = true; }; + } else { + if (s.p != types.I64_MAX) { known = true; }; + }; + if (!known) { + let i: i64 = types.I64_MAX; + if (reverse) { i = types.I64_MIN; }; + let dlen: i64 = 0i64; + let slen: i64 = s.in.len: i64; + + let k: i32 = 0; + for (k < s.delim.len) { + let d: u8 = s.delim[k]; + let ix_found: bool = false; + let ix_val: i32 = 0; + if (reverse) { + match (rindex(s.in, d)) { + case let v: i32 => { ix_found = true; ix_val = v; }; + case void => void; + }; + } else { + match (index(s.in, d)) { + case let v: i32 => { ix_found = true; ix_val = v; }; + case void => void; + }; + }; + if (ix_found) { + if (!reverse) { + if ((ix_val: i64) < i) { i = ix_val: i64; dlen = 1i64; }; + } else { + if ((ix_val: i64) > i) { i = ix_val: i64; dlen = 1i64; }; + }; + } else { + if (!reverse) { + if (slen < i) { i = slen; }; + } else { + if (0i64 > i) { i = 0i64; }; + }; + }; + k += 1; + }; + + if (reverse) { + if (i == slen) { + s.p = -(slen + 1i64); + } else { + s.p = i + dlen - slen - 1i64; + }; + } else { + s.p = i; + }; + }; + + let r: []u8; + if (reverse) { + let start: i32 = (s.in.len: i64 + s.p + 1i64): i32; + r.ptr = s.in.ptr + (start: u64); + r.len = s.in.len - start; + r.cap = r.len; + } else { + let end: i32 = s.p: i32; + r.ptr = s.in.ptr; + r.len = end; + r.cap = end; + }; + return r; +}; + +// next_token — current token, then advance past it and the delim. +// Once the input is exhausted, returns done and zeros `s.delim` so +// subsequent peeks short-circuit. ref/hare/bytes/tokenize.ha:59. +export fn next_token(s: *tokenizer) ([]u8 | done) = { + let b: []u8; + match (peek_token(s)) { + case let v: []u8 => { b = v; }; + case done => { let d: done; return d; }; + }; + + let slen: i64 = s.in.len: i64; + let reverse: bool = s.p < 0i64; + if (reverse) { + if (slen + s.p + 1i64 == 0i64) { + s.delim.len = 0; + s.delim.cap = 0; + s.in.len = 0; + s.in.cap = 0; + } else { + let end: i32 = (slen + s.p + 1i64 - 1i64): i32; + s.in.len = end; + s.in.cap = end; + }; + s.p = types.I64_MIN; + } else { + if (s.p == slen) { + s.delim.len = 0; + s.delim.cap = 0; + s.in.len = 0; + s.in.cap = 0; + } else { + let adv: u64 = (s.p: u64) + 1u64; + let adv_i32: i32 = (s.p: i32) + 1; + s.in.ptr = s.in.ptr + adv; + s.in.len = s.in.len - adv_i32; + s.in.cap = s.in.cap - adv_i32; + }; + s.p = types.I64_MAX; + }; + return b; +}; + +// remaining_tokens — the unconsumed portion of `s.in`. Read-only view. +// ref/hare/bytes/tokenize.ha:145. +export fn remaining_tokens(s: *tokenizer) []u8 = { + return s.in; +}; + // encoding/utf8 — UTF-8 encode/decode. Hare port; see // ref/hare/encoding/utf8/{types,rune,encode,decode,decodetable}.ha. // diff --git a/test/wcc/900_stdlib.c b/test/wcc/900_stdlib.c index e5e0fe35..3e5693c2 100644 --- a/test/wcc/900_stdlib.c +++ b/test/wcc/900_stdlib.c @@ -12,7 +12,6 @@ static const char *modules[] = { "lib/types/types.ww", "lib/ascii/ascii.ww", - "lib/bytes/bytes.ww", "lib/io/io.ww", "lib/errors/errors.ww", "lib/strconv/strconv.ww", @@ -31,17 +30,21 @@ static const char *modules[] = { "lib/math/random/random.ww", "lib/time/time.ww", "lib/c/libc/libc.ww", - /* lib/bufio/bufio.ww, lib/fmt/fmt.ww, lib/os/os.ww, and - * lib/strings/strings.ww moved off this list: each has cross- - * module type refs that only resolve once the driver concatenates - * `use`d modules. bufio / fmt graduated to io.stream-based sinks - * (*io.stream, io.closed, io.eof); lib/os carries time.instant in - * filestat post-Commit B; lib/strings.iterator + lib/strings.next - * reference utf8.decoder / utf8.done. Coverage lives at - * lib/bufio/bufiotest.ww + lib/fmt/fmttest.ww + lib/os/stattest.ww - * + lib/strings/stringstest.ww (wired at 998_bufio_run.c, - * 970_fmt_run.c, 976_stat_run.c, 966_strings_run.c), plus the - * bufio.scanline / fmt.println e2e rows in test/wcc/700_e2e.c. */ + /* lib/bufio/bufio.ww, lib/bytes/bytes.ww, lib/fmt/fmt.ww, + * lib/os/os.ww, and lib/strings/strings.ww moved off this list: + * each has cross-module type refs that only resolve once the + * driver concatenates `use`d modules. bufio / fmt graduated to + * io.stream-based sinks (*io.stream, io.closed, io.eof); lib/os + * carries time.instant in filestat post-Commit B; + * lib/strings.iterator + lib/strings.next reference utf8.decoder + * / utf8.done; lib/bytes.tokenize references os.assert + + * types.I64_MAX/MIN per ref/hare/bytes/tokenize.ha:23-24,42-43. + * Coverage lives at lib/bufio/bufiotest.ww + lib/bytes/bytestest.ww + * + lib/fmt/fmttest.ww + lib/os/stattest.ww + + * lib/strings/stringstest.ww (wired at 998_bufio_run.c, + * 967_bytes_run.c, 970_fmt_run.c, 976_stat_run.c, + * 966_strings_run.c), plus the bufio.scanline / fmt.println e2e + * rows in test/wcc/700_e2e.c. */ "lib/net/net.ww", NULL };