lib/bytes+test: port tokenize family from Hare (#18)

lib/bytes/bytes.ww

@@ -7,11 +7,40 @@
 //     2/3/4-byte needles and falls back to two_way (Crochemore-Perrin)
 //     for longer (ref/hare/bytes/index.ha:61, ref/hare/bytes/two_way.ha).
 //     Correctness equivalent.
+//   - peek_token dispatches index/rindex by branching on `reverse`
+//     rather than a function-pointer `ifunc` (ref/hare/bytes/tokenize.ha:97).
+//     ww has no fn pointers in scope yet — same pattern as lib/strings
+//     `move`. Outwardly identical.
+//   - tokenize / rtokenize zero the `delim` field on the constructed
+//     tokenizer when `in` is empty, rather than mutating the variadic
+//     param before the struct write (ref/hare/bytes/tokenize.ha:26-28).
+//     Semantically identical; the variadic param is borrowed and
+//     captured-by-value into the struct, so mutating either side
+//     yields the same observable state.
+
+package bytes;
+
+import os;
+import types;
+
+// done — iteration sentinel returned by next_token / peek_token at
+// end-of-input. ref/hare/bytes/tokenize.ha uses the built-in `done`
+// token; ww spells it per-package the same way lib/encoding/utf8 does
+// (utf8.ww:36). Plain `void` (not `!void`): continuation signal.
+export type done = void;
+
+// tokenizer — cursor over an input slice. Layout mirrors
+// ref/hare/bytes/tokenize.ha:6-10. `p` is the cached peek-position;
+// I64_MAX (forward) / I64_MIN (reverse) are the unprimed sentinels.
+// p < 0 also identifies a reverse-direction iterator.
+export type tokenizer = struct {
+	in:    []u8,
+	delim: []u8,
+	p:     i64,
+};
 
 // equal — true iff `a` and `b` have the same length and contents.
 // ref/hare/bytes/equal.ha:9.
-package bytes;
-
 export fn equal(a: []u8, b: []u8) bool = {
 	if (a.len != b.len) { return false; };
 	let i: i32 = 0;
@@ -159,3 +188,166 @@ export fn zero(s: []u8) void = {
 		i += 1;
 	};
 };
+
+// tokenize — iterator yielding tokens from `in` separated by any byte
+// in `delim`. Leading / trailing / adjacent delims yield empty tokens.
+// `delim` is borrowed; caller keeps it valid for the tokenizer's
+// lifetime. ref/hare/bytes/tokenize.ha:22.
+export fn tokenize(in: []u8, delim: u8...) tokenizer = {
+	os.assert(delim.len > 0, "bytes.tokenize called with empty slice");
+	os.assert((in.len: i64) < types.I64_MAX,
+		"bytes.tokenize: input length exceeds I64_MAX");
+	let t: tokenizer;
+	t.in = in;
+	t.delim = delim;
+	if (in.len == 0) {
+		t.delim.len = 0;
+		t.delim.cap = 0;
+	};
+	t.p = types.I64_MAX;
+	return t;
+};
+
+// rtokenize — reverse-direction tokenize. First next_token yields the
+// last token, last next_token yields the first. ref/hare/bytes/tokenize.ha:40.
+export fn rtokenize(in: []u8, delim: u8...) tokenizer = {
+	os.assert(delim.len > 0, "bytes.rtokenize called with empty slice");
+	os.assert((in.len: i64) < types.I64_MAX,
+		"bytes.rtokenize: input length exceeds I64_MAX");
+	let t: tokenizer;
+	t.in = in;
+	t.delim = delim;
+	if (in.len == 0) {
+		t.delim.len = 0;
+		t.delim.cap = 0;
+	};
+	t.p = types.I64_MIN;
+	return t;
+};
+
+// peek_token — next token without advancing the cursor. Returns done
+// once `s.delim` has been zeroed by a prior past-end next_token.
+// ref/hare/bytes/tokenize.ha:91.
+export fn peek_token(s: *tokenizer) ([]u8 | done) = {
+	if (s.delim.len == 0) {
+		let d: done; return d;
+	};
+
+	let reverse: bool = s.p < 0i64;
+	let known: bool = false;
+	if (reverse) {
+		if (s.p != types.I64_MIN) { known = true; };
+	} else {
+		if (s.p != types.I64_MAX) { known = true; };
+	};
+	if (!known) {
+		let i: i64 = types.I64_MAX;
+		if (reverse) { i = types.I64_MIN; };
+		let dlen: i64 = 0i64;
+		let slen: i64 = s.in.len: i64;
+
+		let k: i32 = 0;
+		for (k < s.delim.len) {
+			let d: u8 = s.delim[k];
+			let ix_found: bool = false;
+			let ix_val: i32 = 0;
+			if (reverse) {
+				match (rindex(s.in, d)) {
+				case let v: i32 => { ix_found = true; ix_val = v; };
+				case void       => void;
+				};
+			} else {
+				match (index(s.in, d)) {
+				case let v: i32 => { ix_found = true; ix_val = v; };
+				case void       => void;
+				};
+			};
+			if (ix_found) {
+				if (!reverse) {
+					if ((ix_val: i64) < i) { i = ix_val: i64; dlen = 1i64; };
+				} else {
+					if ((ix_val: i64) > i) { i = ix_val: i64; dlen = 1i64; };
+				};
+			} else {
+				if (!reverse) {
+					if (slen < i) { i = slen; };
+				} else {
+					if (0i64 > i) { i = 0i64; };
+				};
+			};
+			k += 1;
+		};
+
+		if (reverse) {
+			if (i == slen) {
+				s.p = -(slen + 1i64);
+			} else {
+				s.p = i + dlen - slen - 1i64;
+			};
+		} else {
+			s.p = i;
+		};
+	};
+
+	let r: []u8;
+	if (reverse) {
+		let start: i32 = (s.in.len: i64 + s.p + 1i64): i32;
+		r.ptr = s.in.ptr + (start: u64);
+		r.len = s.in.len - start;
+		r.cap = r.len;
+	} else {
+		let end: i32 = s.p: i32;
+		r.ptr = s.in.ptr;
+		r.len = end;
+		r.cap = end;
+	};
+	return r;
+};
+
+// next_token — current token, then advance past it and the delim.
+// Once the input is exhausted, returns done and zeros `s.delim` so
+// subsequent peeks short-circuit. ref/hare/bytes/tokenize.ha:59.
+export fn next_token(s: *tokenizer) ([]u8 | done) = {
+	let b: []u8;
+	match (peek_token(s)) {
+	case let v: []u8 => { b = v; };
+	case done        => { let d: done; return d; };
+	};
+
+	let slen: i64 = s.in.len: i64;
+	let reverse: bool = s.p < 0i64;
+	if (reverse) {
+		if (slen + s.p + 1i64 == 0i64) {
+			s.delim.len = 0;
+			s.delim.cap = 0;
+			s.in.len = 0;
+			s.in.cap = 0;
+		} else {
+			let end: i32 = (slen + s.p + 1i64 - 1i64): i32;
+			s.in.len = end;
+			s.in.cap = end;
+		};
+		s.p = types.I64_MIN;
+	} else {
+		if (s.p == slen) {
+			s.delim.len = 0;
+			s.delim.cap = 0;
+			s.in.len = 0;
+			s.in.cap = 0;
+		} else {
+			let adv: u64 = (s.p: u64) + 1u64;
+			let adv_i32: i32 = (s.p: i32) + 1;
+			s.in.ptr = s.in.ptr + adv;
+			s.in.len = s.in.len - adv_i32;
+			s.in.cap = s.in.cap - adv_i32;
+		};
+		s.p = types.I64_MAX;
+	};
+	return b;
+};
+
+// remaining_tokens — the unconsumed portion of `s.in`. Read-only view.
+// ref/hare/bytes/tokenize.ha:145.
+export fn remaining_tokens(s: *tokenizer) []u8 = {
+	return s.in;
+};