lib/strings+test: port tokenize family (Hare cross-module re-export)

lib/strings/strings.ww

@@ -542,3 +542,67 @@ export fn slice(begin: *iterator, end: *iterator) str = {
 export fn position(it: *iterator) i32 = {
 	return it.offs;
 };
+
+// tokenizer — re-export of bytes.tokenizer. ref/hare/strings/tokenize.ha:7.
+// First cross-module type alias in tree; needs #22's transitive
+// alias-chain unwrap (cstage type_chase_named + wwstage
+// structlookupchain) to walk struct fields through the chain.
+export type tokenizer = bytes.tokenizer;
+
+// tokenize — yield substrings of `s` split on any byte in `delim`.
+// Leading / trailing / adjacent delims yield empty tokens. `s` and
+// `delim` are borrowed; caller keeps them live for the tokenizer's
+// lifetime. ref/hare/strings/tokenize.ha:32. ASCII-only delim
+// asserted per Hare lines 35-37: a multibyte rune in delim would
+// split on a single continuation byte and yield invalid UTF-8.
+export fn tokenize(s: str, delim: str) tokenizer = {
+	let d: []u8 = toutf8(delim);
+	let i: i32 = 0;
+	for (i < d.len) {
+		os.assert((d[i] & 0x80u8) == 0u8,
+			"strings.tokenize cannot tokenize on non-ASCII delimiters");
+		i += 1;
+	};
+	return bytes.tokenize(toutf8(s), d...);
+};
+
+// rtokenize — reverse-direction counterpart to [[tokenize]]. First
+// next_token yields the last token, last yields the first.
+// ref/hare/strings/tokenize.ha:44.
+export fn rtokenize(s: str, delim: str) tokenizer = {
+	let d: []u8 = toutf8(delim);
+	let i: i32 = 0;
+	for (i < d.len) {
+		os.assert((d[i] & 0x80u8) == 0u8,
+			"strings.rtokenize cannot tokenize on non-ASCII delimiters");
+		i += 1;
+	};
+	return bytes.rtokenize(toutf8(s), d...);
+};
+
+// next_token — current token, advancing the cursor.
+// ref/hare/strings/tokenize.ha:62.
+export fn next_token(s: *tokenizer) (str | bytes.done) = {
+	let b: *bytes.tokenizer = s: *bytes.tokenizer;
+	match (bytes.next_token(b)) {
+	case let v: []u8 => return fromutf8_unsafe(v);
+	case bytes.done  => { let d: bytes.done; return d; };
+	};
+};
+
+// peek_token — current token without advancing.
+// ref/hare/strings/tokenize.ha:71.
+export fn peek_token(s: *tokenizer) (str | bytes.done) = {
+	let b: *bytes.tokenizer = s: *bytes.tokenizer;
+	match (bytes.peek_token(b)) {
+	case let v: []u8 => return fromutf8_unsafe(v);
+	case bytes.done  => { let d: bytes.done; return d; };
+	};
+};
+
+// remaining_tokens — unconsumed portion of the input ahead of the
+// cursor. ref/hare/strings/tokenize.ha:79.
+export fn remaining_tokens(s: *tokenizer) str = {
+	let b: *bytes.tokenizer = s: *bytes.tokenizer;
+	return fromutf8_unsafe(bytes.remaining_tokens(b));
+};