lib/strings+test: port tokenize family (Hare cross-module re-export)

lib/strings/strings.ww

@@ -542,3 +542,67 @@ export fn slice(begin: *iterator, end: *iterator) str = {
 export fn position(it: *iterator) i32 = {
 	return it.offs;
 };
+
+// tokenizer — re-export of bytes.tokenizer. ref/hare/strings/tokenize.ha:7.
+// First cross-module type alias in tree; needs #22's transitive
+// alias-chain unwrap (cstage type_chase_named + wwstage
+// structlookupchain) to walk struct fields through the chain.
+export type tokenizer = bytes.tokenizer;
+
+// tokenize — yield substrings of `s` split on any byte in `delim`.
+// Leading / trailing / adjacent delims yield empty tokens. `s` and
+// `delim` are borrowed; caller keeps them live for the tokenizer's
+// lifetime. ref/hare/strings/tokenize.ha:32. ASCII-only delim
+// asserted per Hare lines 35-37: a multibyte rune in delim would
+// split on a single continuation byte and yield invalid UTF-8.
+export fn tokenize(s: str, delim: str) tokenizer = {
+	let d: []u8 = toutf8(delim);
+	let i: i32 = 0;
+	for (i < d.len) {
+		os.assert((d[i] & 0x80u8) == 0u8,
+			"strings.tokenize cannot tokenize on non-ASCII delimiters");
+		i += 1;
+	};
+	return bytes.tokenize(toutf8(s), d...);
+};
+
+// rtokenize — reverse-direction counterpart to [[tokenize]]. First
+// next_token yields the last token, last yields the first.
+// ref/hare/strings/tokenize.ha:44.
+export fn rtokenize(s: str, delim: str) tokenizer = {
+	let d: []u8 = toutf8(delim);
+	let i: i32 = 0;
+	for (i < d.len) {
+		os.assert((d[i] & 0x80u8) == 0u8,
+			"strings.rtokenize cannot tokenize on non-ASCII delimiters");
+		i += 1;
+	};
+	return bytes.rtokenize(toutf8(s), d...);
+};
+
+// next_token — current token, advancing the cursor.
+// ref/hare/strings/tokenize.ha:62.
+export fn next_token(s: *tokenizer) (str | bytes.done) = {
+	let b: *bytes.tokenizer = s: *bytes.tokenizer;
+	match (bytes.next_token(b)) {
+	case let v: []u8 => return fromutf8_unsafe(v);
+	case bytes.done  => { let d: bytes.done; return d; };
+	};
+};
+
+// peek_token — current token without advancing.
+// ref/hare/strings/tokenize.ha:71.
+export fn peek_token(s: *tokenizer) (str | bytes.done) = {
+	let b: *bytes.tokenizer = s: *bytes.tokenizer;
+	match (bytes.peek_token(b)) {
+	case let v: []u8 => return fromutf8_unsafe(v);
+	case bytes.done  => { let d: bytes.done; return d; };
+	};
+};
+
+// remaining_tokens — unconsumed portion of the input ahead of the
+// cursor. ref/hare/strings/tokenize.ha:79.
+export fn remaining_tokens(s: *tokenizer) str = {
+	let b: *bytes.tokenizer = s: *bytes.tokenizer;
+	return fromutf8_unsafe(bytes.remaining_tokens(b));
+};

lib/strings/stringstest.ww

@@ -895,6 +895,175 @@ fn streq(a: str, b: str) bool = {
 	if (!streq(strings.iterstr(&rit), "he")) { fail(); };
 };
 
+// ---- tokenize / rtokenize / peek_token / remaining_tokens -----------
+// ref/hare/strings/tokenize.ha:110 @test fn tokenize. Row drivers
+// mirror lib/bytes/bytestest expect_token / expect_done but compare
+// str via streq.
+
+fn expect_str_token(t: *strings.tokenizer, want: str) void = {
+	match (strings.peek_token(t)) {
+	case let p: str    => { if (!streq(p, want)) { fail(); }; };
+	case bytes.done    => { fail(); };
+	};
+	match (strings.next_token(t)) {
+	case let n: str    => { if (!streq(n, want)) { fail(); }; };
+	case bytes.done    => { fail(); };
+	};
+};
+
+fn expect_str_done(t: *strings.tokenizer) void = {
+	match (strings.peek_token(t)) {
+	case let p: str    => { fail(); };
+	case bytes.done    => void;
+	};
+	match (strings.next_token(t)) {
+	case let n: str    => { fail(); };
+	case bytes.done    => void;
+	};
+};
+
+@test fn tokenize_cases() void = {
+	// Hare vector — single-space delim. ref/hare/strings/tokenize.ha:112.
+	signalled = 1700;
+	let t: strings.tokenizer = strings.tokenize(
+		"Hello world! My name is Harriet.", " ");
+	expect_str_token(&t, "Hello");
+	expect_str_token(&t, "world!");
+	expect_str_token(&t, "My");
+	expect_str_token(&t, "name");
+	expect_str_token(&t, "is");
+	expect_str_token(&t, "Harriet.");
+	expect_str_done(&t);
+
+	// Multi-byte delim set — space + tab.
+	// ref/hare/strings/tokenize.ha:124.
+	signalled = 1701;
+	let t2: strings.tokenizer = strings.tokenize(
+		"/dev/sda1\t/ ext4 rw,relatime\t0 0", " \t");
+	expect_str_token(&t2, "/dev/sda1");
+	expect_str_token(&t2, "/");
+	expect_str_token(&t2, "ext4");
+	expect_str_token(&t2, "rw,relatime");
+	expect_str_token(&t2, "0");
+	expect_str_token(&t2, "0");
+	expect_str_done(&t2);
+
+	// Consecutive delimiters — empty interior tokens.
+	// ref/hare/strings/tokenize.ha:136.
+	signalled = 1702;
+	let t3: strings.tokenizer = strings.tokenize("hello    world", " ");
+	expect_str_token(&t3, "hello");
+	expect_str_token(&t3, "");
+	expect_str_token(&t3, "");
+	expect_str_token(&t3, "");
+	expect_str_token(&t3, "world");
+	expect_str_done(&t3);
+
+	// Leading + trailing delimiters yield empty tokens.
+	// ref/hare/strings/tokenize.ha:147.
+	signalled = 1703;
+	let t4: strings.tokenizer = strings.tokenize(" hello world ", " ");
+	expect_str_token(&t4, "");
+	expect_str_token(&t4, "hello");
+	expect_str_token(&t4, "world");
+	expect_str_token(&t4, "");
+	expect_str_done(&t4);
+
+	// No delim hit — single full token.
+	signalled = 1704;
+	let t5: strings.tokenizer = strings.tokenize("abc", " ");
+	expect_str_token(&t5, "abc");
+	expect_str_done(&t5);
+
+	// Empty input — done immediately.
+	signalled = 1705;
+	let t6: strings.tokenizer = strings.tokenize("", " ");
+	expect_str_done(&t6);
+};
+
+@test fn rtokenize_cases() void = {
+	// Reverse direction — first next_token is the last token.
+	signalled = 1710;
+	let t: strings.tokenizer = strings.rtokenize(
+		"Hello world! My name is Harriet.", " ");
+	expect_str_token(&t, "Harriet.");
+	expect_str_token(&t, "is");
+	expect_str_token(&t, "name");
+	expect_str_token(&t, "My");
+	expect_str_token(&t, "world!");
+	expect_str_token(&t, "Hello");
+	expect_str_done(&t);
+
+	// Multi-byte delim set, reverse direction.
+	signalled = 1711;
+	let t2: strings.tokenizer = strings.rtokenize("a b\tc", " \t");
+	expect_str_token(&t2, "c");
+	expect_str_token(&t2, "b");
+	expect_str_token(&t2, "a");
+	expect_str_done(&t2);
+
+	// Empty input — done immediately.
+	signalled = 1712;
+	let t3: strings.tokenizer = strings.rtokenize("", " ");
+	expect_str_done(&t3);
+};
+
+@test fn peek_token_cases() void = {
+	// Two peeks without advancing return the same token.
+	signalled = 1720;
+	let t: strings.tokenizer = strings.tokenize("a b c", " ");
+	match (strings.peek_token(&t)) {
+	case let p: str    => { if (!streq(p, "a")) { fail(); }; };
+	case bytes.done    => { fail(); };
+	};
+	match (strings.peek_token(&t)) {
+	case let p: str    => { if (!streq(p, "a")) { fail(); }; };
+	case bytes.done    => { fail(); };
+	};
+	// Advance once — peek then returns "b".
+	signalled = 1721;
+	match (strings.next_token(&t)) {
+	case let n: str    => { if (!streq(n, "a")) { fail(); }; };
+	case bytes.done    => { fail(); };
+	};
+	match (strings.peek_token(&t)) {
+	case let p: str    => { if (!streq(p, "b")) { fail(); }; };
+	case bytes.done    => { fail(); };
+	};
+
+	// Empty input — peek is done.
+	signalled = 1722;
+	let t2: strings.tokenizer = strings.tokenize("", " ");
+	match (strings.peek_token(&t2)) {
+	case let p: str    => { fail(); };
+	case bytes.done    => void;
+	};
+};
+
+@test fn remaining_tokens_cases() void = {
+	// ref/hare/strings/tokenize.ha:157. After 2 next_tokens, remaining
+	// is "My name is Harriet.".
+	signalled = 1730;
+	let t: strings.tokenizer = strings.tokenize(
+		"Hello world! My name is Harriet.", " ");
+	match (strings.next_token(&t)) {
+	case let n: str    => { if (!streq(n, "Hello")) { fail(); }; };
+	case bytes.done    => { fail(); };
+	};
+	match (strings.next_token(&t)) {
+	case let n: str    => { if (!streq(n, "world!")) { fail(); }; };
+	case bytes.done    => { fail(); };
+	};
+	if (!streq(strings.remaining_tokens(&t), "My name is Harriet.")) {
+		fail();
+	};
+
+	// Fresh tokenizer — remaining_tokens is the whole input.
+	signalled = 1731;
+	let t2: strings.tokenizer = strings.tokenize("a b c", " ");
+	if (!streq(strings.remaining_tokens(&t2), "a b c")) { fail(); };
+};
+
 export fn main() i32 = {
 	signalled = 1;  dup_cases();
 	signalled = 2;  concat_cases();
@@ -926,5 +1095,9 @@ export fn main() i32 = {
 	signalled = 25; iter_position_cases();
 	signalled = 26; iter_iterstr_reverse_cases();
 	signalled = 27; iter_slice_cases();
+	signalled = 31; tokenize_cases();
+	signalled = 32; rtokenize_cases();
+	signalled = 33; peek_token_cases();
+	signalled = 34; remaining_tokens_cases();
 	return 0;
 };

selfhost/cmd/w6c/main.combined.ww

@@ -2340,6 +2340,70 @@ export fn position(it: *iterator) i32 = {
 	return it.offs;
 };
 
+// tokenizer — re-export of bytes.tokenizer. ref/hare/strings/tokenize.ha:7.
+// First cross-module type alias in tree; needs #22's transitive
+// alias-chain unwrap (cstage type_chase_named + wwstage
+// structlookupchain) to walk struct fields through the chain.
+export type tokenizer = bytes.tokenizer;
+
+// tokenize — yield substrings of `s` split on any byte in `delim`.
+// Leading / trailing / adjacent delims yield empty tokens. `s` and
+// `delim` are borrowed; caller keeps them live for the tokenizer's
+// lifetime. ref/hare/strings/tokenize.ha:32. ASCII-only delim
+// asserted per Hare lines 35-37: a multibyte rune in delim would
+// split on a single continuation byte and yield invalid UTF-8.
+export fn tokenize(s: str, delim: str) tokenizer = {
+	let d: []u8 = toutf8(delim);
+	let i: i32 = 0;
+	for (i < d.len) {
+		os.assert((d[i] & 0x80u8) == 0u8,
+			"strings.tokenize cannot tokenize on non-ASCII delimiters");
+		i += 1;
+	};
+	return bytes.tokenize(toutf8(s), d...);
+};
+
+// rtokenize — reverse-direction counterpart to [[tokenize]]. First
+// next_token yields the last token, last yields the first.
+// ref/hare/strings/tokenize.ha:44.
+export fn rtokenize(s: str, delim: str) tokenizer = {
+	let d: []u8 = toutf8(delim);
+	let i: i32 = 0;
+	for (i < d.len) {
+		os.assert((d[i] & 0x80u8) == 0u8,
+			"strings.rtokenize cannot tokenize on non-ASCII delimiters");
+		i += 1;
+	};
+	return bytes.rtokenize(toutf8(s), d...);
+};
+
+// next_token — current token, advancing the cursor.
+// ref/hare/strings/tokenize.ha:62.
+export fn next_token(s: *tokenizer) (str | bytes.done) = {
+	let b: *bytes.tokenizer = s: *bytes.tokenizer;
+	match (bytes.next_token(b)) {
+	case let v: []u8 => return fromutf8_unsafe(v);
+	case bytes.done  => { let d: bytes.done; return d; };
+	};
+};
+
+// peek_token — current token without advancing.
+// ref/hare/strings/tokenize.ha:71.
+export fn peek_token(s: *tokenizer) (str | bytes.done) = {
+	let b: *bytes.tokenizer = s: *bytes.tokenizer;
+	match (bytes.peek_token(b)) {
+	case let v: []u8 => return fromutf8_unsafe(v);
+	case bytes.done  => { let d: bytes.done; return d; };
+	};
+};
+
+// remaining_tokens — unconsumed portion of the input ahead of the
+// cursor. ref/hare/strings/tokenize.ha:79.
+export fn remaining_tokens(s: *tokenizer) str = {
+	let b: *bytes.tokenizer = s: *bytes.tokenizer;
+	return fromutf8_unsafe(bytes.remaining_tokens(b));
+};
+
 // strconv — number↔string conversions.
 //
 // Mirrors Hare's strconv:: surface. The *tos functions return a

selfhost/cmd/wwdump/main.combined.ww

@@ -2340,6 +2340,70 @@ export fn position(it: *iterator) i32 = {
 	return it.offs;
 };
 
+// tokenizer — re-export of bytes.tokenizer. ref/hare/strings/tokenize.ha:7.
+// First cross-module type alias in tree; needs #22's transitive
+// alias-chain unwrap (cstage type_chase_named + wwstage
+// structlookupchain) to walk struct fields through the chain.
+export type tokenizer = bytes.tokenizer;
+
+// tokenize — yield substrings of `s` split on any byte in `delim`.
+// Leading / trailing / adjacent delims yield empty tokens. `s` and
+// `delim` are borrowed; caller keeps them live for the tokenizer's
+// lifetime. ref/hare/strings/tokenize.ha:32. ASCII-only delim
+// asserted per Hare lines 35-37: a multibyte rune in delim would
+// split on a single continuation byte and yield invalid UTF-8.
+export fn tokenize(s: str, delim: str) tokenizer = {
+	let d: []u8 = toutf8(delim);
+	let i: i32 = 0;
+	for (i < d.len) {
+		os.assert((d[i] & 0x80u8) == 0u8,
+			"strings.tokenize cannot tokenize on non-ASCII delimiters");
+		i += 1;
+	};
+	return bytes.tokenize(toutf8(s), d...);
+};
+
+// rtokenize — reverse-direction counterpart to [[tokenize]]. First
+// next_token yields the last token, last yields the first.
+// ref/hare/strings/tokenize.ha:44.
+export fn rtokenize(s: str, delim: str) tokenizer = {
+	let d: []u8 = toutf8(delim);
+	let i: i32 = 0;
+	for (i < d.len) {
+		os.assert((d[i] & 0x80u8) == 0u8,
+			"strings.rtokenize cannot tokenize on non-ASCII delimiters");
+		i += 1;
+	};
+	return bytes.rtokenize(toutf8(s), d...);
+};
+
+// next_token — current token, advancing the cursor.
+// ref/hare/strings/tokenize.ha:62.
+export fn next_token(s: *tokenizer) (str | bytes.done) = {
+	let b: *bytes.tokenizer = s: *bytes.tokenizer;
+	match (bytes.next_token(b)) {
+	case let v: []u8 => return fromutf8_unsafe(v);
+	case bytes.done  => { let d: bytes.done; return d; };
+	};
+};
+
+// peek_token — current token without advancing.
+// ref/hare/strings/tokenize.ha:71.
+export fn peek_token(s: *tokenizer) (str | bytes.done) = {
+	let b: *bytes.tokenizer = s: *bytes.tokenizer;
+	match (bytes.peek_token(b)) {
+	case let v: []u8 => return fromutf8_unsafe(v);
+	case bytes.done  => { let d: bytes.done; return d; };
+	};
+};
+
+// remaining_tokens — unconsumed portion of the input ahead of the
+// cursor. ref/hare/strings/tokenize.ha:79.
+export fn remaining_tokens(s: *tokenizer) str = {
+	let b: *bytes.tokenizer = s: *bytes.tokenizer;
+	return fromutf8_unsafe(bytes.remaining_tokens(b));
+};
+
 // strconv — number↔string conversions.
 //
 // Mirrors Hare's strconv:: surface. The *tos functions return a

selfhost/test/smoke.combined.ww

@@ -2231,6 +2231,70 @@ export fn position(it: *iterator) i32 = {
 	return it.offs;
 };
 
+// tokenizer — re-export of bytes.tokenizer. ref/hare/strings/tokenize.ha:7.
+// First cross-module type alias in tree; needs #22's transitive
+// alias-chain unwrap (cstage type_chase_named + wwstage
+// structlookupchain) to walk struct fields through the chain.
+export type tokenizer = bytes.tokenizer;
+
+// tokenize — yield substrings of `s` split on any byte in `delim`.
+// Leading / trailing / adjacent delims yield empty tokens. `s` and
+// `delim` are borrowed; caller keeps them live for the tokenizer's
+// lifetime. ref/hare/strings/tokenize.ha:32. ASCII-only delim
+// asserted per Hare lines 35-37: a multibyte rune in delim would
+// split on a single continuation byte and yield invalid UTF-8.
+export fn tokenize(s: str, delim: str) tokenizer = {
+	let d: []u8 = toutf8(delim);
+	let i: i32 = 0;
+	for (i < d.len) {
+		os.assert((d[i] & 0x80u8) == 0u8,
+			"strings.tokenize cannot tokenize on non-ASCII delimiters");
+		i += 1;
+	};
+	return bytes.tokenize(toutf8(s), d...);
+};
+
+// rtokenize — reverse-direction counterpart to [[tokenize]]. First
+// next_token yields the last token, last yields the first.
+// ref/hare/strings/tokenize.ha:44.
+export fn rtokenize(s: str, delim: str) tokenizer = {
+	let d: []u8 = toutf8(delim);
+	let i: i32 = 0;
+	for (i < d.len) {
+		os.assert((d[i] & 0x80u8) == 0u8,
+			"strings.rtokenize cannot tokenize on non-ASCII delimiters");
+		i += 1;
+	};
+	return bytes.rtokenize(toutf8(s), d...);
+};
+
+// next_token — current token, advancing the cursor.
+// ref/hare/strings/tokenize.ha:62.
+export fn next_token(s: *tokenizer) (str | bytes.done) = {
+	let b: *bytes.tokenizer = s: *bytes.tokenizer;
+	match (bytes.next_token(b)) {
+	case let v: []u8 => return fromutf8_unsafe(v);
+	case bytes.done  => { let d: bytes.done; return d; };
+	};
+};
+
+// peek_token — current token without advancing.
+// ref/hare/strings/tokenize.ha:71.
+export fn peek_token(s: *tokenizer) (str | bytes.done) = {
+	let b: *bytes.tokenizer = s: *bytes.tokenizer;
+	match (bytes.peek_token(b)) {
+	case let v: []u8 => return fromutf8_unsafe(v);
+	case bytes.done  => { let d: bytes.done; return d; };
+	};
+};
+
+// remaining_tokens — unconsumed portion of the input ahead of the
+// cursor. ref/hare/strings/tokenize.ha:79.
+export fn remaining_tokens(s: *tokenizer) str = {
+	let b: *bytes.tokenizer = s: *bytes.tokenizer;
+	return fromutf8_unsafe(bytes.remaining_tokens(b));
+};
+
 // strconv — number↔string conversions.
 //
 // Mirrors Hare's strconv:: surface. The *tos functions return a