lib/strings+test: port tokenize family (Hare cross-module re-export)

lib/strings/stringstest.ww

@@ -895,6 +895,175 @@ fn streq(a: str, b: str) bool = {
 	if (!streq(strings.iterstr(&rit), "he")) { fail(); };
 };
 
+// ---- tokenize / rtokenize / peek_token / remaining_tokens -----------
+// ref/hare/strings/tokenize.ha:110 @test fn tokenize. Row drivers
+// mirror lib/bytes/bytestest expect_token / expect_done but compare
+// str via streq.
+
+fn expect_str_token(t: *strings.tokenizer, want: str) void = {
+	match (strings.peek_token(t)) {
+	case let p: str    => { if (!streq(p, want)) { fail(); }; };
+	case bytes.done    => { fail(); };
+	};
+	match (strings.next_token(t)) {
+	case let n: str    => { if (!streq(n, want)) { fail(); }; };
+	case bytes.done    => { fail(); };
+	};
+};
+
+fn expect_str_done(t: *strings.tokenizer) void = {
+	match (strings.peek_token(t)) {
+	case let p: str    => { fail(); };
+	case bytes.done    => void;
+	};
+	match (strings.next_token(t)) {
+	case let n: str    => { fail(); };
+	case bytes.done    => void;
+	};
+};
+
+@test fn tokenize_cases() void = {
+	// Hare vector — single-space delim. ref/hare/strings/tokenize.ha:112.
+	signalled = 1700;
+	let t: strings.tokenizer = strings.tokenize(
+		"Hello world! My name is Harriet.", " ");
+	expect_str_token(&t, "Hello");
+	expect_str_token(&t, "world!");
+	expect_str_token(&t, "My");
+	expect_str_token(&t, "name");
+	expect_str_token(&t, "is");
+	expect_str_token(&t, "Harriet.");
+	expect_str_done(&t);
+
+	// Multi-byte delim set — space + tab.
+	// ref/hare/strings/tokenize.ha:124.
+	signalled = 1701;
+	let t2: strings.tokenizer = strings.tokenize(
+		"/dev/sda1\t/ ext4 rw,relatime\t0 0", " \t");
+	expect_str_token(&t2, "/dev/sda1");
+	expect_str_token(&t2, "/");
+	expect_str_token(&t2, "ext4");
+	expect_str_token(&t2, "rw,relatime");
+	expect_str_token(&t2, "0");
+	expect_str_token(&t2, "0");
+	expect_str_done(&t2);
+
+	// Consecutive delimiters — empty interior tokens.
+	// ref/hare/strings/tokenize.ha:136.
+	signalled = 1702;
+	let t3: strings.tokenizer = strings.tokenize("hello    world", " ");
+	expect_str_token(&t3, "hello");
+	expect_str_token(&t3, "");
+	expect_str_token(&t3, "");
+	expect_str_token(&t3, "");
+	expect_str_token(&t3, "world");
+	expect_str_done(&t3);
+
+	// Leading + trailing delimiters yield empty tokens.
+	// ref/hare/strings/tokenize.ha:147.
+	signalled = 1703;
+	let t4: strings.tokenizer = strings.tokenize(" hello world ", " ");
+	expect_str_token(&t4, "");
+	expect_str_token(&t4, "hello");
+	expect_str_token(&t4, "world");
+	expect_str_token(&t4, "");
+	expect_str_done(&t4);
+
+	// No delim hit — single full token.
+	signalled = 1704;
+	let t5: strings.tokenizer = strings.tokenize("abc", " ");
+	expect_str_token(&t5, "abc");
+	expect_str_done(&t5);
+
+	// Empty input — done immediately.
+	signalled = 1705;
+	let t6: strings.tokenizer = strings.tokenize("", " ");
+	expect_str_done(&t6);
+};
+
+@test fn rtokenize_cases() void = {
+	// Reverse direction — first next_token is the last token.
+	signalled = 1710;
+	let t: strings.tokenizer = strings.rtokenize(
+		"Hello world! My name is Harriet.", " ");
+	expect_str_token(&t, "Harriet.");
+	expect_str_token(&t, "is");
+	expect_str_token(&t, "name");
+	expect_str_token(&t, "My");
+	expect_str_token(&t, "world!");
+	expect_str_token(&t, "Hello");
+	expect_str_done(&t);
+
+	// Multi-byte delim set, reverse direction.
+	signalled = 1711;
+	let t2: strings.tokenizer = strings.rtokenize("a b\tc", " \t");
+	expect_str_token(&t2, "c");
+	expect_str_token(&t2, "b");
+	expect_str_token(&t2, "a");
+	expect_str_done(&t2);
+
+	// Empty input — done immediately.
+	signalled = 1712;
+	let t3: strings.tokenizer = strings.rtokenize("", " ");
+	expect_str_done(&t3);
+};
+
+@test fn peek_token_cases() void = {
+	// Two peeks without advancing return the same token.
+	signalled = 1720;
+	let t: strings.tokenizer = strings.tokenize("a b c", " ");
+	match (strings.peek_token(&t)) {
+	case let p: str    => { if (!streq(p, "a")) { fail(); }; };
+	case bytes.done    => { fail(); };
+	};
+	match (strings.peek_token(&t)) {
+	case let p: str    => { if (!streq(p, "a")) { fail(); }; };
+	case bytes.done    => { fail(); };
+	};
+	// Advance once — peek then returns "b".
+	signalled = 1721;
+	match (strings.next_token(&t)) {
+	case let n: str    => { if (!streq(n, "a")) { fail(); }; };
+	case bytes.done    => { fail(); };
+	};
+	match (strings.peek_token(&t)) {
+	case let p: str    => { if (!streq(p, "b")) { fail(); }; };
+	case bytes.done    => { fail(); };
+	};
+
+	// Empty input — peek is done.
+	signalled = 1722;
+	let t2: strings.tokenizer = strings.tokenize("", " ");
+	match (strings.peek_token(&t2)) {
+	case let p: str    => { fail(); };
+	case bytes.done    => void;
+	};
+};
+
+@test fn remaining_tokens_cases() void = {
+	// ref/hare/strings/tokenize.ha:157. After 2 next_tokens, remaining
+	// is "My name is Harriet.".
+	signalled = 1730;
+	let t: strings.tokenizer = strings.tokenize(
+		"Hello world! My name is Harriet.", " ");
+	match (strings.next_token(&t)) {
+	case let n: str    => { if (!streq(n, "Hello")) { fail(); }; };
+	case bytes.done    => { fail(); };
+	};
+	match (strings.next_token(&t)) {
+	case let n: str    => { if (!streq(n, "world!")) { fail(); }; };
+	case bytes.done    => { fail(); };
+	};
+	if (!streq(strings.remaining_tokens(&t), "My name is Harriet.")) {
+		fail();
+	};
+
+	// Fresh tokenizer — remaining_tokens is the whole input.
+	signalled = 1731;
+	let t2: strings.tokenizer = strings.tokenize("a b c", " ");
+	if (!streq(strings.remaining_tokens(&t2), "a b c")) { fail(); };
+};
+
 export fn main() i32 = {
 	signalled = 1;  dup_cases();
 	signalled = 2;  concat_cases();
@@ -926,5 +1095,9 @@ export fn main() i32 = {
 	signalled = 25; iter_position_cases();
 	signalled = 26; iter_iterstr_reverse_cases();
 	signalled = 27; iter_slice_cases();
+	signalled = 31; tokenize_cases();
+	signalled = 32; rtokenize_cases();
+	signalled = 33; peek_token_cases();
+	signalled = 34; remaining_tokens_cases();
 	return 0;
 };