// Vectors mirror ref/hare/strings/tokenize.ha where ww can express them. // External strings_test package per the Go foo_test idiom (rule-5/9 // carve-out, task #16); the shared streq helper lives in // helpers_test.ww — same-package test files compose in dir-mode // (the package, not the file, is the unit of testing). package strings_test; import bytes; import strings; import os; import test; // ref/hare/strings/tokenize.ha:110 @test fn tokenize. Row drivers // mirror lib/bytes/bytestest expect_token / expect_done but compare // str via streq. fn expect_str_token(t: *strings.tokenizer, want: str) void = { match (strings.peektoken(t)) { case let p: str => { assert(!(!streq(p, want))); }; case bytes.done => { abort(); }; }; match (strings.nexttoken(t)) { case let n: str => { assert(!(!streq(n, want))); }; case bytes.done => { abort(); }; }; }; fn expect_str_done(t: *strings.tokenizer) void = { match (strings.peektoken(t)) { case let p: str => { abort(); }; case bytes.done => void; }; match (strings.nexttoken(t)) { case let n: str => { abort(); }; case bytes.done => void; }; }; @test fn tokenize_cases() void = { // Hare vector — single-space delim. ref/hare/strings/tokenize.ha:112. let t: strings.tokenizer = strings.tokenize( "Hello world! My name is Harriet.", " "); expect_str_token(&t, "Hello"); expect_str_token(&t, "world!"); expect_str_token(&t, "My"); expect_str_token(&t, "name"); expect_str_token(&t, "is"); expect_str_token(&t, "Harriet."); expect_str_done(&t); // Multi-byte delim set — space + tab. // ref/hare/strings/tokenize.ha:124. let t2: strings.tokenizer = strings.tokenize( "/dev/sda1\t/ ext4 rw,relatime\t0 0", " \t"); expect_str_token(&t2, "/dev/sda1"); expect_str_token(&t2, "/"); expect_str_token(&t2, "ext4"); expect_str_token(&t2, "rw,relatime"); expect_str_token(&t2, "0"); expect_str_token(&t2, "0"); expect_str_done(&t2); // Consecutive delimiters — empty interior tokens. // ref/hare/strings/tokenize.ha:136. let t3: strings.tokenizer = strings.tokenize("hello world", " "); expect_str_token(&t3, "hello"); expect_str_token(&t3, ""); expect_str_token(&t3, ""); expect_str_token(&t3, ""); expect_str_token(&t3, "world"); expect_str_done(&t3); // Leading + trailing delimiters yield empty tokens. // ref/hare/strings/tokenize.ha:147. let t4: strings.tokenizer = strings.tokenize(" hello world ", " "); expect_str_token(&t4, ""); expect_str_token(&t4, "hello"); expect_str_token(&t4, "world"); expect_str_token(&t4, ""); expect_str_done(&t4); // No delim hit — single full token. let t5: strings.tokenizer = strings.tokenize("abc", " "); expect_str_token(&t5, "abc"); expect_str_done(&t5); // Empty input — done immediately. let t6: strings.tokenizer = strings.tokenize("", " "); expect_str_done(&t6); }; @test fn rtokenize_cases() void = { // Reverse direction — first nexttoken is the last token. let t: strings.tokenizer = strings.rtokenize( "Hello world! My name is Harriet.", " "); expect_str_token(&t, "Harriet."); expect_str_token(&t, "is"); expect_str_token(&t, "name"); expect_str_token(&t, "My"); expect_str_token(&t, "world!"); expect_str_token(&t, "Hello"); expect_str_done(&t); // Multi-byte delim set, reverse direction. let t2: strings.tokenizer = strings.rtokenize("a b\tc", " \t"); expect_str_token(&t2, "c"); expect_str_token(&t2, "b"); expect_str_token(&t2, "a"); expect_str_done(&t2); // Empty input — done immediately. let t3: strings.tokenizer = strings.rtokenize("", " "); expect_str_done(&t3); }; @test fn peektoken_cases() void = { // Two peeks without advancing return the same token. let t: strings.tokenizer = strings.tokenize("a b c", " "); match (strings.peektoken(&t)) { case let p: str => { assert(!(!streq(p, "a"))); }; case bytes.done => { abort(); }; }; match (strings.peektoken(&t)) { case let p: str => { assert(!(!streq(p, "a"))); }; case bytes.done => { abort(); }; }; // Advance once — peek then returns "b". match (strings.nexttoken(&t)) { case let n: str => { assert(!(!streq(n, "a"))); }; case bytes.done => { abort(); }; }; match (strings.peektoken(&t)) { case let p: str => { assert(!(!streq(p, "b"))); }; case bytes.done => { abort(); }; }; // Empty input — peek is done. let t2: strings.tokenizer = strings.tokenize("", " "); match (strings.peektoken(&t2)) { case let p: str => { abort(); }; case bytes.done => void; }; }; @test fn remainingtokens_cases() void = { // ref/hare/strings/tokenize.ha:157. After 2 nexttokens, remaining // is "My name is Harriet.". let t: strings.tokenizer = strings.tokenize( "Hello world! My name is Harriet.", " "); match (strings.nexttoken(&t)) { case let n: str => { assert(!(!streq(n, "Hello"))); }; case bytes.done => { abort(); }; }; match (strings.nexttoken(&t)) { case let n: str => { assert(!(!streq(n, "world!"))); }; case bytes.done => { abort(); }; }; if (!streq(strings.remainingtokens(&t), "My name is Harriet.")) { abort(); }; // Fresh tokenizer — remainingtokens is the whole input. let t2: strings.tokenizer = strings.tokenize("a b c", " "); assert(!(!streq(strings.remainingtokens(&t2), "a b c"))); }; // ref/hare/strings/tokenize.ha:245 @test fn split. Hare's vectors // mirrored here directly; element reads go through `&toks.ptr[i]: *str` // rather than `toks[i]` so the 16B str element copy stays out of the // multi-word-store gap noted at cmd/w6c/cgen.c:6515. fn expect_str(toks: []str, i: i32, want: str) void = { assert(!(i >= toks.len)); let p: *str = &toks.ptr[i]; assert(!(p.len != want.len)); assert(!(p.cap != p.len)); let j: i32 = 0; for (j < want.len) { assert(!(p.ptr[j] != want[j])); j += 1; }; }; @test fn splitn_cases() void = { // ref/hare/strings/tokenize.ha:247 — n=4 buckets the trailing // "is Drew" as the remainder slot. let t1: []str = strings.splitn("Hello, my name is Drew", " ", 4); assert(!(t1.len != 4)); expect_str(t1, 0, "Hello,"); expect_str(t1, 1, "my"); expect_str(t1, 2, "name"); expect_str(t1, 3, "is Drew"); os.free(t1.ptr: *void, (t1.cap: u64) * size(str): u64); // ref/hare/strings/tokenize.ha:263 — n > tokens leaves a single // slot holding the unchanged input (delim not found). let t2: []str = strings.splitn("one", "=", 2); assert(!(t2.len != 1)); expect_str(t2, 0, "one"); os.free(t2.ptr: *void, (t2.cap: u64) * size(str): u64); // n == 1 — single slot holding the whole input as remainder. let t3: []str = strings.splitn("a b c", " ", 1); assert(!(t3.len != 1)); expect_str(t3, 0, "a b c"); os.free(t3.ptr: *void, (t3.cap: u64) * size(str): u64); // Empty input — empty result. let t4: []str = strings.splitn("", " ", 5); assert(!(t4.len != 0)); if (t4.cap > 0) { os.free(t4.ptr: *void, (t4.cap: u64) * size(str): u64); }; // Multi-byte delim set (byte-set semantics per // ref/hare/strings/tokenize.ha:35) — split on ',' OR ':' OR ';'. let t5: []str = strings.splitn("hello;world,foo:bar", ",:;", 10); assert(!(t5.len != 4)); expect_str(t5, 0, "hello"); expect_str(t5, 1, "world"); expect_str(t5, 2, "foo"); expect_str(t5, 3, "bar"); os.free(t5.ptr: *void, (t5.cap: u64) * size(str): u64); let t6: []str = strings.splitn("a b", " ", 0); assert(!(t6.len != 0 || t6.cap != 0)); }; @test fn splitn_negative_aborts() void = { test.expectabort(); strings.splitn("a b", " ", -1); }; @test fn rsplitn_cases() void = { // ref/hare/strings/tokenize.ha:271 — reverse n=4 with the // "Hello, my" prefix as the remainder slot at index 0. let t1: []str = strings.rsplitn("Hello, my name is Drew", " ", 4); assert(!(t1.len != 4)); expect_str(t1, 0, "Hello, my"); expect_str(t1, 1, "name"); expect_str(t1, 2, "is"); expect_str(t1, 3, "Drew"); os.free(t1.ptr: *void, (t1.cap: u64) * size(str): u64); // n > token count still returns input order. let t2: []str = strings.rsplitn("a b c", " ", 10); assert(!(t2.len != 3)); expect_str(t2, 0, "a"); expect_str(t2, 1, "b"); expect_str(t2, 2, "c"); os.free(t2.ptr: *void, (t2.cap: u64) * size(str): u64); // n == 1 — single slot holding the whole input as remainder. let t3: []str = strings.rsplitn("a b c", " ", 1); assert(!(t3.len != 1)); expect_str(t3, 0, "a b c"); os.free(t3.ptr: *void, (t3.cap: u64) * size(str): u64); // delim absent — first nexttoken yields the entire input as the // sole token; second iter sees done and short-circuits with the // 1-elem toks un-reversed (single element, reverse is a no-op). let t4: []str = strings.rsplitn("abc", "=", 5); assert(!(t4.len != 1)); expect_str(t4, 0, "abc"); os.free(t4.ptr: *void, (t4.cap: u64) * size(str): u64); let t5: []str = strings.rsplitn("a b", " ", 0); assert(!(t5.len != 0 || t5.cap != 0)); }; @test fn rsplitn_negative_aborts() void = { test.expectabort(); strings.rsplitn("a b", " ", -1); }; @test fn split_cases() void = { // ref/hare/strings/tokenize.ha:255 — full split, every delim hit // is a boundary. let t1: []str = strings.split("Hello, my name is Drew", " "); assert(!(t1.len != 5)); expect_str(t1, 0, "Hello,"); expect_str(t1, 1, "my"); expect_str(t1, 2, "name"); expect_str(t1, 3, "is"); expect_str(t1, 4, "Drew"); os.free(t1.ptr: *void, (t1.cap: u64) * size(str): u64); // Leading + trailing delim — empty tokens at ends. let t2: []str = strings.split(" a b ", " "); assert(!(t2.len != 4)); expect_str(t2, 0, ""); expect_str(t2, 1, "a"); expect_str(t2, 2, "b"); expect_str(t2, 3, ""); os.free(t2.ptr: *void, (t2.cap: u64) * size(str): u64); // Multi-byte delim set, byte-set semantics matching Hare's // strings::split example at ref/hare/strings/tokenize.ha:235. let t3: []str = strings.split("hello;world,foo:bar", ",:;"); assert(!(t3.len != 4)); expect_str(t3, 0, "hello"); expect_str(t3, 1, "world"); expect_str(t3, 2, "foo"); expect_str(t3, 3, "bar"); os.free(t3.ptr: *void, (t3.cap: u64) * size(str): u64); }; // ref/hare/strings/tokenize.ha:316 (@test fn cut). str wrappers over // bytes.cut/rcut. The `let (a, b) = cut(...)` destructure drives the // over-cap tuple-return (sret) path end-to-end (a 2nd #10 witness). @test fn cut_cases() void = { // both halves present. let (a0, b0) = strings.cut("hello=world", "="); assert(!(!streq(a0, "hello"))); assert(!(!streq(b0, "world"))); // only first instance is cut; rest stays in the second half. let (a1, b1) = strings.cut("hello=world=foobar", "="); assert(!(!streq(a1, "hello"))); assert(!(!streq(b1, "world=foobar"))); // delim absent -> (whole input, ""). let (a2, b2) = strings.cut("hello world", "="); assert(!(!streq(a2, "hello world"))); assert(!(!streq(b2, ""))); // delim at start -> empty before. let (a3, b3) = strings.cut("=world", "="); assert(!(!streq(a3, ""))); assert(!(!streq(b3, "world"))); // delim at end -> empty after. let (a4, b4) = strings.cut("hello=", "="); assert(!(!streq(a4, "hello"))); assert(!(!streq(b4, ""))); // empty input -> ("", ""). let (a5, b5) = strings.cut("", "="); assert(!(!streq(a5, ""))); assert(!(!streq(b5, ""))); // multi-byte delim present. let (a6, b6) = strings.cut("aXYbXYc", "XY"); assert(!(!streq(a6, "a"))); assert(!(!streq(b6, "bXYc"))); // multi-byte delim absent. let (a7, b7) = strings.cut("abc", "XY"); assert(!(!streq(a7, "abc"))); assert(!(!streq(b7, ""))); // borrowed, not copied: first half aliases the input bytes. let in: str = "hello=world"; let (a8, b8) = strings.cut(in, "="); assert(!(a8.ptr != in.ptr)); }; @test fn rcut_cases() void = { // rcut splits along the LAST instance. let (a0, b0) = strings.rcut("hello=world=foobar", "="); assert(!(!streq(a0, "hello=world"))); assert(!(!streq(b0, "foobar"))); // single instance == cut. let (a1, b1) = strings.rcut("hello=world", "="); assert(!(!streq(a1, "hello"))); assert(!(!streq(b1, "world"))); // delim absent -> (whole input, ""). let (a2, b2) = strings.rcut("hello world", "="); assert(!(!streq(a2, "hello world"))); assert(!(!streq(b2, ""))); // delim at end -> empty after. let (a3, b3) = strings.rcut("hello=", "="); assert(!(!streq(a3, "hello"))); assert(!(!streq(b3, ""))); // delim at start -> empty before. let (a4, b4) = strings.rcut("=world", "="); assert(!(!streq(a4, ""))); assert(!(!streq(b4, "world"))); // empty input -> ("", ""). let (a5, b5) = strings.rcut("", "="); assert(!(!streq(a5, ""))); assert(!(!streq(b5, ""))); // multi-byte delim, two instances -> cut at LAST. let (a6, b6) = strings.rcut("XYaXYb", "XY"); assert(!(!streq(a6, "XYa"))); assert(!(!streq(b6, "b"))); // multi-byte delim absent. let (a7, b7) = strings.rcut("abc", "XY"); assert(!(!streq(a7, "abc"))); assert(!(!streq(b7, ""))); };