Hojun-Cho
51cb8e7d64
Unblocked by the package-mode ruling: same-package test files now
compose everywhere (coordinator dir-mode owns test-library and the
libbyteid dir entry owns byte identity), so the per-file standalone
constraint is gone and Go's stdlib layout applies directly. The
15-banner strings_test.ww monolith dissolves into per-ref-file
family files — dup, concat, trim, sub, utf8, iter, tokenize
(tokenize+splitn+cut, all of ref tokenize.ha), pad, replace — plus
helpers_test.ww holding exactly the one cross-family helper (streq),
Go's shared test-helper idiom. Pure moves; family-local helpers
(expect_str_token et al.) stay in their family file. Leading banners
that merely restated the filename dropped; 6 interior sub-family
boundaries remain. lib/ census 141 -> 132.
libbyteid gains the dx() dir-mode entry form decided in B2: compose
every *_test.ww under moddir exactly as pkgcombined does
(module-reset separators, byte-lex order) and byte-id the composed
root through both driver stages. The five per-file strings fx
entries retire into one dx("lib/strings") — every source line they
covered is inside the composed unit, and the completeness scan still
keys the module. NENTEXPECT 60 -> 56.
Recorded, not split: fmt_test's 13 banners are scenario groups over
the one fprint surface and ref/hare/fmt itself keeps a single
+test.ha — a print/wrappers split would fight the ref; memio stays
per the standing ruling (everything ww ports lives in ref
stream.ha).
386 lines
12 KiB
Plaintext
386 lines
12 KiB
Plaintext
// Vectors mirror ref/hare/strings/tokenize.ha where ww can express them.
|
|
// External strings_test package per the Go foo_test idiom (rule-5/9
|
|
// carve-out, task #16); the shared streq helper lives in
|
|
// helpers_test.ww — same-package test files compose in dir-mode
|
|
// (the package, not the file, is the unit of testing).
|
|
package strings_test;
|
|
|
|
import strings;
|
|
import os;
|
|
|
|
// ref/hare/strings/tokenize.ha:110 @test fn tokenize. Row drivers
|
|
// mirror lib/bytes/bytestest expect_token / expect_done but compare
|
|
// str via streq.
|
|
|
|
fn expect_str_token(t: *strings.tokenizer, want: str) void = {
|
|
match (strings.peektoken(t)) {
|
|
case let p: str => { assert(!(!streq(p, want))); };
|
|
case bytes.done => { abort(); };
|
|
};
|
|
match (strings.nexttoken(t)) {
|
|
case let n: str => { assert(!(!streq(n, want))); };
|
|
case bytes.done => { abort(); };
|
|
};
|
|
};
|
|
|
|
fn expect_str_done(t: *strings.tokenizer) void = {
|
|
match (strings.peektoken(t)) {
|
|
case let p: str => { abort(); };
|
|
case bytes.done => void;
|
|
};
|
|
match (strings.nexttoken(t)) {
|
|
case let n: str => { abort(); };
|
|
case bytes.done => void;
|
|
};
|
|
};
|
|
|
|
@test fn tokenize_cases() void = {
|
|
// Hare vector — single-space delim. ref/hare/strings/tokenize.ha:112.
|
|
let t: strings.tokenizer = strings.tokenize(
|
|
"Hello world! My name is Harriet.", " ");
|
|
expect_str_token(&t, "Hello");
|
|
expect_str_token(&t, "world!");
|
|
expect_str_token(&t, "My");
|
|
expect_str_token(&t, "name");
|
|
expect_str_token(&t, "is");
|
|
expect_str_token(&t, "Harriet.");
|
|
expect_str_done(&t);
|
|
|
|
// Multi-byte delim set — space + tab.
|
|
// ref/hare/strings/tokenize.ha:124.
|
|
let t2: strings.tokenizer = strings.tokenize(
|
|
"/dev/sda1\t/ ext4 rw,relatime\t0 0", " \t");
|
|
expect_str_token(&t2, "/dev/sda1");
|
|
expect_str_token(&t2, "/");
|
|
expect_str_token(&t2, "ext4");
|
|
expect_str_token(&t2, "rw,relatime");
|
|
expect_str_token(&t2, "0");
|
|
expect_str_token(&t2, "0");
|
|
expect_str_done(&t2);
|
|
|
|
// Consecutive delimiters — empty interior tokens.
|
|
// ref/hare/strings/tokenize.ha:136.
|
|
let t3: strings.tokenizer = strings.tokenize("hello world", " ");
|
|
expect_str_token(&t3, "hello");
|
|
expect_str_token(&t3, "");
|
|
expect_str_token(&t3, "");
|
|
expect_str_token(&t3, "");
|
|
expect_str_token(&t3, "world");
|
|
expect_str_done(&t3);
|
|
|
|
// Leading + trailing delimiters yield empty tokens.
|
|
// ref/hare/strings/tokenize.ha:147.
|
|
let t4: strings.tokenizer = strings.tokenize(" hello world ", " ");
|
|
expect_str_token(&t4, "");
|
|
expect_str_token(&t4, "hello");
|
|
expect_str_token(&t4, "world");
|
|
expect_str_token(&t4, "");
|
|
expect_str_done(&t4);
|
|
|
|
// No delim hit — single full token.
|
|
let t5: strings.tokenizer = strings.tokenize("abc", " ");
|
|
expect_str_token(&t5, "abc");
|
|
expect_str_done(&t5);
|
|
|
|
// Empty input — done immediately.
|
|
let t6: strings.tokenizer = strings.tokenize("", " ");
|
|
expect_str_done(&t6);
|
|
};
|
|
|
|
@test fn rtokenize_cases() void = {
|
|
// Reverse direction — first nexttoken is the last token.
|
|
let t: strings.tokenizer = strings.rtokenize(
|
|
"Hello world! My name is Harriet.", " ");
|
|
expect_str_token(&t, "Harriet.");
|
|
expect_str_token(&t, "is");
|
|
expect_str_token(&t, "name");
|
|
expect_str_token(&t, "My");
|
|
expect_str_token(&t, "world!");
|
|
expect_str_token(&t, "Hello");
|
|
expect_str_done(&t);
|
|
|
|
// Multi-byte delim set, reverse direction.
|
|
let t2: strings.tokenizer = strings.rtokenize("a b\tc", " \t");
|
|
expect_str_token(&t2, "c");
|
|
expect_str_token(&t2, "b");
|
|
expect_str_token(&t2, "a");
|
|
expect_str_done(&t2);
|
|
|
|
// Empty input — done immediately.
|
|
let t3: strings.tokenizer = strings.rtokenize("", " ");
|
|
expect_str_done(&t3);
|
|
};
|
|
|
|
@test fn peektoken_cases() void = {
|
|
// Two peeks without advancing return the same token.
|
|
let t: strings.tokenizer = strings.tokenize("a b c", " ");
|
|
match (strings.peektoken(&t)) {
|
|
case let p: str => { assert(!(!streq(p, "a"))); };
|
|
case bytes.done => { abort(); };
|
|
};
|
|
match (strings.peektoken(&t)) {
|
|
case let p: str => { assert(!(!streq(p, "a"))); };
|
|
case bytes.done => { abort(); };
|
|
};
|
|
// Advance once — peek then returns "b".
|
|
match (strings.nexttoken(&t)) {
|
|
case let n: str => { assert(!(!streq(n, "a"))); };
|
|
case bytes.done => { abort(); };
|
|
};
|
|
match (strings.peektoken(&t)) {
|
|
case let p: str => { assert(!(!streq(p, "b"))); };
|
|
case bytes.done => { abort(); };
|
|
};
|
|
|
|
// Empty input — peek is done.
|
|
let t2: strings.tokenizer = strings.tokenize("", " ");
|
|
match (strings.peektoken(&t2)) {
|
|
case let p: str => { abort(); };
|
|
case bytes.done => void;
|
|
};
|
|
};
|
|
|
|
@test fn remainingtokens_cases() void = {
|
|
// ref/hare/strings/tokenize.ha:157. After 2 nexttokens, remaining
|
|
// is "My name is Harriet.".
|
|
let t: strings.tokenizer = strings.tokenize(
|
|
"Hello world! My name is Harriet.", " ");
|
|
match (strings.nexttoken(&t)) {
|
|
case let n: str => { assert(!(!streq(n, "Hello"))); };
|
|
case bytes.done => { abort(); };
|
|
};
|
|
match (strings.nexttoken(&t)) {
|
|
case let n: str => { assert(!(!streq(n, "world!"))); };
|
|
case bytes.done => { abort(); };
|
|
};
|
|
if (!streq(strings.remainingtokens(&t), "My name is Harriet.")) {
|
|
abort();
|
|
};
|
|
|
|
// Fresh tokenizer — remainingtokens is the whole input.
|
|
let t2: strings.tokenizer = strings.tokenize("a b c", " ");
|
|
assert(!(!streq(strings.remainingtokens(&t2), "a b c")));
|
|
};
|
|
|
|
// ---- splitn / rsplitn / split ----------------------------------------
|
|
// ref/hare/strings/tokenize.ha:245 @test fn split. Hare's vectors
|
|
// mirrored here directly; element reads go through `&toks.ptr[i]: *str`
|
|
// rather than `toks[i]` so the 16B str element copy stays out of the
|
|
// multi-word-store gap noted at cmd/w6c/cgen.c:6515.
|
|
|
|
fn expect_str(toks: []str, i: i32, want: str) void = {
|
|
assert(!(i >= toks.len));
|
|
let p: *str = &toks.ptr[i];
|
|
assert(!(p.len != want.len));
|
|
let j: i32 = 0;
|
|
for (j < want.len) {
|
|
assert(!(p.ptr[j] != want[j]));
|
|
j += 1;
|
|
};
|
|
};
|
|
|
|
@test fn splitn_cases() void = {
|
|
// ref/hare/strings/tokenize.ha:247 — n=4 buckets the trailing
|
|
// "is Drew" as the remainder slot.
|
|
let t1: []str = strings.splitn("Hello, my name is Drew", " ", 4);
|
|
assert(!(t1.len != 4));
|
|
expect_str(t1, 0, "Hello,");
|
|
expect_str(t1, 1, "my");
|
|
expect_str(t1, 2, "name");
|
|
expect_str(t1, 3, "is Drew");
|
|
os.free(t1.ptr: *void, (t1.cap: u64) * size(str): u64);
|
|
|
|
// ref/hare/strings/tokenize.ha:263 — n > tokens leaves a single
|
|
// slot holding the unchanged input (delim not found).
|
|
let t2: []str = strings.splitn("one", "=", 2);
|
|
assert(!(t2.len != 1));
|
|
expect_str(t2, 0, "one");
|
|
os.free(t2.ptr: *void, (t2.cap: u64) * size(str): u64);
|
|
|
|
// n == 1 — single slot holding the whole input as remainder.
|
|
let t3: []str = strings.splitn("a b c", " ", 1);
|
|
assert(!(t3.len != 1));
|
|
expect_str(t3, 0, "a b c");
|
|
os.free(t3.ptr: *void, (t3.cap: u64) * size(str): u64);
|
|
|
|
// Empty input — empty result.
|
|
let t4: []str = strings.splitn("", " ", 5);
|
|
assert(!(t4.len != 0));
|
|
if (t4.cap > 0) {
|
|
os.free(t4.ptr: *void, (t4.cap: u64) * size(str): u64);
|
|
};
|
|
|
|
// Multi-byte delim set (byte-set semantics per
|
|
// ref/hare/strings/tokenize.ha:35) — split on ',' OR ':' OR ';'.
|
|
let t5: []str = strings.splitn("hello;world,foo:bar", ",:;", 10);
|
|
assert(!(t5.len != 4));
|
|
expect_str(t5, 0, "hello");
|
|
expect_str(t5, 1, "world");
|
|
expect_str(t5, 2, "foo");
|
|
expect_str(t5, 3, "bar");
|
|
os.free(t5.ptr: *void, (t5.cap: u64) * size(str): u64);
|
|
};
|
|
|
|
@test fn rsplitn_cases() void = {
|
|
// ref/hare/strings/tokenize.ha:271 — reverse n=4 with the
|
|
// "Hello, my" prefix as the remainder slot at index 0.
|
|
let t1: []str = strings.rsplitn("Hello, my name is Drew", " ", 4);
|
|
assert(!(t1.len != 4));
|
|
expect_str(t1, 0, "Hello, my");
|
|
expect_str(t1, 1, "name");
|
|
expect_str(t1, 2, "is");
|
|
expect_str(t1, 3, "Drew");
|
|
os.free(t1.ptr: *void, (t1.cap: u64) * size(str): u64);
|
|
|
|
// n > token count — done short-circuit returns toks UN-reversed
|
|
// (last-token-first order). Mirrors bytes.rsplitn (Hare's
|
|
// ref/hare/strings/tokenize.ha:219-224 reverse step is gated
|
|
// behind the n-1 loop completion).
|
|
let t2: []str = strings.rsplitn("a b c", " ", 10);
|
|
assert(!(t2.len != 3));
|
|
expect_str(t2, 0, "c");
|
|
expect_str(t2, 1, "b");
|
|
expect_str(t2, 2, "a");
|
|
os.free(t2.ptr: *void, (t2.cap: u64) * size(str): u64);
|
|
|
|
// n == 1 — single slot holding the whole input as remainder.
|
|
let t3: []str = strings.rsplitn("a b c", " ", 1);
|
|
assert(!(t3.len != 1));
|
|
expect_str(t3, 0, "a b c");
|
|
os.free(t3.ptr: *void, (t3.cap: u64) * size(str): u64);
|
|
|
|
// delim absent — first nexttoken yields the entire input as the
|
|
// sole token; second iter sees done and short-circuits with the
|
|
// 1-elem toks un-reversed (single element, reverse is a no-op).
|
|
let t4: []str = strings.rsplitn("abc", "=", 5);
|
|
assert(!(t4.len != 1));
|
|
expect_str(t4, 0, "abc");
|
|
os.free(t4.ptr: *void, (t4.cap: u64) * size(str): u64);
|
|
};
|
|
|
|
@test fn split_cases() void = {
|
|
// ref/hare/strings/tokenize.ha:255 — full split, every delim hit
|
|
// is a boundary.
|
|
let t1: []str = strings.split("Hello, my name is Drew", " ");
|
|
assert(!(t1.len != 5));
|
|
expect_str(t1, 0, "Hello,");
|
|
expect_str(t1, 1, "my");
|
|
expect_str(t1, 2, "name");
|
|
expect_str(t1, 3, "is");
|
|
expect_str(t1, 4, "Drew");
|
|
os.free(t1.ptr: *void, (t1.cap: u64) * size(str): u64);
|
|
|
|
// Leading + trailing delim — empty tokens at ends.
|
|
let t2: []str = strings.split(" a b ", " ");
|
|
assert(!(t2.len != 4));
|
|
expect_str(t2, 0, "");
|
|
expect_str(t2, 1, "a");
|
|
expect_str(t2, 2, "b");
|
|
expect_str(t2, 3, "");
|
|
os.free(t2.ptr: *void, (t2.cap: u64) * size(str): u64);
|
|
|
|
// Multi-byte delim set, byte-set semantics matching Hare's
|
|
// strings::split example at ref/hare/strings/tokenize.ha:235.
|
|
let t3: []str = strings.split("hello;world,foo:bar", ",:;");
|
|
assert(!(t3.len != 4));
|
|
expect_str(t3, 0, "hello");
|
|
expect_str(t3, 1, "world");
|
|
expect_str(t3, 2, "foo");
|
|
expect_str(t3, 3, "bar");
|
|
os.free(t3.ptr: *void, (t3.cap: u64) * size(str): u64);
|
|
};
|
|
|
|
// ---- cut / rcut -------------------------------------------------------
|
|
// ref/hare/strings/tokenize.ha:316 (@test fn cut). str wrappers over
|
|
// bytes.cut/rcut. The `let (a, b) = cut(...)` destructure drives the
|
|
// over-cap tuple-return (sret) path end-to-end (a 2nd #10 witness).
|
|
|
|
@test fn cut_cases() void = {
|
|
// both halves present.
|
|
let (a0, b0) = strings.cut("hello=world", "=");
|
|
assert(!(!streq(a0, "hello")));
|
|
assert(!(!streq(b0, "world")));
|
|
|
|
// only first instance is cut; rest stays in the second half.
|
|
let (a1, b1) = strings.cut("hello=world=foobar", "=");
|
|
assert(!(!streq(a1, "hello")));
|
|
assert(!(!streq(b1, "world=foobar")));
|
|
|
|
// delim absent -> (whole input, "").
|
|
let (a2, b2) = strings.cut("hello world", "=");
|
|
assert(!(!streq(a2, "hello world")));
|
|
assert(!(!streq(b2, "")));
|
|
|
|
// delim at start -> empty before.
|
|
let (a3, b3) = strings.cut("=world", "=");
|
|
assert(!(!streq(a3, "")));
|
|
assert(!(!streq(b3, "world")));
|
|
|
|
// delim at end -> empty after.
|
|
let (a4, b4) = strings.cut("hello=", "=");
|
|
assert(!(!streq(a4, "hello")));
|
|
assert(!(!streq(b4, "")));
|
|
|
|
// empty input -> ("", "").
|
|
let (a5, b5) = strings.cut("", "=");
|
|
assert(!(!streq(a5, "")));
|
|
assert(!(!streq(b5, "")));
|
|
|
|
// multi-byte delim present.
|
|
let (a6, b6) = strings.cut("aXYbXYc", "XY");
|
|
assert(!(!streq(a6, "a")));
|
|
assert(!(!streq(b6, "bXYc")));
|
|
|
|
// multi-byte delim absent.
|
|
let (a7, b7) = strings.cut("abc", "XY");
|
|
assert(!(!streq(a7, "abc")));
|
|
assert(!(!streq(b7, "")));
|
|
|
|
// borrowed, not copied: first half aliases the input bytes.
|
|
let in: str = "hello=world";
|
|
let (a8, b8) = strings.cut(in, "=");
|
|
assert(!(a8.ptr != in.ptr));
|
|
};
|
|
|
|
@test fn rcut_cases() void = {
|
|
// rcut splits along the LAST instance.
|
|
let (a0, b0) = strings.rcut("hello=world=foobar", "=");
|
|
assert(!(!streq(a0, "hello=world")));
|
|
assert(!(!streq(b0, "foobar")));
|
|
|
|
// single instance == cut.
|
|
let (a1, b1) = strings.rcut("hello=world", "=");
|
|
assert(!(!streq(a1, "hello")));
|
|
assert(!(!streq(b1, "world")));
|
|
|
|
// delim absent -> (whole input, "").
|
|
let (a2, b2) = strings.rcut("hello world", "=");
|
|
assert(!(!streq(a2, "hello world")));
|
|
assert(!(!streq(b2, "")));
|
|
|
|
// delim at end -> empty after.
|
|
let (a3, b3) = strings.rcut("hello=", "=");
|
|
assert(!(!streq(a3, "hello")));
|
|
assert(!(!streq(b3, "")));
|
|
|
|
// delim at start -> empty before.
|
|
let (a4, b4) = strings.rcut("=world", "=");
|
|
assert(!(!streq(a4, "")));
|
|
assert(!(!streq(b4, "world")));
|
|
|
|
// empty input -> ("", "").
|
|
let (a5, b5) = strings.rcut("", "=");
|
|
assert(!(!streq(a5, "")));
|
|
assert(!(!streq(b5, "")));
|
|
|
|
// multi-byte delim, two instances -> cut at LAST.
|
|
let (a6, b6) = strings.rcut("XYaXYb", "XY");
|
|
assert(!(!streq(a6, "XYa")));
|
|
assert(!(!streq(b6, "b")));
|
|
|
|
// multi-byte delim absent.
|
|
let (a7, b7) = strings.rcut("abc", "XY");
|
|
assert(!(!streq(a7, "abc")));
|
|
assert(!(!streq(b7, "")));
|
|
};
|