From 8e2e6ae162aac8f55366ddb46dafd16202f707b4 Mon Sep 17 00:00:00 2001 From: Hojun-Cho Date: Sun, 9 Aug 2026 17:46:22 +0900 Subject: [PATCH] strings: preserve UTF-8 and boundary invariants --- lib/CLAUDE.md | 6 +-- lib/strings/pad_test.ww | 36 ++++++++----- lib/strings/replace_test.ww | 5 ++ lib/strings/strings.ww | 99 ++++++++++++++++++++++-------------- lib/strings/sub_test.ww | 10 ++++ lib/strings/tokenize_test.ww | 27 +++++++--- lib/strings/utf8_test.ww | 2 +- 7 files changed, 124 insertions(+), 61 deletions(-) diff --git a/lib/CLAUDE.md b/lib/CLAUDE.md index 3fd0a999..a253fcc5 100644 --- a/lib/CLAUDE.md +++ b/lib/CLAUDE.md @@ -40,9 +40,9 @@ Signatures mirror Hare too, modulo: - `(T | U)` sum-typed parameters dispatch via `match` inside the callee. `strings.byteindex(haystack: str, needle: (str | rune))`, `bytes.index(s: []u8, needle: (u8 | []u8))`, and `rbyteindex`/ - `rindex` follow Hare's shape directly. The rune-indexed - `strings.index` (rune-wise position) isn't shipped yet — we don't - have UTF-8 rune iteration in the language stack. + `rindex` follow Hare's shape directly. `strings.index`/`rindex` expose + rune positions; `byteindex`/`rbyteindex` expose byte offsets. UTF-8 + iterators and both indexing axes are shipped and tested. Don't ship a richer surface than Hare has. A documented subset is fine; an extension, rename, or convenience-wrapper is not — callers diff --git a/lib/strings/pad_test.ww b/lib/strings/pad_test.ww index 999a8622..72e7cee3 100644 --- a/lib/strings/pad_test.ww +++ b/lib/strings/pad_test.ww @@ -7,14 +7,13 @@ package strings_test; import strings; import os; +import encoding.utf8; +import test; // ref/hare/strings/pad.ha:23 (@test fn lpad), :54 (@test fn rpad). Hare's // row triple (smaxlen (early return path), multibyte s with byte-length -// counting, and multibyte pad rune (encoded width >1 byte). The -// multibyte-pad rows pin Hare's `[..maxlen]` slice contract — a -// multibyte trailing pad may be truncated mid-codepoint, exactly as -// Hare does. +// counting, and multibyte pad rune (encoded width >1 byte). @test fn lpad_cases() void = { // Hare row 1: shorter s, ASCII pad. @@ -49,19 +48,26 @@ import os; os.free(r5.ptr: *void, r5.len: u64); // ww row 3: multibyte pad rune. 'α' U+03B1 is 2 bytes (0xCE 0xB1). - // s="x" (1 byte), maxlen=5 → npads = 5 - 1 = 4, pad_bytes=2, - // total writes = 4*2 + 1 = 9 bytes; result `[..5]` = "αα" (4 bytes) - // + 0xCE (truncated leading byte of next α) — exactly Hare's slice. + // s="x" (1 byte), maxlen=5 leaves exactly two complete pad runes. let r6: str = strings.lpad("x", 0x03B1u32: rune, 5); - assert(!(r6.len != 5)); - assert(!(r6.ptr[0] != 0xCEu8)); // α byte 0 - assert(!(r6.ptr[1] != 0xB1u8)); // α byte 1 - assert(!(r6.ptr[2] != 0xCEu8)); // α byte 0 - assert(!(r6.ptr[3] != 0xB1u8)); // α byte 1 - assert(!(r6.ptr[4] != 0xCEu8)); // truncated α byte 0 + assert(!(!streq(r6, "ααx"))); + match (utf8.validate(strings.toutf8(r6))) { + case void => void; + case utf8.invalid => abort(); + }; os.free(r6.ptr: *void, r6.len: u64); }; +@test fn lpad_split_rune_aborts() void = { + test.expectabort(); + strings.lpad("x", 0x03B1u32: rune, 4); +}; + +@test fn lpad_negative_length_aborts() void = { + test.expectabort(); + strings.lpad("x", ' ', -1); +}; + @test fn rpad_cases() void = { // Hare row 1: shorter s, ASCII pad. let r1: str = strings.rpad("2", '0', 5); @@ -105,3 +111,7 @@ import os; os.free(r6.ptr: *void, r6.len: u64); }; +@test fn rpad_split_rune_aborts() void = { + test.expectabort(); + strings.rpad("x", 0x03B1u32: rune, 4); +}; diff --git a/lib/strings/replace_test.ww b/lib/strings/replace_test.ww index 09b9dc6a..887402b6 100644 --- a/lib/strings/replace_test.ww +++ b/lib/strings/replace_test.ww @@ -7,6 +7,7 @@ package strings_test; import strings; import os; +import test; // ref/hare/strings/replace.ha:46 (#4). Hare rows 1-5 (target found // once / multiple times / shorter / longer / multibyte) plus ww rows @@ -108,3 +109,7 @@ import os; }; }; +@test fn replace_empty_needle_aborts() void = { + test.expectabort(); + strings.replace("abc", "", "x"); +}; diff --git a/lib/strings/strings.ww b/lib/strings/strings.ww index 95af719b..d2f85ee0 100644 --- a/lib/strings/strings.ww +++ b/lib/strings/strings.ww @@ -39,12 +39,14 @@ export fn toutf8(s: str) []u8 = { return r; }; -// Pure reinterpret per CLAUDE.md rule 9 carve-out; -// ref/hare/strings/utf8.ha:10. +// Pure borrowed reinterpret per CLAUDE.md rule 9 carve-out; +// ref/hare/strings/utf8.ha:10. Mutating the view is valid only when the +// caller owns mutable backing storage. export fn frombytes(in: []u8) str = { let r: str; r.ptr = in.ptr; r.len = in.len; + r.cap = in.len; return r; }; @@ -67,6 +69,7 @@ export fn dup(s: str) str = { let r: str; r.ptr = nil; r.len = 0; + r.cap = 0; if (s.len == 0) { return r; }; let buf: []u8 = alloc([], s.len: u64)!; let i: i32 = 0; @@ -133,12 +136,19 @@ export fn freeall(s: []str) void = { // ref/hare/strings/concat.ha:5. Hare's `nomem` return is dropped: // `os.alloc` aborts on OOM. export fn concat(strs: str...) str = { - let total: i32 = 0; + let total64: i64 = 0; let i: i32 = 0; - for (i < strs.len) { total += strs[i].len; i += 1; }; + for (i < strs.len) { + total64 += strs[i].len: i64; + assert(total64 <= types.I32_MAX: i64, + "strings.concat: result exceeds maximum string length"); + i += 1; + }; + let total: i32 = total64: i32; let r: str; r.ptr = nil; r.len = 0; + r.cap = 0; if (total == 0) { return r; }; let buf: []u8 = alloc([], total: u64)!; let off: i32 = 0; @@ -160,16 +170,20 @@ export fn concat(strs: str...) str = { // ref/hare/strings/concat.ha:46. Hare's `nomem` return is dropped: // `os.alloc` aborts on OOM. export fn join(delim: str, strs: str...) str = { - let total: i32 = 0; + let total64: i64 = 0; let i: i32 = 0; for (i < strs.len) { - total += strs[i].len; - if (i + 1 < strs.len) { total += delim.len; }; + total64 += strs[i].len: i64; + if (i + 1 < strs.len) { total64 += delim.len: i64; }; + assert(total64 <= types.I32_MAX: i64, + "strings.join: result exceeds maximum string length"); i += 1; }; + let total: i32 = total64: i32; let r: str; r.ptr = nil; r.len = 0; + r.cap = 0; if (total == 0) { return r; }; let buf: []u8 = alloc([], total: u64)!; let off: i32 = 0; @@ -215,6 +229,8 @@ fn utf8bytelenbounded(it: *iterator, end: i32) i32 = { // defaulting end=END is omitted: ww has no default-parameter syntax // (filed as #37). export fn sub(s: str, start: i32, end: i32) str = { + assert(start >= 0 && end >= 0, + "strings.sub: indexes must not be negative"); assert(start <= end, "strings.sub: start is higher than end"); let it: iterator = iter(s); let starti: i32 = utf8bytelenbounded(&it, start); @@ -222,6 +238,7 @@ export fn sub(s: str, start: i32, end: i32) str = { let r: str; r.ptr = s.ptr + (starti: u64); r.len = endi - starti; + r.cap = r.len; return r; }; @@ -232,6 +249,8 @@ export fn sub(s: str, start: i32, end: i32) str = { // codepoint); the equivalent Hare predicate is `s[i] & 0xc0 == 0x80` // at ref/hare/strings/sub.ha:72-73. export fn bytesub(s: str, start: i32, end: i32) (str | utf8.invalid) = { + assert(start >= 0 && end >= 0, + "strings.bytesub: indexes must not be negative"); assert(start <= end, "strings.bytesub: start is higher than end"); assert(end <= s.len, "strings.bytesub: end exceeds string length"); if (start < s.len && (s[start] & 0xC0u8) == 0x80u8) { @@ -243,6 +262,7 @@ export fn bytesub(s: str, start: i32, end: i32) (str | utf8.invalid) = { let r: str; r.ptr = s.ptr + (start: u64); r.len = end - start; + r.cap = r.len; return r; }; @@ -438,6 +458,7 @@ export fn trimprefix(input: str, prefix: str) str = { let r: str; r.ptr = input.ptr + (prefix.len: u64); r.len = input.len - prefix.len; + r.cap = r.len; return r; }; @@ -447,6 +468,7 @@ export fn trimsuffix(input: str, suffix: str) str = { let r: str; r.ptr = input.ptr; r.len = input.len - suffix.len; + r.cap = r.len; return r; }; @@ -706,10 +728,12 @@ export fn rcut(in: str, delim: str) (str, str) = { // // ref/hare/strings/tokenize.ha:172. export fn splitn(in: str, delim: str, n: i32) []str = { + assert(n >= 0, "strings.splitn: token limit must not be negative"); let toks: []str; toks.ptr = nil: *str; toks.len = 0; toks.cap = 0; + if (n == 0) { return toks; }; let tok: tokenizer = tokenize(in, delim); let i: i32 = 0; for (i < n - 1) { @@ -732,23 +756,20 @@ export fn splitn(in: str, delim: str, n: i32) []str = { // The trailing slot holds the unconsumed prefix (everything before // the n-th-from-last delim hit). // -// When the input has fewer than n tokens, the `done` short-circuit -// returns toks UN-reversed (in last-token-first order). Mirrors Hare -// at ref/hare/strings/tokenize.ha:219-224 where the in-place reverse -// step is gated behind the n-1 loop running to completion. -// // ref/hare/strings/tokenize.ha:200. export fn rsplitn(in: str, delim: str, n: i32) []str = { + assert(n >= 0, "strings.rsplitn: token limit must not be negative"); let toks: []str; toks.ptr = nil: *str; toks.len = 0; toks.cap = 0; + if (n == 0) { return toks; }; let tok: tokenizer = rtokenize(in, delim); let i: i32 = 0; for (i < n - 1) { match (nexttoken(&tok)) { case let s: str => { append(toks, s); }; - case bytes.done => { return toks; }; + case bytes.done => { break; }; }; i += 1; }; @@ -762,7 +783,7 @@ export fn rsplitn(in: str, delim: str, n: i32) []str = { // In-place reverse so callers see argv-order, matching Hare // (ref/hare/strings/tokenize.ha:220). Element copy is field-wise - // through `*str` because `toks[i] = toks[j]` (full 16B str store) + // through `*str` because `toks[i] = toks[j]` (full str-header store) // lands in the multi-word-store gap noted at cmd/w6c/cgen.c:6515. let a: i32 = 0; let b: i32 = toks.len - 1; @@ -771,10 +792,13 @@ export fn rsplitn(in: str, delim: str, n: i32) []str = { let pb: *str = &toks.ptr[b]; let tp: *u8 = pa.ptr; let tl: i32 = pa.len; + let tc: i32 = pa.cap; pa.ptr = pb.ptr; pa.len = pb.len; + pa.cap = pb.cap; pb.ptr = tp; pb.len = tl; + pb.cap = tc; a += 1; b -= 1; }; @@ -789,35 +813,26 @@ export fn split(in: str, delim: str) []str = { return splitn(in, delim, types.I32_MAX); }; -// Length comparison is BYTES, mirroring Hare's `len(s) >= maxlen` -// at ref/hare/strings/pad.ha:9. A multibyte `p` whose encoded width -// doesn't divide `maxlen - s.len` evenly leaves a trailing pad byte -// pair sliced mid-codepoint at byte `maxlen-1`, exactly as Hare's -// `res[..maxlen]` does (ref/hare/strings/pad.ha:20). When -// `(maxlen - s.len) * pad.len >= maxlen` (multibyte pad overflows the -// budget), `s` is entirely sliced off — same as Hare. Caller releases -// with `os.free(r.ptr, r.len: u64)`. Hare's `nomem` return is dropped: -// `os.alloc` aborts on OOM. Buf size == r.len keeps the free-contract -// shape of [[dup]] / [[concat]] / [[join]]; Hare's `alloc([], maxlen)!` -// over-allocs via append then slices, but Hare's slice-free recovers -// the true capacity from the heap allocator (rt/ensure.ha:24), which -// ww's munmap-based `os.free` cannot do. +// Length is measured in bytes, matching Hare's `len(s) >= maxlen` at +// ref/hare/strings/pad.ha:9. The exact byte target must hold a whole number of +// pad runes; otherwise no valid UTF-8 result of that length exists. Caller +// releases with `os.free(r.ptr, r.len: u64)`. export fn lpad(s: str, p: rune, maxlen: i32) str = { + assert(maxlen >= 0, "strings.lpad: length must not be negative"); if (s.len >= maxlen) { return dup(s); }; let scratch: [4]u8; let pad: []u8 = runebytes(scratch[0:4], p); + let gap: i32 = maxlen - s.len; + assert(gap % pad.len == 0, + "strings.lpad: byte length would split the pad rune"); let buf: []u8 = alloc([], maxlen: u64)!; - let padwrite: i32 = (maxlen - s.len) * pad.len; - if (padwrite > maxlen) { padwrite = maxlen; }; let off: i32 = 0; - for (off < padwrite) { + for (off < gap) { buf[off] = pad.ptr[off % pad.len]; off += 1; }; let k: i32 = 0; - let srem: i32 = maxlen - off; - if (srem > s.len) { srem = s.len; }; - for (k < srem) { + for (k < s.len) { buf[off + k] = s[k]; k += 1; }; @@ -836,9 +851,10 @@ export fn lpad(s: str, p: rune, maxlen: i32) str = { // Single nomem path (the `alloc([], total)?`) preserves Hare's // signature without a per-write `append(...)?` (ww's append builtin // aborts on OOM, #11). Empty `needle` would hasprefix-match every -// position with a zero stride — same infinite loop Hare exhibits at -// ref/hare/strings/replace.ha:31; not gated. +// position with a zero stride, so it is rejected as an explicit +// precondition rather than hanging. export fn replace(s: str, needle: str, target: str) (str | nomem) = { + assert(needle.len > 0, "strings.replace: needle must not be empty"); let sb: []u8 = toutf8(s); let nb: []u8 = toutf8(needle); let tb: []u8 = toutf8(target); @@ -852,11 +868,15 @@ export fn replace(s: str, needle: str, target: str) (str | nomem) = { i += 1; }; }; - let total: i32 = sb.len + count * (tb.len - nb.len); + let total64: i64 = (sb.len: i64) + (count: i64) * + ((tb.len: i64) - (nb.len: i64)); + if (total64 > types.I32_MAX: i64) { let e: nomem; return e; }; + let total: i32 = total64: i32; if (total == 0) { let r: str; r.ptr = nil; r.len = 0; + r.cap = 0; return r; }; let res: []u8 = alloc([], total)?; @@ -883,18 +903,21 @@ export fn replace(s: str, needle: str, target: str) (str | nomem) = { // Symmetric with [[lpad]]. ref/hare/strings/pad.ha:39. export fn rpad(s: str, p: rune, maxlen: i32) str = { + assert(maxlen >= 0, "strings.rpad: length must not be negative"); if (s.len >= maxlen) { return dup(s); }; let scratch: [4]u8; let pad: []u8 = runebytes(scratch[0:4], p); + let gap: i32 = maxlen - s.len; + assert(gap % pad.len == 0, + "strings.rpad: byte length would split the pad rune"); let buf: []u8 = alloc([], maxlen: u64)!; let k: i32 = 0; for (k < s.len) { buf[k] = s[k]; k += 1; }; - let padwrite: i32 = maxlen - s.len; let i: i32 = 0; - for (i < padwrite) { + for (i < gap) { buf[s.len + i] = pad.ptr[i % pad.len]; i += 1; }; diff --git a/lib/strings/sub_test.ww b/lib/strings/sub_test.ww index 991132f6..4382da27 100644 --- a/lib/strings/sub_test.ww +++ b/lib/strings/sub_test.ww @@ -7,6 +7,7 @@ package strings_test; import strings; import encoding.utf8; +import test; // ref/hare/strings/sub.ha:44 (@test fn sub), :79 (@test fn bytesub). Hare's // 2-arg `sub(s, start)` rows are omitted: ww has no default-parameter @@ -29,6 +30,11 @@ import encoding.utf8; assert(!(!streq(strings.sub("héllo", 0, 5), "héllo"))); }; +@test fn sub_negative_aborts() void = { + test.expectabort(); + strings.sub("abc", -1, 0); +}; + // Match-shape mirrors Hare's `bytesub(...)!` at ref/hare/strings/sub.ha: // 80-86. Inlined at each call site (rather than a helper that takes // `(str | utf8.invalid)` by value) because the union-by-value path @@ -94,3 +100,7 @@ import encoding.utf8; }; }; +@test fn bytesub_negative_aborts() void = { + test.expectabort(); + strings.bytesub("abc", -1, 0); +}; diff --git a/lib/strings/tokenize_test.ww b/lib/strings/tokenize_test.ww index 784b1a03..1c55a42e 100644 --- a/lib/strings/tokenize_test.ww +++ b/lib/strings/tokenize_test.ww @@ -7,6 +7,7 @@ package strings_test; import strings; import os; +import test; // ref/hare/strings/tokenize.ha:110 @test fn tokenize. Row drivers // mirror lib/bytes/bytestest expect_token / expect_done but compare @@ -171,6 +172,7 @@ fn expect_str(toks: []str, i: i32, want: str) void = { assert(!(i >= toks.len)); let p: *str = &toks.ptr[i]; assert(!(p.len != want.len)); + assert(!(p.cap != p.len)); let j: i32 = 0; for (j < want.len) { assert(!(p.ptr[j] != want[j])); @@ -218,6 +220,14 @@ fn expect_str(toks: []str, i: i32, want: str) void = { expect_str(t5, 2, "foo"); expect_str(t5, 3, "bar"); os.free(t5.ptr: *void, (t5.cap: u64) * size(str): u64); + + let t6: []str = strings.splitn("a b", " ", 0); + assert(!(t6.len != 0 || t6.cap != 0)); +}; + +@test fn splitn_negative_aborts() void = { + test.expectabort(); + strings.splitn("a b", " ", -1); }; @test fn rsplitn_cases() void = { @@ -231,15 +241,12 @@ fn expect_str(toks: []str, i: i32, want: str) void = { expect_str(t1, 3, "Drew"); os.free(t1.ptr: *void, (t1.cap: u64) * size(str): u64); - // n > token count — done short-circuit returns toks UN-reversed - // (last-token-first order). Mirrors bytes.rsplitn (Hare's - // ref/hare/strings/tokenize.ha:219-224 reverse step is gated - // behind the n-1 loop completion). + // n > token count still returns input order. let t2: []str = strings.rsplitn("a b c", " ", 10); assert(!(t2.len != 3)); - expect_str(t2, 0, "c"); + expect_str(t2, 0, "a"); expect_str(t2, 1, "b"); - expect_str(t2, 2, "a"); + expect_str(t2, 2, "c"); os.free(t2.ptr: *void, (t2.cap: u64) * size(str): u64); // n == 1 — single slot holding the whole input as remainder. @@ -255,6 +262,14 @@ fn expect_str(toks: []str, i: i32, want: str) void = { assert(!(t4.len != 1)); expect_str(t4, 0, "abc"); os.free(t4.ptr: *void, (t4.cap: u64) * size(str): u64); + + let t5: []str = strings.rsplitn("a b", " ", 0); + assert(!(t5.len != 0 || t5.cap != 0)); +}; + +@test fn rsplitn_negative_aborts() void = { + test.expectabort(); + strings.rsplitn("a b", " ", -1); }; @test fn split_cases() void = { diff --git a/lib/strings/utf8_test.ww b/lib/strings/utf8_test.ww index b14ad432..da6845d2 100644 --- a/lib/strings/utf8_test.ww +++ b/lib/strings/utf8_test.ww @@ -19,5 +19,5 @@ import encoding.utf8; let r: str = strings.frombytes(b); assert(!(!streq(r, "hello"))); assert(!(r.ptr != s.ptr)); // borrowed, not copied + assert(!(r.cap != r.len)); }; -