Hojun-Cho/ww
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

strings: preserve UTF-8 and boundary invariants

Browse Source
This commit is contained in:
Hojun-Cho
2026-08-09 17:46:22 +09:00
parent 2d947c469e
commit 8e2e6ae162
7 changed files with 124 additions and 61 deletions
Download Patch File Download Diff File Expand all files Collapse all files

99
lib/strings/strings.ww
View File

@@ -39,12 +39,14 @@ export fn toutf8(s: str) []u8 = {
return r;
};
// Pure reinterpret per CLAUDE.md rule 9 carve-out;
// ref/hare/strings/utf8.ha:10.
// Pure borrowed reinterpret per CLAUDE.md rule 9 carve-out;
// ref/hare/strings/utf8.ha:10. Mutating the view is valid only when the
// caller owns mutable backing storage.
export fn frombytes(in: []u8) str = {
let r: str;
r.ptr = in.ptr;
r.len = in.len;
r.cap = in.len;
return r;
};
@@ -67,6 +69,7 @@ export fn dup(s: str) str = {
let r: str;
r.ptr = nil;
r.len = 0;
r.cap = 0;
if (s.len == 0) { return r; };
let buf: []u8 = alloc([], s.len: u64)!;
let i: i32 = 0;
@@ -133,12 +136,19 @@ export fn freeall(s: []str) void = {
// ref/hare/strings/concat.ha:5. Hare's `nomem` return is dropped:
// `os.alloc` aborts on OOM.
export fn concat(strs: str...) str = {
let total: i32 = 0;
let total64: i64 = 0;
let i: i32 = 0;
for (i < strs.len) { total += strs[i].len; i += 1; };
for (i < strs.len) {
total64 += strs[i].len: i64;
assert(total64 <= types.I32_MAX: i64,
"strings.concat: result exceeds maximum string length");
i += 1;
};
let total: i32 = total64: i32;
let r: str;
r.ptr = nil;
r.len = 0;
r.cap = 0;
if (total == 0) { return r; };
let buf: []u8 = alloc([], total: u64)!;
let off: i32 = 0;
@@ -160,16 +170,20 @@ export fn concat(strs: str...) str = {
// ref/hare/strings/concat.ha:46. Hare's `nomem` return is dropped:
// `os.alloc` aborts on OOM.
export fn join(delim: str, strs: str...) str = {
let total: i32 = 0;
let total64: i64 = 0;
let i: i32 = 0;
for (i < strs.len) {
total += strs[i].len;
if (i + 1 < strs.len) { total += delim.len; };
total64 += strs[i].len: i64;
if (i + 1 < strs.len) { total64 += delim.len: i64; };
assert(total64 <= types.I32_MAX: i64,
"strings.join: result exceeds maximum string length");
i += 1;
};
let total: i32 = total64: i32;
let r: str;
r.ptr = nil;
r.len = 0;
r.cap = 0;
if (total == 0) { return r; };
let buf: []u8 = alloc([], total: u64)!;
let off: i32 = 0;
@@ -215,6 +229,8 @@ fn utf8bytelenbounded(it: *iterator, end: i32) i32 = {
// defaulting end=END is omitted: ww has no default-parameter syntax
// (filed as #37).
export fn sub(s: str, start: i32, end: i32) str = {
assert(start >= 0 && end >= 0,
"strings.sub: indexes must not be negative");
assert(start <= end, "strings.sub: start is higher than end");
let it: iterator = iter(s);
let starti: i32 = utf8bytelenbounded(&it, start);
@@ -222,6 +238,7 @@ export fn sub(s: str, start: i32, end: i32) str = {
let r: str;
r.ptr = s.ptr + (starti: u64);
r.len = endi - starti;
r.cap = r.len;
return r;
};
@@ -232,6 +249,8 @@ export fn sub(s: str, start: i32, end: i32) str = {
// codepoint); the equivalent Hare predicate is `s[i] & 0xc0 == 0x80`
// at ref/hare/strings/sub.ha:72-73.
export fn bytesub(s: str, start: i32, end: i32) (str | utf8.invalid) = {
assert(start >= 0 && end >= 0,
"strings.bytesub: indexes must not be negative");
assert(start <= end, "strings.bytesub: start is higher than end");
assert(end <= s.len, "strings.bytesub: end exceeds string length");
if (start < s.len && (s[start] & 0xC0u8) == 0x80u8) {
@@ -243,6 +262,7 @@ export fn bytesub(s: str, start: i32, end: i32) (str | utf8.invalid) = {
let r: str;
r.ptr = s.ptr + (start: u64);
r.len = end - start;
r.cap = r.len;
return r;
};
@@ -438,6 +458,7 @@ export fn trimprefix(input: str, prefix: str) str = {
let r: str;
r.ptr = input.ptr + (prefix.len: u64);
r.len = input.len - prefix.len;
r.cap = r.len;
return r;
};
@@ -447,6 +468,7 @@ export fn trimsuffix(input: str, suffix: str) str = {
let r: str;
r.ptr = input.ptr;
r.len = input.len - suffix.len;
r.cap = r.len;
return r;
};
@@ -706,10 +728,12 @@ export fn rcut(in: str, delim: str) (str, str) = {
//
// ref/hare/strings/tokenize.ha:172.
export fn splitn(in: str, delim: str, n: i32) []str = {
assert(n >= 0, "strings.splitn: token limit must not be negative");
let toks: []str;
toks.ptr = nil: *str;
toks.len = 0;
toks.cap = 0;
if (n == 0) { return toks; };
let tok: tokenizer = tokenize(in, delim);
let i: i32 = 0;
for (i < n - 1) {
@@ -732,23 +756,20 @@ export fn splitn(in: str, delim: str, n: i32) []str = {
// The trailing slot holds the unconsumed prefix (everything before
// the n-th-from-last delim hit).
//
// When the input has fewer than n tokens, the `done` short-circuit
// returns toks UN-reversed (in last-token-first order). Mirrors Hare
// at ref/hare/strings/tokenize.ha:219-224 where the in-place reverse
// step is gated behind the n-1 loop running to completion.
//
// ref/hare/strings/tokenize.ha:200.
export fn rsplitn(in: str, delim: str, n: i32) []str = {
assert(n >= 0, "strings.rsplitn: token limit must not be negative");
let toks: []str;
toks.ptr = nil: *str;
toks.len = 0;
toks.cap = 0;
if (n == 0) { return toks; };
let tok: tokenizer = rtokenize(in, delim);
let i: i32 = 0;
for (i < n - 1) {
match (nexttoken(&tok)) {
case let s: str => { append(toks, s); };
case bytes.done => { return toks; };
case bytes.done => { break; };
};
i += 1;
};
@@ -762,7 +783,7 @@ export fn rsplitn(in: str, delim: str, n: i32) []str = {
// In-place reverse so callers see argv-order, matching Hare
// (ref/hare/strings/tokenize.ha:220). Element copy is field-wise
// through `*str` because `toks[i] = toks[j]` (full 16B str store)
// through `*str` because `toks[i] = toks[j]` (full str-header store)
// lands in the multi-word-store gap noted at cmd/w6c/cgen.c:6515.
let a: i32 = 0;
let b: i32 = toks.len - 1;
@@ -771,10 +792,13 @@ export fn rsplitn(in: str, delim: str, n: i32) []str = {
let pb: *str = &toks.ptr[b];
let tp: *u8 = pa.ptr;
let tl: i32 = pa.len;
let tc: i32 = pa.cap;
pa.ptr = pb.ptr;
pa.len = pb.len;
pa.cap = pb.cap;
pb.ptr = tp;
pb.len = tl;
pb.cap = tc;
a += 1;
b -= 1;
};
@@ -789,35 +813,26 @@ export fn split(in: str, delim: str) []str = {
return splitn(in, delim, types.I32_MAX);
};
// Length comparison is BYTES, mirroring Hare's `len(s) >= maxlen`
// at ref/hare/strings/pad.ha:9. A multibyte `p` whose encoded width
// doesn't divide `maxlen - s.len` evenly leaves a trailing pad byte
// pair sliced mid-codepoint at byte `maxlen-1`, exactly as Hare's
// `res[..maxlen]` does (ref/hare/strings/pad.ha:20). When
// `(maxlen - s.len) * pad.len >= maxlen` (multibyte pad overflows the
// budget), `s` is entirely sliced off — same as Hare. Caller releases
// with `os.free(r.ptr, r.len: u64)`. Hare's `nomem` return is dropped:
// `os.alloc` aborts on OOM. Buf size == r.len keeps the free-contract
// shape of [[dup]] / [[concat]] / [[join]]; Hare's `alloc([], maxlen)!`
// over-allocs via append then slices, but Hare's slice-free recovers
// the true capacity from the heap allocator (rt/ensure.ha:24), which
// ww's munmap-based `os.free` cannot do.
// Length is measured in bytes, matching Hare's `len(s) >= maxlen` at
// ref/hare/strings/pad.ha:9. The exact byte target must hold a whole number of
// pad runes; otherwise no valid UTF-8 result of that length exists. Caller
// releases with `os.free(r.ptr, r.len: u64)`.
export fn lpad(s: str, p: rune, maxlen: i32) str = {
assert(maxlen >= 0, "strings.lpad: length must not be negative");
if (s.len >= maxlen) { return dup(s); };
let scratch: [4]u8;
let pad: []u8 = runebytes(scratch[0:4], p);
let gap: i32 = maxlen - s.len;
assert(gap % pad.len == 0,
"strings.lpad: byte length would split the pad rune");
let buf: []u8 = alloc([], maxlen: u64)!;
let padwrite: i32 = (maxlen - s.len) * pad.len;
if (padwrite > maxlen) { padwrite = maxlen; };
let off: i32 = 0;
for (off < padwrite) {
for (off < gap) {
buf[off] = pad.ptr[off % pad.len];
off += 1;
};
let k: i32 = 0;
let srem: i32 = maxlen - off;
if (srem > s.len) { srem = s.len; };
for (k < srem) {
for (k < s.len) {
buf[off + k] = s[k];
k += 1;
};
@@ -836,9 +851,10 @@ export fn lpad(s: str, p: rune, maxlen: i32) str = {
// Single nomem path (the `alloc([], total)?`) preserves Hare's
// signature without a per-write `append(...)?` (ww's append builtin
// aborts on OOM, #11). Empty `needle` would hasprefix-match every
// position with a zero stride — same infinite loop Hare exhibits at
// ref/hare/strings/replace.ha:31; not gated.
// position with a zero stride, so it is rejected as an explicit
// precondition rather than hanging.
export fn replace(s: str, needle: str, target: str) (str | nomem) = {
assert(needle.len > 0, "strings.replace: needle must not be empty");
let sb: []u8 = toutf8(s);
let nb: []u8 = toutf8(needle);
let tb: []u8 = toutf8(target);
@@ -852,11 +868,15 @@ export fn replace(s: str, needle: str, target: str) (str | nomem) = {
i += 1;
};
};
let total: i32 = sb.len + count * (tb.len - nb.len);
let total64: i64 = (sb.len: i64) + (count: i64) *
((tb.len: i64) - (nb.len: i64));
if (total64 > types.I32_MAX: i64) { let e: nomem; return e; };
let total: i32 = total64: i32;
if (total == 0) {
let r: str;
r.ptr = nil;
r.len = 0;
r.cap = 0;
return r;
};
let res: []u8 = alloc([], total)?;
@@ -883,18 +903,21 @@ export fn replace(s: str, needle: str, target: str) (str | nomem) = {
// Symmetric with [[lpad]]. ref/hare/strings/pad.ha:39.
export fn rpad(s: str, p: rune, maxlen: i32) str = {
assert(maxlen >= 0, "strings.rpad: length must not be negative");
if (s.len >= maxlen) { return dup(s); };
let scratch: [4]u8;
let pad: []u8 = runebytes(scratch[0:4], p);
let gap: i32 = maxlen - s.len;
assert(gap % pad.len == 0,
"strings.rpad: byte length would split the pad rune");
let buf: []u8 = alloc([], maxlen: u64)!;
let k: i32 = 0;
for (k < s.len) {
buf[k] = s[k];
k += 1;
};
let padwrite: i32 = maxlen - s.len;
let i: i32 = 0;
for (i < padwrite) {
for (i < gap) {
buf[s.len + i] = pad.ptr[i % pad.len];
i += 1;
};