Hojun-Cho/ww
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

lib: drop _unsafe convention; rename fromutf8_unsafe → frombytes; strings α-batch (concat/join/lpad/rpad)

Browse Source 
CLAUDE.md rule 9 amended with the explicit carve-out: ww is C/Plan-9-
lineage — no GC, no "safe" baseline to be unsafe relative to — so the
Hare `_unsafe` suffix flags an axis ww doesn't have. The convention
is dropped wholesale in lib/.

Concrete changes:
- lib/strings: `fromutf8_unsafe` → `frombytes` (pure reinterpret). The
  validating sibling `fromutf8` is deleted entirely (28 lines, plus its
  84-line fromutf8_cases test). Callers that need validation write the
  two lines inline at the IO source: `utf8.validate(b)?;
  let s = strings.frombytes(b);`. `fromutf8` name reserved for a future
  true validating helper.
- lib/strings α-batch: concat/join/lpad/rpad migrate from
  `rt.malloc(N): *u8` to `alloc([], N)!` + `buf.len = N;` +
  `return frombytes(buf);`. Same dup-pilot pattern (4c07ef0). Task #41.
- lib/memio header comment trimmed: drops a stale reference to
  "lib has no fromutf8 today"; cites the rule-9 carve-out instead.
- Caller renames across selfhost combined.ww files (auto-regen) +
  cgenutil.ww comment ref.

Rule-11 disclosure on the bundle: the rename and the α-batch are
nominally separable concerns (symbol-naming policy vs amalloc→
alloc-slice migration), but they touch the same 4 functions in
lib/strings/strings.ww — the α-batch's first emission of `frombytes`
postdates the rename. The α-batch was applied on top of the rename
sweep mid-flight by the pre-commit reviewer; splitting them back
out is fiddly text surgery for marginal bisect value. The rename is
the primary concern; α-batch is one entry in #8's sized-slice
migration.

Verified: make test 132/132, 995_self_rebuild byte-identity holds.
Closes #42; advances #41.
This commit is contained in:
Hojun-Cho
2026-05-21 00:35:14 +09:00
parent 4c07ef0552
commit 47918d3ced
8 changed files with 109 additions and 340 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
lib/memio/memio.ww
View File

@@ -20,11 +20,11 @@
//
// Subset of Hare's surface: io.stream's variants are {eof, closed},
// so memio drops Hare's NONBLOCK flag (would need an `again` variant
// in lib/io) and string()'s utf8 validation (lib has no fromutf8
// today). Hare's seek / copy callbacks are likewise absent: lib/io's
// stream vtable has only read/write/close slots, so memio can't wire
// a seeker or copier even if we wanted to. All three come back when
// their dependencies do.
// in lib/io). string()'s utf8-validating constructor is omitted per
// CLAUDE.md rule 9 carve-out. Hare's seek / copy callbacks are
// likewise absent: lib/io's stream vtable has only read/write/close
// slots, so memio can't wire a seeker or copier even if we wanted
// to. All three come back when their dependencies do.
package memio;

83
lib/strings/strings.ww
View File

@@ -40,44 +40,15 @@ export fn toutf8(s: str) []u8 = {
return r;
};
// fromutf8_unsafe — borrowed str view of `in`. Does not validate.
// ref/hare/strings/utf8.ha:10.
export fn fromutf8_unsafe(in: []u8) str = {
// frombytes — borrowed str view of `in`. Pure reinterpret per
// CLAUDE.md rule 9 carve-out; ref/hare/strings/utf8.ha:10.
export fn frombytes(in: []u8) str = {
let r: str;
r.ptr = in.ptr;
r.len = in.len;
return r;
};
// fromutf8 — validating sibling of [[fromutf8_unsafe]]. Returns the
// borrowed str view on success, or `utf8.invalid` if `in` contains a
// malformed UTF-8 sequence. ref/hare/strings/utf8.ha:22 (#7).
//
// Hare's spelling is `utf8::validate(in)?; return fromutf8_unsafe(in)`.
// ww open-codes the same byte-by-byte DFA walk (ref/hare/encoding/utf8/
// decode.ha:207) because the cross-shape `(void | invalid) →
// (str | invalid)` propagation `?` needs is #19. The decoder-walk
// form's `(rune | done | more | invalid)` arm matches the failure
// axis of fromutf8 and lifts cleanly. Collapses to `utf8.validate(in)?`
// once #19 lands.
export fn fromutf8(in: []u8) (str | utf8.invalid) = {
let d: utf8.decoder = utf8.decode(in);
for (true) {
match (utf8.next(&d)) {
case let r: rune => void;
case utf8.done => {
let r: str = fromutf8_unsafe(in);
return r;
};
case utf8.more => {
let e: utf8.invalid; return e;
};
case let e: utf8.invalid => { return e; };
};
};
let e: utf8.invalid; return e;
};
// compare — three-way bytewise codepoint-order comparison. Return is
// a sign (neg/zero/pos), not an index, so it tracks Hare's `int`
// rather than the str-index i32 (#8). ref/hare/strings/compare.ha:12.
@@ -103,7 +74,7 @@ export fn dup(s: str) str = {
let i: i32 = 0;
for (i < s.len) { buf[i] = s[i]; i += 1; };
buf.len = s.len;
return fromutf8_unsafe(buf);
return frombytes(buf);
};
// dupall — fresh `[]str` whose elements are independent copies of
@@ -176,7 +147,7 @@ export fn concat(strs: str...) str = {
r.ptr = nil;
r.len = 0;
if (total == 0) { return r; };
let buf: *u8 = rt.malloc(total: u64): *u8;
let buf: []u8 = alloc([], total: u64)!;
let off: i32 = 0;
i = 0;
for (i < strs.len) {
@@ -188,9 +159,8 @@ export fn concat(strs: str...) str = {
off += strs[i].len;
i += 1;
};
r.ptr = buf;
r.len = total;
return r;
buf.len = total;
return frombytes(buf);
};
// join — fresh allocation with `delim` placed between each element of
@@ -209,7 +179,7 @@ export fn join(delim: str, strs: str...) str = {
r.ptr = nil;
r.len = 0;
if (total == 0) { return r; };
let buf: *u8 = rt.malloc(total: u64): *u8;
let buf: []u8 = alloc([], total: u64)!;
let off: i32 = 0;
i = 0;
for (i < strs.len) {
@@ -229,9 +199,8 @@ export fn join(delim: str, strs: str...) str = {
};
i += 1;
};
r.ptr = buf;
r.len = total;
return r;
buf.len = total;
return frombytes(buf);
};
// utf8bytelenbounded — walk `it` forward `end` runes and return the
@@ -518,7 +487,7 @@ let whitespace: [4]u8 = [0x20u8, 0x0Au8, 0x09u8, 0x0Du8];
// the working field-assign path.
export fn ltrim(input: str, trim: rune...) str = {
if (trim.len == 0) {
return fromutf8_unsafe(bytes.ltrim(toutf8(input), whitespace[0:4]...));
return frombytes(bytes.ltrim(toutf8(input), whitespace[0:4]...));
};
let it: iterator = iter(input);
for (true) {
@@ -550,7 +519,7 @@ export fn ltrim(input: str, trim: rune...) str = {
// ref/hare/strings/trim.ha:32.
export fn rtrim(input: str, trim: rune...) str = {
if (trim.len == 0) {
return fromutf8_unsafe(bytes.rtrim(toutf8(input), whitespace[0:4]...));
return frombytes(bytes.rtrim(toutf8(input), whitespace[0:4]...));
};
let it: iterator = riter(input);
for (true) {
@@ -663,7 +632,7 @@ export fn iterstr(it: *iterator) str = {
} else {
r = it.src[it.offs:it.src.len];
};
return fromutf8_unsafe(r);
return frombytes(r);
};
// slice — borrowed substring between two iterator positions.
@@ -678,7 +647,7 @@ export fn slice(begin: *iterator, end: *iterator) str = {
let e: utf8.decoder;
e.src = end.src;
e.offs = end.offs;
return fromutf8_unsafe(utf8.slice(&b, &e));
return frombytes(utf8.slice(&b, &e));
};
// position — byte-wise offset of the iterator in its source.
@@ -729,7 +698,7 @@ export fn rtokenize(s: str, delim: str) tokenizer = {
export fn next_token(s: *tokenizer) (str | bytes.done) = {
let b: *bytes.tokenizer = s: *bytes.tokenizer;
match (bytes.next_token(b)) {
case let v: []u8 => return fromutf8_unsafe(v);
case let v: []u8 => return frombytes(v);
case bytes.done => { let d: bytes.done; return d; };
};
};
@@ -739,7 +708,7 @@ export fn next_token(s: *tokenizer) (str | bytes.done) = {
export fn peek_token(s: *tokenizer) (str | bytes.done) = {
let b: *bytes.tokenizer = s: *bytes.tokenizer;
match (bytes.peek_token(b)) {
case let v: []u8 => return fromutf8_unsafe(v);
case let v: []u8 => return frombytes(v);
case bytes.done => { let d: bytes.done; return d; };
};
};
@@ -748,7 +717,7 @@ export fn peek_token(s: *tokenizer) (str | bytes.done) = {
// cursor. ref/hare/strings/tokenize.ha:79.
export fn remaining_tokens(s: *tokenizer) str = {
let b: *bytes.tokenizer = s: *bytes.tokenizer;
return fromutf8_unsafe(bytes.remaining_tokens(b));
return frombytes(bytes.remaining_tokens(b));
};
// rt_ensure is the runtime slice-growth helper invoked by the
@@ -886,7 +855,7 @@ export fn lpad(s: str, p: rune, maxlen: i32) str = {
if (s.len >= maxlen) { return dup(s); };
let scratch: [4]u8;
let pad: []u8 = runebytes(scratch[0:4], p);
let buf: *u8 = rt.malloc(maxlen: u64): *u8;
let buf: []u8 = alloc([], maxlen: u64)!;
let padwrite: i32 = (maxlen - s.len) * pad.len;
if (padwrite > maxlen) { padwrite = maxlen; };
let off: i32 = 0;
@@ -901,10 +870,8 @@ export fn lpad(s: str, p: rune, maxlen: i32) str = {
buf[off + k] = s[k];
k += 1;
};
let r: str;
r.ptr = buf;
r.len = maxlen;
return r;
buf.len = maxlen;
return frombytes(buf);
};
// replace — fresh allocation of `s` with every non-overlapping
@@ -960,7 +927,7 @@ export fn replace(s: str, needle: str, target: str) (str | nomem) = {
};
};
res.len = total;
return fromutf8_unsafe(res);
return frombytes(res);
};
// rpad — right-pad `s` with `p` rune until the result reaches `maxlen`
@@ -969,7 +936,7 @@ export fn rpad(s: str, p: rune, maxlen: i32) str = {
if (s.len >= maxlen) { return dup(s); };
let scratch: [4]u8;
let pad: []u8 = runebytes(scratch[0:4], p);
let buf: *u8 = rt.malloc(maxlen: u64): *u8;
let buf: []u8 = alloc([], maxlen: u64)!;
let k: i32 = 0;
for (k < s.len) {
buf[k] = s[k];
@@ -981,8 +948,6 @@ export fn rpad(s: str, p: rune, maxlen: i32) str = {
buf[s.len + i] = pad.ptr[i % pad.len];
i += 1;
};
let r: str;
r.ptr = buf;
r.len = maxlen;
return r;
buf.len = maxlen;
return frombytes(buf);
};

99
lib/strings/stringstest.ww
View File

@@ -845,110 +845,20 @@ fn streq(a: str, b: str) bool = {
};
};
// ---- toutf8 / fromutf8_unsafe roundtrip -------------------------------
// ref/hare/strings/utf8.ha:31.
// ---- toutf8 / frombytes roundtrip -------------------------------
// ref/hare/strings/utf8.ha:31. Validation-half coverage lives in
// lib/encoding/utf8 (utf8test) per CLAUDE.md rule 9 carve-out.
@test fn utf8_roundtrip_cases() void = {
let s: str = "hello";
let b: []u8 = strings.toutf8(s);
if (b.len != 5) { fail(); };
if (b[0] != 104u8) { fail(); }; // 'h'
let r: str = strings.fromutf8_unsafe(b);
let r: str = strings.frombytes(b);
if (!streq(r, "hello")) { fail(); };
if (r.ptr != s.ptr) { fail(); }; // borrowed, not copied
};
// ---- fromutf8 ---------------------------------------------------------
// ref/hare/strings/utf8.ha:22 + @test fn utf8 (line 31). Validating
// sibling of fromutf8_unsafe — walks via [[utf8.validate]] and returns
// `utf8.invalid` on any malformed byte sequence (#7).
@test fn fromutf8_cases() void = {
// Hare vector — "hello world" round-trips.
signalled = 1900;
let hb: [11]u8;
hb[0] = 0x68u8; hb[1] = 0x65u8; hb[2] = 0x6Cu8; hb[3] = 0x6Cu8;
hb[4] = 0x6Fu8; hb[5] = 0x20u8; hb[6] = 0x77u8; hb[7] = 0x6Fu8;
hb[8] = 0x72u8; hb[9] = 0x6Cu8; hb[10] = 0x64u8;
match (strings.fromutf8(hb[0:11])) {
case let s: str => { if (!streq(s, "hello world")) { fail(); }; };
case let e: utf8.invalid => { fail(); };
};
// Empty input — Hare's `fromutf8([])! == ""`.
signalled = 1901;
let eb: [1]u8;
match (strings.fromutf8(eb[0:0])) {
case let s: str => { if (s.len != 0) { fail(); }; };
case let e: utf8.invalid => { fail(); };
};
// ASCII-only borrowed view: ptr aliases input.
signalled = 1902;
let s: str = "abc";
let bs: []u8 = strings.toutf8(s);
match (strings.fromutf8(bs)) {
case let r: str => {
if (!streq(r, "abc")) { fail(); };
if (r.ptr != s.ptr) { fail(); };
};
case let e: utf8.invalid => { fail(); };
};
// Multibyte Japanese — full UTF-8 round-trip.
signalled = 1903;
let jb: [15]u8;
jb[0] = 0xE3u8; jb[1] = 0x81u8; jb[2] = 0x93u8; // こ
jb[3] = 0xE3u8; jb[4] = 0x82u8; jb[5] = 0x93u8; // ん
jb[6] = 0xE3u8; jb[7] = 0x81u8; jb[8] = 0xABu8; // に
jb[9] = 0xE3u8; jb[10] = 0x81u8; jb[11] = 0xA1u8; // ち
jb[12] = 0xE3u8; jb[13] = 0x81u8; jb[14] = 0xAFu8; // は
match (strings.fromutf8(jb[0:15])) {
case let s: str => { if (!streq(s, "こんにちは")) { fail(); }; };
case let e: utf8.invalid => { fail(); };
};
// Multibyte French — mixed-width "héllo".
signalled = 1904;
let fb: [6]u8;
fb[0] = 0x68u8; // h
fb[1] = 0xC3u8; fb[2] = 0xA9u8; // é
fb[3] = 0x6Cu8; fb[4] = 0x6Cu8; fb[5] = 0x6Fu8; // llo
match (strings.fromutf8(fb[0:6])) {
case let s: str => { if (!streq(s, "héllo")) { fail(); }; };
case let e: utf8.invalid => { fail(); };
};
// Lone 0x80 continuation byte — invalid.
signalled = 1905;
let c: [1]u8;
c[0] = 0x80u8;
match (strings.fromutf8(c[0:1])) {
case let r: str => { fail(); };
case let e: utf8.invalid => void;
};
// Overlong 2-byte encoding of NUL (0xC0 0x80) — invalid per
// ref/hare/encoding/utf8/decodetable.ha (state 0 rejects 0xC0).
signalled = 1906;
let ov: [2]u8;
ov[0] = 0xC0u8; ov[1] = 0x80u8;
match (strings.fromutf8(ov[0:2])) {
case let r: str => { fail(); };
case let e: utf8.invalid => void;
};
// Truncated multibyte — 0xE3 0x81 (2 of 3 bytes for U+3053) is
// rejected as invalid by [[utf8.validate]] (state != 0 at EOI).
signalled = 1907;
let tr: [2]u8;
tr[0] = 0xE3u8; tr[1] = 0x81u8;
match (strings.fromutf8(tr[0:2])) {
case let r: str => { fail(); };
case let e: utf8.invalid => void;
};
};
// ---- iter / next ------------------------------------------------------
// ref/hare/strings/iter.ha:84-108. Hare's @test fn iter() uses prev +
// riter heavily; both are deferred (no `utf8.prev`). Rebuild forward-
@@ -1797,7 +1707,6 @@ export fn main() i32 = {
signalled = 40; sub_cases();
signalled = 41; bytesub_cases();
signalled = 15; utf8_roundtrip_cases();
signalled = 43; fromutf8_cases();
signalled = 16; iter_empty_cases();
signalled = 17; iter_ascii_cases();
signalled = 18; iter_twobyte_cases();