Hojun-Cho/ww
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

lib: drop _unsafe convention; rename fromutf8_unsafe → frombytes; strings α-batch (concat/join/lpad/rpad)

Browse Source 
CLAUDE.md rule 9 amended with the explicit carve-out: ww is C/Plan-9-
lineage — no GC, no "safe" baseline to be unsafe relative to — so the
Hare `_unsafe` suffix flags an axis ww doesn't have. The convention
is dropped wholesale in lib/.

Concrete changes:
- lib/strings: `fromutf8_unsafe` → `frombytes` (pure reinterpret). The
  validating sibling `fromutf8` is deleted entirely (28 lines, plus its
  84-line fromutf8_cases test). Callers that need validation write the
  two lines inline at the IO source: `utf8.validate(b)?;
  let s = strings.frombytes(b);`. `fromutf8` name reserved for a future
  true validating helper.
- lib/strings α-batch: concat/join/lpad/rpad migrate from
  `rt.malloc(N): *u8` to `alloc([], N)!` + `buf.len = N;` +
  `return frombytes(buf);`. Same dup-pilot pattern (4c07ef0). Task #41.
- lib/memio header comment trimmed: drops a stale reference to
  "lib has no fromutf8 today"; cites the rule-9 carve-out instead.
- Caller renames across selfhost combined.ww files (auto-regen) +
  cgenutil.ww comment ref.

Rule-11 disclosure on the bundle: the rename and the α-batch are
nominally separable concerns (symbol-naming policy vs amalloc→
alloc-slice migration), but they touch the same 4 functions in
lib/strings/strings.ww — the α-batch's first emission of `frombytes`
postdates the rename. The α-batch was applied on top of the rename
sweep mid-flight by the pre-commit reviewer; splitting them back
out is fiddly text surgery for marginal bisect value. The rename is
the primary concern; α-batch is one entry in #8's sized-slice
migration.

Verified: make test 132/132, 995_self_rebuild byte-identity holds.
Closes #42; advances #41.
This commit is contained in:
Hojun-Cho
2026-05-21 00:35:14 +09:00
parent 4c07ef0552
commit 47918d3ced
8 changed files with 109 additions and 340 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
CLAUDE.md
View File

@@ -6,7 +6,7 @@
6. ref/hare and ref/plan9front are read-only references — consult before inventing data shapes or syntax
7. No workarounds. If a bug forces a workaround, STOP and report with a precise repro. Document any retained divergence at the site with a pointer to the filed task. Never silent.
8. Comments are WHY-only. Never narrate WHAT the code does — names carry the WHAT. Comment only non-obvious WHY: a constraint, a divergence from a reference, a citation to a filed task.
9. Hare-fidelity over convenience. No ad-hoc extensions, renames, or convenience wrappers in lib/. Cite ref/hare/<module>/<file> for every signature ported.
9. Hare-fidelity over convenience. No ad-hoc extensions, renames, or convenience wrappers in lib/. Cite ref/hare/<module>/<file> for every signature ported. Carve-out: the Hare `_unsafe` suffix convention is dropped wholesale (ww is C/Plan-9-lineage, an unmanaged systems language — no GC, no "safe" baseline to be unsafe relative to). bytes→str is a pure reinterpret (`strings.frombytes`, renamed from Hare's `fromutf8_unsafe`); validation is opt-in via `utf8.validate(b)?` at the IO source, never wrapped per-construction. Rationale: ref/hare/strings/utf8.ha:10,22 — the suffix flags Hare's managed-bytes-safety axis, which ww doesn't have. The honest name (`frombytes`) reserves `fromutf8` for a future true validating helper.
10. Symmetric stages. cstage and wwstage MUST emit byte-identical asm for the same input. When inference power differs, align the richer side DOWN to the leaner side, not the other way.
11. Split commits when they bundle unrelated concerns. Bisect-cleanliness is the default. Multi-fix commits need a body paragraph explaining why they couldn't split.
12. Simple data, simple algorithms. Sea-of-stars style. Mirror Hare's structural choices over clever alternatives.

10
lib/memio/memio.ww
View File

@@ -20,11 +20,11 @@
//
// Subset of Hare's surface: io.stream's variants are {eof, closed},
// so memio drops Hare's NONBLOCK flag (would need an `again` variant
// in lib/io) and string()'s utf8 validation (lib has no fromutf8
// today). Hare's seek / copy callbacks are likewise absent: lib/io's
// stream vtable has only read/write/close slots, so memio can't wire
// a seeker or copier even if we wanted to. All three come back when
// their dependencies do.
// in lib/io). string()'s utf8-validating constructor is omitted per
// CLAUDE.md rule 9 carve-out. Hare's seek / copy callbacks are
// likewise absent: lib/io's stream vtable has only read/write/close
// slots, so memio can't wire a seeker or copier even if we wanted
// to. All three come back when their dependencies do.
package memio;

83
lib/strings/strings.ww
View File

@@ -40,44 +40,15 @@ export fn toutf8(s: str) []u8 = {
return r;
};
// fromutf8_unsafe — borrowed str view of `in`. Does not validate.
// ref/hare/strings/utf8.ha:10.
export fn fromutf8_unsafe(in: []u8) str = {
// frombytes — borrowed str view of `in`. Pure reinterpret per
// CLAUDE.md rule 9 carve-out; ref/hare/strings/utf8.ha:10.
export fn frombytes(in: []u8) str = {
let r: str;
r.ptr = in.ptr;
r.len = in.len;
return r;
};
// fromutf8 — validating sibling of [[fromutf8_unsafe]]. Returns the
// borrowed str view on success, or `utf8.invalid` if `in` contains a
// malformed UTF-8 sequence. ref/hare/strings/utf8.ha:22 (#7).
//
// Hare's spelling is `utf8::validate(in)?; return fromutf8_unsafe(in)`.
// ww open-codes the same byte-by-byte DFA walk (ref/hare/encoding/utf8/
// decode.ha:207) because the cross-shape `(void | invalid) →
// (str | invalid)` propagation `?` needs is #19. The decoder-walk
// form's `(rune | done | more | invalid)` arm matches the failure
// axis of fromutf8 and lifts cleanly. Collapses to `utf8.validate(in)?`
// once #19 lands.
export fn fromutf8(in: []u8) (str | utf8.invalid) = {
let d: utf8.decoder = utf8.decode(in);
for (true) {
match (utf8.next(&d)) {
case let r: rune => void;
case utf8.done => {
let r: str = fromutf8_unsafe(in);
return r;
};
case utf8.more => {
let e: utf8.invalid; return e;
};
case let e: utf8.invalid => { return e; };
};
};
let e: utf8.invalid; return e;
};
// compare — three-way bytewise codepoint-order comparison. Return is
// a sign (neg/zero/pos), not an index, so it tracks Hare's `int`
// rather than the str-index i32 (#8). ref/hare/strings/compare.ha:12.
@@ -103,7 +74,7 @@ export fn dup(s: str) str = {
let i: i32 = 0;
for (i < s.len) { buf[i] = s[i]; i += 1; };
buf.len = s.len;
return fromutf8_unsafe(buf);
return frombytes(buf);
};
// dupall — fresh `[]str` whose elements are independent copies of
@@ -176,7 +147,7 @@ export fn concat(strs: str...) str = {
r.ptr = nil;
r.len = 0;
if (total == 0) { return r; };
let buf: *u8 = rt.malloc(total: u64): *u8;
let buf: []u8 = alloc([], total: u64)!;
let off: i32 = 0;
i = 0;
for (i < strs.len) {
@@ -188,9 +159,8 @@ export fn concat(strs: str...) str = {
off += strs[i].len;
i += 1;
};
r.ptr = buf;
r.len = total;
return r;
buf.len = total;
return frombytes(buf);
};
// join — fresh allocation with `delim` placed between each element of
@@ -209,7 +179,7 @@ export fn join(delim: str, strs: str...) str = {
r.ptr = nil;
r.len = 0;
if (total == 0) { return r; };
let buf: *u8 = rt.malloc(total: u64): *u8;
let buf: []u8 = alloc([], total: u64)!;
let off: i32 = 0;
i = 0;
for (i < strs.len) {
@@ -229,9 +199,8 @@ export fn join(delim: str, strs: str...) str = {
};
i += 1;
};
r.ptr = buf;
r.len = total;
return r;
buf.len = total;
return frombytes(buf);
};
// utf8bytelenbounded — walk `it` forward `end` runes and return the
@@ -518,7 +487,7 @@ let whitespace: [4]u8 = [0x20u8, 0x0Au8, 0x09u8, 0x0Du8];
// the working field-assign path.
export fn ltrim(input: str, trim: rune...) str = {
if (trim.len == 0) {
return fromutf8_unsafe(bytes.ltrim(toutf8(input), whitespace[0:4]...));
return frombytes(bytes.ltrim(toutf8(input), whitespace[0:4]...));
};
let it: iterator = iter(input);
for (true) {
@@ -550,7 +519,7 @@ export fn ltrim(input: str, trim: rune...) str = {
// ref/hare/strings/trim.ha:32.
export fn rtrim(input: str, trim: rune...) str = {
if (trim.len == 0) {
return fromutf8_unsafe(bytes.rtrim(toutf8(input), whitespace[0:4]...));
return frombytes(bytes.rtrim(toutf8(input), whitespace[0:4]...));
};
let it: iterator = riter(input);
for (true) {
@@ -663,7 +632,7 @@ export fn iterstr(it: *iterator) str = {
} else {
r = it.src[it.offs:it.src.len];
};
return fromutf8_unsafe(r);
return frombytes(r);
};
// slice — borrowed substring between two iterator positions.
@@ -678,7 +647,7 @@ export fn slice(begin: *iterator, end: *iterator) str = {
let e: utf8.decoder;
e.src = end.src;
e.offs = end.offs;
return fromutf8_unsafe(utf8.slice(&b, &e));
return frombytes(utf8.slice(&b, &e));
};
// position — byte-wise offset of the iterator in its source.
@@ -729,7 +698,7 @@ export fn rtokenize(s: str, delim: str) tokenizer = {
export fn next_token(s: *tokenizer) (str | bytes.done) = {
let b: *bytes.tokenizer = s: *bytes.tokenizer;
match (bytes.next_token(b)) {
case let v: []u8 => return fromutf8_unsafe(v);
case let v: []u8 => return frombytes(v);
case bytes.done => { let d: bytes.done; return d; };
};
};
@@ -739,7 +708,7 @@ export fn next_token(s: *tokenizer) (str | bytes.done) = {
export fn peek_token(s: *tokenizer) (str | bytes.done) = {
let b: *bytes.tokenizer = s: *bytes.tokenizer;
match (bytes.peek_token(b)) {
case let v: []u8 => return fromutf8_unsafe(v);
case let v: []u8 => return frombytes(v);
case bytes.done => { let d: bytes.done; return d; };
};
};
@@ -748,7 +717,7 @@ export fn peek_token(s: *tokenizer) (str | bytes.done) = {
// cursor. ref/hare/strings/tokenize.ha:79.
export fn remaining_tokens(s: *tokenizer) str = {
let b: *bytes.tokenizer = s: *bytes.tokenizer;
return fromutf8_unsafe(bytes.remaining_tokens(b));
return frombytes(bytes.remaining_tokens(b));
};
// rt_ensure is the runtime slice-growth helper invoked by the
@@ -886,7 +855,7 @@ export fn lpad(s: str, p: rune, maxlen: i32) str = {
if (s.len >= maxlen) { return dup(s); };
let scratch: [4]u8;
let pad: []u8 = runebytes(scratch[0:4], p);
let buf: *u8 = rt.malloc(maxlen: u64): *u8;
let buf: []u8 = alloc([], maxlen: u64)!;
let padwrite: i32 = (maxlen - s.len) * pad.len;
if (padwrite > maxlen) { padwrite = maxlen; };
let off: i32 = 0;
@@ -901,10 +870,8 @@ export fn lpad(s: str, p: rune, maxlen: i32) str = {
buf[off + k] = s[k];
k += 1;
};
let r: str;
r.ptr = buf;
r.len = maxlen;
return r;
buf.len = maxlen;
return frombytes(buf);
};
// replace — fresh allocation of `s` with every non-overlapping
@@ -960,7 +927,7 @@ export fn replace(s: str, needle: str, target: str) (str | nomem) = {
};
};
res.len = total;
return fromutf8_unsafe(res);
return frombytes(res);
};
// rpad — right-pad `s` with `p` rune until the result reaches `maxlen`
@@ -969,7 +936,7 @@ export fn rpad(s: str, p: rune, maxlen: i32) str = {
if (s.len >= maxlen) { return dup(s); };
let scratch: [4]u8;
let pad: []u8 = runebytes(scratch[0:4], p);
let buf: *u8 = rt.malloc(maxlen: u64): *u8;
let buf: []u8 = alloc([], maxlen: u64)!;
let k: i32 = 0;
for (k < s.len) {
buf[k] = s[k];
@@ -981,8 +948,6 @@ export fn rpad(s: str, p: rune, maxlen: i32) str = {
buf[s.len + i] = pad.ptr[i % pad.len];
i += 1;
};
let r: str;
r.ptr = buf;
r.len = maxlen;
return r;
buf.len = maxlen;
return frombytes(buf);
};

99
lib/strings/stringstest.ww
View File

@@ -845,110 +845,20 @@ fn streq(a: str, b: str) bool = {
};
};
// ---- toutf8 / fromutf8_unsafe roundtrip -------------------------------
// ref/hare/strings/utf8.ha:31.
// ---- toutf8 / frombytes roundtrip -------------------------------
// ref/hare/strings/utf8.ha:31. Validation-half coverage lives in
// lib/encoding/utf8 (utf8test) per CLAUDE.md rule 9 carve-out.
@test fn utf8_roundtrip_cases() void = {
let s: str = "hello";
let b: []u8 = strings.toutf8(s);
if (b.len != 5) { fail(); };
if (b[0] != 104u8) { fail(); }; // 'h'
let r: str = strings.fromutf8_unsafe(b);
let r: str = strings.frombytes(b);
if (!streq(r, "hello")) { fail(); };
if (r.ptr != s.ptr) { fail(); }; // borrowed, not copied
};
// ---- fromutf8 ---------------------------------------------------------
// ref/hare/strings/utf8.ha:22 + @test fn utf8 (line 31). Validating
// sibling of fromutf8_unsafe — walks via [[utf8.validate]] and returns
// `utf8.invalid` on any malformed byte sequence (#7).
@test fn fromutf8_cases() void = {
// Hare vector — "hello world" round-trips.
signalled = 1900;
let hb: [11]u8;
hb[0] = 0x68u8; hb[1] = 0x65u8; hb[2] = 0x6Cu8; hb[3] = 0x6Cu8;
hb[4] = 0x6Fu8; hb[5] = 0x20u8; hb[6] = 0x77u8; hb[7] = 0x6Fu8;
hb[8] = 0x72u8; hb[9] = 0x6Cu8; hb[10] = 0x64u8;
match (strings.fromutf8(hb[0:11])) {
case let s: str => { if (!streq(s, "hello world")) { fail(); }; };
case let e: utf8.invalid => { fail(); };
};
// Empty input — Hare's `fromutf8([])! == ""`.
signalled = 1901;
let eb: [1]u8;
match (strings.fromutf8(eb[0:0])) {
case let s: str => { if (s.len != 0) { fail(); }; };
case let e: utf8.invalid => { fail(); };
};
// ASCII-only borrowed view: ptr aliases input.
signalled = 1902;
let s: str = "abc";
let bs: []u8 = strings.toutf8(s);
match (strings.fromutf8(bs)) {
case let r: str => {
if (!streq(r, "abc")) { fail(); };
if (r.ptr != s.ptr) { fail(); };
};
case let e: utf8.invalid => { fail(); };
};
// Multibyte Japanese — full UTF-8 round-trip.
signalled = 1903;
let jb: [15]u8;
jb[0] = 0xE3u8; jb[1] = 0x81u8; jb[2] = 0x93u8; // こ
jb[3] = 0xE3u8; jb[4] = 0x82u8; jb[5] = 0x93u8; // ん
jb[6] = 0xE3u8; jb[7] = 0x81u8; jb[8] = 0xABu8; // に
jb[9] = 0xE3u8; jb[10] = 0x81u8; jb[11] = 0xA1u8; // ち
jb[12] = 0xE3u8; jb[13] = 0x81u8; jb[14] = 0xAFu8; // は
match (strings.fromutf8(jb[0:15])) {
case let s: str => { if (!streq(s, "こんにちは")) { fail(); }; };
case let e: utf8.invalid => { fail(); };
};
// Multibyte French — mixed-width "héllo".
signalled = 1904;
let fb: [6]u8;
fb[0] = 0x68u8; // h
fb[1] = 0xC3u8; fb[2] = 0xA9u8; // é
fb[3] = 0x6Cu8; fb[4] = 0x6Cu8; fb[5] = 0x6Fu8; // llo
match (strings.fromutf8(fb[0:6])) {
case let s: str => { if (!streq(s, "héllo")) { fail(); }; };
case let e: utf8.invalid => { fail(); };
};
// Lone 0x80 continuation byte — invalid.
signalled = 1905;
let c: [1]u8;
c[0] = 0x80u8;
match (strings.fromutf8(c[0:1])) {
case let r: str => { fail(); };
case let e: utf8.invalid => void;
};
// Overlong 2-byte encoding of NUL (0xC0 0x80) — invalid per
// ref/hare/encoding/utf8/decodetable.ha (state 0 rejects 0xC0).
signalled = 1906;
let ov: [2]u8;
ov[0] = 0xC0u8; ov[1] = 0x80u8;
match (strings.fromutf8(ov[0:2])) {
case let r: str => { fail(); };
case let e: utf8.invalid => void;
};
// Truncated multibyte — 0xE3 0x81 (2 of 3 bytes for U+3053) is
// rejected as invalid by [[utf8.validate]] (state != 0 at EOI).
signalled = 1907;
let tr: [2]u8;
tr[0] = 0xE3u8; tr[1] = 0x81u8;
match (strings.fromutf8(tr[0:2])) {
case let r: str => { fail(); };
case let e: utf8.invalid => void;
};
};
// ---- iter / next ------------------------------------------------------
// ref/hare/strings/iter.ha:84-108. Hare's @test fn iter() uses prev +
// riter heavily; both are deferred (no `utf8.prev`). Rebuild forward-
@@ -1797,7 +1707,6 @@ export fn main() i32 = {
signalled = 40; sub_cases();
signalled = 41; bytesub_cases();
signalled = 15; utf8_roundtrip_cases();
signalled = 43; fromutf8_cases();
signalled = 16; iter_empty_cases();
signalled = 17; iter_ascii_cases();
signalled = 18; iter_twobyte_cases();

85
selfhost/cmd/w6c/main.combined.ww
View File

@@ -1879,44 +1879,15 @@ export fn toutf8(s: str) []u8 = {
return r;
};
// fromutf8_unsafe — borrowed str view of `in`. Does not validate.
// ref/hare/strings/utf8.ha:10.
export fn fromutf8_unsafe(in: []u8) str = {
// frombytes — borrowed str view of `in`. Pure reinterpret per
// CLAUDE.md rule 9 carve-out; ref/hare/strings/utf8.ha:10.
export fn frombytes(in: []u8) str = {
let r: str;
r.ptr = in.ptr;
r.len = in.len;
return r;
};
// fromutf8 — validating sibling of [[fromutf8_unsafe]]. Returns the
// borrowed str view on success, or `utf8.invalid` if `in` contains a
// malformed UTF-8 sequence. ref/hare/strings/utf8.ha:22 (#7).
//
// Hare's spelling is `utf8::validate(in)?; return fromutf8_unsafe(in)`.
// ww open-codes the same byte-by-byte DFA walk (ref/hare/encoding/utf8/
// decode.ha:207) because the cross-shape `(void | invalid) →
// (str | invalid)` propagation `?` needs is #19. The decoder-walk
// form's `(rune | done | more | invalid)` arm matches the failure
// axis of fromutf8 and lifts cleanly. Collapses to `utf8.validate(in)?`
// once #19 lands.
export fn fromutf8(in: []u8) (str | utf8.invalid) = {
let d: utf8.decoder = utf8.decode(in);
for (true) {
match (utf8.next(&d)) {
case let r: rune => void;
case utf8.done => {
let r: str = fromutf8_unsafe(in);
return r;
};
case utf8.more => {
let e: utf8.invalid; return e;
};
case let e: utf8.invalid => { return e; };
};
};
let e: utf8.invalid; return e;
};
// compare — three-way bytewise codepoint-order comparison. Return is
// a sign (neg/zero/pos), not an index, so it tracks Hare's `int`
// rather than the str-index i32 (#8). ref/hare/strings/compare.ha:12.
@@ -1942,7 +1913,7 @@ export fn dup(s: str) str = {
let i: i32 = 0;
for (i < s.len) { buf[i] = s[i]; i += 1; };
buf.len = s.len;
return fromutf8_unsafe(buf);
return frombytes(buf);
};
// dupall — fresh `[]str` whose elements are independent copies of
@@ -2015,7 +1986,7 @@ export fn concat(strs: str...) str = {
r.ptr = nil;
r.len = 0;
if (total == 0) { return r; };
let buf: *u8 = rt.malloc(total: u64): *u8;
let buf: []u8 = alloc([], total: u64)!;
let off: i32 = 0;
i = 0;
for (i < strs.len) {
@@ -2027,9 +1998,8 @@ export fn concat(strs: str...) str = {
off += strs[i].len;
i += 1;
};
r.ptr = buf;
r.len = total;
return r;
buf.len = total;
return frombytes(buf);
};
// join — fresh allocation with `delim` placed between each element of
@@ -2048,7 +2018,7 @@ export fn join(delim: str, strs: str...) str = {
r.ptr = nil;
r.len = 0;
if (total == 0) { return r; };
let buf: *u8 = rt.malloc(total: u64): *u8;
let buf: []u8 = alloc([], total: u64)!;
let off: i32 = 0;
i = 0;
for (i < strs.len) {
@@ -2068,9 +2038,8 @@ export fn join(delim: str, strs: str...) str = {
};
i += 1;
};
r.ptr = buf;
r.len = total;
return r;
buf.len = total;
return frombytes(buf);
};
// utf8bytelenbounded — walk `it` forward `end` runes and return the
@@ -2357,7 +2326,7 @@ let whitespace: [4]u8 = [0x20u8, 0x0Au8, 0x09u8, 0x0Du8];
// the working field-assign path.
export fn ltrim(input: str, trim: rune...) str = {
if (trim.len == 0) {
return fromutf8_unsafe(bytes.ltrim(toutf8(input), whitespace[0:4]...));
return frombytes(bytes.ltrim(toutf8(input), whitespace[0:4]...));
};
let it: iterator = iter(input);
for (true) {
@@ -2389,7 +2358,7 @@ export fn ltrim(input: str, trim: rune...) str = {
// ref/hare/strings/trim.ha:32.
export fn rtrim(input: str, trim: rune...) str = {
if (trim.len == 0) {
return fromutf8_unsafe(bytes.rtrim(toutf8(input), whitespace[0:4]...));
return frombytes(bytes.rtrim(toutf8(input), whitespace[0:4]...));
};
let it: iterator = riter(input);
for (true) {
@@ -2502,7 +2471,7 @@ export fn iterstr(it: *iterator) str = {
} else {
r = it.src[it.offs:it.src.len];
};
return fromutf8_unsafe(r);
return frombytes(r);
};
// slice — borrowed substring between two iterator positions.
@@ -2517,7 +2486,7 @@ export fn slice(begin: *iterator, end: *iterator) str = {
let e: utf8.decoder;
e.src = end.src;
e.offs = end.offs;
return fromutf8_unsafe(utf8.slice(&b, &e));
return frombytes(utf8.slice(&b, &e));
};
// position — byte-wise offset of the iterator in its source.
@@ -2568,7 +2537,7 @@ export fn rtokenize(s: str, delim: str) tokenizer = {
export fn next_token(s: *tokenizer) (str | bytes.done) = {
let b: *bytes.tokenizer = s: *bytes.tokenizer;
match (bytes.next_token(b)) {
case let v: []u8 => return fromutf8_unsafe(v);
case let v: []u8 => return frombytes(v);
case bytes.done => { let d: bytes.done; return d; };
};
};
@@ -2578,7 +2547,7 @@ export fn next_token(s: *tokenizer) (str | bytes.done) = {
export fn peek_token(s: *tokenizer) (str | bytes.done) = {
let b: *bytes.tokenizer = s: *bytes.tokenizer;
match (bytes.peek_token(b)) {
case let v: []u8 => return fromutf8_unsafe(v);
case let v: []u8 => return frombytes(v);
case bytes.done => { let d: bytes.done; return d; };
};
};
@@ -2587,7 +2556,7 @@ export fn peek_token(s: *tokenizer) (str | bytes.done) = {
// cursor. ref/hare/strings/tokenize.ha:79.
export fn remaining_tokens(s: *tokenizer) str = {
let b: *bytes.tokenizer = s: *bytes.tokenizer;
return fromutf8_unsafe(bytes.remaining_tokens(b));
return frombytes(bytes.remaining_tokens(b));
};
// rt_ensure is the runtime slice-growth helper invoked by the
@@ -2725,7 +2694,7 @@ export fn lpad(s: str, p: rune, maxlen: i32) str = {
if (s.len >= maxlen) { return dup(s); };
let scratch: [4]u8;
let pad: []u8 = runebytes(scratch[0:4], p);
let buf: *u8 = rt.malloc(maxlen: u64): *u8;
let buf: []u8 = alloc([], maxlen: u64)!;
let padwrite: i32 = (maxlen - s.len) * pad.len;
if (padwrite > maxlen) { padwrite = maxlen; };
let off: i32 = 0;
@@ -2740,10 +2709,8 @@ export fn lpad(s: str, p: rune, maxlen: i32) str = {
buf[off + k] = s[k];
k += 1;
};
let r: str;
r.ptr = buf;
r.len = maxlen;
return r;
buf.len = maxlen;
return frombytes(buf);
};
// replace — fresh allocation of `s` with every non-overlapping
@@ -2799,7 +2766,7 @@ export fn replace(s: str, needle: str, target: str) (str | nomem) = {
};
};
res.len = total;
return fromutf8_unsafe(res);
return frombytes(res);
};
// rpad — right-pad `s` with `p` rune until the result reaches `maxlen`
@@ -2808,7 +2775,7 @@ export fn rpad(s: str, p: rune, maxlen: i32) str = {
if (s.len >= maxlen) { return dup(s); };
let scratch: [4]u8;
let pad: []u8 = runebytes(scratch[0:4], p);
let buf: *u8 = rt.malloc(maxlen: u64): *u8;
let buf: []u8 = alloc([], maxlen: u64)!;
let k: i32 = 0;
for (k < s.len) {
buf[k] = s[k];
@@ -2820,10 +2787,8 @@ export fn rpad(s: str, p: rune, maxlen: i32) str = {
buf[s.len + i] = pad.ptr[i % pad.len];
i += 1;
};
let r: str;
r.ptr = buf;
r.len = maxlen;
return r;
buf.len = maxlen;
return frombytes(buf);
};
// strconv — number↔string conversions.
@@ -9754,7 +9719,7 @@ fn nodeisslice(c: *cgen, n: *node) bool = {
// N_DOT (cross-module callee, #34): route through fnretlookupmod
// so a same-leaf caller-module fn with diverging return shape
// doesn't shadow the explicit `mod.f()` qualifier — surfaced by
// strings.slice returning `fromutf8_unsafe(utf8.slice(...))`
// strings.slice returning `frombytes(utf8.slice(...))`
// where strings.slice itself returns str.
if (k == nkind.N_CALL) {
let callee: *node = n.lhs;

2
selfhost/cmd/wcc/cgenutil.ww
View File

@@ -541,7 +541,7 @@ fn nodeisslice(c: *cgen, n: *node) bool = {
// N_DOT (cross-module callee, #34): route through fnretlookupmod
// so a same-leaf caller-module fn with diverging return shape
// doesn't shadow the explicit `mod.f()` qualifier — surfaced by
// strings.slice returning `fromutf8_unsafe(utf8.slice(...))`
// strings.slice returning `frombytes(utf8.slice(...))`
// where strings.slice itself returns str.
if (k == nkind.N_CALL) {
let callee: *node = n.lhs;

85
selfhost/cmd/wwdump/main.combined.ww
View File

@@ -1879,44 +1879,15 @@ export fn toutf8(s: str) []u8 = {
return r;
};
// fromutf8_unsafe — borrowed str view of `in`. Does not validate.
// ref/hare/strings/utf8.ha:10.
export fn fromutf8_unsafe(in: []u8) str = {
// frombytes — borrowed str view of `in`. Pure reinterpret per
// CLAUDE.md rule 9 carve-out; ref/hare/strings/utf8.ha:10.
export fn frombytes(in: []u8) str = {
let r: str;
r.ptr = in.ptr;
r.len = in.len;
return r;
};
// fromutf8 — validating sibling of [[fromutf8_unsafe]]. Returns the
// borrowed str view on success, or `utf8.invalid` if `in` contains a
// malformed UTF-8 sequence. ref/hare/strings/utf8.ha:22 (#7).
//
// Hare's spelling is `utf8::validate(in)?; return fromutf8_unsafe(in)`.
// ww open-codes the same byte-by-byte DFA walk (ref/hare/encoding/utf8/
// decode.ha:207) because the cross-shape `(void | invalid) →
// (str | invalid)` propagation `?` needs is #19. The decoder-walk
// form's `(rune | done | more | invalid)` arm matches the failure
// axis of fromutf8 and lifts cleanly. Collapses to `utf8.validate(in)?`
// once #19 lands.
export fn fromutf8(in: []u8) (str | utf8.invalid) = {
let d: utf8.decoder = utf8.decode(in);
for (true) {
match (utf8.next(&d)) {
case let r: rune => void;
case utf8.done => {
let r: str = fromutf8_unsafe(in);
return r;
};
case utf8.more => {
let e: utf8.invalid; return e;
};
case let e: utf8.invalid => { return e; };
};
};
let e: utf8.invalid; return e;
};
// compare — three-way bytewise codepoint-order comparison. Return is
// a sign (neg/zero/pos), not an index, so it tracks Hare's `int`
// rather than the str-index i32 (#8). ref/hare/strings/compare.ha:12.
@@ -1942,7 +1913,7 @@ export fn dup(s: str) str = {
let i: i32 = 0;
for (i < s.len) { buf[i] = s[i]; i += 1; };
buf.len = s.len;
return fromutf8_unsafe(buf);
return frombytes(buf);
};
// dupall — fresh `[]str` whose elements are independent copies of
@@ -2015,7 +1986,7 @@ export fn concat(strs: str...) str = {
r.ptr = nil;
r.len = 0;
if (total == 0) { return r; };
let buf: *u8 = rt.malloc(total: u64): *u8;
let buf: []u8 = alloc([], total: u64)!;
let off: i32 = 0;
i = 0;
for (i < strs.len) {
@@ -2027,9 +1998,8 @@ export fn concat(strs: str...) str = {
off += strs[i].len;
i += 1;
};
r.ptr = buf;
r.len = total;
return r;
buf.len = total;
return frombytes(buf);
};
// join — fresh allocation with `delim` placed between each element of
@@ -2048,7 +2018,7 @@ export fn join(delim: str, strs: str...) str = {
r.ptr = nil;
r.len = 0;
if (total == 0) { return r; };
let buf: *u8 = rt.malloc(total: u64): *u8;
let buf: []u8 = alloc([], total: u64)!;
let off: i32 = 0;
i = 0;
for (i < strs.len) {
@@ -2068,9 +2038,8 @@ export fn join(delim: str, strs: str...) str = {
};
i += 1;
};
r.ptr = buf;
r.len = total;
return r;
buf.len = total;
return frombytes(buf);
};
// utf8bytelenbounded — walk `it` forward `end` runes and return the
@@ -2357,7 +2326,7 @@ let whitespace: [4]u8 = [0x20u8, 0x0Au8, 0x09u8, 0x0Du8];
// the working field-assign path.
export fn ltrim(input: str, trim: rune...) str = {
if (trim.len == 0) {
return fromutf8_unsafe(bytes.ltrim(toutf8(input), whitespace[0:4]...));
return frombytes(bytes.ltrim(toutf8(input), whitespace[0:4]...));
};
let it: iterator = iter(input);
for (true) {
@@ -2389,7 +2358,7 @@ export fn ltrim(input: str, trim: rune...) str = {
// ref/hare/strings/trim.ha:32.
export fn rtrim(input: str, trim: rune...) str = {
if (trim.len == 0) {
return fromutf8_unsafe(bytes.rtrim(toutf8(input), whitespace[0:4]...));
return frombytes(bytes.rtrim(toutf8(input), whitespace[0:4]...));
};
let it: iterator = riter(input);
for (true) {
@@ -2502,7 +2471,7 @@ export fn iterstr(it: *iterator) str = {
} else {
r = it.src[it.offs:it.src.len];
};
return fromutf8_unsafe(r);
return frombytes(r);
};
// slice — borrowed substring between two iterator positions.
@@ -2517,7 +2486,7 @@ export fn slice(begin: *iterator, end: *iterator) str = {
let e: utf8.decoder;
e.src = end.src;
e.offs = end.offs;
return fromutf8_unsafe(utf8.slice(&b, &e));
return frombytes(utf8.slice(&b, &e));
};
// position — byte-wise offset of the iterator in its source.
@@ -2568,7 +2537,7 @@ export fn rtokenize(s: str, delim: str) tokenizer = {
export fn next_token(s: *tokenizer) (str | bytes.done) = {
let b: *bytes.tokenizer = s: *bytes.tokenizer;
match (bytes.next_token(b)) {
case let v: []u8 => return fromutf8_unsafe(v);
case let v: []u8 => return frombytes(v);
case bytes.done => { let d: bytes.done; return d; };
};
};
@@ -2578,7 +2547,7 @@ export fn next_token(s: *tokenizer) (str | bytes.done) = {
export fn peek_token(s: *tokenizer) (str | bytes.done) = {
let b: *bytes.tokenizer = s: *bytes.tokenizer;
match (bytes.peek_token(b)) {
case let v: []u8 => return fromutf8_unsafe(v);
case let v: []u8 => return frombytes(v);
case bytes.done => { let d: bytes.done; return d; };
};
};
@@ -2587,7 +2556,7 @@ export fn peek_token(s: *tokenizer) (str | bytes.done) = {
// cursor. ref/hare/strings/tokenize.ha:79.
export fn remaining_tokens(s: *tokenizer) str = {
let b: *bytes.tokenizer = s: *bytes.tokenizer;
return fromutf8_unsafe(bytes.remaining_tokens(b));
return frombytes(bytes.remaining_tokens(b));
};
// rt_ensure is the runtime slice-growth helper invoked by the
@@ -2725,7 +2694,7 @@ export fn lpad(s: str, p: rune, maxlen: i32) str = {
if (s.len >= maxlen) { return dup(s); };
let scratch: [4]u8;
let pad: []u8 = runebytes(scratch[0:4], p);
let buf: *u8 = rt.malloc(maxlen: u64): *u8;
let buf: []u8 = alloc([], maxlen: u64)!;
let padwrite: i32 = (maxlen - s.len) * pad.len;
if (padwrite > maxlen) { padwrite = maxlen; };
let off: i32 = 0;
@@ -2740,10 +2709,8 @@ export fn lpad(s: str, p: rune, maxlen: i32) str = {
buf[off + k] = s[k];
k += 1;
};
let r: str;
r.ptr = buf;
r.len = maxlen;
return r;
buf.len = maxlen;
return frombytes(buf);
};
// replace — fresh allocation of `s` with every non-overlapping
@@ -2799,7 +2766,7 @@ export fn replace(s: str, needle: str, target: str) (str | nomem) = {
};
};
res.len = total;
return fromutf8_unsafe(res);
return frombytes(res);
};
// rpad — right-pad `s` with `p` rune until the result reaches `maxlen`
@@ -2808,7 +2775,7 @@ export fn rpad(s: str, p: rune, maxlen: i32) str = {
if (s.len >= maxlen) { return dup(s); };
let scratch: [4]u8;
let pad: []u8 = runebytes(scratch[0:4], p);
let buf: *u8 = rt.malloc(maxlen: u64): *u8;
let buf: []u8 = alloc([], maxlen: u64)!;
let k: i32 = 0;
for (k < s.len) {
buf[k] = s[k];
@@ -2820,10 +2787,8 @@ export fn rpad(s: str, p: rune, maxlen: i32) str = {
buf[s.len + i] = pad.ptr[i % pad.len];
i += 1;
};
let r: str;
r.ptr = buf;
r.len = maxlen;
return r;
buf.len = maxlen;
return frombytes(buf);
};
// strconv — number↔string conversions.
@@ -9754,7 +9719,7 @@ fn nodeisslice(c: *cgen, n: *node) bool = {
// N_DOT (cross-module callee, #34): route through fnretlookupmod
// so a same-leaf caller-module fn with diverging return shape
// doesn't shadow the explicit `mod.f()` qualifier — surfaced by
// strings.slice returning `fromutf8_unsafe(utf8.slice(...))`
// strings.slice returning `frombytes(utf8.slice(...))`
// where strings.slice itself returns str.
if (k == nkind.N_CALL) {
let callee: *node = n.lhs;

83
selfhost/test/smoke.combined.ww
View File

@@ -1769,44 +1769,15 @@ export fn toutf8(s: str) []u8 = {
return r;
};
// fromutf8_unsafe — borrowed str view of `in`. Does not validate.
// ref/hare/strings/utf8.ha:10.
export fn fromutf8_unsafe(in: []u8) str = {
// frombytes — borrowed str view of `in`. Pure reinterpret per
// CLAUDE.md rule 9 carve-out; ref/hare/strings/utf8.ha:10.
export fn frombytes(in: []u8) str = {
let r: str;
r.ptr = in.ptr;
r.len = in.len;
return r;
};
// fromutf8 — validating sibling of [[fromutf8_unsafe]]. Returns the
// borrowed str view on success, or `utf8.invalid` if `in` contains a
// malformed UTF-8 sequence. ref/hare/strings/utf8.ha:22 (#7).
//
// Hare's spelling is `utf8::validate(in)?; return fromutf8_unsafe(in)`.
// ww open-codes the same byte-by-byte DFA walk (ref/hare/encoding/utf8/
// decode.ha:207) because the cross-shape `(void | invalid) →
// (str | invalid)` propagation `?` needs is #19. The decoder-walk
// form's `(rune | done | more | invalid)` arm matches the failure
// axis of fromutf8 and lifts cleanly. Collapses to `utf8.validate(in)?`
// once #19 lands.
export fn fromutf8(in: []u8) (str | utf8.invalid) = {
let d: utf8.decoder = utf8.decode(in);
for (true) {
match (utf8.next(&d)) {
case let r: rune => void;
case utf8.done => {
let r: str = fromutf8_unsafe(in);
return r;
};
case utf8.more => {
let e: utf8.invalid; return e;
};
case let e: utf8.invalid => { return e; };
};
};
let e: utf8.invalid; return e;
};
// compare — three-way bytewise codepoint-order comparison. Return is
// a sign (neg/zero/pos), not an index, so it tracks Hare's `int`
// rather than the str-index i32 (#8). ref/hare/strings/compare.ha:12.
@@ -1832,7 +1803,7 @@ export fn dup(s: str) str = {
let i: i32 = 0;
for (i < s.len) { buf[i] = s[i]; i += 1; };
buf.len = s.len;
return fromutf8_unsafe(buf);
return frombytes(buf);
};
// dupall — fresh `[]str` whose elements are independent copies of
@@ -1905,7 +1876,7 @@ export fn concat(strs: str...) str = {
r.ptr = nil;
r.len = 0;
if (total == 0) { return r; };
let buf: *u8 = rt.malloc(total: u64): *u8;
let buf: []u8 = alloc([], total: u64)!;
let off: i32 = 0;
i = 0;
for (i < strs.len) {
@@ -1917,9 +1888,8 @@ export fn concat(strs: str...) str = {
off += strs[i].len;
i += 1;
};
r.ptr = buf;
r.len = total;
return r;
buf.len = total;
return frombytes(buf);
};
// join — fresh allocation with `delim` placed between each element of
@@ -1938,7 +1908,7 @@ export fn join(delim: str, strs: str...) str = {
r.ptr = nil;
r.len = 0;
if (total == 0) { return r; };
let buf: *u8 = rt.malloc(total: u64): *u8;
let buf: []u8 = alloc([], total: u64)!;
let off: i32 = 0;
i = 0;
for (i < strs.len) {
@@ -1958,9 +1928,8 @@ export fn join(delim: str, strs: str...) str = {
};
i += 1;
};
r.ptr = buf;
r.len = total;
return r;
buf.len = total;
return frombytes(buf);
};
// utf8bytelenbounded — walk `it` forward `end` runes and return the
@@ -2247,7 +2216,7 @@ let whitespace: [4]u8 = [0x20u8, 0x0Au8, 0x09u8, 0x0Du8];
// the working field-assign path.
export fn ltrim(input: str, trim: rune...) str = {
if (trim.len == 0) {
return fromutf8_unsafe(bytes.ltrim(toutf8(input), whitespace[0:4]...));
return frombytes(bytes.ltrim(toutf8(input), whitespace[0:4]...));
};
let it: iterator = iter(input);
for (true) {
@@ -2279,7 +2248,7 @@ export fn ltrim(input: str, trim: rune...) str = {
// ref/hare/strings/trim.ha:32.
export fn rtrim(input: str, trim: rune...) str = {
if (trim.len == 0) {
return fromutf8_unsafe(bytes.rtrim(toutf8(input), whitespace[0:4]...));
return frombytes(bytes.rtrim(toutf8(input), whitespace[0:4]...));
};
let it: iterator = riter(input);
for (true) {
@@ -2392,7 +2361,7 @@ export fn iterstr(it: *iterator) str = {
} else {
r = it.src[it.offs:it.src.len];
};
return fromutf8_unsafe(r);
return frombytes(r);
};
// slice — borrowed substring between two iterator positions.
@@ -2407,7 +2376,7 @@ export fn slice(begin: *iterator, end: *iterator) str = {
let e: utf8.decoder;
e.src = end.src;
e.offs = end.offs;
return fromutf8_unsafe(utf8.slice(&b, &e));
return frombytes(utf8.slice(&b, &e));
};
// position — byte-wise offset of the iterator in its source.
@@ -2458,7 +2427,7 @@ export fn rtokenize(s: str, delim: str) tokenizer = {
export fn next_token(s: *tokenizer) (str | bytes.done) = {
let b: *bytes.tokenizer = s: *bytes.tokenizer;
match (bytes.next_token(b)) {
case let v: []u8 => return fromutf8_unsafe(v);
case let v: []u8 => return frombytes(v);
case bytes.done => { let d: bytes.done; return d; };
};
};
@@ -2468,7 +2437,7 @@ export fn next_token(s: *tokenizer) (str | bytes.done) = {
export fn peek_token(s: *tokenizer) (str | bytes.done) = {
let b: *bytes.tokenizer = s: *bytes.tokenizer;
match (bytes.peek_token(b)) {
case let v: []u8 => return fromutf8_unsafe(v);
case let v: []u8 => return frombytes(v);
case bytes.done => { let d: bytes.done; return d; };
};
};
@@ -2477,7 +2446,7 @@ export fn peek_token(s: *tokenizer) (str | bytes.done) = {
// cursor. ref/hare/strings/tokenize.ha:79.
export fn remaining_tokens(s: *tokenizer) str = {
let b: *bytes.tokenizer = s: *bytes.tokenizer;
return fromutf8_unsafe(bytes.remaining_tokens(b));
return frombytes(bytes.remaining_tokens(b));
};
// rt_ensure is the runtime slice-growth helper invoked by the
@@ -2615,7 +2584,7 @@ export fn lpad(s: str, p: rune, maxlen: i32) str = {
if (s.len >= maxlen) { return dup(s); };
let scratch: [4]u8;
let pad: []u8 = runebytes(scratch[0:4], p);
let buf: *u8 = rt.malloc(maxlen: u64): *u8;
let buf: []u8 = alloc([], maxlen: u64)!;
let padwrite: i32 = (maxlen - s.len) * pad.len;
if (padwrite > maxlen) { padwrite = maxlen; };
let off: i32 = 0;
@@ -2630,10 +2599,8 @@ export fn lpad(s: str, p: rune, maxlen: i32) str = {
buf[off + k] = s[k];
k += 1;
};
let r: str;
r.ptr = buf;
r.len = maxlen;
return r;
buf.len = maxlen;
return frombytes(buf);
};
// replace — fresh allocation of `s` with every non-overlapping
@@ -2689,7 +2656,7 @@ export fn replace(s: str, needle: str, target: str) (str | nomem) = {
};
};
res.len = total;
return fromutf8_unsafe(res);
return frombytes(res);
};
// rpad — right-pad `s` with `p` rune until the result reaches `maxlen`
@@ -2698,7 +2665,7 @@ export fn rpad(s: str, p: rune, maxlen: i32) str = {
if (s.len >= maxlen) { return dup(s); };
let scratch: [4]u8;
let pad: []u8 = runebytes(scratch[0:4], p);
let buf: *u8 = rt.malloc(maxlen: u64): *u8;
let buf: []u8 = alloc([], maxlen: u64)!;
let k: i32 = 0;
for (k < s.len) {
buf[k] = s[k];
@@ -2710,10 +2677,8 @@ export fn rpad(s: str, p: rune, maxlen: i32) str = {
buf[s.len + i] = pad.ptr[i % pad.len];
i += 1;
};
let r: str;
r.ptr = buf;
r.len = maxlen;
return r;
buf.len = maxlen;
return frombytes(buf);
};
// strconv — number↔string conversions.