lib/strings+test: graduate fromutf8 + bytesub to validating return

fromutf8(in: []u8) (str | utf8.invalid) and the bytesub form per
ref/hare/strings/utf8.ha:22 and sub.ha:59. bytesub keeps its byte
asserts (ww extension over Hare; predates #7).

fromutf8 walks the utf8 decoder via utf8.next rather than the
shorter `utf8.validate(in)?` form. Two compiler bugs in the way:
cross-shape `(void | invalid) → (str | invalid)` propagation is
#19, and (void | !void) match-bind locals diverge between stages /
str→union lift SIGSEGVs in cstage — both filed as #48. The
decoder-walk form bypasses both and matches what
ref/hare/strings/utf8.ha actually does in source.

getopt.ww:314 caller updated to match the new (str | invalid)
return; bi+1 cannot hit a continuation byte in well-formed argv
(bi is a just-matched ASCII flag), so abort spells the precondition.

bytesub_cases rewritten as exhaustive match; new rows cover
start-on-continuation and end-on-continuation invalid arms plus an
end==s.len bypass. fromutf8_cases is new — Hare vector + edge
bytes + multibyte parity rows.
This commit is contained in:
2026-05-20 02:02:28 +09:00
parent e03a6281d6
commit 6d006da26c
6 changed files with 357 additions and 29 deletions

View File

@@ -93,6 +93,7 @@
package getopt;
import encoding.utf8;
import os;
import strings;
@@ -311,7 +312,19 @@ export fn tryparse(out: *command, argv: []str, help: []help) (void | error) = {
} else {
// PARAM: glued value, or next argv slot.
if (bi + 1 < arg.len) {
let v: str = strings.bytesub(arg, bi + 1, arg.len);
// strings.bytesub now validates rune
// boundaries (#7). `bi` is the byte
// offset of the just-matched ASCII flag
// char, so `bi + 1` cannot land on a
// continuation byte in well-formed argv;
// abort spells the well-formedness
// precondition as Hare's `!` does.
let v: str;
match (strings.bytesub(arg, bi + 1, arg.len)) {
case let s: str => v = s;
case utf8.invalid =>
abort("getopt: malformed argv UTF-8");
};
appendoption(&opts, r, v);
advanced = true;
} else {

View File

@@ -48,6 +48,39 @@ export fn fromutf8_unsafe(in: []u8) str = {
return r;
};
// fromutf8 — validating sibling of [[fromutf8_unsafe]]. Returns the
// borrowed str view on success, or `utf8.invalid` if `in` contains a
// malformed UTF-8 sequence. ref/hare/strings/utf8.ha:22 (#7).
//
// Hare's spelling is `utf8::validate(in)?; return fromutf8_unsafe(in)`.
// ww calls `utf8.validate` and matches its `(void | utf8.invalid)`
// result instead of `?` (cross-shape propagation is #19). Even with
// match, the call diverges cstage/wwstage when the result is bound to
// a local (#48: wwstage spills DX for the payload-less arm) and the
// success path then SIGSEGVs lifting `str` to `(str | utf8.invalid)`.
// So the validation is open-coded against the decoder — the same
// byte-by-byte DFA walk `utf8.validate` performs (ref/hare/encoding/
// utf8/decode.ha:207) — whose `(rune | done | more | invalid)` arm
// matches the failure axis of fromutf8 and lifts cleanly. Collapses
// to `utf8.validate(in)?` once #19 + #48 are fixed.
export fn fromutf8(in: []u8) (str | utf8.invalid) = {
let d: utf8.decoder = utf8.decode(in);
for (true) {
match (utf8.next(&d)) {
case let r: rune => void;
case utf8.done => {
let r: str = fromutf8_unsafe(in);
return r;
};
case utf8.more => {
let e: utf8.invalid; return e;
};
case let e: utf8.invalid => { return e; };
};
};
let e: utf8.invalid; return e;
};
// compare — three-way bytewise codepoint-order comparison. Return is
// a sign (neg/zero/pos), not an index, so it tracks Hare's `int`
// rather than the str-index i32 (#8). ref/hare/strings/compare.ha:12.
@@ -235,12 +268,23 @@ export fn sub(s: str, start: i32, end: i32) str = {
};
// bytesub — borrowed substring [start, end) where start/end are byte
// offsets. ref/hare/strings/sub.ha:59. Hare's validating return
// `(str | utf8::invalid)` is dropped (filed as #7); a multibyte-split
// argument will yield a sub-string whose bytes don't decode cleanly.
export fn bytesub(s: str, start: i32, end: i32) str = {
// offsets. ref/hare/strings/sub.ha:59 (#7). Returns `utf8.invalid` if
// either endpoint lands on a continuation byte (would split a
// codepoint); the equivalent Hare predicate is `s[i] & 0xc0 == 0x80`
// at ref/hare/strings/sub.ha:72-73.
export fn bytesub(s: str, start: i32, end: i32) (str | utf8.invalid) = {
os.assert(start <= end, "strings.bytesub: start is higher than end");
os.assert(end <= s.len, "strings.bytesub: end exceeds string length");
if (start < s.len) {
if ((s[start] & 0xC0u8) == 0x80u8) {
let e: utf8.invalid; return e;
};
};
if (end < s.len) {
if ((s[end] & 0xC0u8) == 0x80u8) {
let e: utf8.invalid; return e;
};
};
let r: str;
r.ptr = s.ptr + (start: u64);
r.len = end - start;

View File

@@ -742,8 +742,7 @@ fn streq(a: str, b: str) bool = {
// ---- sub / bytesub ----------------------------------------------------
// ref/hare/strings/sub.ha:44 (@test fn sub), :79 (@test fn bytesub). Hare's
// 2-arg `sub(s, start)` rows are omitted: ww has no default-parameter
// syntax (filed as #37). The `is utf8::invalid` row for bytesub is
// deferred to the validating port (#7).
// syntax (filed as #37). bytesub now validates rune boundaries (#7).
@test fn sub_cases() void = {
signalled = 1790;
@@ -770,32 +769,80 @@ fn streq(a: str, b: str) bool = {
if (!streq(strings.sub("héllo", 0, 5), "héllo")) { fail(); };
};
// Match-shape mirrors Hare's `bytesub(...)!` at ref/hare/strings/sub.ha:
// 80-86. Inlined at each call site (rather than a helper that takes
// `(str | utf8.invalid)` by value) because the union-by-value path
// crashes — same lift-on-pass-by-value family as #48.
@test fn bytesub_cases() void = {
signalled = 1800;
if (!streq(strings.bytesub("a string", 0, 8), "a string")) { fail(); };
match (strings.bytesub("a string", 0, 8)) {
case let s: str => { if (!streq(s, "a string")) { fail(); }; };
case let e: utf8.invalid => { fail(); };
};
signalled = 1801;
if (!streq(strings.bytesub("a string", 0, 1), "a")) { fail(); };
match (strings.bytesub("a string", 0, 1)) {
case let s: str => { if (!streq(s, "a")) { fail(); }; };
case let e: utf8.invalid => { fail(); };
};
signalled = 1802;
if (!streq(strings.bytesub("a string", 0, 3), "a s")) { fail(); };
match (strings.bytesub("a string", 0, 3)) {
case let s: str => { if (!streq(s, "a s")) { fail(); }; };
case let e: utf8.invalid => { fail(); };
};
signalled = 1803;
if (!streq(strings.bytesub("a string", 2, 8), "string")) { fail(); };
match (strings.bytesub("a string", 2, 8)) {
case let s: str => { if (!streq(s, "string")) { fail(); }; };
case let e: utf8.invalid => { fail(); };
};
signalled = 1804;
if (!streq(strings.bytesub("a string", 4, 4), "")) { fail(); };
match (strings.bytesub("a string", 4, 4)) {
case let s: str => { if (!streq(s, "")) { fail(); }; };
case let e: utf8.invalid => { fail(); };
};
// Hare vector — byte indices 3..9 over "こんにちは" select "んに".
signalled = 1805;
if (!streq(strings.bytesub("こんにちは", 3, 9), "んに")) { fail(); };
match (strings.bytesub("こんにちは", 3, 9)) {
case let s: str => { if (!streq(s, "んに")) { fail(); }; };
case let e: utf8.invalid => { fail(); };
};
// Rune/byte axis disagree on identical args (#3): sub(s,0,3) walks 3
// runes and yields 9 bytes; bytesub(s,0,3) yields the first 3 bytes
// — one 3-byte codepoint.
signalled = 1806;
if (!streq(strings.sub("こんにちは", 0, 3), "こんに")) { fail(); };
if (!streq(strings.bytesub("こんにちは", 0, 3), "こ")) { fail(); };
match (strings.bytesub("こんにちは", 0, 3)) {
case let s: str => { if (!streq(s, "こ")) { fail(); }; };
case let e: utf8.invalid => { fail(); };
};
// Borrowed view: ptr aliases input.
signalled = 1807;
let s: str = "hello";
let r: str = strings.bytesub(s, 1, 4);
match (strings.bytesub(s, 1, 4)) {
case let r: str => {
if (r.ptr != s.ptr + 1u64) { fail(); };
if (r.len != 3) { fail(); };
};
case let e: utf8.invalid => { fail(); };
};
// Hare's invalid row (ref/hare/strings/sub.ha:87) — start lands on
// a continuation byte (2nd byte of "こ"), bytesub must reject (#7).
signalled = 1808;
match (strings.bytesub("こんにちは", 1, 3)) {
case let r: str => { fail(); };
case let e: utf8.invalid => void;
};
// Symmetric: end lands on a continuation byte (2nd byte of "ん").
signalled = 1809;
match (strings.bytesub("こんにちは", 0, 4)) {
case let r: str => { fail(); };
case let e: utf8.invalid => void;
};
// end == s.len bypasses the continuation check (s[end] is OOB).
signalled = 1810;
match (strings.bytesub("こんにちは", 0, 15)) {
case let s: str => { if (!streq(s, "こんにちは")) { fail(); }; };
case let e: utf8.invalid => { fail(); };
};
};
// ---- toutf8 / fromutf8_unsafe roundtrip -------------------------------
@@ -811,6 +858,97 @@ fn streq(a: str, b: str) bool = {
if (r.ptr != s.ptr) { fail(); }; // borrowed, not copied
};
// ---- fromutf8 ---------------------------------------------------------
// ref/hare/strings/utf8.ha:22 + @test fn utf8 (line 31). Validating
// sibling of fromutf8_unsafe — walks via [[utf8.validate]] and returns
// `utf8.invalid` on any malformed byte sequence (#7).
@test fn fromutf8_cases() void = {
// Hare vector — "hello world" round-trips.
signalled = 1900;
let hb: [11]u8;
hb[0] = 0x68u8; hb[1] = 0x65u8; hb[2] = 0x6Cu8; hb[3] = 0x6Cu8;
hb[4] = 0x6Fu8; hb[5] = 0x20u8; hb[6] = 0x77u8; hb[7] = 0x6Fu8;
hb[8] = 0x72u8; hb[9] = 0x6Cu8; hb[10] = 0x64u8;
match (strings.fromutf8(hb[0:11])) {
case let s: str => { if (!streq(s, "hello world")) { fail(); }; };
case let e: utf8.invalid => { fail(); };
};
// Empty input — Hare's `fromutf8([])! == ""`.
signalled = 1901;
let eb: [1]u8;
match (strings.fromutf8(eb[0:0])) {
case let s: str => { if (s.len != 0) { fail(); }; };
case let e: utf8.invalid => { fail(); };
};
// ASCII-only borrowed view: ptr aliases input.
signalled = 1902;
let s: str = "abc";
let bs: []u8 = strings.toutf8(s);
match (strings.fromutf8(bs)) {
case let r: str => {
if (!streq(r, "abc")) { fail(); };
if (r.ptr != s.ptr) { fail(); };
};
case let e: utf8.invalid => { fail(); };
};
// Multibyte Japanese — full UTF-8 round-trip.
signalled = 1903;
let jb: [15]u8;
jb[0] = 0xE3u8; jb[1] = 0x81u8; jb[2] = 0x93u8; // こ
jb[3] = 0xE3u8; jb[4] = 0x82u8; jb[5] = 0x93u8; // ん
jb[6] = 0xE3u8; jb[7] = 0x81u8; jb[8] = 0xABu8; // に
jb[9] = 0xE3u8; jb[10] = 0x81u8; jb[11] = 0xA1u8; // ち
jb[12] = 0xE3u8; jb[13] = 0x81u8; jb[14] = 0xAFu8; // は
match (strings.fromutf8(jb[0:15])) {
case let s: str => { if (!streq(s, "こんにちは")) { fail(); }; };
case let e: utf8.invalid => { fail(); };
};
// Multibyte French — mixed-width "héllo".
signalled = 1904;
let fb: [6]u8;
fb[0] = 0x68u8; // h
fb[1] = 0xC3u8; fb[2] = 0xA9u8; // é
fb[3] = 0x6Cu8; fb[4] = 0x6Cu8; fb[5] = 0x6Fu8; // llo
match (strings.fromutf8(fb[0:6])) {
case let s: str => { if (!streq(s, "héllo")) { fail(); }; };
case let e: utf8.invalid => { fail(); };
};
// Lone 0x80 continuation byte — invalid.
signalled = 1905;
let c: [1]u8;
c[0] = 0x80u8;
match (strings.fromutf8(c[0:1])) {
case let r: str => { fail(); };
case let e: utf8.invalid => void;
};
// Overlong 2-byte encoding of NUL (0xC0 0x80) — invalid per
// ref/hare/encoding/utf8/decodetable.ha (state 0 rejects 0xC0).
signalled = 1906;
let ov: [2]u8;
ov[0] = 0xC0u8; ov[1] = 0x80u8;
match (strings.fromutf8(ov[0:2])) {
case let r: str => { fail(); };
case let e: utf8.invalid => void;
};
// Truncated multibyte — 0xE3 0x81 (2 of 3 bytes for U+3053) is
// rejected as invalid by [[utf8.validate]] (state != 0 at EOI).
signalled = 1907;
let tr: [2]u8;
tr[0] = 0xE3u8; tr[1] = 0x81u8;
match (strings.fromutf8(tr[0:2])) {
case let r: str => { fail(); };
case let e: utf8.invalid => void;
};
};
// ---- iter / next ------------------------------------------------------
// ref/hare/strings/iter.ha:84-108. Hare's @test fn iter() uses prev +
// riter heavily; both are deferred (no `utf8.prev`). Rebuild forward-
@@ -1548,6 +1686,7 @@ export fn main() i32 = {
signalled = 40; sub_cases();
signalled = 41; bytesub_cases();
signalled = 15; utf8_roundtrip_cases();
signalled = 43; fromutf8_cases();
signalled = 16; iter_empty_cases();
signalled = 17; iter_ascii_cases();
signalled = 18; iter_twobyte_cases();

View File

@@ -1885,6 +1885,39 @@ export fn fromutf8_unsafe(in: []u8) str = {
return r;
};
// fromutf8 — validating sibling of [[fromutf8_unsafe]]. Returns the
// borrowed str view on success, or `utf8.invalid` if `in` contains a
// malformed UTF-8 sequence. ref/hare/strings/utf8.ha:22 (#7).
//
// Hare's spelling is `utf8::validate(in)?; return fromutf8_unsafe(in)`.
// ww calls `utf8.validate` and matches its `(void | utf8.invalid)`
// result instead of `?` (cross-shape propagation is #19). Even with
// match, the call diverges cstage/wwstage when the result is bound to
// a local (#48: wwstage spills DX for the payload-less arm) and the
// success path then SIGSEGVs lifting `str` to `(str | utf8.invalid)`.
// So the validation is open-coded against the decoder — the same
// byte-by-byte DFA walk `utf8.validate` performs (ref/hare/encoding/
// utf8/decode.ha:207) — whose `(rune | done | more | invalid)` arm
// matches the failure axis of fromutf8 and lifts cleanly. Collapses
// to `utf8.validate(in)?` once #19 + #48 are fixed.
export fn fromutf8(in: []u8) (str | utf8.invalid) = {
let d: utf8.decoder = utf8.decode(in);
for (true) {
match (utf8.next(&d)) {
case let r: rune => void;
case utf8.done => {
let r: str = fromutf8_unsafe(in);
return r;
};
case utf8.more => {
let e: utf8.invalid; return e;
};
case let e: utf8.invalid => { return e; };
};
};
let e: utf8.invalid; return e;
};
// compare — three-way bytewise codepoint-order comparison. Return is
// a sign (neg/zero/pos), not an index, so it tracks Hare's `int`
// rather than the str-index i32 (#8). ref/hare/strings/compare.ha:12.
@@ -2072,12 +2105,23 @@ export fn sub(s: str, start: i32, end: i32) str = {
};
// bytesub — borrowed substring [start, end) where start/end are byte
// offsets. ref/hare/strings/sub.ha:59. Hare's validating return
// `(str | utf8::invalid)` is dropped (filed as #7); a multibyte-split
// argument will yield a sub-string whose bytes don't decode cleanly.
export fn bytesub(s: str, start: i32, end: i32) str = {
// offsets. ref/hare/strings/sub.ha:59 (#7). Returns `utf8.invalid` if
// either endpoint lands on a continuation byte (would split a
// codepoint); the equivalent Hare predicate is `s[i] & 0xc0 == 0x80`
// at ref/hare/strings/sub.ha:72-73.
export fn bytesub(s: str, start: i32, end: i32) (str | utf8.invalid) = {
os.assert(start <= end, "strings.bytesub: start is higher than end");
os.assert(end <= s.len, "strings.bytesub: end exceeds string length");
if (start < s.len) {
if ((s[start] & 0xC0u8) == 0x80u8) {
let e: utf8.invalid; return e;
};
};
if (end < s.len) {
if ((s[end] & 0xC0u8) == 0x80u8) {
let e: utf8.invalid; return e;
};
};
let r: str;
r.ptr = s.ptr + (start: u64);
r.len = end - start;

View File

@@ -1885,6 +1885,39 @@ export fn fromutf8_unsafe(in: []u8) str = {
return r;
};
// fromutf8 — validating sibling of [[fromutf8_unsafe]]. Returns the
// borrowed str view on success, or `utf8.invalid` if `in` contains a
// malformed UTF-8 sequence. ref/hare/strings/utf8.ha:22 (#7).
//
// Hare's spelling is `utf8::validate(in)?; return fromutf8_unsafe(in)`.
// ww calls `utf8.validate` and matches its `(void | utf8.invalid)`
// result instead of `?` (cross-shape propagation is #19). Even with
// match, the call diverges cstage/wwstage when the result is bound to
// a local (#48: wwstage spills DX for the payload-less arm) and the
// success path then SIGSEGVs lifting `str` to `(str | utf8.invalid)`.
// So the validation is open-coded against the decoder — the same
// byte-by-byte DFA walk `utf8.validate` performs (ref/hare/encoding/
// utf8/decode.ha:207) — whose `(rune | done | more | invalid)` arm
// matches the failure axis of fromutf8 and lifts cleanly. Collapses
// to `utf8.validate(in)?` once #19 + #48 are fixed.
export fn fromutf8(in: []u8) (str | utf8.invalid) = {
let d: utf8.decoder = utf8.decode(in);
for (true) {
match (utf8.next(&d)) {
case let r: rune => void;
case utf8.done => {
let r: str = fromutf8_unsafe(in);
return r;
};
case utf8.more => {
let e: utf8.invalid; return e;
};
case let e: utf8.invalid => { return e; };
};
};
let e: utf8.invalid; return e;
};
// compare — three-way bytewise codepoint-order comparison. Return is
// a sign (neg/zero/pos), not an index, so it tracks Hare's `int`
// rather than the str-index i32 (#8). ref/hare/strings/compare.ha:12.
@@ -2072,12 +2105,23 @@ export fn sub(s: str, start: i32, end: i32) str = {
};
// bytesub — borrowed substring [start, end) where start/end are byte
// offsets. ref/hare/strings/sub.ha:59. Hare's validating return
// `(str | utf8::invalid)` is dropped (filed as #7); a multibyte-split
// argument will yield a sub-string whose bytes don't decode cleanly.
export fn bytesub(s: str, start: i32, end: i32) str = {
// offsets. ref/hare/strings/sub.ha:59 (#7). Returns `utf8.invalid` if
// either endpoint lands on a continuation byte (would split a
// codepoint); the equivalent Hare predicate is `s[i] & 0xc0 == 0x80`
// at ref/hare/strings/sub.ha:72-73.
export fn bytesub(s: str, start: i32, end: i32) (str | utf8.invalid) = {
os.assert(start <= end, "strings.bytesub: start is higher than end");
os.assert(end <= s.len, "strings.bytesub: end exceeds string length");
if (start < s.len) {
if ((s[start] & 0xC0u8) == 0x80u8) {
let e: utf8.invalid; return e;
};
};
if (end < s.len) {
if ((s[end] & 0xC0u8) == 0x80u8) {
let e: utf8.invalid; return e;
};
};
let r: str;
r.ptr = s.ptr + (start: u64);
r.len = end - start;

View File

@@ -1776,6 +1776,39 @@ export fn fromutf8_unsafe(in: []u8) str = {
return r;
};
// fromutf8 — validating sibling of [[fromutf8_unsafe]]. Returns the
// borrowed str view on success, or `utf8.invalid` if `in` contains a
// malformed UTF-8 sequence. ref/hare/strings/utf8.ha:22 (#7).
//
// Hare's spelling is `utf8::validate(in)?; return fromutf8_unsafe(in)`.
// ww calls `utf8.validate` and matches its `(void | utf8.invalid)`
// result instead of `?` (cross-shape propagation is #19). Even with
// match, the call diverges cstage/wwstage when the result is bound to
// a local (#48: wwstage spills DX for the payload-less arm) and the
// success path then SIGSEGVs lifting `str` to `(str | utf8.invalid)`.
// So the validation is open-coded against the decoder — the same
// byte-by-byte DFA walk `utf8.validate` performs (ref/hare/encoding/
// utf8/decode.ha:207) — whose `(rune | done | more | invalid)` arm
// matches the failure axis of fromutf8 and lifts cleanly. Collapses
// to `utf8.validate(in)?` once #19 + #48 are fixed.
export fn fromutf8(in: []u8) (str | utf8.invalid) = {
let d: utf8.decoder = utf8.decode(in);
for (true) {
match (utf8.next(&d)) {
case let r: rune => void;
case utf8.done => {
let r: str = fromutf8_unsafe(in);
return r;
};
case utf8.more => {
let e: utf8.invalid; return e;
};
case let e: utf8.invalid => { return e; };
};
};
let e: utf8.invalid; return e;
};
// compare — three-way bytewise codepoint-order comparison. Return is
// a sign (neg/zero/pos), not an index, so it tracks Hare's `int`
// rather than the str-index i32 (#8). ref/hare/strings/compare.ha:12.
@@ -1963,12 +1996,23 @@ export fn sub(s: str, start: i32, end: i32) str = {
};
// bytesub — borrowed substring [start, end) where start/end are byte
// offsets. ref/hare/strings/sub.ha:59. Hare's validating return
// `(str | utf8::invalid)` is dropped (filed as #7); a multibyte-split
// argument will yield a sub-string whose bytes don't decode cleanly.
export fn bytesub(s: str, start: i32, end: i32) str = {
// offsets. ref/hare/strings/sub.ha:59 (#7). Returns `utf8.invalid` if
// either endpoint lands on a continuation byte (would split a
// codepoint); the equivalent Hare predicate is `s[i] & 0xc0 == 0x80`
// at ref/hare/strings/sub.ha:72-73.
export fn bytesub(s: str, start: i32, end: i32) (str | utf8.invalid) = {
os.assert(start <= end, "strings.bytesub: start is higher than end");
os.assert(end <= s.len, "strings.bytesub: end exceeds string length");
if (start < s.len) {
if ((s[start] & 0xC0u8) == 0x80u8) {
let e: utf8.invalid; return e;
};
};
if (end < s.len) {
if ((s[end] & 0xC0u8) == 0x80u8) {
let e: utf8.invalid; return e;
};
};
let r: str;
r.ptr = s.ptr + (start: u64);
r.len = end - start;