lib/strings+test: graduate fromutf8 + bytesub to validating return
fromutf8(in: []u8) (str | utf8.invalid) and the bytesub form per ref/hare/strings/utf8.ha:22 and sub.ha:59. bytesub keeps its byte asserts (ww extension over Hare; predates #7). fromutf8 walks the utf8 decoder via utf8.next rather than the shorter `utf8.validate(in)?` form. Two compiler bugs in the way: cross-shape `(void | invalid) → (str | invalid)` propagation is #19, and (void | !void) match-bind locals diverge between stages / str→union lift SIGSEGVs in cstage — both filed as #48. The decoder-walk form bypasses both and matches what ref/hare/strings/utf8.ha actually does in source. getopt.ww:314 caller updated to match the new (str | invalid) return; bi+1 cannot hit a continuation byte in well-formed argv (bi is a just-matched ASCII flag), so abort spells the precondition. bytesub_cases rewritten as exhaustive match; new rows cover start-on-continuation and end-on-continuation invalid arms plus an end==s.len bypass. fromutf8_cases is new — Hare vector + edge bytes + multibyte parity rows.
This commit is contained in:
@@ -93,6 +93,7 @@
|
||||
|
||||
package getopt;
|
||||
|
||||
import encoding.utf8;
|
||||
import os;
|
||||
import strings;
|
||||
|
||||
@@ -311,7 +312,19 @@ export fn tryparse(out: *command, argv: []str, help: []help) (void | error) = {
|
||||
} else {
|
||||
// PARAM: glued value, or next argv slot.
|
||||
if (bi + 1 < arg.len) {
|
||||
let v: str = strings.bytesub(arg, bi + 1, arg.len);
|
||||
// strings.bytesub now validates rune
|
||||
// boundaries (#7). `bi` is the byte
|
||||
// offset of the just-matched ASCII flag
|
||||
// char, so `bi + 1` cannot land on a
|
||||
// continuation byte in well-formed argv;
|
||||
// abort spells the well-formedness
|
||||
// precondition as Hare's `!` does.
|
||||
let v: str;
|
||||
match (strings.bytesub(arg, bi + 1, arg.len)) {
|
||||
case let s: str => v = s;
|
||||
case utf8.invalid =>
|
||||
abort("getopt: malformed argv UTF-8");
|
||||
};
|
||||
appendoption(&opts, r, v);
|
||||
advanced = true;
|
||||
} else {
|
||||
|
||||
@@ -48,6 +48,39 @@ export fn fromutf8_unsafe(in: []u8) str = {
|
||||
return r;
|
||||
};
|
||||
|
||||
// fromutf8 — validating sibling of [[fromutf8_unsafe]]. Returns the
|
||||
// borrowed str view on success, or `utf8.invalid` if `in` contains a
|
||||
// malformed UTF-8 sequence. ref/hare/strings/utf8.ha:22 (#7).
|
||||
//
|
||||
// Hare's spelling is `utf8::validate(in)?; return fromutf8_unsafe(in)`.
|
||||
// ww calls `utf8.validate` and matches its `(void | utf8.invalid)`
|
||||
// result instead of `?` (cross-shape propagation is #19). Even with
|
||||
// match, the call diverges cstage/wwstage when the result is bound to
|
||||
// a local (#48: wwstage spills DX for the payload-less arm) and the
|
||||
// success path then SIGSEGVs lifting `str` to `(str | utf8.invalid)`.
|
||||
// So the validation is open-coded against the decoder — the same
|
||||
// byte-by-byte DFA walk `utf8.validate` performs (ref/hare/encoding/
|
||||
// utf8/decode.ha:207) — whose `(rune | done | more | invalid)` arm
|
||||
// matches the failure axis of fromutf8 and lifts cleanly. Collapses
|
||||
// to `utf8.validate(in)?` once #19 + #48 are fixed.
|
||||
export fn fromutf8(in: []u8) (str | utf8.invalid) = {
|
||||
let d: utf8.decoder = utf8.decode(in);
|
||||
for (true) {
|
||||
match (utf8.next(&d)) {
|
||||
case let r: rune => void;
|
||||
case utf8.done => {
|
||||
let r: str = fromutf8_unsafe(in);
|
||||
return r;
|
||||
};
|
||||
case utf8.more => {
|
||||
let e: utf8.invalid; return e;
|
||||
};
|
||||
case let e: utf8.invalid => { return e; };
|
||||
};
|
||||
};
|
||||
let e: utf8.invalid; return e;
|
||||
};
|
||||
|
||||
// compare — three-way bytewise codepoint-order comparison. Return is
|
||||
// a sign (neg/zero/pos), not an index, so it tracks Hare's `int`
|
||||
// rather than the str-index i32 (#8). ref/hare/strings/compare.ha:12.
|
||||
@@ -235,12 +268,23 @@ export fn sub(s: str, start: i32, end: i32) str = {
|
||||
};
|
||||
|
||||
// bytesub — borrowed substring [start, end) where start/end are byte
|
||||
// offsets. ref/hare/strings/sub.ha:59. Hare's validating return
|
||||
// `(str | utf8::invalid)` is dropped (filed as #7); a multibyte-split
|
||||
// argument will yield a sub-string whose bytes don't decode cleanly.
|
||||
export fn bytesub(s: str, start: i32, end: i32) str = {
|
||||
// offsets. ref/hare/strings/sub.ha:59 (#7). Returns `utf8.invalid` if
|
||||
// either endpoint lands on a continuation byte (would split a
|
||||
// codepoint); the equivalent Hare predicate is `s[i] & 0xc0 == 0x80`
|
||||
// at ref/hare/strings/sub.ha:72-73.
|
||||
export fn bytesub(s: str, start: i32, end: i32) (str | utf8.invalid) = {
|
||||
os.assert(start <= end, "strings.bytesub: start is higher than end");
|
||||
os.assert(end <= s.len, "strings.bytesub: end exceeds string length");
|
||||
if (start < s.len) {
|
||||
if ((s[start] & 0xC0u8) == 0x80u8) {
|
||||
let e: utf8.invalid; return e;
|
||||
};
|
||||
};
|
||||
if (end < s.len) {
|
||||
if ((s[end] & 0xC0u8) == 0x80u8) {
|
||||
let e: utf8.invalid; return e;
|
||||
};
|
||||
};
|
||||
let r: str;
|
||||
r.ptr = s.ptr + (start: u64);
|
||||
r.len = end - start;
|
||||
|
||||
@@ -742,8 +742,7 @@ fn streq(a: str, b: str) bool = {
|
||||
// ---- sub / bytesub ----------------------------------------------------
|
||||
// ref/hare/strings/sub.ha:44 (@test fn sub), :79 (@test fn bytesub). Hare's
|
||||
// 2-arg `sub(s, start)` rows are omitted: ww has no default-parameter
|
||||
// syntax (filed as #37). The `is utf8::invalid` row for bytesub is
|
||||
// deferred to the validating port (#7).
|
||||
// syntax (filed as #37). bytesub now validates rune boundaries (#7).
|
||||
|
||||
@test fn sub_cases() void = {
|
||||
signalled = 1790;
|
||||
@@ -770,32 +769,80 @@ fn streq(a: str, b: str) bool = {
|
||||
if (!streq(strings.sub("héllo", 0, 5), "héllo")) { fail(); };
|
||||
};
|
||||
|
||||
// Match-shape mirrors Hare's `bytesub(...)!` at ref/hare/strings/sub.ha:
|
||||
// 80-86. Inlined at each call site (rather than a helper that takes
|
||||
// `(str | utf8.invalid)` by value) because the union-by-value path
|
||||
// crashes — same lift-on-pass-by-value family as #48.
|
||||
@test fn bytesub_cases() void = {
|
||||
signalled = 1800;
|
||||
if (!streq(strings.bytesub("a string", 0, 8), "a string")) { fail(); };
|
||||
match (strings.bytesub("a string", 0, 8)) {
|
||||
case let s: str => { if (!streq(s, "a string")) { fail(); }; };
|
||||
case let e: utf8.invalid => { fail(); };
|
||||
};
|
||||
signalled = 1801;
|
||||
if (!streq(strings.bytesub("a string", 0, 1), "a")) { fail(); };
|
||||
match (strings.bytesub("a string", 0, 1)) {
|
||||
case let s: str => { if (!streq(s, "a")) { fail(); }; };
|
||||
case let e: utf8.invalid => { fail(); };
|
||||
};
|
||||
signalled = 1802;
|
||||
if (!streq(strings.bytesub("a string", 0, 3), "a s")) { fail(); };
|
||||
match (strings.bytesub("a string", 0, 3)) {
|
||||
case let s: str => { if (!streq(s, "a s")) { fail(); }; };
|
||||
case let e: utf8.invalid => { fail(); };
|
||||
};
|
||||
signalled = 1803;
|
||||
if (!streq(strings.bytesub("a string", 2, 8), "string")) { fail(); };
|
||||
match (strings.bytesub("a string", 2, 8)) {
|
||||
case let s: str => { if (!streq(s, "string")) { fail(); }; };
|
||||
case let e: utf8.invalid => { fail(); };
|
||||
};
|
||||
signalled = 1804;
|
||||
if (!streq(strings.bytesub("a string", 4, 4), "")) { fail(); };
|
||||
match (strings.bytesub("a string", 4, 4)) {
|
||||
case let s: str => { if (!streq(s, "")) { fail(); }; };
|
||||
case let e: utf8.invalid => { fail(); };
|
||||
};
|
||||
// Hare vector — byte indices 3..9 over "こんにちは" select "んに".
|
||||
signalled = 1805;
|
||||
if (!streq(strings.bytesub("こんにちは", 3, 9), "んに")) { fail(); };
|
||||
match (strings.bytesub("こんにちは", 3, 9)) {
|
||||
case let s: str => { if (!streq(s, "んに")) { fail(); }; };
|
||||
case let e: utf8.invalid => { fail(); };
|
||||
};
|
||||
// Rune/byte axis disagree on identical args (#3): sub(s,0,3) walks 3
|
||||
// runes and yields 9 bytes; bytesub(s,0,3) yields the first 3 bytes
|
||||
// — one 3-byte codepoint.
|
||||
signalled = 1806;
|
||||
if (!streq(strings.sub("こんにちは", 0, 3), "こんに")) { fail(); };
|
||||
if (!streq(strings.bytesub("こんにちは", 0, 3), "こ")) { fail(); };
|
||||
match (strings.bytesub("こんにちは", 0, 3)) {
|
||||
case let s: str => { if (!streq(s, "こ")) { fail(); }; };
|
||||
case let e: utf8.invalid => { fail(); };
|
||||
};
|
||||
// Borrowed view: ptr aliases input.
|
||||
signalled = 1807;
|
||||
let s: str = "hello";
|
||||
let r: str = strings.bytesub(s, 1, 4);
|
||||
if (r.ptr != s.ptr + 1u64) { fail(); };
|
||||
if (r.len != 3) { fail(); };
|
||||
match (strings.bytesub(s, 1, 4)) {
|
||||
case let r: str => {
|
||||
if (r.ptr != s.ptr + 1u64) { fail(); };
|
||||
if (r.len != 3) { fail(); };
|
||||
};
|
||||
case let e: utf8.invalid => { fail(); };
|
||||
};
|
||||
// Hare's invalid row (ref/hare/strings/sub.ha:87) — start lands on
|
||||
// a continuation byte (2nd byte of "こ"), bytesub must reject (#7).
|
||||
signalled = 1808;
|
||||
match (strings.bytesub("こんにちは", 1, 3)) {
|
||||
case let r: str => { fail(); };
|
||||
case let e: utf8.invalid => void;
|
||||
};
|
||||
// Symmetric: end lands on a continuation byte (2nd byte of "ん").
|
||||
signalled = 1809;
|
||||
match (strings.bytesub("こんにちは", 0, 4)) {
|
||||
case let r: str => { fail(); };
|
||||
case let e: utf8.invalid => void;
|
||||
};
|
||||
// end == s.len bypasses the continuation check (s[end] is OOB).
|
||||
signalled = 1810;
|
||||
match (strings.bytesub("こんにちは", 0, 15)) {
|
||||
case let s: str => { if (!streq(s, "こんにちは")) { fail(); }; };
|
||||
case let e: utf8.invalid => { fail(); };
|
||||
};
|
||||
};
|
||||
|
||||
// ---- toutf8 / fromutf8_unsafe roundtrip -------------------------------
|
||||
@@ -811,6 +858,97 @@ fn streq(a: str, b: str) bool = {
|
||||
if (r.ptr != s.ptr) { fail(); }; // borrowed, not copied
|
||||
};
|
||||
|
||||
// ---- fromutf8 ---------------------------------------------------------
|
||||
// ref/hare/strings/utf8.ha:22 + @test fn utf8 (line 31). Validating
|
||||
// sibling of fromutf8_unsafe — walks via [[utf8.validate]] and returns
|
||||
// `utf8.invalid` on any malformed byte sequence (#7).
|
||||
|
||||
@test fn fromutf8_cases() void = {
|
||||
// Hare vector — "hello world" round-trips.
|
||||
signalled = 1900;
|
||||
let hb: [11]u8;
|
||||
hb[0] = 0x68u8; hb[1] = 0x65u8; hb[2] = 0x6Cu8; hb[3] = 0x6Cu8;
|
||||
hb[4] = 0x6Fu8; hb[5] = 0x20u8; hb[6] = 0x77u8; hb[7] = 0x6Fu8;
|
||||
hb[8] = 0x72u8; hb[9] = 0x6Cu8; hb[10] = 0x64u8;
|
||||
match (strings.fromutf8(hb[0:11])) {
|
||||
case let s: str => { if (!streq(s, "hello world")) { fail(); }; };
|
||||
case let e: utf8.invalid => { fail(); };
|
||||
};
|
||||
|
||||
// Empty input — Hare's `fromutf8([])! == ""`.
|
||||
signalled = 1901;
|
||||
let eb: [1]u8;
|
||||
match (strings.fromutf8(eb[0:0])) {
|
||||
case let s: str => { if (s.len != 0) { fail(); }; };
|
||||
case let e: utf8.invalid => { fail(); };
|
||||
};
|
||||
|
||||
// ASCII-only borrowed view: ptr aliases input.
|
||||
signalled = 1902;
|
||||
let s: str = "abc";
|
||||
let bs: []u8 = strings.toutf8(s);
|
||||
match (strings.fromutf8(bs)) {
|
||||
case let r: str => {
|
||||
if (!streq(r, "abc")) { fail(); };
|
||||
if (r.ptr != s.ptr) { fail(); };
|
||||
};
|
||||
case let e: utf8.invalid => { fail(); };
|
||||
};
|
||||
|
||||
// Multibyte Japanese — full UTF-8 round-trip.
|
||||
signalled = 1903;
|
||||
let jb: [15]u8;
|
||||
jb[0] = 0xE3u8; jb[1] = 0x81u8; jb[2] = 0x93u8; // こ
|
||||
jb[3] = 0xE3u8; jb[4] = 0x82u8; jb[5] = 0x93u8; // ん
|
||||
jb[6] = 0xE3u8; jb[7] = 0x81u8; jb[8] = 0xABu8; // に
|
||||
jb[9] = 0xE3u8; jb[10] = 0x81u8; jb[11] = 0xA1u8; // ち
|
||||
jb[12] = 0xE3u8; jb[13] = 0x81u8; jb[14] = 0xAFu8; // は
|
||||
match (strings.fromutf8(jb[0:15])) {
|
||||
case let s: str => { if (!streq(s, "こんにちは")) { fail(); }; };
|
||||
case let e: utf8.invalid => { fail(); };
|
||||
};
|
||||
|
||||
// Multibyte French — mixed-width "héllo".
|
||||
signalled = 1904;
|
||||
let fb: [6]u8;
|
||||
fb[0] = 0x68u8; // h
|
||||
fb[1] = 0xC3u8; fb[2] = 0xA9u8; // é
|
||||
fb[3] = 0x6Cu8; fb[4] = 0x6Cu8; fb[5] = 0x6Fu8; // llo
|
||||
match (strings.fromutf8(fb[0:6])) {
|
||||
case let s: str => { if (!streq(s, "héllo")) { fail(); }; };
|
||||
case let e: utf8.invalid => { fail(); };
|
||||
};
|
||||
|
||||
// Lone 0x80 continuation byte — invalid.
|
||||
signalled = 1905;
|
||||
let c: [1]u8;
|
||||
c[0] = 0x80u8;
|
||||
match (strings.fromutf8(c[0:1])) {
|
||||
case let r: str => { fail(); };
|
||||
case let e: utf8.invalid => void;
|
||||
};
|
||||
|
||||
// Overlong 2-byte encoding of NUL (0xC0 0x80) — invalid per
|
||||
// ref/hare/encoding/utf8/decodetable.ha (state 0 rejects 0xC0).
|
||||
signalled = 1906;
|
||||
let ov: [2]u8;
|
||||
ov[0] = 0xC0u8; ov[1] = 0x80u8;
|
||||
match (strings.fromutf8(ov[0:2])) {
|
||||
case let r: str => { fail(); };
|
||||
case let e: utf8.invalid => void;
|
||||
};
|
||||
|
||||
// Truncated multibyte — 0xE3 0x81 (2 of 3 bytes for U+3053) is
|
||||
// rejected as invalid by [[utf8.validate]] (state != 0 at EOI).
|
||||
signalled = 1907;
|
||||
let tr: [2]u8;
|
||||
tr[0] = 0xE3u8; tr[1] = 0x81u8;
|
||||
match (strings.fromutf8(tr[0:2])) {
|
||||
case let r: str => { fail(); };
|
||||
case let e: utf8.invalid => void;
|
||||
};
|
||||
};
|
||||
|
||||
// ---- iter / next ------------------------------------------------------
|
||||
// ref/hare/strings/iter.ha:84-108. Hare's @test fn iter() uses prev +
|
||||
// riter heavily; both are deferred (no `utf8.prev`). Rebuild forward-
|
||||
@@ -1548,6 +1686,7 @@ export fn main() i32 = {
|
||||
signalled = 40; sub_cases();
|
||||
signalled = 41; bytesub_cases();
|
||||
signalled = 15; utf8_roundtrip_cases();
|
||||
signalled = 43; fromutf8_cases();
|
||||
signalled = 16; iter_empty_cases();
|
||||
signalled = 17; iter_ascii_cases();
|
||||
signalled = 18; iter_twobyte_cases();
|
||||
|
||||
Reference in New Issue
Block a user