From 6d006da26c7b2a84aa0feabaa6ef5ad9dd10ced2 Mon Sep 17 00:00:00 2001 From: Hojun-Cho Date: Wed, 20 May 2026 02:02:28 +0900 Subject: [PATCH] lib/strings+test: graduate fromutf8 + bytesub to validating return MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit fromutf8(in: []u8) (str | utf8.invalid) and the bytesub form per ref/hare/strings/utf8.ha:22 and sub.ha:59. bytesub keeps its byte asserts (ww extension over Hare; predates #7). fromutf8 walks the utf8 decoder via utf8.next rather than the shorter `utf8.validate(in)?` form. Two compiler bugs in the way: cross-shape `(void | invalid) → (str | invalid)` propagation is #19, and (void | !void) match-bind locals diverge between stages / str→union lift SIGSEGVs in cstage — both filed as #48. The decoder-walk form bypasses both and matches what ref/hare/strings/utf8.ha actually does in source. getopt.ww:314 caller updated to match the new (str | invalid) return; bi+1 cannot hit a continuation byte in well-formed argv (bi is a just-matched ASCII flag), so abort spells the precondition. bytesub_cases rewritten as exhaustive match; new rows cover start-on-continuation and end-on-continuation invalid arms plus an end==s.len bypass. fromutf8_cases is new — Hare vector + edge bytes + multibyte parity rows. --- lib/getopt/getopt.ww | 15 ++- lib/strings/strings.ww | 52 ++++++++- lib/strings/stringstest.ww | 163 +++++++++++++++++++++++++-- selfhost/cmd/w6c/main.combined.ww | 52 ++++++++- selfhost/cmd/wwdump/main.combined.ww | 52 ++++++++- selfhost/test/smoke.combined.ww | 52 ++++++++- 6 files changed, 357 insertions(+), 29 deletions(-) diff --git a/lib/getopt/getopt.ww b/lib/getopt/getopt.ww index 1e478570..22721d16 100644 --- a/lib/getopt/getopt.ww +++ b/lib/getopt/getopt.ww @@ -93,6 +93,7 @@ package getopt; +import encoding.utf8; import os; import strings; @@ -311,7 +312,19 @@ export fn tryparse(out: *command, argv: []str, help: []help) (void | error) = { } else { // PARAM: glued value, or next argv slot. if (bi + 1 < arg.len) { - let v: str = strings.bytesub(arg, bi + 1, arg.len); + // strings.bytesub now validates rune + // boundaries (#7). `bi` is the byte + // offset of the just-matched ASCII flag + // char, so `bi + 1` cannot land on a + // continuation byte in well-formed argv; + // abort spells the well-formedness + // precondition as Hare's `!` does. + let v: str; + match (strings.bytesub(arg, bi + 1, arg.len)) { + case let s: str => v = s; + case utf8.invalid => + abort("getopt: malformed argv UTF-8"); + }; appendoption(&opts, r, v); advanced = true; } else { diff --git a/lib/strings/strings.ww b/lib/strings/strings.ww index 7d5ba2f0..af527d89 100644 --- a/lib/strings/strings.ww +++ b/lib/strings/strings.ww @@ -48,6 +48,39 @@ export fn fromutf8_unsafe(in: []u8) str = { return r; }; +// fromutf8 — validating sibling of [[fromutf8_unsafe]]. Returns the +// borrowed str view on success, or `utf8.invalid` if `in` contains a +// malformed UTF-8 sequence. ref/hare/strings/utf8.ha:22 (#7). +// +// Hare's spelling is `utf8::validate(in)?; return fromutf8_unsafe(in)`. +// ww calls `utf8.validate` and matches its `(void | utf8.invalid)` +// result instead of `?` (cross-shape propagation is #19). Even with +// match, the call diverges cstage/wwstage when the result is bound to +// a local (#48: wwstage spills DX for the payload-less arm) and the +// success path then SIGSEGVs lifting `str` to `(str | utf8.invalid)`. +// So the validation is open-coded against the decoder — the same +// byte-by-byte DFA walk `utf8.validate` performs (ref/hare/encoding/ +// utf8/decode.ha:207) — whose `(rune | done | more | invalid)` arm +// matches the failure axis of fromutf8 and lifts cleanly. Collapses +// to `utf8.validate(in)?` once #19 + #48 are fixed. +export fn fromutf8(in: []u8) (str | utf8.invalid) = { + let d: utf8.decoder = utf8.decode(in); + for (true) { + match (utf8.next(&d)) { + case let r: rune => void; + case utf8.done => { + let r: str = fromutf8_unsafe(in); + return r; + }; + case utf8.more => { + let e: utf8.invalid; return e; + }; + case let e: utf8.invalid => { return e; }; + }; + }; + let e: utf8.invalid; return e; +}; + // compare — three-way bytewise codepoint-order comparison. Return is // a sign (neg/zero/pos), not an index, so it tracks Hare's `int` // rather than the str-index i32 (#8). ref/hare/strings/compare.ha:12. @@ -235,12 +268,23 @@ export fn sub(s: str, start: i32, end: i32) str = { }; // bytesub — borrowed substring [start, end) where start/end are byte -// offsets. ref/hare/strings/sub.ha:59. Hare's validating return -// `(str | utf8::invalid)` is dropped (filed as #7); a multibyte-split -// argument will yield a sub-string whose bytes don't decode cleanly. -export fn bytesub(s: str, start: i32, end: i32) str = { +// offsets. ref/hare/strings/sub.ha:59 (#7). Returns `utf8.invalid` if +// either endpoint lands on a continuation byte (would split a +// codepoint); the equivalent Hare predicate is `s[i] & 0xc0 == 0x80` +// at ref/hare/strings/sub.ha:72-73. +export fn bytesub(s: str, start: i32, end: i32) (str | utf8.invalid) = { os.assert(start <= end, "strings.bytesub: start is higher than end"); os.assert(end <= s.len, "strings.bytesub: end exceeds string length"); + if (start < s.len) { + if ((s[start] & 0xC0u8) == 0x80u8) { + let e: utf8.invalid; return e; + }; + }; + if (end < s.len) { + if ((s[end] & 0xC0u8) == 0x80u8) { + let e: utf8.invalid; return e; + }; + }; let r: str; r.ptr = s.ptr + (start: u64); r.len = end - start; diff --git a/lib/strings/stringstest.ww b/lib/strings/stringstest.ww index 893e4bc3..dc8a1210 100644 --- a/lib/strings/stringstest.ww +++ b/lib/strings/stringstest.ww @@ -742,8 +742,7 @@ fn streq(a: str, b: str) bool = { // ---- sub / bytesub ---------------------------------------------------- // ref/hare/strings/sub.ha:44 (@test fn sub), :79 (@test fn bytesub). Hare's // 2-arg `sub(s, start)` rows are omitted: ww has no default-parameter -// syntax (filed as #37). The `is utf8::invalid` row for bytesub is -// deferred to the validating port (#7). +// syntax (filed as #37). bytesub now validates rune boundaries (#7). @test fn sub_cases() void = { signalled = 1790; @@ -770,32 +769,80 @@ fn streq(a: str, b: str) bool = { if (!streq(strings.sub("héllo", 0, 5), "héllo")) { fail(); }; }; +// Match-shape mirrors Hare's `bytesub(...)!` at ref/hare/strings/sub.ha: +// 80-86. Inlined at each call site (rather than a helper that takes +// `(str | utf8.invalid)` by value) because the union-by-value path +// crashes — same lift-on-pass-by-value family as #48. @test fn bytesub_cases() void = { signalled = 1800; - if (!streq(strings.bytesub("a string", 0, 8), "a string")) { fail(); }; + match (strings.bytesub("a string", 0, 8)) { + case let s: str => { if (!streq(s, "a string")) { fail(); }; }; + case let e: utf8.invalid => { fail(); }; + }; signalled = 1801; - if (!streq(strings.bytesub("a string", 0, 1), "a")) { fail(); }; + match (strings.bytesub("a string", 0, 1)) { + case let s: str => { if (!streq(s, "a")) { fail(); }; }; + case let e: utf8.invalid => { fail(); }; + }; signalled = 1802; - if (!streq(strings.bytesub("a string", 0, 3), "a s")) { fail(); }; + match (strings.bytesub("a string", 0, 3)) { + case let s: str => { if (!streq(s, "a s")) { fail(); }; }; + case let e: utf8.invalid => { fail(); }; + }; signalled = 1803; - if (!streq(strings.bytesub("a string", 2, 8), "string")) { fail(); }; + match (strings.bytesub("a string", 2, 8)) { + case let s: str => { if (!streq(s, "string")) { fail(); }; }; + case let e: utf8.invalid => { fail(); }; + }; signalled = 1804; - if (!streq(strings.bytesub("a string", 4, 4), "")) { fail(); }; + match (strings.bytesub("a string", 4, 4)) { + case let s: str => { if (!streq(s, "")) { fail(); }; }; + case let e: utf8.invalid => { fail(); }; + }; // Hare vector — byte indices 3..9 over "こんにちは" select "んに". signalled = 1805; - if (!streq(strings.bytesub("こんにちは", 3, 9), "んに")) { fail(); }; + match (strings.bytesub("こんにちは", 3, 9)) { + case let s: str => { if (!streq(s, "んに")) { fail(); }; }; + case let e: utf8.invalid => { fail(); }; + }; // Rune/byte axis disagree on identical args (#3): sub(s,0,3) walks 3 // runes and yields 9 bytes; bytesub(s,0,3) yields the first 3 bytes // — one 3-byte codepoint. signalled = 1806; if (!streq(strings.sub("こんにちは", 0, 3), "こんに")) { fail(); }; - if (!streq(strings.bytesub("こんにちは", 0, 3), "こ")) { fail(); }; + match (strings.bytesub("こんにちは", 0, 3)) { + case let s: str => { if (!streq(s, "こ")) { fail(); }; }; + case let e: utf8.invalid => { fail(); }; + }; // Borrowed view: ptr aliases input. signalled = 1807; let s: str = "hello"; - let r: str = strings.bytesub(s, 1, 4); - if (r.ptr != s.ptr + 1u64) { fail(); }; - if (r.len != 3) { fail(); }; + match (strings.bytesub(s, 1, 4)) { + case let r: str => { + if (r.ptr != s.ptr + 1u64) { fail(); }; + if (r.len != 3) { fail(); }; + }; + case let e: utf8.invalid => { fail(); }; + }; + // Hare's invalid row (ref/hare/strings/sub.ha:87) — start lands on + // a continuation byte (2nd byte of "こ"), bytesub must reject (#7). + signalled = 1808; + match (strings.bytesub("こんにちは", 1, 3)) { + case let r: str => { fail(); }; + case let e: utf8.invalid => void; + }; + // Symmetric: end lands on a continuation byte (2nd byte of "ん"). + signalled = 1809; + match (strings.bytesub("こんにちは", 0, 4)) { + case let r: str => { fail(); }; + case let e: utf8.invalid => void; + }; + // end == s.len bypasses the continuation check (s[end] is OOB). + signalled = 1810; + match (strings.bytesub("こんにちは", 0, 15)) { + case let s: str => { if (!streq(s, "こんにちは")) { fail(); }; }; + case let e: utf8.invalid => { fail(); }; + }; }; // ---- toutf8 / fromutf8_unsafe roundtrip ------------------------------- @@ -811,6 +858,97 @@ fn streq(a: str, b: str) bool = { if (r.ptr != s.ptr) { fail(); }; // borrowed, not copied }; +// ---- fromutf8 --------------------------------------------------------- +// ref/hare/strings/utf8.ha:22 + @test fn utf8 (line 31). Validating +// sibling of fromutf8_unsafe — walks via [[utf8.validate]] and returns +// `utf8.invalid` on any malformed byte sequence (#7). + +@test fn fromutf8_cases() void = { + // Hare vector — "hello world" round-trips. + signalled = 1900; + let hb: [11]u8; + hb[0] = 0x68u8; hb[1] = 0x65u8; hb[2] = 0x6Cu8; hb[3] = 0x6Cu8; + hb[4] = 0x6Fu8; hb[5] = 0x20u8; hb[6] = 0x77u8; hb[7] = 0x6Fu8; + hb[8] = 0x72u8; hb[9] = 0x6Cu8; hb[10] = 0x64u8; + match (strings.fromutf8(hb[0:11])) { + case let s: str => { if (!streq(s, "hello world")) { fail(); }; }; + case let e: utf8.invalid => { fail(); }; + }; + + // Empty input — Hare's `fromutf8([])! == ""`. + signalled = 1901; + let eb: [1]u8; + match (strings.fromutf8(eb[0:0])) { + case let s: str => { if (s.len != 0) { fail(); }; }; + case let e: utf8.invalid => { fail(); }; + }; + + // ASCII-only borrowed view: ptr aliases input. + signalled = 1902; + let s: str = "abc"; + let bs: []u8 = strings.toutf8(s); + match (strings.fromutf8(bs)) { + case let r: str => { + if (!streq(r, "abc")) { fail(); }; + if (r.ptr != s.ptr) { fail(); }; + }; + case let e: utf8.invalid => { fail(); }; + }; + + // Multibyte Japanese — full UTF-8 round-trip. + signalled = 1903; + let jb: [15]u8; + jb[0] = 0xE3u8; jb[1] = 0x81u8; jb[2] = 0x93u8; // こ + jb[3] = 0xE3u8; jb[4] = 0x82u8; jb[5] = 0x93u8; // ん + jb[6] = 0xE3u8; jb[7] = 0x81u8; jb[8] = 0xABu8; // に + jb[9] = 0xE3u8; jb[10] = 0x81u8; jb[11] = 0xA1u8; // ち + jb[12] = 0xE3u8; jb[13] = 0x81u8; jb[14] = 0xAFu8; // は + match (strings.fromutf8(jb[0:15])) { + case let s: str => { if (!streq(s, "こんにちは")) { fail(); }; }; + case let e: utf8.invalid => { fail(); }; + }; + + // Multibyte French — mixed-width "héllo". + signalled = 1904; + let fb: [6]u8; + fb[0] = 0x68u8; // h + fb[1] = 0xC3u8; fb[2] = 0xA9u8; // é + fb[3] = 0x6Cu8; fb[4] = 0x6Cu8; fb[5] = 0x6Fu8; // llo + match (strings.fromutf8(fb[0:6])) { + case let s: str => { if (!streq(s, "héllo")) { fail(); }; }; + case let e: utf8.invalid => { fail(); }; + }; + + // Lone 0x80 continuation byte — invalid. + signalled = 1905; + let c: [1]u8; + c[0] = 0x80u8; + match (strings.fromutf8(c[0:1])) { + case let r: str => { fail(); }; + case let e: utf8.invalid => void; + }; + + // Overlong 2-byte encoding of NUL (0xC0 0x80) — invalid per + // ref/hare/encoding/utf8/decodetable.ha (state 0 rejects 0xC0). + signalled = 1906; + let ov: [2]u8; + ov[0] = 0xC0u8; ov[1] = 0x80u8; + match (strings.fromutf8(ov[0:2])) { + case let r: str => { fail(); }; + case let e: utf8.invalid => void; + }; + + // Truncated multibyte — 0xE3 0x81 (2 of 3 bytes for U+3053) is + // rejected as invalid by [[utf8.validate]] (state != 0 at EOI). + signalled = 1907; + let tr: [2]u8; + tr[0] = 0xE3u8; tr[1] = 0x81u8; + match (strings.fromutf8(tr[0:2])) { + case let r: str => { fail(); }; + case let e: utf8.invalid => void; + }; +}; + // ---- iter / next ------------------------------------------------------ // ref/hare/strings/iter.ha:84-108. Hare's @test fn iter() uses prev + // riter heavily; both are deferred (no `utf8.prev`). Rebuild forward- @@ -1548,6 +1686,7 @@ export fn main() i32 = { signalled = 40; sub_cases(); signalled = 41; bytesub_cases(); signalled = 15; utf8_roundtrip_cases(); + signalled = 43; fromutf8_cases(); signalled = 16; iter_empty_cases(); signalled = 17; iter_ascii_cases(); signalled = 18; iter_twobyte_cases(); diff --git a/selfhost/cmd/w6c/main.combined.ww b/selfhost/cmd/w6c/main.combined.ww index dfc6cef9..f88d7d48 100644 --- a/selfhost/cmd/w6c/main.combined.ww +++ b/selfhost/cmd/w6c/main.combined.ww @@ -1885,6 +1885,39 @@ export fn fromutf8_unsafe(in: []u8) str = { return r; }; +// fromutf8 — validating sibling of [[fromutf8_unsafe]]. Returns the +// borrowed str view on success, or `utf8.invalid` if `in` contains a +// malformed UTF-8 sequence. ref/hare/strings/utf8.ha:22 (#7). +// +// Hare's spelling is `utf8::validate(in)?; return fromutf8_unsafe(in)`. +// ww calls `utf8.validate` and matches its `(void | utf8.invalid)` +// result instead of `?` (cross-shape propagation is #19). Even with +// match, the call diverges cstage/wwstage when the result is bound to +// a local (#48: wwstage spills DX for the payload-less arm) and the +// success path then SIGSEGVs lifting `str` to `(str | utf8.invalid)`. +// So the validation is open-coded against the decoder — the same +// byte-by-byte DFA walk `utf8.validate` performs (ref/hare/encoding/ +// utf8/decode.ha:207) — whose `(rune | done | more | invalid)` arm +// matches the failure axis of fromutf8 and lifts cleanly. Collapses +// to `utf8.validate(in)?` once #19 + #48 are fixed. +export fn fromutf8(in: []u8) (str | utf8.invalid) = { + let d: utf8.decoder = utf8.decode(in); + for (true) { + match (utf8.next(&d)) { + case let r: rune => void; + case utf8.done => { + let r: str = fromutf8_unsafe(in); + return r; + }; + case utf8.more => { + let e: utf8.invalid; return e; + }; + case let e: utf8.invalid => { return e; }; + }; + }; + let e: utf8.invalid; return e; +}; + // compare — three-way bytewise codepoint-order comparison. Return is // a sign (neg/zero/pos), not an index, so it tracks Hare's `int` // rather than the str-index i32 (#8). ref/hare/strings/compare.ha:12. @@ -2072,12 +2105,23 @@ export fn sub(s: str, start: i32, end: i32) str = { }; // bytesub — borrowed substring [start, end) where start/end are byte -// offsets. ref/hare/strings/sub.ha:59. Hare's validating return -// `(str | utf8::invalid)` is dropped (filed as #7); a multibyte-split -// argument will yield a sub-string whose bytes don't decode cleanly. -export fn bytesub(s: str, start: i32, end: i32) str = { +// offsets. ref/hare/strings/sub.ha:59 (#7). Returns `utf8.invalid` if +// either endpoint lands on a continuation byte (would split a +// codepoint); the equivalent Hare predicate is `s[i] & 0xc0 == 0x80` +// at ref/hare/strings/sub.ha:72-73. +export fn bytesub(s: str, start: i32, end: i32) (str | utf8.invalid) = { os.assert(start <= end, "strings.bytesub: start is higher than end"); os.assert(end <= s.len, "strings.bytesub: end exceeds string length"); + if (start < s.len) { + if ((s[start] & 0xC0u8) == 0x80u8) { + let e: utf8.invalid; return e; + }; + }; + if (end < s.len) { + if ((s[end] & 0xC0u8) == 0x80u8) { + let e: utf8.invalid; return e; + }; + }; let r: str; r.ptr = s.ptr + (start: u64); r.len = end - start; diff --git a/selfhost/cmd/wwdump/main.combined.ww b/selfhost/cmd/wwdump/main.combined.ww index 56d57ca3..e825417d 100644 --- a/selfhost/cmd/wwdump/main.combined.ww +++ b/selfhost/cmd/wwdump/main.combined.ww @@ -1885,6 +1885,39 @@ export fn fromutf8_unsafe(in: []u8) str = { return r; }; +// fromutf8 — validating sibling of [[fromutf8_unsafe]]. Returns the +// borrowed str view on success, or `utf8.invalid` if `in` contains a +// malformed UTF-8 sequence. ref/hare/strings/utf8.ha:22 (#7). +// +// Hare's spelling is `utf8::validate(in)?; return fromutf8_unsafe(in)`. +// ww calls `utf8.validate` and matches its `(void | utf8.invalid)` +// result instead of `?` (cross-shape propagation is #19). Even with +// match, the call diverges cstage/wwstage when the result is bound to +// a local (#48: wwstage spills DX for the payload-less arm) and the +// success path then SIGSEGVs lifting `str` to `(str | utf8.invalid)`. +// So the validation is open-coded against the decoder — the same +// byte-by-byte DFA walk `utf8.validate` performs (ref/hare/encoding/ +// utf8/decode.ha:207) — whose `(rune | done | more | invalid)` arm +// matches the failure axis of fromutf8 and lifts cleanly. Collapses +// to `utf8.validate(in)?` once #19 + #48 are fixed. +export fn fromutf8(in: []u8) (str | utf8.invalid) = { + let d: utf8.decoder = utf8.decode(in); + for (true) { + match (utf8.next(&d)) { + case let r: rune => void; + case utf8.done => { + let r: str = fromutf8_unsafe(in); + return r; + }; + case utf8.more => { + let e: utf8.invalid; return e; + }; + case let e: utf8.invalid => { return e; }; + }; + }; + let e: utf8.invalid; return e; +}; + // compare — three-way bytewise codepoint-order comparison. Return is // a sign (neg/zero/pos), not an index, so it tracks Hare's `int` // rather than the str-index i32 (#8). ref/hare/strings/compare.ha:12. @@ -2072,12 +2105,23 @@ export fn sub(s: str, start: i32, end: i32) str = { }; // bytesub — borrowed substring [start, end) where start/end are byte -// offsets. ref/hare/strings/sub.ha:59. Hare's validating return -// `(str | utf8::invalid)` is dropped (filed as #7); a multibyte-split -// argument will yield a sub-string whose bytes don't decode cleanly. -export fn bytesub(s: str, start: i32, end: i32) str = { +// offsets. ref/hare/strings/sub.ha:59 (#7). Returns `utf8.invalid` if +// either endpoint lands on a continuation byte (would split a +// codepoint); the equivalent Hare predicate is `s[i] & 0xc0 == 0x80` +// at ref/hare/strings/sub.ha:72-73. +export fn bytesub(s: str, start: i32, end: i32) (str | utf8.invalid) = { os.assert(start <= end, "strings.bytesub: start is higher than end"); os.assert(end <= s.len, "strings.bytesub: end exceeds string length"); + if (start < s.len) { + if ((s[start] & 0xC0u8) == 0x80u8) { + let e: utf8.invalid; return e; + }; + }; + if (end < s.len) { + if ((s[end] & 0xC0u8) == 0x80u8) { + let e: utf8.invalid; return e; + }; + }; let r: str; r.ptr = s.ptr + (start: u64); r.len = end - start; diff --git a/selfhost/test/smoke.combined.ww b/selfhost/test/smoke.combined.ww index 40310474..39e9693c 100644 --- a/selfhost/test/smoke.combined.ww +++ b/selfhost/test/smoke.combined.ww @@ -1776,6 +1776,39 @@ export fn fromutf8_unsafe(in: []u8) str = { return r; }; +// fromutf8 — validating sibling of [[fromutf8_unsafe]]. Returns the +// borrowed str view on success, or `utf8.invalid` if `in` contains a +// malformed UTF-8 sequence. ref/hare/strings/utf8.ha:22 (#7). +// +// Hare's spelling is `utf8::validate(in)?; return fromutf8_unsafe(in)`. +// ww calls `utf8.validate` and matches its `(void | utf8.invalid)` +// result instead of `?` (cross-shape propagation is #19). Even with +// match, the call diverges cstage/wwstage when the result is bound to +// a local (#48: wwstage spills DX for the payload-less arm) and the +// success path then SIGSEGVs lifting `str` to `(str | utf8.invalid)`. +// So the validation is open-coded against the decoder — the same +// byte-by-byte DFA walk `utf8.validate` performs (ref/hare/encoding/ +// utf8/decode.ha:207) — whose `(rune | done | more | invalid)` arm +// matches the failure axis of fromutf8 and lifts cleanly. Collapses +// to `utf8.validate(in)?` once #19 + #48 are fixed. +export fn fromutf8(in: []u8) (str | utf8.invalid) = { + let d: utf8.decoder = utf8.decode(in); + for (true) { + match (utf8.next(&d)) { + case let r: rune => void; + case utf8.done => { + let r: str = fromutf8_unsafe(in); + return r; + }; + case utf8.more => { + let e: utf8.invalid; return e; + }; + case let e: utf8.invalid => { return e; }; + }; + }; + let e: utf8.invalid; return e; +}; + // compare — three-way bytewise codepoint-order comparison. Return is // a sign (neg/zero/pos), not an index, so it tracks Hare's `int` // rather than the str-index i32 (#8). ref/hare/strings/compare.ha:12. @@ -1963,12 +1996,23 @@ export fn sub(s: str, start: i32, end: i32) str = { }; // bytesub — borrowed substring [start, end) where start/end are byte -// offsets. ref/hare/strings/sub.ha:59. Hare's validating return -// `(str | utf8::invalid)` is dropped (filed as #7); a multibyte-split -// argument will yield a sub-string whose bytes don't decode cleanly. -export fn bytesub(s: str, start: i32, end: i32) str = { +// offsets. ref/hare/strings/sub.ha:59 (#7). Returns `utf8.invalid` if +// either endpoint lands on a continuation byte (would split a +// codepoint); the equivalent Hare predicate is `s[i] & 0xc0 == 0x80` +// at ref/hare/strings/sub.ha:72-73. +export fn bytesub(s: str, start: i32, end: i32) (str | utf8.invalid) = { os.assert(start <= end, "strings.bytesub: start is higher than end"); os.assert(end <= s.len, "strings.bytesub: end exceeds string length"); + if (start < s.len) { + if ((s[start] & 0xC0u8) == 0x80u8) { + let e: utf8.invalid; return e; + }; + }; + if (end < s.len) { + if ((s[end] & 0xC0u8) == 0x80u8) { + let e: utf8.invalid; return e; + }; + }; let r: str; r.ptr = s.ptr + (start: u64); r.len = end - start;