diff --git a/lib/encoding/utf8/utf8.ww b/lib/encoding/utf8/utf8.ww index 402e9ace..05adc89c 100644 --- a/lib/encoding/utf8/utf8.ww +++ b/lib/encoding/utf8/utf8.ww @@ -14,10 +14,10 @@ // the caller-buffer form mirrors lib/encoding/hex.encode and // skips the static-buffer/slice-return pair. // -// Deferred (no in-tree caller, follow-up tasks): `prev`, `slice`, -// `position`, `remaining`, `appendrune`, `strencode`, `strdecode`. -// Hare's string-iteration surface (`strings::iterator`/`strings::next` -// — ref/hare/strings/iter.ha) lives under lib/strings, not here. +// Deferred (no in-tree caller, follow-up tasks): `appendrune`, +// `strencode`, `strdecode`. Hare's string-iteration surface +// (`strings::iterator`/`strings::next` — ref/hare/strings/iter.ha) +// lives under lib/strings, not here. // ref/hare/encoding/utf8/types.ha:6 — incomplete trailing sequence. // Plain `void` (not `!void`): a truncated tail is a control-flow @@ -339,3 +339,92 @@ export fn encoderune(out: []u8, r: rune) i32 = { return n; }; +// ref/hare/encoding/utf8/decode.ha:52. Walks back from `d.offs` to a +// byte that could start a codepoint (state-0 dfa cell != -1), re-decodes +// forward from there, and confirms the forward decode lands back at the +// original offset. Returns `done` at start-of-input; `invalid` if no +// initial byte appears within 4 steps (no legal UTF-8 codepoint exceeds +// 4 bytes), if the forward decode returns `more`/`invalid`, or if it +// lands at a different offset than expected. Returns `more` when the +// walk reaches byte 0 without finding any initial byte. +// +// Hare's `for (d.offs < len(d.src); d.offs -= 1)` relies on size_t +// wrap-around to exit when offs underflows past 0; ww's offs is i32, +// so we spell the same exit as `d.offs >= 0`. Hare's `defer d.offs = t` +// is inlined in each match arm — ww has no defer. +export fn prev(d: *decoder) (rune | done | more | invalid) = { + if (d.offs == 0) { + let dn: done; return dn; + }; + let n: i32 = d.offs; + d.offs -= 1; + for (d.offs >= 0) { + let b: u8 = d.src[d.offs]; + let bi: i32 = b: i32; + let cell: i8 = dfa[bi]; + if (cell: i32 != -1) { + let t: i32 = d.offs; + match (next(d)) { + case let r: rune => { + let landed: i32 = d.offs; + d.offs = t; + if (landed != n) { + let e: invalid; return e; + }; + return r; + }; + case let dn: done => { + d.offs = t; + let e: invalid; return e; + }; + case let m: more => { + d.offs = t; + let e: invalid; return e; + }; + case let e: invalid => { + d.offs = t; + let e2: invalid; return e2; + }; + }; + }; + if (n - d.offs == 4) { + let e: invalid; return e; + }; + d.offs -= 1; + }; + let mr: more; return mr; +}; + +// ref/hare/encoding/utf8/decode.ha:74. Borrowed view of the bytes from +// the decoder's current position to the end of its source. +export fn remaining(d: *decoder) []u8 = { + let r: []u8; + r.ptr = d.src.ptr + (d.offs: u64); + r.len = d.src.len - d.offs; + r.cap = d.src.len - d.offs; + return r; +}; + +// ref/hare/encoding/utf8/decode.ha:80. Borrowed view of the bytes +// between two decoders' positions. Precondition (Hare asserts both): +// the decoders share the same source, and `begin.offs <= end.offs`. +export fn slice(begin: *decoder, end: *decoder) []u8 = { + if (begin.src.ptr != end.src.ptr) { + abort("utf8.slice: decoders from different sources"); + }; + if (begin.offs > end.offs) { + abort("utf8.slice: begin past end"); + }; + let r: []u8; + r.ptr = begin.src.ptr + (begin.offs: u64); + r.len = end.offs - begin.offs; + r.cap = end.offs - begin.offs; + return r; +}; + +// ref/hare/encoding/utf8/decode.ha:203. Byte position of the decoder +// in its source. +export fn position(d: *decoder) i32 = { + return d.offs; +}; + diff --git a/lib/encoding/utf8/utf8test.ww b/lib/encoding/utf8/utf8test.ww index e40c3cf2..78cbefca 100644 --- a/lib/encoding/utf8/utf8test.ww +++ b/lib/encoding/utf8/utf8test.ww @@ -5,6 +5,7 @@ package utf8; +import bytes; import encoding.utf8; import os; @@ -292,6 +293,288 @@ fn fail() void = { os.exit(signalled + 10); }; }; }; +// ---- decoder.prev: done at start-of-input ----------------------------- +// ref/hare/encoding/utf8/decode.ha:53-55. Cursor at offs=0 has no prior +// codepoint. + +@test fn prev_done_at_start() void = { + let src: [4]u8; + src[0] = 0x41u8; + let d: utf8.decoder = utf8.decode(src[0:1]); + match (utf8.prev(&d)) { + case utf8.done => void; + case let r: rune => { fail(); }; + case utf8.more => { fail(); }; + case let e: utf8.invalid => { fail(); }; + }; +}; + +// ---- decoder.prev: each codepoint width round-trips ------------------- +// ref/hare/encoding/utf8/decode.ha:56-71. Forward-decode one rune, +// reverse-decode back to the same rune. + +@test fn prev_one_byte() void = { + let src: [1]u8; + src[0] = 0x41u8; + let d: utf8.decoder = utf8.decode(src[0:1]); + match (utf8.next(&d)) { + case let r: rune => void; + case => { fail(); }; + }; + match (utf8.prev(&d)) { + case let r: rune => { if (r != 0x41u32: rune) { fail(); }; }; + case => { fail(); }; + }; +}; + +@test fn prev_two_byte() void = { + let src: [2]u8; + src[0] = 0xC3u8; src[1] = 0xA9u8; // 'é' + let d: utf8.decoder = utf8.decode(src[0:2]); + match (utf8.next(&d)) { + case let r: rune => void; + case => { fail(); }; + }; + match (utf8.prev(&d)) { + case let r: rune => { if (r != 0xE9u32: rune) { fail(); }; }; + case => { fail(); }; + }; +}; + +@test fn prev_three_byte() void = { + let src: [3]u8; + src[0] = 0xE2u8; src[1] = 0x82u8; src[2] = 0xACu8; // '€' + let d: utf8.decoder = utf8.decode(src[0:3]); + match (utf8.next(&d)) { + case let r: rune => void; + case => { fail(); }; + }; + match (utf8.prev(&d)) { + case let r: rune => { if (r != 0x20ACu32: rune) { fail(); }; }; + case => { fail(); }; + }; +}; + +@test fn prev_four_byte() void = { + let src: [4]u8; + src[0] = 0xF0u8; src[1] = 0x9Fu8; src[2] = 0xA6u8; src[3] = 0x80u8; // '🦀' + let d: utf8.decoder = utf8.decode(src[0:4]); + match (utf8.next(&d)) { + case let r: rune => void; + case => { fail(); }; + }; + match (utf8.prev(&d)) { + case let r: rune => { if (r != 0x1F980u32: rune) { fail(); }; }; + case => { fail(); }; + }; +}; + +// ---- decoder.prev: full forward + full reverse on mixed input --------- +// ref/hare/encoding/utf8/decode.ha:85-111. After consuming all runes +// forward, prev walks back through them in reverse order; prev at the +// start-of-input then returns done. + +@test fn prev_mixed_roundtrip() void = { + let src: [16]u8; + src[0] = 0xE3u8; src[1] = 0x81u8; src[2] = 0x93u8; // こ + src[3] = 0xE3u8; src[4] = 0x82u8; src[5] = 0x93u8; // ん + src[6] = 0xE3u8; src[7] = 0x81u8; src[8] = 0xABu8; // に + src[9] = 0xE3u8; src[10] = 0x81u8; src[11] = 0xA1u8; // ち + src[12] = 0xE3u8; src[13] = 0x81u8; src[14] = 0xAFu8; // は + src[15] = 0u8; + let d: utf8.decoder = utf8.decode(src[0:16]); + let fwd: [6]u32; + let i: i32 = 0; + for (i < 6) { + match (utf8.next(&d)) { + case let r: rune => { fwd[i] = r: u32; }; + case => { fail(); }; + }; + i += 1; + }; + if (utf8.position(&d) != 16) { fail(); }; + i = 0; + for (i < 6) { + match (utf8.prev(&d)) { + case let r: rune => { + if ((r: u32) != fwd[5 - i]) { fail(); }; + }; + case => { fail(); }; + }; + i += 1; + }; + match (utf8.prev(&d)) { + case utf8.done => void; + case => { fail(); }; + }; +}; + +// ---- decoder.prev: invalid inputs ------------------------------------- +// ref/hare/encoding/utf8/decode.ha:113-150. Two-continuation walk +// reaches offs=0 without an initial byte → more; otherwise the forward +// re-decode catches the malformed run as invalid. + +// ref/hare/encoding/utf8/decode.ha:117 — `[0xA0, 0xA1]` is all +// continuation bytes; walking back from offs=2 finds no initial byte +// within the 4-step bound nor before offs underflows, so the loop +// exits via the `>= 0` guard and returns `more`. +@test fn prev_continuation_only_more() void = { + let src: [2]u8; + src[0] = 0xA0u8; src[1] = 0xA1u8; + let d: utf8.decoder = utf8.decode(src[0:2]); + d.offs = 2; + match (utf8.prev(&d)) { + case utf8.more => void; + case let r: rune => { fail(); }; + case utf8.done => { fail(); }; + case let e: utf8.invalid => { fail(); }; + }; +}; + +@test fn prev_incomplete_invalid() void = { + let src: [2]u8; + src[0] = 0xE3u8; src[1] = 0x81u8; + let d: utf8.decoder = utf8.decode(src[0:2]); + d.offs = 2; + match (utf8.prev(&d)) { + case let e: utf8.invalid => void; + case let r: rune => { fail(); }; + case utf8.done => { fail(); }; + case utf8.more => { fail(); }; + }; +}; + +@test fn prev_surrogate_invalid() void = { + let src: [3]u8; + src[0] = 0xEDu8; src[1] = 0xA0u8; src[2] = 0x80u8; + let d: utf8.decoder = utf8.decode(src[0:3]); + d.offs = 3; + match (utf8.prev(&d)) { + case let e: utf8.invalid => void; + case let r: rune => { fail(); }; + case utf8.done => { fail(); }; + case utf8.more => { fail(); }; + }; +}; + +@test fn prev_overlong_invalid() void = { + let src: [4]u8; + src[0] = 0xF0u8; src[1] = 0x82u8; src[2] = 0x82u8; src[3] = 0xACu8; + let d: utf8.decoder = utf8.decode(src[0:4]); + d.offs = 4; + match (utf8.prev(&d)) { + case let e: utf8.invalid => void; + case let r: rune => { fail(); }; + case utf8.done => { fail(); }; + case utf8.more => { fail(); }; + }; +}; + +// ref/hare/encoding/utf8/decode.ha:146 — extra continuation byte after +// a complete 2-byte codepoint. prev from offs=3 finds 0xC2 (initial +// byte), forward-decodes back to offs=2, but n=3 ≠ 2 → invalid. +@test fn prev_extracont_invalid() void = { + let src: [3]u8; + src[0] = 0xC2u8; src[1] = 0xA3u8; src[2] = 0x95u8; + let d: utf8.decoder = utf8.decode(src[0:3]); + d.offs = 3; + match (utf8.prev(&d)) { + case let e: utf8.invalid => void; + case let r: rune => { fail(); }; + case utf8.done => { fail(); }; + case utf8.more => { fail(); }; + }; +}; + +// ref/hare/encoding/utf8/decode.ha:158-163 — max in-range codepoint +// reverse-decoded. Pins state-7 acceptance via prev: from offs=4 the +// walk skips 3 continuations, lands on 0xF4 (state-7 lead), and the +// forward re-decode reproduces U+10FFFF. +@test fn prev_max_in_range() void = { + let src: [4]u8; + src[0] = 0xF4u8; src[1] = 0x8Fu8; src[2] = 0xBFu8; src[3] = 0xBFu8; + let d: utf8.decoder = utf8.decode(src[0:4]); + d.offs = 4; + match (utf8.prev(&d)) { + case let r: rune => { if (r != 0x10FFFFu32: rune) { fail(); }; }; + case let e: utf8.invalid => { fail(); }; + case utf8.done => { fail(); }; + case utf8.more => { fail(); }; + }; +}; + +// ref/hare/encoding/utf8/decode.ha:166-169 — `[0xF5, 0x94, 0x80, 0x80]`: +// 0xF5 is not a legal initial byte (dfa state-0 cell is -1). prev from +// offs=4 walks 4 bytes without finding an initial byte, trips the +// `n - d.offs == 4` bound, and returns invalid. The only test that +// exercises this branch (prev_continuation_only_more underflows past +// 0 before reaching 4). +@test fn prev_min_out_of_range() void = { + let src: [4]u8; + src[0] = 0xF5u8; src[1] = 0x94u8; src[2] = 0x80u8; src[3] = 0x80u8; + let d: utf8.decoder = utf8.decode(src[0:4]); + d.offs = 4; + match (utf8.prev(&d)) { + case let e: utf8.invalid => void; + case let r: rune => { fail(); }; + case utf8.done => { fail(); }; + case utf8.more => { fail(); }; + }; +}; + +// ---- remaining / slice / position ------------------------------------- +// ref/hare/encoding/utf8/decode.ha:172-198. + +@test fn remaining_slice_position() void = { + let src: [16]u8; + src[0] = 0xE3u8; src[1] = 0x81u8; src[2] = 0x93u8; // こ + src[3] = 0xE3u8; src[4] = 0x82u8; src[5] = 0x93u8; // ん + src[6] = 0xE3u8; src[7] = 0x81u8; src[8] = 0xABu8; // に + src[9] = 0xE3u8; src[10] = 0x81u8; src[11] = 0xA1u8; // ち + src[12] = 0xE3u8; src[13] = 0x81u8; src[14] = 0xAFu8; // は + src[15] = 0u8; + // Two decode() calls instead of `let d2 = d1` per Hare style — + // let-copy on struct >8B is a Class A cstage+wwstage miscompile + // (task #32; rhs ident is dropped, slot zero-inits). Both calls + // take the same src[0:16] slice so d1.src.ptr == d2.src.ptr, + // matching Hare's `d2 = d1` for slice()'s same-source precondition. + let d1: utf8.decoder = utf8.decode(src[0:16]); + let d2: utf8.decoder = utf8.decode(src[0:16]); + + if (!bytes.equal(utf8.remaining(&d1), src[0:16])) { fail(); }; + if (utf8.slice(&d1, &d2).len != 0) { fail(); }; + if (utf8.slice(&d2, &d1).len != 0) { fail(); }; + if (utf8.position(&d1) != 0) { fail(); }; + + let i: i32 = 0; + for (i < 2) { + match (utf8.next(&d1)) { case let r: rune => void; case => { fail(); }; }; + match (utf8.next(&d2)) { case let r: rune => void; case => { fail(); }; }; + i += 1; + }; + if (utf8.position(&d1) != 6) { fail(); }; + if (!bytes.equal(utf8.remaining(&d1), src[6:16])) { fail(); }; + if (utf8.slice(&d1, &d2).len != 0) { fail(); }; + + i = 0; + for (i < 3) { + match (utf8.next(&d2)) { case let r: rune => void; case => { fail(); }; }; + i += 1; + }; + if (utf8.position(&d2) != 15) { fail(); }; + if (!bytes.equal(utf8.remaining(&d2), src[15:16])) { fail(); }; + if (!bytes.equal(utf8.slice(&d1, &d2), src[6:15])) { fail(); }; + + i = 0; + for (i < 3) { + match (utf8.next(&d1)) { case let r: rune => void; case => { fail(); }; }; + i += 1; + }; + if (utf8.slice(&d1, &d2).len != 0) { fail(); }; + match (utf8.next(&d1)) { case let r: rune => void; case => { fail(); }; }; + if (utf8.remaining(&d1).len != 0) { fail(); }; +}; + // ---- round-trip: encode → decode → equal rune -------------------------- @test fn roundtrip() void = { @@ -339,5 +622,19 @@ export fn main() i32 = { signalled = 19; validate_malformed(); signalled = 20; validate_empty_ok(); signalled = 21; roundtrip(); + signalled = 22; prev_done_at_start(); + signalled = 23; prev_one_byte(); + signalled = 24; prev_two_byte(); + signalled = 25; prev_three_byte(); + signalled = 26; prev_four_byte(); + signalled = 27; prev_mixed_roundtrip(); + signalled = 28; prev_continuation_only_more(); + signalled = 29; prev_incomplete_invalid(); + signalled = 30; prev_surrogate_invalid(); + signalled = 31; prev_overlong_invalid(); + signalled = 32; prev_extracont_invalid(); + signalled = 33; prev_max_in_range(); + signalled = 34; prev_min_out_of_range(); + signalled = 35; remaining_slice_position(); return 0; }; diff --git a/selfhost/cmd/w6c/main.combined.ww b/selfhost/cmd/w6c/main.combined.ww index e5cf6590..14db4f3b 100644 --- a/selfhost/cmd/w6c/main.combined.ww +++ b/selfhost/cmd/w6c/main.combined.ww @@ -1025,10 +1025,10 @@ export fn zero(s: []u8) void = { // the caller-buffer form mirrors lib/encoding/hex.encode and // skips the static-buffer/slice-return pair. // -// Deferred (no in-tree caller, follow-up tasks): `prev`, `slice`, -// `position`, `remaining`, `appendrune`, `strencode`, `strdecode`. -// Hare's string-iteration surface (`strings::iterator`/`strings::next` -// — ref/hare/strings/iter.ha) lives under lib/strings, not here. +// Deferred (no in-tree caller, follow-up tasks): `appendrune`, +// `strencode`, `strdecode`. Hare's string-iteration surface +// (`strings::iterator`/`strings::next` — ref/hare/strings/iter.ha) +// lives under lib/strings, not here. // ref/hare/encoding/utf8/types.ha:6 — incomplete trailing sequence. // Plain `void` (not `!void`): a truncated tail is a control-flow @@ -1350,6 +1350,95 @@ export fn encoderune(out: []u8, r: rune) i32 = { return n; }; +// ref/hare/encoding/utf8/decode.ha:52. Walks back from `d.offs` to a +// byte that could start a codepoint (state-0 dfa cell != -1), re-decodes +// forward from there, and confirms the forward decode lands back at the +// original offset. Returns `done` at start-of-input; `invalid` if no +// initial byte appears within 4 steps (no legal UTF-8 codepoint exceeds +// 4 bytes), if the forward decode returns `more`/`invalid`, or if it +// lands at a different offset than expected. Returns `more` when the +// walk reaches byte 0 without finding any initial byte. +// +// Hare's `for (d.offs < len(d.src); d.offs -= 1)` relies on size_t +// wrap-around to exit when offs underflows past 0; ww's offs is i32, +// so we spell the same exit as `d.offs >= 0`. Hare's `defer d.offs = t` +// is inlined in each match arm — ww has no defer. +export fn prev(d: *decoder) (rune | done | more | invalid) = { + if (d.offs == 0) { + let dn: done; return dn; + }; + let n: i32 = d.offs; + d.offs -= 1; + for (d.offs >= 0) { + let b: u8 = d.src[d.offs]; + let bi: i32 = b: i32; + let cell: i8 = dfa[bi]; + if (cell: i32 != -1) { + let t: i32 = d.offs; + match (next(d)) { + case let r: rune => { + let landed: i32 = d.offs; + d.offs = t; + if (landed != n) { + let e: invalid; return e; + }; + return r; + }; + case let dn: done => { + d.offs = t; + let e: invalid; return e; + }; + case let m: more => { + d.offs = t; + let e: invalid; return e; + }; + case let e: invalid => { + d.offs = t; + let e2: invalid; return e2; + }; + }; + }; + if (n - d.offs == 4) { + let e: invalid; return e; + }; + d.offs -= 1; + }; + let mr: more; return mr; +}; + +// ref/hare/encoding/utf8/decode.ha:74. Borrowed view of the bytes from +// the decoder's current position to the end of its source. +export fn remaining(d: *decoder) []u8 = { + let r: []u8; + r.ptr = d.src.ptr + (d.offs: u64); + r.len = d.src.len - d.offs; + r.cap = d.src.len - d.offs; + return r; +}; + +// ref/hare/encoding/utf8/decode.ha:80. Borrowed view of the bytes +// between two decoders' positions. Precondition (Hare asserts both): +// the decoders share the same source, and `begin.offs <= end.offs`. +export fn slice(begin: *decoder, end: *decoder) []u8 = { + if (begin.src.ptr != end.src.ptr) { + abort("utf8.slice: decoders from different sources"); + }; + if (begin.offs > end.offs) { + abort("utf8.slice: begin past end"); + }; + let r: []u8; + r.ptr = begin.src.ptr + (begin.offs: u64); + r.len = end.offs - begin.offs; + r.cap = end.offs - begin.offs; + return r; +}; + +// ref/hare/encoding/utf8/decode.ha:203. Byte position of the decoder +// in its source. +export fn position(d: *decoder) i32 = { + return d.offs; +}; + // strings — operations over str ({ptr,len}). Hare port; see // ref/hare/strings/. diff --git a/selfhost/cmd/wwdump/main.combined.ww b/selfhost/cmd/wwdump/main.combined.ww index 4932c433..d3e64751 100644 --- a/selfhost/cmd/wwdump/main.combined.ww +++ b/selfhost/cmd/wwdump/main.combined.ww @@ -1025,10 +1025,10 @@ export fn zero(s: []u8) void = { // the caller-buffer form mirrors lib/encoding/hex.encode and // skips the static-buffer/slice-return pair. // -// Deferred (no in-tree caller, follow-up tasks): `prev`, `slice`, -// `position`, `remaining`, `appendrune`, `strencode`, `strdecode`. -// Hare's string-iteration surface (`strings::iterator`/`strings::next` -// — ref/hare/strings/iter.ha) lives under lib/strings, not here. +// Deferred (no in-tree caller, follow-up tasks): `appendrune`, +// `strencode`, `strdecode`. Hare's string-iteration surface +// (`strings::iterator`/`strings::next` — ref/hare/strings/iter.ha) +// lives under lib/strings, not here. // ref/hare/encoding/utf8/types.ha:6 — incomplete trailing sequence. // Plain `void` (not `!void`): a truncated tail is a control-flow @@ -1350,6 +1350,95 @@ export fn encoderune(out: []u8, r: rune) i32 = { return n; }; +// ref/hare/encoding/utf8/decode.ha:52. Walks back from `d.offs` to a +// byte that could start a codepoint (state-0 dfa cell != -1), re-decodes +// forward from there, and confirms the forward decode lands back at the +// original offset. Returns `done` at start-of-input; `invalid` if no +// initial byte appears within 4 steps (no legal UTF-8 codepoint exceeds +// 4 bytes), if the forward decode returns `more`/`invalid`, or if it +// lands at a different offset than expected. Returns `more` when the +// walk reaches byte 0 without finding any initial byte. +// +// Hare's `for (d.offs < len(d.src); d.offs -= 1)` relies on size_t +// wrap-around to exit when offs underflows past 0; ww's offs is i32, +// so we spell the same exit as `d.offs >= 0`. Hare's `defer d.offs = t` +// is inlined in each match arm — ww has no defer. +export fn prev(d: *decoder) (rune | done | more | invalid) = { + if (d.offs == 0) { + let dn: done; return dn; + }; + let n: i32 = d.offs; + d.offs -= 1; + for (d.offs >= 0) { + let b: u8 = d.src[d.offs]; + let bi: i32 = b: i32; + let cell: i8 = dfa[bi]; + if (cell: i32 != -1) { + let t: i32 = d.offs; + match (next(d)) { + case let r: rune => { + let landed: i32 = d.offs; + d.offs = t; + if (landed != n) { + let e: invalid; return e; + }; + return r; + }; + case let dn: done => { + d.offs = t; + let e: invalid; return e; + }; + case let m: more => { + d.offs = t; + let e: invalid; return e; + }; + case let e: invalid => { + d.offs = t; + let e2: invalid; return e2; + }; + }; + }; + if (n - d.offs == 4) { + let e: invalid; return e; + }; + d.offs -= 1; + }; + let mr: more; return mr; +}; + +// ref/hare/encoding/utf8/decode.ha:74. Borrowed view of the bytes from +// the decoder's current position to the end of its source. +export fn remaining(d: *decoder) []u8 = { + let r: []u8; + r.ptr = d.src.ptr + (d.offs: u64); + r.len = d.src.len - d.offs; + r.cap = d.src.len - d.offs; + return r; +}; + +// ref/hare/encoding/utf8/decode.ha:80. Borrowed view of the bytes +// between two decoders' positions. Precondition (Hare asserts both): +// the decoders share the same source, and `begin.offs <= end.offs`. +export fn slice(begin: *decoder, end: *decoder) []u8 = { + if (begin.src.ptr != end.src.ptr) { + abort("utf8.slice: decoders from different sources"); + }; + if (begin.offs > end.offs) { + abort("utf8.slice: begin past end"); + }; + let r: []u8; + r.ptr = begin.src.ptr + (begin.offs: u64); + r.len = end.offs - begin.offs; + r.cap = end.offs - begin.offs; + return r; +}; + +// ref/hare/encoding/utf8/decode.ha:203. Byte position of the decoder +// in its source. +export fn position(d: *decoder) i32 = { + return d.offs; +}; + // strings — operations over str ({ptr,len}). Hare port; see // ref/hare/strings/. diff --git a/selfhost/test/smoke.combined.ww b/selfhost/test/smoke.combined.ww index d8a51b2a..355cd4e3 100644 --- a/selfhost/test/smoke.combined.ww +++ b/selfhost/test/smoke.combined.ww @@ -916,10 +916,10 @@ export fn zero(s: []u8) void = { // the caller-buffer form mirrors lib/encoding/hex.encode and // skips the static-buffer/slice-return pair. // -// Deferred (no in-tree caller, follow-up tasks): `prev`, `slice`, -// `position`, `remaining`, `appendrune`, `strencode`, `strdecode`. -// Hare's string-iteration surface (`strings::iterator`/`strings::next` -// — ref/hare/strings/iter.ha) lives under lib/strings, not here. +// Deferred (no in-tree caller, follow-up tasks): `appendrune`, +// `strencode`, `strdecode`. Hare's string-iteration surface +// (`strings::iterator`/`strings::next` — ref/hare/strings/iter.ha) +// lives under lib/strings, not here. // ref/hare/encoding/utf8/types.ha:6 — incomplete trailing sequence. // Plain `void` (not `!void`): a truncated tail is a control-flow @@ -1241,6 +1241,95 @@ export fn encoderune(out: []u8, r: rune) i32 = { return n; }; +// ref/hare/encoding/utf8/decode.ha:52. Walks back from `d.offs` to a +// byte that could start a codepoint (state-0 dfa cell != -1), re-decodes +// forward from there, and confirms the forward decode lands back at the +// original offset. Returns `done` at start-of-input; `invalid` if no +// initial byte appears within 4 steps (no legal UTF-8 codepoint exceeds +// 4 bytes), if the forward decode returns `more`/`invalid`, or if it +// lands at a different offset than expected. Returns `more` when the +// walk reaches byte 0 without finding any initial byte. +// +// Hare's `for (d.offs < len(d.src); d.offs -= 1)` relies on size_t +// wrap-around to exit when offs underflows past 0; ww's offs is i32, +// so we spell the same exit as `d.offs >= 0`. Hare's `defer d.offs = t` +// is inlined in each match arm — ww has no defer. +export fn prev(d: *decoder) (rune | done | more | invalid) = { + if (d.offs == 0) { + let dn: done; return dn; + }; + let n: i32 = d.offs; + d.offs -= 1; + for (d.offs >= 0) { + let b: u8 = d.src[d.offs]; + let bi: i32 = b: i32; + let cell: i8 = dfa[bi]; + if (cell: i32 != -1) { + let t: i32 = d.offs; + match (next(d)) { + case let r: rune => { + let landed: i32 = d.offs; + d.offs = t; + if (landed != n) { + let e: invalid; return e; + }; + return r; + }; + case let dn: done => { + d.offs = t; + let e: invalid; return e; + }; + case let m: more => { + d.offs = t; + let e: invalid; return e; + }; + case let e: invalid => { + d.offs = t; + let e2: invalid; return e2; + }; + }; + }; + if (n - d.offs == 4) { + let e: invalid; return e; + }; + d.offs -= 1; + }; + let mr: more; return mr; +}; + +// ref/hare/encoding/utf8/decode.ha:74. Borrowed view of the bytes from +// the decoder's current position to the end of its source. +export fn remaining(d: *decoder) []u8 = { + let r: []u8; + r.ptr = d.src.ptr + (d.offs: u64); + r.len = d.src.len - d.offs; + r.cap = d.src.len - d.offs; + return r; +}; + +// ref/hare/encoding/utf8/decode.ha:80. Borrowed view of the bytes +// between two decoders' positions. Precondition (Hare asserts both): +// the decoders share the same source, and `begin.offs <= end.offs`. +export fn slice(begin: *decoder, end: *decoder) []u8 = { + if (begin.src.ptr != end.src.ptr) { + abort("utf8.slice: decoders from different sources"); + }; + if (begin.offs > end.offs) { + abort("utf8.slice: begin past end"); + }; + let r: []u8; + r.ptr = begin.src.ptr + (begin.offs: u64); + r.len = end.offs - begin.offs; + r.cap = end.offs - begin.offs; + return r; +}; + +// ref/hare/encoding/utf8/decode.ha:203. Byte position of the decoder +// in its source. +export fn position(d: *decoder) i32 = { + return d.offs; +}; + // strings — operations over str ({ptr,len}). Hare port; see // ref/hare/strings/.