Hojun-Cho/ww
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

lib/encoding/utf8+test: Hare port (prev + slice + position + remaining)

Browse Source 
Port four deferred utf8 functions per c2 backlog (utf8.ww:18
pre-port). All are non-trivial enough that the test rows mirror
Hare's @test fn decode/slice bodies (ref/hare/encoding/utf8/
decode.ha:85-198) row-for-row.

  - prev       ref/hare/encoding/utf8/decode.ha:52-71
  - remaining  ref/hare/encoding/utf8/decode.ha:74
  - slice      ref/hare/encoding/utf8/decode.ha:80-83
  - position   ref/hare/encoding/utf8/decode.ha:203

prev walks back from d.offs to a byte that could start a codepoint
(state-0 dfa cell != -1), re-decodes forward from there, and
confirms the forward decode lands back at the original offset.
Returns done at start-of-input; invalid if no initial byte appears
within 4 steps (no legal UTF-8 codepoint exceeds 4 bytes) or if
the forward decode shortcircuits to more/invalid or lands at a
different offset than expected.

Two structural deltas from the Hare source:

  - Hare's `for (d.offs < len(d.src); d.offs -= 1)` relies on
    size_t wrap-around to exit when offs underflows past 0; ww's
    offs is i32 (utf8.ww:204), so the same exit is spelled
    `d.offs >= 0`.

  - Hare's `defer d.offs = t` restores offs after the return; ww
    has no defer, so the restore is inlined in each match arm.

slice asserts the Hare precondition (same source + begin.offs <=
end.offs) via abort; Hare uses assert(). position is a one-liner
returning d.offs (Hare uses size, ww uses i32 per lib/CLAUDE.md
"indices use the underlying length type").

The `bi: i32 = b: i32;` indirection in prev's dfa lookup is
required because `dfa[b: i32]` parses as a slice expression
`dfa[b : i32]` where `i32` becomes the upper bound. The let-binding
matches the existing pattern at utf8.ww:236-238 in the c1 next()
port.

Tests: 14 new @test fns in utf8test.ww (signalled 22-35):
prev_done_at_start, prev_one/two/three/four_byte (round-trip
forward+reverse), prev_mixed_roundtrip (full forward then full
reverse on the same こんにちは+NUL input Hare uses at
decode.ha:85-111), prev_continuation_only_more (Hare's
[0xA0,0xA1] more case at decode.ha:117), prev_incomplete /
surrogate / overlong / extracont_invalid (decode.ha:120-150),
prev_max_in_range (decode.ha:158-163; pins state-7 acceptance via
reverse decode), prev_min_out_of_range (decode.ha:166-169; the
only case that trips prev's 4-step-bound arm), and
remaining_slice_position mirroring decode.ha:172-198.

The Hare slice @test idiom `let d2 = d1` (struct copy) miscompiles
in both stages (cstage + wwstage zero-init the rhs instead of
copying — task #32, Class A but bootstrap-byte-id-symmetric, so
995 doesn't catch it). The ww test uses two parallel
`decode(src[0:16])` calls to produce two decoders with the same
src.ptr; coverage on slice() is equivalent (exercises the same
source-different-offs pattern). Divergence cited inline.

117/117 ok. 995_self_rebuild stays green (ww2==ww3==ww4 byte-id).
This commit is contained in:
Hojun-Cho
2026-05-18 22:00:15 +09:00
parent 3bd9b1d56f
commit 9526d21007
5 changed files with 669 additions and 16 deletions
Download Patch File Download Diff File Expand all files Collapse all files

97
lib/encoding/utf8/utf8.ww
View File

@@ -14,10 +14,10 @@
// the caller-buffer form mirrors lib/encoding/hex.encode and
// skips the static-buffer/slice-return pair.
//
// Deferred (no in-tree caller, follow-up tasks): `prev`, `slice`,
// `position`, `remaining`, `appendrune`, `strencode`, `strdecode`.
// Hare's string-iteration surface (`strings::iterator`/`strings::next`
// — ref/hare/strings/iter.ha) lives under lib/strings, not here.
// Deferred (no in-tree caller, follow-up tasks): `appendrune`,
// `strencode`, `strdecode`. Hare's string-iteration surface
// (`strings::iterator`/`strings::next` — ref/hare/strings/iter.ha)
// lives under lib/strings, not here.
// ref/hare/encoding/utf8/types.ha:6 — incomplete trailing sequence.
// Plain `void` (not `!void`): a truncated tail is a control-flow
@@ -339,3 +339,92 @@ export fn encoderune(out: []u8, r: rune) i32 = {
return n;
};
// ref/hare/encoding/utf8/decode.ha:52. Walks back from `d.offs` to a
// byte that could start a codepoint (state-0 dfa cell != -1), re-decodes
// forward from there, and confirms the forward decode lands back at the
// original offset. Returns `done` at start-of-input; `invalid` if no
// initial byte appears within 4 steps (no legal UTF-8 codepoint exceeds
// 4 bytes), if the forward decode returns `more`/`invalid`, or if it
// lands at a different offset than expected. Returns `more` when the
// walk reaches byte 0 without finding any initial byte.
//
// Hare's `for (d.offs < len(d.src); d.offs -= 1)` relies on size_t
// wrap-around to exit when offs underflows past 0; ww's offs is i32,
// so we spell the same exit as `d.offs >= 0`. Hare's `defer d.offs = t`
// is inlined in each match arm — ww has no defer.
export fn prev(d: *decoder) (rune | done | more | invalid) = {
if (d.offs == 0) {
let dn: done; return dn;
};
let n: i32 = d.offs;
d.offs -= 1;
for (d.offs >= 0) {
let b: u8 = d.src[d.offs];
let bi: i32 = b: i32;
let cell: i8 = dfa[bi];
if (cell: i32 != -1) {
let t: i32 = d.offs;
match (next(d)) {
case let r: rune => {
let landed: i32 = d.offs;
d.offs = t;
if (landed != n) {
let e: invalid; return e;
};
return r;
};
case let dn: done => {
d.offs = t;
let e: invalid; return e;
};
case let m: more => {
d.offs = t;
let e: invalid; return e;
};
case let e: invalid => {
d.offs = t;
let e2: invalid; return e2;
};
};
};
if (n - d.offs == 4) {
let e: invalid; return e;
};
d.offs -= 1;
};
let mr: more; return mr;
};
// ref/hare/encoding/utf8/decode.ha:74. Borrowed view of the bytes from
// the decoder's current position to the end of its source.
export fn remaining(d: *decoder) []u8 = {
let r: []u8;
r.ptr = d.src.ptr + (d.offs: u64);
r.len = d.src.len - d.offs;
r.cap = d.src.len - d.offs;
return r;
};
// ref/hare/encoding/utf8/decode.ha:80. Borrowed view of the bytes
// between two decoders' positions. Precondition (Hare asserts both):
// the decoders share the same source, and `begin.offs <= end.offs`.
export fn slice(begin: *decoder, end: *decoder) []u8 = {
if (begin.src.ptr != end.src.ptr) {
abort("utf8.slice: decoders from different sources");
};
if (begin.offs > end.offs) {
abort("utf8.slice: begin past end");
};
let r: []u8;
r.ptr = begin.src.ptr + (begin.offs: u64);
r.len = end.offs - begin.offs;
r.cap = end.offs - begin.offs;
return r;
};
// ref/hare/encoding/utf8/decode.ha:203. Byte position of the decoder
// in its source.
export fn position(d: *decoder) i32 = {
return d.offs;
};

297
lib/encoding/utf8/utf8test.ww
View File

@@ -5,6 +5,7 @@
package utf8;
import bytes;
import encoding.utf8;
import os;
@@ -292,6 +293,288 @@ fn fail() void = { os.exit(signalled + 10); };
};
};
// ---- decoder.prev: done at start-of-input -----------------------------
// ref/hare/encoding/utf8/decode.ha:53-55. Cursor at offs=0 has no prior
// codepoint.
@test fn prev_done_at_start() void = {
let src: [4]u8;
src[0] = 0x41u8;
let d: utf8.decoder = utf8.decode(src[0:1]);
match (utf8.prev(&d)) {
case utf8.done => void;
case let r: rune => { fail(); };
case utf8.more => { fail(); };
case let e: utf8.invalid => { fail(); };
};
};
// ---- decoder.prev: each codepoint width round-trips -------------------
// ref/hare/encoding/utf8/decode.ha:56-71. Forward-decode one rune,
// reverse-decode back to the same rune.
@test fn prev_one_byte() void = {
let src: [1]u8;
src[0] = 0x41u8;
let d: utf8.decoder = utf8.decode(src[0:1]);
match (utf8.next(&d)) {
case let r: rune => void;
case => { fail(); };
};
match (utf8.prev(&d)) {
case let r: rune => { if (r != 0x41u32: rune) { fail(); }; };
case => { fail(); };
};
};
@test fn prev_two_byte() void = {
let src: [2]u8;
src[0] = 0xC3u8; src[1] = 0xA9u8; // 'é'
let d: utf8.decoder = utf8.decode(src[0:2]);
match (utf8.next(&d)) {
case let r: rune => void;
case => { fail(); };
};
match (utf8.prev(&d)) {
case let r: rune => { if (r != 0xE9u32: rune) { fail(); }; };
case => { fail(); };
};
};
@test fn prev_three_byte() void = {
let src: [3]u8;
src[0] = 0xE2u8; src[1] = 0x82u8; src[2] = 0xACu8; // '€'
let d: utf8.decoder = utf8.decode(src[0:3]);
match (utf8.next(&d)) {
case let r: rune => void;
case => { fail(); };
};
match (utf8.prev(&d)) {
case let r: rune => { if (r != 0x20ACu32: rune) { fail(); }; };
case => { fail(); };
};
};
@test fn prev_four_byte() void = {
let src: [4]u8;
src[0] = 0xF0u8; src[1] = 0x9Fu8; src[2] = 0xA6u8; src[3] = 0x80u8; // '🦀'
let d: utf8.decoder = utf8.decode(src[0:4]);
match (utf8.next(&d)) {
case let r: rune => void;
case => { fail(); };
};
match (utf8.prev(&d)) {
case let r: rune => { if (r != 0x1F980u32: rune) { fail(); }; };
case => { fail(); };
};
};
// ---- decoder.prev: full forward + full reverse on mixed input ---------
// ref/hare/encoding/utf8/decode.ha:85-111. After consuming all runes
// forward, prev walks back through them in reverse order; prev at the
// start-of-input then returns done.
@test fn prev_mixed_roundtrip() void = {
let src: [16]u8;
src[0] = 0xE3u8; src[1] = 0x81u8; src[2] = 0x93u8; // こ
src[3] = 0xE3u8; src[4] = 0x82u8; src[5] = 0x93u8; // ん
src[6] = 0xE3u8; src[7] = 0x81u8; src[8] = 0xABu8; // に
src[9] = 0xE3u8; src[10] = 0x81u8; src[11] = 0xA1u8; // ち
src[12] = 0xE3u8; src[13] = 0x81u8; src[14] = 0xAFu8; // は
src[15] = 0u8;
let d: utf8.decoder = utf8.decode(src[0:16]);
let fwd: [6]u32;
let i: i32 = 0;
for (i < 6) {
match (utf8.next(&d)) {
case let r: rune => { fwd[i] = r: u32; };
case => { fail(); };
};
i += 1;
};
if (utf8.position(&d) != 16) { fail(); };
i = 0;
for (i < 6) {
match (utf8.prev(&d)) {
case let r: rune => {
if ((r: u32) != fwd[5 - i]) { fail(); };
};
case => { fail(); };
};
i += 1;
};
match (utf8.prev(&d)) {
case utf8.done => void;
case => { fail(); };
};
};
// ---- decoder.prev: invalid inputs -------------------------------------
// ref/hare/encoding/utf8/decode.ha:113-150. Two-continuation walk
// reaches offs=0 without an initial byte → more; otherwise the forward
// re-decode catches the malformed run as invalid.
// ref/hare/encoding/utf8/decode.ha:117 — `[0xA0, 0xA1]` is all
// continuation bytes; walking back from offs=2 finds no initial byte
// within the 4-step bound nor before offs underflows, so the loop
// exits via the `>= 0` guard and returns `more`.
@test fn prev_continuation_only_more() void = {
let src: [2]u8;
src[0] = 0xA0u8; src[1] = 0xA1u8;
let d: utf8.decoder = utf8.decode(src[0:2]);
d.offs = 2;
match (utf8.prev(&d)) {
case utf8.more => void;
case let r: rune => { fail(); };
case utf8.done => { fail(); };
case let e: utf8.invalid => { fail(); };
};
};
@test fn prev_incomplete_invalid() void = {
let src: [2]u8;
src[0] = 0xE3u8; src[1] = 0x81u8;
let d: utf8.decoder = utf8.decode(src[0:2]);
d.offs = 2;
match (utf8.prev(&d)) {
case let e: utf8.invalid => void;
case let r: rune => { fail(); };
case utf8.done => { fail(); };
case utf8.more => { fail(); };
};
};
@test fn prev_surrogate_invalid() void = {
let src: [3]u8;
src[0] = 0xEDu8; src[1] = 0xA0u8; src[2] = 0x80u8;
let d: utf8.decoder = utf8.decode(src[0:3]);
d.offs = 3;
match (utf8.prev(&d)) {
case let e: utf8.invalid => void;
case let r: rune => { fail(); };
case utf8.done => { fail(); };
case utf8.more => { fail(); };
};
};
@test fn prev_overlong_invalid() void = {
let src: [4]u8;
src[0] = 0xF0u8; src[1] = 0x82u8; src[2] = 0x82u8; src[3] = 0xACu8;
let d: utf8.decoder = utf8.decode(src[0:4]);
d.offs = 4;
match (utf8.prev(&d)) {
case let e: utf8.invalid => void;
case let r: rune => { fail(); };
case utf8.done => { fail(); };
case utf8.more => { fail(); };
};
};
// ref/hare/encoding/utf8/decode.ha:146 — extra continuation byte after
// a complete 2-byte codepoint. prev from offs=3 finds 0xC2 (initial
// byte), forward-decodes back to offs=2, but n=3 ≠ 2 → invalid.
@test fn prev_extracont_invalid() void = {
let src: [3]u8;
src[0] = 0xC2u8; src[1] = 0xA3u8; src[2] = 0x95u8;
let d: utf8.decoder = utf8.decode(src[0:3]);
d.offs = 3;
match (utf8.prev(&d)) {
case let e: utf8.invalid => void;
case let r: rune => { fail(); };
case utf8.done => { fail(); };
case utf8.more => { fail(); };
};
};
// ref/hare/encoding/utf8/decode.ha:158-163 — max in-range codepoint
// reverse-decoded. Pins state-7 acceptance via prev: from offs=4 the
// walk skips 3 continuations, lands on 0xF4 (state-7 lead), and the
// forward re-decode reproduces U+10FFFF.
@test fn prev_max_in_range() void = {
let src: [4]u8;
src[0] = 0xF4u8; src[1] = 0x8Fu8; src[2] = 0xBFu8; src[3] = 0xBFu8;
let d: utf8.decoder = utf8.decode(src[0:4]);
d.offs = 4;
match (utf8.prev(&d)) {
case let r: rune => { if (r != 0x10FFFFu32: rune) { fail(); }; };
case let e: utf8.invalid => { fail(); };
case utf8.done => { fail(); };
case utf8.more => { fail(); };
};
};
// ref/hare/encoding/utf8/decode.ha:166-169 — `[0xF5, 0x94, 0x80, 0x80]`:
// 0xF5 is not a legal initial byte (dfa state-0 cell is -1). prev from
// offs=4 walks 4 bytes without finding an initial byte, trips the
// `n - d.offs == 4` bound, and returns invalid. The only test that
// exercises this branch (prev_continuation_only_more underflows past
// 0 before reaching 4).
@test fn prev_min_out_of_range() void = {
let src: [4]u8;
src[0] = 0xF5u8; src[1] = 0x94u8; src[2] = 0x80u8; src[3] = 0x80u8;
let d: utf8.decoder = utf8.decode(src[0:4]);
d.offs = 4;
match (utf8.prev(&d)) {
case let e: utf8.invalid => void;
case let r: rune => { fail(); };
case utf8.done => { fail(); };
case utf8.more => { fail(); };
};
};
// ---- remaining / slice / position -------------------------------------
// ref/hare/encoding/utf8/decode.ha:172-198.
@test fn remaining_slice_position() void = {
let src: [16]u8;
src[0] = 0xE3u8; src[1] = 0x81u8; src[2] = 0x93u8; // こ
src[3] = 0xE3u8; src[4] = 0x82u8; src[5] = 0x93u8; // ん
src[6] = 0xE3u8; src[7] = 0x81u8; src[8] = 0xABu8; // に
src[9] = 0xE3u8; src[10] = 0x81u8; src[11] = 0xA1u8; // ち
src[12] = 0xE3u8; src[13] = 0x81u8; src[14] = 0xAFu8; // は
src[15] = 0u8;
// Two decode() calls instead of `let d2 = d1` per Hare style —
// let-copy on struct >8B is a Class A cstage+wwstage miscompile
// (task #32; rhs ident is dropped, slot zero-inits). Both calls
// take the same src[0:16] slice so d1.src.ptr == d2.src.ptr,
// matching Hare's `d2 = d1` for slice()'s same-source precondition.
let d1: utf8.decoder = utf8.decode(src[0:16]);
let d2: utf8.decoder = utf8.decode(src[0:16]);
if (!bytes.equal(utf8.remaining(&d1), src[0:16])) { fail(); };
if (utf8.slice(&d1, &d2).len != 0) { fail(); };
if (utf8.slice(&d2, &d1).len != 0) { fail(); };
if (utf8.position(&d1) != 0) { fail(); };
let i: i32 = 0;
for (i < 2) {
match (utf8.next(&d1)) { case let r: rune => void; case => { fail(); }; };
match (utf8.next(&d2)) { case let r: rune => void; case => { fail(); }; };
i += 1;
};
if (utf8.position(&d1) != 6) { fail(); };
if (!bytes.equal(utf8.remaining(&d1), src[6:16])) { fail(); };
if (utf8.slice(&d1, &d2).len != 0) { fail(); };
i = 0;
for (i < 3) {
match (utf8.next(&d2)) { case let r: rune => void; case => { fail(); }; };
i += 1;
};
if (utf8.position(&d2) != 15) { fail(); };
if (!bytes.equal(utf8.remaining(&d2), src[15:16])) { fail(); };
if (!bytes.equal(utf8.slice(&d1, &d2), src[6:15])) { fail(); };
i = 0;
for (i < 3) {
match (utf8.next(&d1)) { case let r: rune => void; case => { fail(); }; };
i += 1;
};
if (utf8.slice(&d1, &d2).len != 0) { fail(); };
match (utf8.next(&d1)) { case let r: rune => void; case => { fail(); }; };
if (utf8.remaining(&d1).len != 0) { fail(); };
};
// ---- round-trip: encode → decode → equal rune --------------------------
@test fn roundtrip() void = {
@@ -339,5 +622,19 @@ export fn main() i32 = {
signalled = 19; validate_malformed();
signalled = 20; validate_empty_ok();
signalled = 21; roundtrip();
signalled = 22; prev_done_at_start();
signalled = 23; prev_one_byte();
signalled = 24; prev_two_byte();
signalled = 25; prev_three_byte();
signalled = 26; prev_four_byte();
signalled = 27; prev_mixed_roundtrip();
signalled = 28; prev_continuation_only_more();
signalled = 29; prev_incomplete_invalid();
signalled = 30; prev_surrogate_invalid();
signalled = 31; prev_overlong_invalid();
signalled = 32; prev_extracont_invalid();
signalled = 33; prev_max_in_range();
signalled = 34; prev_min_out_of_range();
signalled = 35; remaining_slice_position();
return 0;
};