lib/encoding/utf8+test: Hare port (prev + slice + position + remaining)

Port four deferred utf8 functions per c2 backlog (utf8.ww:18
pre-port). All are non-trivial enough that the test rows mirror
Hare's @test fn decode/slice bodies (ref/hare/encoding/utf8/
decode.ha:85-198) row-for-row.

  - prev       ref/hare/encoding/utf8/decode.ha:52-71
  - remaining  ref/hare/encoding/utf8/decode.ha:74
  - slice      ref/hare/encoding/utf8/decode.ha:80-83
  - position   ref/hare/encoding/utf8/decode.ha:203

prev walks back from d.offs to a byte that could start a codepoint
(state-0 dfa cell != -1), re-decodes forward from there, and
confirms the forward decode lands back at the original offset.
Returns done at start-of-input; invalid if no initial byte appears
within 4 steps (no legal UTF-8 codepoint exceeds 4 bytes) or if
the forward decode shortcircuits to more/invalid or lands at a
different offset than expected.

Two structural deltas from the Hare source:

  - Hare's `for (d.offs < len(d.src); d.offs -= 1)` relies on
    size_t wrap-around to exit when offs underflows past 0; ww's
    offs is i32 (utf8.ww:204), so the same exit is spelled
    `d.offs >= 0`.

  - Hare's `defer d.offs = t` restores offs after the return; ww
    has no defer, so the restore is inlined in each match arm.

slice asserts the Hare precondition (same source + begin.offs <=
end.offs) via abort; Hare uses assert(). position is a one-liner
returning d.offs (Hare uses size, ww uses i32 per lib/CLAUDE.md
"indices use the underlying length type").

The `bi: i32 = b: i32;` indirection in prev's dfa lookup is
required because `dfa[b: i32]` parses as a slice expression
`dfa[b : i32]` where `i32` becomes the upper bound. The let-binding
matches the existing pattern at utf8.ww:236-238 in the c1 next()
port.

Tests: 14 new @test fns in utf8test.ww (signalled 22-35):
prev_done_at_start, prev_one/two/three/four_byte (round-trip
forward+reverse), prev_mixed_roundtrip (full forward then full
reverse on the same こんにちは+NUL input Hare uses at
decode.ha:85-111), prev_continuation_only_more (Hare's
[0xA0,0xA1] more case at decode.ha:117), prev_incomplete /
surrogate / overlong / extracont_invalid (decode.ha:120-150),
prev_max_in_range (decode.ha:158-163; pins state-7 acceptance via
reverse decode), prev_min_out_of_range (decode.ha:166-169; the
only case that trips prev's 4-step-bound arm), and
remaining_slice_position mirroring decode.ha:172-198.

The Hare slice @test idiom `let d2 = d1` (struct copy) miscompiles
in both stages (cstage + wwstage zero-init the rhs instead of
copying — task #32, Class A but bootstrap-byte-id-symmetric, so
995 doesn't catch it). The ww test uses two parallel
`decode(src[0:16])` calls to produce two decoders with the same
src.ptr; coverage on slice() is equivalent (exercises the same
source-different-offs pattern). Divergence cited inline.

117/117 ok. 995_self_rebuild stays green (ww2==ww3==ww4 byte-id).
This commit is contained in:
2026-05-18 22:00:15 +09:00
parent 3bd9b1d56f
commit 9526d21007
5 changed files with 669 additions and 16 deletions

View File

@@ -14,10 +14,10 @@
// the caller-buffer form mirrors lib/encoding/hex.encode and
// skips the static-buffer/slice-return pair.
//
// Deferred (no in-tree caller, follow-up tasks): `prev`, `slice`,
// `position`, `remaining`, `appendrune`, `strencode`, `strdecode`.
// Hare's string-iteration surface (`strings::iterator`/`strings::next`
// — ref/hare/strings/iter.ha) lives under lib/strings, not here.
// Deferred (no in-tree caller, follow-up tasks): `appendrune`,
// `strencode`, `strdecode`. Hare's string-iteration surface
// (`strings::iterator`/`strings::next` — ref/hare/strings/iter.ha)
// lives under lib/strings, not here.
// ref/hare/encoding/utf8/types.ha:6 — incomplete trailing sequence.
// Plain `void` (not `!void`): a truncated tail is a control-flow
@@ -339,3 +339,92 @@ export fn encoderune(out: []u8, r: rune) i32 = {
return n;
};
// ref/hare/encoding/utf8/decode.ha:52. Walks back from `d.offs` to a
// byte that could start a codepoint (state-0 dfa cell != -1), re-decodes
// forward from there, and confirms the forward decode lands back at the
// original offset. Returns `done` at start-of-input; `invalid` if no
// initial byte appears within 4 steps (no legal UTF-8 codepoint exceeds
// 4 bytes), if the forward decode returns `more`/`invalid`, or if it
// lands at a different offset than expected. Returns `more` when the
// walk reaches byte 0 without finding any initial byte.
//
// Hare's `for (d.offs < len(d.src); d.offs -= 1)` relies on size_t
// wrap-around to exit when offs underflows past 0; ww's offs is i32,
// so we spell the same exit as `d.offs >= 0`. Hare's `defer d.offs = t`
// is inlined in each match arm — ww has no defer.
export fn prev(d: *decoder) (rune | done | more | invalid) = {
if (d.offs == 0) {
let dn: done; return dn;
};
let n: i32 = d.offs;
d.offs -= 1;
for (d.offs >= 0) {
let b: u8 = d.src[d.offs];
let bi: i32 = b: i32;
let cell: i8 = dfa[bi];
if (cell: i32 != -1) {
let t: i32 = d.offs;
match (next(d)) {
case let r: rune => {
let landed: i32 = d.offs;
d.offs = t;
if (landed != n) {
let e: invalid; return e;
};
return r;
};
case let dn: done => {
d.offs = t;
let e: invalid; return e;
};
case let m: more => {
d.offs = t;
let e: invalid; return e;
};
case let e: invalid => {
d.offs = t;
let e2: invalid; return e2;
};
};
};
if (n - d.offs == 4) {
let e: invalid; return e;
};
d.offs -= 1;
};
let mr: more; return mr;
};
// ref/hare/encoding/utf8/decode.ha:74. Borrowed view of the bytes from
// the decoder's current position to the end of its source.
export fn remaining(d: *decoder) []u8 = {
let r: []u8;
r.ptr = d.src.ptr + (d.offs: u64);
r.len = d.src.len - d.offs;
r.cap = d.src.len - d.offs;
return r;
};
// ref/hare/encoding/utf8/decode.ha:80. Borrowed view of the bytes
// between two decoders' positions. Precondition (Hare asserts both):
// the decoders share the same source, and `begin.offs <= end.offs`.
export fn slice(begin: *decoder, end: *decoder) []u8 = {
if (begin.src.ptr != end.src.ptr) {
abort("utf8.slice: decoders from different sources");
};
if (begin.offs > end.offs) {
abort("utf8.slice: begin past end");
};
let r: []u8;
r.ptr = begin.src.ptr + (begin.offs: u64);
r.len = end.offs - begin.offs;
r.cap = end.offs - begin.offs;
return r;
};
// ref/hare/encoding/utf8/decode.ha:203. Byte position of the decoder
// in its source.
export fn position(d: *decoder) i32 = {
return d.offs;
};