lib/encoding/utf8+test: Hare port (prev + slice + position + remaining)
Port four deferred utf8 functions per c2 backlog (utf8.ww:18
pre-port). All are non-trivial enough that the test rows mirror
Hare's @test fn decode/slice bodies (ref/hare/encoding/utf8/
decode.ha:85-198) row-for-row.
- prev ref/hare/encoding/utf8/decode.ha:52-71
- remaining ref/hare/encoding/utf8/decode.ha:74
- slice ref/hare/encoding/utf8/decode.ha:80-83
- position ref/hare/encoding/utf8/decode.ha:203
prev walks back from d.offs to a byte that could start a codepoint
(state-0 dfa cell != -1), re-decodes forward from there, and
confirms the forward decode lands back at the original offset.
Returns done at start-of-input; invalid if no initial byte appears
within 4 steps (no legal UTF-8 codepoint exceeds 4 bytes) or if
the forward decode shortcircuits to more/invalid or lands at a
different offset than expected.
Two structural deltas from the Hare source:
- Hare's `for (d.offs < len(d.src); d.offs -= 1)` relies on
size_t wrap-around to exit when offs underflows past 0; ww's
offs is i32 (utf8.ww:204), so the same exit is spelled
`d.offs >= 0`.
- Hare's `defer d.offs = t` restores offs after the return; ww
has no defer, so the restore is inlined in each match arm.
slice asserts the Hare precondition (same source + begin.offs <=
end.offs) via abort; Hare uses assert(). position is a one-liner
returning d.offs (Hare uses size, ww uses i32 per lib/CLAUDE.md
"indices use the underlying length type").
The `bi: i32 = b: i32;` indirection in prev's dfa lookup is
required because `dfa[b: i32]` parses as a slice expression
`dfa[b : i32]` where `i32` becomes the upper bound. The let-binding
matches the existing pattern at utf8.ww:236-238 in the c1 next()
port.
Tests: 14 new @test fns in utf8test.ww (signalled 22-35):
prev_done_at_start, prev_one/two/three/four_byte (round-trip
forward+reverse), prev_mixed_roundtrip (full forward then full
reverse on the same こんにちは+NUL input Hare uses at
decode.ha:85-111), prev_continuation_only_more (Hare's
[0xA0,0xA1] more case at decode.ha:117), prev_incomplete /
surrogate / overlong / extracont_invalid (decode.ha:120-150),
prev_max_in_range (decode.ha:158-163; pins state-7 acceptance via
reverse decode), prev_min_out_of_range (decode.ha:166-169; the
only case that trips prev's 4-step-bound arm), and
remaining_slice_position mirroring decode.ha:172-198.
The Hare slice @test idiom `let d2 = d1` (struct copy) miscompiles
in both stages (cstage + wwstage zero-init the rhs instead of
copying — task #32, Class A but bootstrap-byte-id-symmetric, so
995 doesn't catch it). The ww test uses two parallel
`decode(src[0:16])` calls to produce two decoders with the same
src.ptr; coverage on slice() is equivalent (exercises the same
source-different-offs pattern). Divergence cited inline.
117/117 ok. 995_self_rebuild stays green (ww2==ww3==ww4 byte-id).
This commit is contained in:
@@ -14,10 +14,10 @@
|
||||
// the caller-buffer form mirrors lib/encoding/hex.encode and
|
||||
// skips the static-buffer/slice-return pair.
|
||||
//
|
||||
// Deferred (no in-tree caller, follow-up tasks): `prev`, `slice`,
|
||||
// `position`, `remaining`, `appendrune`, `strencode`, `strdecode`.
|
||||
// Hare's string-iteration surface (`strings::iterator`/`strings::next`
|
||||
// — ref/hare/strings/iter.ha) lives under lib/strings, not here.
|
||||
// Deferred (no in-tree caller, follow-up tasks): `appendrune`,
|
||||
// `strencode`, `strdecode`. Hare's string-iteration surface
|
||||
// (`strings::iterator`/`strings::next` — ref/hare/strings/iter.ha)
|
||||
// lives under lib/strings, not here.
|
||||
|
||||
// ref/hare/encoding/utf8/types.ha:6 — incomplete trailing sequence.
|
||||
// Plain `void` (not `!void`): a truncated tail is a control-flow
|
||||
@@ -339,3 +339,92 @@ export fn encoderune(out: []u8, r: rune) i32 = {
|
||||
return n;
|
||||
};
|
||||
|
||||
// ref/hare/encoding/utf8/decode.ha:52. Walks back from `d.offs` to a
|
||||
// byte that could start a codepoint (state-0 dfa cell != -1), re-decodes
|
||||
// forward from there, and confirms the forward decode lands back at the
|
||||
// original offset. Returns `done` at start-of-input; `invalid` if no
|
||||
// initial byte appears within 4 steps (no legal UTF-8 codepoint exceeds
|
||||
// 4 bytes), if the forward decode returns `more`/`invalid`, or if it
|
||||
// lands at a different offset than expected. Returns `more` when the
|
||||
// walk reaches byte 0 without finding any initial byte.
|
||||
//
|
||||
// Hare's `for (d.offs < len(d.src); d.offs -= 1)` relies on size_t
|
||||
// wrap-around to exit when offs underflows past 0; ww's offs is i32,
|
||||
// so we spell the same exit as `d.offs >= 0`. Hare's `defer d.offs = t`
|
||||
// is inlined in each match arm — ww has no defer.
|
||||
export fn prev(d: *decoder) (rune | done | more | invalid) = {
|
||||
if (d.offs == 0) {
|
||||
let dn: done; return dn;
|
||||
};
|
||||
let n: i32 = d.offs;
|
||||
d.offs -= 1;
|
||||
for (d.offs >= 0) {
|
||||
let b: u8 = d.src[d.offs];
|
||||
let bi: i32 = b: i32;
|
||||
let cell: i8 = dfa[bi];
|
||||
if (cell: i32 != -1) {
|
||||
let t: i32 = d.offs;
|
||||
match (next(d)) {
|
||||
case let r: rune => {
|
||||
let landed: i32 = d.offs;
|
||||
d.offs = t;
|
||||
if (landed != n) {
|
||||
let e: invalid; return e;
|
||||
};
|
||||
return r;
|
||||
};
|
||||
case let dn: done => {
|
||||
d.offs = t;
|
||||
let e: invalid; return e;
|
||||
};
|
||||
case let m: more => {
|
||||
d.offs = t;
|
||||
let e: invalid; return e;
|
||||
};
|
||||
case let e: invalid => {
|
||||
d.offs = t;
|
||||
let e2: invalid; return e2;
|
||||
};
|
||||
};
|
||||
};
|
||||
if (n - d.offs == 4) {
|
||||
let e: invalid; return e;
|
||||
};
|
||||
d.offs -= 1;
|
||||
};
|
||||
let mr: more; return mr;
|
||||
};
|
||||
|
||||
// ref/hare/encoding/utf8/decode.ha:74. Borrowed view of the bytes from
|
||||
// the decoder's current position to the end of its source.
|
||||
export fn remaining(d: *decoder) []u8 = {
|
||||
let r: []u8;
|
||||
r.ptr = d.src.ptr + (d.offs: u64);
|
||||
r.len = d.src.len - d.offs;
|
||||
r.cap = d.src.len - d.offs;
|
||||
return r;
|
||||
};
|
||||
|
||||
// ref/hare/encoding/utf8/decode.ha:80. Borrowed view of the bytes
|
||||
// between two decoders' positions. Precondition (Hare asserts both):
|
||||
// the decoders share the same source, and `begin.offs <= end.offs`.
|
||||
export fn slice(begin: *decoder, end: *decoder) []u8 = {
|
||||
if (begin.src.ptr != end.src.ptr) {
|
||||
abort("utf8.slice: decoders from different sources");
|
||||
};
|
||||
if (begin.offs > end.offs) {
|
||||
abort("utf8.slice: begin past end");
|
||||
};
|
||||
let r: []u8;
|
||||
r.ptr = begin.src.ptr + (begin.offs: u64);
|
||||
r.len = end.offs - begin.offs;
|
||||
r.cap = end.offs - begin.offs;
|
||||
return r;
|
||||
};
|
||||
|
||||
// ref/hare/encoding/utf8/decode.ha:203. Byte position of the decoder
|
||||
// in its source.
|
||||
export fn position(d: *decoder) i32 = {
|
||||
return d.offs;
|
||||
};
|
||||
|
||||
|
||||
Reference in New Issue
Block a user