lib/encoding/utf8+test: Hare port (decoder / next / encoderune / runesz / utf8sz / validate)
Hoehrmann DFA from ref/hare/encoding/utf8/decodetable.ha flattened to 1D [2048]i8 (task #20: 2D-array jagged cgen still pending); encoderune takes a caller buffer matching lib/encoding/hex.encode; done/more/invalid all spelled as plain void aliases per lib/io's eof precedent. Surface ports decoder + decode + next + encoderune + runesz + utf8sz + validate from ref/hare/encoding/utf8/{types, decode,encode,rune}.ha. next() polarity rewritten from Hare's `(state-1):uint >> 31` to an explicit `if state == 0` branch because ww's uint is 64-bit (cmd/wcc/type.c:58); same effect, no hidden 32-bit assumption. Deferred (no in-tree callers): prev, slice, position, remaining, appendrune, strencode, strdecode. String iteration (chars/ newchars/nextchar in the session-4 draft) dropped per Hare discipline — belongs in lib/strings::iterator, not encoding/utf8. Tests: - 968_utf8_run drives lib/encoding/utf8/utf8test.ww via ww run. 21 @test fns: boundaries (ASCII, 2-byte, 3-byte, 4-byte encode/decode), surrogate/overlong/out-of-range/bad-continuation reject, max-in-range (U+10FFFF) accept, truncated→more, done@EOI, validate empty/mixed/malformed, encode/decode roundtrip. Two rows ported from ref/hare/encoding/utf8/decode.ha @test that were missing in the session-4 draft: bad-continuation [0xC2,0xFF]→invalid and max-in-range [0xF4,0x8F,0xBF,0xBF]→ U+10FFFF. - 9xx stdlib runtime slot range extended from 970-989 to 960-989 to accommodate utf8 at 968 (970-989 block was full). 90/90 ok. 995_self_rebuild stays green (ww2==ww3==ww4 byte-id).
This commit is contained in:
6
Makefile
6
Makefile
@@ -260,7 +260,7 @@ TESTS = $(BIN)/test_smoke $(BIN)/test_lex $(BIN)/test_parse $(BIN)/test_check \
|
||||
$(BIN)/test_shlex_run $(BIN)/test_getenv_run $(BIN)/test_dirs_run \
|
||||
$(BIN)/test_stat_run $(BIN)/test_time_run \
|
||||
$(BIN)/test_intdiv_signed \
|
||||
$(BIN)/test_hex_run \
|
||||
$(BIN)/test_hex_run $(BIN)/test_utf8_run \
|
||||
$(BIN)/test_memio_run $(BIN)/test_temp_run $(BIN)/test_getopt_run \
|
||||
$(BIN)/test_base32_run $(BIN)/test_base64_run \
|
||||
$(BIN)/test_adler32_run $(BIN)/test_crc16_run \
|
||||
@@ -634,6 +634,10 @@ $(BIN)/test_hex_run: test/wcc/979_hex_run.c $(BIN)/ww $(BIN)/w6c \
|
||||
$(BIN)/w6a $(BIN)/w6l $(LIB)/libwwrt.a | $(BIN)
|
||||
$(CC) $(CFLAGS) -o $@ $<
|
||||
|
||||
$(BIN)/test_utf8_run: test/wcc/968_utf8_run.c $(BIN)/ww $(BIN)/w6c \
|
||||
$(BIN)/w6a $(BIN)/w6l $(LIB)/libwwrt.a | $(BIN)
|
||||
$(CC) $(CFLAGS) -o $@ $<
|
||||
|
||||
$(BIN)/test_memio_run: test/wcc/980_memio_run.c $(BIN)/ww $(BIN)/w6c \
|
||||
$(BIN)/w6a $(BIN)/w6l $(LIB)/libwwrt.a | $(BIN)
|
||||
$(CC) $(CFLAGS) -o $@ $<
|
||||
|
||||
@@ -1,15 +1,339 @@
|
||||
// encoding/utf8 — UTF-8 helpers. RFC 3629; we only handle the legal
|
||||
// subset (no over-long encodings, no surrogates).
|
||||
// encoding/utf8 — UTF-8 encode/decode. Hare port; see
|
||||
// ref/hare/encoding/utf8/{types,rune,encode,decode,decodetable}.ha.
|
||||
//
|
||||
// The decoder is Hoehrmann's branchless DFA, originally published
|
||||
// at <https://bjoern.hoehrmann.de/utf-8/decoder/dfa/>. Hare's
|
||||
// ref/hare/encoding/utf8/decodetable.ha:4 restructures Hoehrmann's
|
||||
// flat table to 2D `[8][256]i8`; we flatten back to 1D `[2048]i8`
|
||||
// because ww cgen does not yet ship 2D arrays (task #20).
|
||||
//
|
||||
// Surface deviation from ref/hare/encoding/utf8:
|
||||
//
|
||||
// - `encoderune` takes a caller-supplied `out: []u8` and returns
|
||||
// the byte count. Hare returns a slice into a `static let buf`;
|
||||
// the caller-buffer form mirrors lib/encoding/hex.encode and
|
||||
// skips the static-buffer/slice-return pair.
|
||||
//
|
||||
// Deferred (no in-tree caller, follow-up tasks): `prev`, `slice`,
|
||||
// `position`, `remaining`, `appendrune`, `strencode`, `strdecode`.
|
||||
// Hare's string-iteration surface (`strings::iterator`/`strings::next`
|
||||
// — ref/hare/strings/iter.ha) lives under lib/strings, not here.
|
||||
|
||||
def MAX: rune = 1114111; // 0x10FFFF
|
||||
// ref/hare/encoding/utf8/types.ha:6 — incomplete trailing sequence.
|
||||
// Plain `void` (not `!void`): a truncated tail is a control-flow
|
||||
// signal, not an error caller can ignore.
|
||||
export type more = void;
|
||||
|
||||
// runesz — encoded byte length of `r` as UTF-8. void variant means
|
||||
// `r` is outside the legal range (negative, > 0x10FFFF).
|
||||
export fn runesz(r: rune) (i32 | void) = {
|
||||
if (r < 0) { return; };
|
||||
if (r < 128) { return 1; };
|
||||
if (r < 2048) { return 2; };
|
||||
if (r < 65536) { return 3; };
|
||||
if (r <= MAX) { return 4; };
|
||||
return;
|
||||
// ref/hare/encoding/utf8/types.ha:9 — invalid UTF-8 sequence.
|
||||
export type invalid = !void;
|
||||
|
||||
// `done` is not a built-in singleton in ww (Hare ships it as part of
|
||||
// the type system). Plain `void` (not `!void`): end-of-input is a
|
||||
// continuation signal, not an error. lib/io spells its EOF the same
|
||||
// way (lib/io/io.ww:8-11).
|
||||
export type done = void;
|
||||
|
||||
// ref/hare/encoding/utf8/decodetable.ha:4 — Hoehrmann's UTF-8 DFA,
|
||||
// flat 1D `[2048]i8`. Layout: dfa[state*256 + byte] gives the next
|
||||
// state (>0), the accept transition (0 — emit rune), or invalid (-1).
|
||||
// Values match ref/hare/encoding/utf8/decodetable.ha verbatim.
|
||||
let dfa: [2048]i8 = [
|
||||
// state 0 — initial byte: ASCII accepts (0), continuation/illegal
|
||||
// byte rejects (-1), legal multibyte start emits a state.
|
||||
0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8,
|
||||
0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8,
|
||||
0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8,
|
||||
0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8,
|
||||
0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8,
|
||||
0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8,
|
||||
0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8,
|
||||
0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8,
|
||||
-1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8,
|
||||
-1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8,
|
||||
-1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8,
|
||||
-1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8,
|
||||
-1i8, -1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8,
|
||||
1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8,
|
||||
3i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 4i8, 2i8, 2i8,
|
||||
5i8, 6i8, 6i8, 6i8, 7i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8,
|
||||
|
||||
// state 1 — expecting one continuation byte (0x80..0xBF).
|
||||
-1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8,
|
||||
-1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8,
|
||||
-1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8,
|
||||
-1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8,
|
||||
-1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8,
|
||||
-1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8,
|
||||
-1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8,
|
||||
-1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8,
|
||||
0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8,
|
||||
0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8,
|
||||
0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8,
|
||||
0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8,
|
||||
-1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8,
|
||||
-1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8,
|
||||
-1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8,
|
||||
-1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8,
|
||||
|
||||
// state 2 — expecting one continuation byte (full 0x80..0xBF range).
|
||||
-1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8,
|
||||
-1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8,
|
||||
-1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8,
|
||||
-1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8,
|
||||
-1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8,
|
||||
-1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8,
|
||||
-1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8,
|
||||
-1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8,
|
||||
1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8,
|
||||
1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8,
|
||||
1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8,
|
||||
1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8,
|
||||
-1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8,
|
||||
-1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8,
|
||||
-1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8,
|
||||
-1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8,
|
||||
|
||||
// state 3 — first byte was 0xE0; continuation byte must be 0xA0..0xBF
|
||||
// (rejects overlong 3-byte encodings).
|
||||
-1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8,
|
||||
-1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8,
|
||||
-1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8,
|
||||
-1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8,
|
||||
-1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8,
|
||||
-1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8,
|
||||
-1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8,
|
||||
-1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8,
|
||||
-1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8,
|
||||
-1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8,
|
||||
1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8,
|
||||
1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8,
|
||||
-1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8,
|
||||
-1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8,
|
||||
-1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8,
|
||||
-1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8,
|
||||
|
||||
// state 4 — first byte was 0xED; continuation byte must be 0x80..0x9F
|
||||
// (rejects UTF-16 surrogate codepoints U+D800..U+DFFF).
|
||||
-1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8,
|
||||
-1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8,
|
||||
-1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8,
|
||||
-1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8,
|
||||
-1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8,
|
||||
-1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8,
|
||||
-1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8,
|
||||
-1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8,
|
||||
1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8,
|
||||
1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8,
|
||||
-1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8,
|
||||
-1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8,
|
||||
-1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8,
|
||||
-1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8,
|
||||
-1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8,
|
||||
-1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8,
|
||||
|
||||
// state 5 — first byte was 0xF0; continuation byte must be 0x90..0xBF
|
||||
// (rejects overlong 4-byte encodings).
|
||||
-1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8,
|
||||
-1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8,
|
||||
-1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8,
|
||||
-1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8,
|
||||
-1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8,
|
||||
-1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8,
|
||||
-1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8,
|
||||
-1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8,
|
||||
-1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8,
|
||||
2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8,
|
||||
2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8,
|
||||
2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8,
|
||||
-1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8,
|
||||
-1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8,
|
||||
-1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8,
|
||||
-1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8,
|
||||
|
||||
// state 6 — middle continuation byte of a 4-byte sequence (0x80..0xBF).
|
||||
-1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8,
|
||||
-1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8,
|
||||
-1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8,
|
||||
-1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8,
|
||||
-1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8,
|
||||
-1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8,
|
||||
-1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8,
|
||||
-1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8,
|
||||
2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8,
|
||||
2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8,
|
||||
2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8,
|
||||
2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8,
|
||||
-1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8,
|
||||
-1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8,
|
||||
-1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8,
|
||||
-1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8,
|
||||
|
||||
// state 7 — first byte was 0xF4; continuation byte must be 0x80..0x8F
|
||||
// (rejects codepoints above U+10FFFF).
|
||||
-1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8,
|
||||
-1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8,
|
||||
-1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8,
|
||||
-1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8,
|
||||
-1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8,
|
||||
-1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8,
|
||||
-1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8,
|
||||
-1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8,
|
||||
2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8,
|
||||
-1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8,
|
||||
-1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8,
|
||||
-1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8,
|
||||
-1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8,
|
||||
-1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8,
|
||||
-1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8,
|
||||
-1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8,
|
||||
];
|
||||
|
||||
// ref/hare/encoding/utf8/decode.ha:17 — payload-bit masks. Hare's
|
||||
// [2][8]u8 flattened to 1D [16]u8; row 0 (offsets 0..7) is the
|
||||
// continuation-byte mask (always 0x3F), row 1 (offsets 8..15) is the
|
||||
// initial-byte payload mask indexed by the transition class.
|
||||
let masks: [16]u8 = [
|
||||
0x3fu8, 0x3fu8, 0x3fu8, 0x3fu8, 0x3fu8, 0x3fu8, 0x3fu8, 0x3fu8,
|
||||
0x7fu8, 0x1fu8, 0x0fu8, 0x0fu8, 0x0fu8, 0x07u8, 0x07u8, 0x07u8,
|
||||
];
|
||||
|
||||
// ref/hare/encoding/utf8/decode.ha:6 — incremental decoder state.
|
||||
export type decoder = struct {
|
||||
offs: i32,
|
||||
src: []u8,
|
||||
};
|
||||
|
||||
// ref/hare/encoding/utf8/decode.ha:12.
|
||||
export fn decode(src: []u8) decoder = {
|
||||
let d: decoder;
|
||||
d.src = src;
|
||||
d.offs = 0;
|
||||
return d;
|
||||
};
|
||||
|
||||
// ref/hare/encoding/utf8/decode.ha:27. Returns the next rune from a
|
||||
// decoder, `done` at end-of-input, `more` on truncated trailing
|
||||
// sequence, `invalid` on malformed input (overlong, surrogate,
|
||||
// out-of-range, bad continuation).
|
||||
//
|
||||
// Algorithm is verbatim Hoehrmann (see file header). One structural
|
||||
// rewrite: Hare encodes the "initial vs continuation byte" decision
|
||||
// as the branchless `(state - 1): uint >> 31`, which assumes a 32-bit
|
||||
// uint. ww's uint is 64-bit (cmd/wcc/type.c:58), so the shift answer
|
||||
// would be 0x1_ffff_ffff rather than 1. We spell the same predicate
|
||||
// with an explicit conditional.
|
||||
export fn next(d: *decoder) (rune | done | more | invalid) = {
|
||||
if (d.offs == d.src.len) {
|
||||
let dn: done; return dn;
|
||||
};
|
||||
let nx: i32 = 0;
|
||||
let state: i32 = 0;
|
||||
let r: u32 = 0u32;
|
||||
for (d.offs < d.src.len) {
|
||||
let b: u8 = d.src[d.offs];
|
||||
let bi: i32 = b: i32;
|
||||
let row: i32 = state * 256 + bi;
|
||||
let cell: i8 = dfa[row];
|
||||
nx = cell: i32;
|
||||
let mi: i32 = 0;
|
||||
if (state == 0) { mi = 1; };
|
||||
let m: u8 = masks[mi * 8 + (nx & 7)];
|
||||
r = (r << 6u32) | ((b & m): u32);
|
||||
if (nx <= 0) {
|
||||
d.offs += 1;
|
||||
if (nx == 0) { return r: rune; };
|
||||
let e: invalid; return e;
|
||||
};
|
||||
state = nx;
|
||||
d.offs += 1;
|
||||
};
|
||||
let mr: more; return mr;
|
||||
};
|
||||
|
||||
// ref/hare/encoding/utf8/decode.ha:207. Strict whole-input check.
|
||||
// The hot path: tight DFA loop, no rune assembly. Bails the moment
|
||||
// the table returns -1 so malformed inputs don't pay for the rest
|
||||
// of the buffer.
|
||||
export fn validate(src: []u8) (void | invalid) = {
|
||||
let state: i32 = 0;
|
||||
let i: i32 = 0;
|
||||
for (i < src.len) {
|
||||
if (state < 0) { break; };
|
||||
let bi: i32 = src[i]: i32;
|
||||
let cell: i8 = dfa[state * 256 + bi];
|
||||
state = cell: i32;
|
||||
i += 1;
|
||||
};
|
||||
if (state == 0) { return; };
|
||||
let e: invalid; return e;
|
||||
};
|
||||
|
||||
// ref/hare/encoding/utf8/rune.ha:5. Encoded byte length of `r` as
|
||||
// UTF-8. Callers in ww use this to size the buffer they hand to
|
||||
// [[encoderune]]; values >0x10FFFF or negative are not legal Unicode
|
||||
// codepoints and Hare aborts on them in `encoderune` itself, so we
|
||||
// keep `runesz` infallible (matches Hare).
|
||||
export fn runesz(r: rune) i32 = {
|
||||
let ch: u32 = r: u32;
|
||||
if (ch < 128u32) { return 1; };
|
||||
if (ch < 2048u32) { return 2; };
|
||||
if (ch < 65536u32) { return 3; };
|
||||
return 4;
|
||||
};
|
||||
|
||||
// ref/hare/encoding/utf8/rune.ha:15. Expected byte length of the
|
||||
// codepoint that starts with `c`, or `invalid` if `c` cannot start
|
||||
// a legal UTF-8 sequence. Constants written in decimal because ww
|
||||
// doesn't accept Hare's `0b1000_0000` binary syntax: 0x80=128,
|
||||
// 0xC2=194, 0xE0=224, 0xF0=240, 0xF8=248.
|
||||
export fn utf8sz(c: u8) (i32 | invalid) = {
|
||||
if (c < 128u8) { return 1; };
|
||||
if (c < 194u8) { let e: invalid; return e; };
|
||||
if (c >= 248u8) { let e: invalid; return e; };
|
||||
if (c < 224u8) { return 2; };
|
||||
if (c < 240u8) { return 3; };
|
||||
return 4;
|
||||
};
|
||||
|
||||
// ref/hare/encoding/utf8/encode.ha:7. Encode `r` into `out` (caller-
|
||||
// supplied; must hold at least [[runesz]](r) bytes) and return the
|
||||
// byte count. ABORT if `r` is a UTF-16 surrogate or above U+10FFFF —
|
||||
// same precondition Hare asserts at ref/hare/encoding/utf8/encode.ha:9.
|
||||
//
|
||||
// Surface deviation: Hare returns `[]u8` (slice into a static buf).
|
||||
// ww uses the caller-buffer form (matches lib/encoding/hex.encode);
|
||||
// caller can reuse a [4]u8 stack scratch across encodes.
|
||||
export fn encoderune(out: []u8, r: rune) i32 = {
|
||||
let ch: u32 = r: u32;
|
||||
if (ch >= 0xD800u32) {
|
||||
if (ch <= 0xDFFFu32) {
|
||||
abort("utf8.encoderune: surrogate codepoint");
|
||||
};
|
||||
};
|
||||
if (ch > 0x10FFFFu32) {
|
||||
abort("utf8.encoderune: codepoint > U+10FFFF");
|
||||
};
|
||||
|
||||
let n: i32 = 0;
|
||||
let first: u8 = 0u8;
|
||||
if (ch < 0x80u32) {
|
||||
first = 0u8; n = 1;
|
||||
} else if (ch < 0x800u32) {
|
||||
first = 0xC0u8; n = 2;
|
||||
} else if (ch < 0x10000u32) {
|
||||
first = 0xE0u8; n = 3;
|
||||
} else {
|
||||
first = 0xF0u8; n = 4;
|
||||
};
|
||||
|
||||
let v: u32 = ch;
|
||||
let i: i32 = n - 1;
|
||||
for (i > 0) {
|
||||
out[i] = ((v: u8) & 0x3Fu8) | 0x80u8;
|
||||
v = v >> 6u32;
|
||||
i -= 1;
|
||||
};
|
||||
out[0] = (v: u8) | first;
|
||||
return n;
|
||||
};
|
||||
|
||||
|
||||
351
lib/encoding/utf8/utf8test.ww
Normal file
351
lib/encoding/utf8/utf8test.ww
Normal file
@@ -0,0 +1,351 @@
|
||||
// utf8test — exercises lib/encoding/utf8. Run with
|
||||
// `out/bin/ww run lib/encoding/utf8/utf8test.ww`. Same signalled-
|
||||
// then-fail()-with-+10 pattern as hex / base32 / time tests:
|
||||
// non-zero exit pinpoints the failing scenario.
|
||||
|
||||
use utf8;
|
||||
use os;
|
||||
|
||||
let signalled: i32 = 0;
|
||||
fn fail() void = { os.exit(signalled + 10); };
|
||||
|
||||
fn beq(a: []u8, b: []u8) bool = {
|
||||
if (a.len != b.len) { return false; };
|
||||
let i: i32 = 0;
|
||||
for (i < a.len) {
|
||||
if (a[i] != b[i]) { return false; };
|
||||
i += 1;
|
||||
};
|
||||
return true;
|
||||
};
|
||||
|
||||
// ---- runesz: byte length per range ------------------------------------
|
||||
// ref/hare/encoding/utf8/rune.ha:5. Boundaries: 0x7F → 1, 0x80 → 2,
|
||||
// 0x7FF → 2, 0x800 → 3, 0xFFFF → 3, 0x10000 → 4, 0x10FFFF → 4.
|
||||
|
||||
@test fn runesz_ranges() void = {
|
||||
if (utf8.runesz(0u32: rune) != 1) { fail(); };
|
||||
if (utf8.runesz(0x7Fu32: rune) != 1) { fail(); };
|
||||
if (utf8.runesz(0x80u32: rune) != 2) { fail(); };
|
||||
if (utf8.runesz(0x7FFu32: rune) != 2) { fail(); };
|
||||
if (utf8.runesz(0x800u32: rune) != 3) { fail(); };
|
||||
if (utf8.runesz(0xFFFFu32: rune) != 3) { fail(); };
|
||||
if (utf8.runesz(0x10000u32: rune) != 4) { fail(); };
|
||||
if (utf8.runesz(0x10FFFFu32: rune) != 4) { fail(); };
|
||||
};
|
||||
|
||||
// ---- utf8sz: start-byte classification --------------------------------
|
||||
// ref/hare/encoding/utf8/rune.ha:15. ASCII → 1; legal multibyte
|
||||
// leads → 2/3/4; continuation and >0xF7 → invalid.
|
||||
|
||||
@test fn utf8sz_classify() void = {
|
||||
match (utf8.utf8sz(0u8)) {
|
||||
case let n: i32 => { if (n != 1) { fail(); }; };
|
||||
case let e: utf8.invalid => { fail(); };
|
||||
};
|
||||
match (utf8.utf8sz(0x7Fu8)) {
|
||||
case let n: i32 => { if (n != 1) { fail(); }; };
|
||||
case let e: utf8.invalid => { fail(); };
|
||||
};
|
||||
match (utf8.utf8sz(0x80u8)) { // continuation
|
||||
case let n: i32 => { fail(); };
|
||||
case let e: utf8.invalid => void;
|
||||
};
|
||||
match (utf8.utf8sz(0xC1u8)) { // overlong 2-byte lead
|
||||
case let n: i32 => { fail(); };
|
||||
case let e: utf8.invalid => void;
|
||||
};
|
||||
match (utf8.utf8sz(0xC2u8)) {
|
||||
case let n: i32 => { if (n != 2) { fail(); }; };
|
||||
case let e: utf8.invalid => { fail(); };
|
||||
};
|
||||
match (utf8.utf8sz(0xE0u8)) {
|
||||
case let n: i32 => { if (n != 3) { fail(); }; };
|
||||
case let e: utf8.invalid => { fail(); };
|
||||
};
|
||||
match (utf8.utf8sz(0xF0u8)) {
|
||||
case let n: i32 => { if (n != 4) { fail(); }; };
|
||||
case let e: utf8.invalid => { fail(); };
|
||||
};
|
||||
match (utf8.utf8sz(0xF8u8)) { // 5-byte lead — illegal in modern UTF-8
|
||||
case let n: i32 => { fail(); };
|
||||
case let e: utf8.invalid => void;
|
||||
};
|
||||
match (utf8.utf8sz(0xFFu8)) {
|
||||
case let n: i32 => { fail(); };
|
||||
case let e: utf8.invalid => void;
|
||||
};
|
||||
};
|
||||
|
||||
// ---- encoderune: all four widths --------------------------------------
|
||||
// ref/hare/encoding/utf8/encode.ha:7. Byte vectors come from the
|
||||
// Unicode specification (UAX standard examples).
|
||||
|
||||
@test fn encode_ascii() void = {
|
||||
let out: [4]u8;
|
||||
let n: i32 = utf8.encoderune(out[0:4], 0x41u32: rune); // 'A'
|
||||
if (n != 1) { fail(); };
|
||||
if (out[0] != 0x41u8) { fail(); };
|
||||
};
|
||||
|
||||
@test fn encode_two_byte() void = {
|
||||
let out: [4]u8;
|
||||
let n: i32 = utf8.encoderune(out[0:4], 0xE9u32: rune); // 'é' U+00E9
|
||||
if (n != 2) { fail(); };
|
||||
if (out[0] != 0xC3u8) { fail(); };
|
||||
if (out[1] != 0xA9u8) { fail(); };
|
||||
};
|
||||
|
||||
@test fn encode_three_byte() void = {
|
||||
let out: [4]u8;
|
||||
let n: i32 = utf8.encoderune(out[0:4], 0x20ACu32: rune); // '€' U+20AC
|
||||
if (n != 3) { fail(); };
|
||||
if (out[0] != 0xE2u8) { fail(); };
|
||||
if (out[1] != 0x82u8) { fail(); };
|
||||
if (out[2] != 0xACu8) { fail(); };
|
||||
};
|
||||
|
||||
@test fn encode_four_byte() void = {
|
||||
let out: [4]u8;
|
||||
let n: i32 = utf8.encoderune(out[0:4], 0x1F980u32: rune); // '🦀' U+1F980
|
||||
if (n != 4) { fail(); };
|
||||
if (out[0] != 0xF0u8) { fail(); };
|
||||
if (out[1] != 0x9Fu8) { fail(); };
|
||||
if (out[2] != 0xA6u8) { fail(); };
|
||||
if (out[3] != 0x80u8) { fail(); };
|
||||
};
|
||||
|
||||
// ---- decoder.next: valid 1/2/3/4-byte ---------------------------------
|
||||
// ref/hare/encoding/utf8/decode.ha:27. Round-trip via the same byte
|
||||
// vectors used in encode.
|
||||
|
||||
@test fn decode_one_byte() void = {
|
||||
let src: [1]u8;
|
||||
src[0] = 0x41u8;
|
||||
let d: utf8.decoder = utf8.decode(src[0:1]);
|
||||
match (utf8.next(&d)) {
|
||||
case utf8.done => { fail(); };
|
||||
case utf8.more => { fail(); };
|
||||
case let r: utf8.invalid => { fail(); };
|
||||
case let r: rune => { if (r != 0x41u32: rune) { fail(); }; };
|
||||
};
|
||||
};
|
||||
|
||||
@test fn decode_two_byte() void = {
|
||||
let src: [2]u8;
|
||||
src[0] = 0xC3u8; src[1] = 0xA9u8;
|
||||
let d: utf8.decoder = utf8.decode(src[0:2]);
|
||||
match (utf8.next(&d)) {
|
||||
case utf8.done => { fail(); };
|
||||
case utf8.more => { fail(); };
|
||||
case let r: utf8.invalid => { fail(); };
|
||||
case let r: rune => { if (r != 0xE9u32: rune) { fail(); }; };
|
||||
};
|
||||
};
|
||||
|
||||
@test fn decode_three_byte() void = {
|
||||
let src: [3]u8;
|
||||
src[0] = 0xE2u8; src[1] = 0x82u8; src[2] = 0xACu8;
|
||||
let d: utf8.decoder = utf8.decode(src[0:3]);
|
||||
match (utf8.next(&d)) {
|
||||
case utf8.done => { fail(); };
|
||||
case utf8.more => { fail(); };
|
||||
case let r: utf8.invalid => { fail(); };
|
||||
case let r: rune => { if (r != 0x20ACu32: rune) { fail(); }; };
|
||||
};
|
||||
};
|
||||
|
||||
@test fn decode_four_byte() void = {
|
||||
let src: [4]u8;
|
||||
src[0] = 0xF0u8; src[1] = 0x9Fu8; src[2] = 0xA6u8; src[3] = 0x80u8;
|
||||
let d: utf8.decoder = utf8.decode(src[0:4]);
|
||||
match (utf8.next(&d)) {
|
||||
case utf8.done => { fail(); };
|
||||
case utf8.more => { fail(); };
|
||||
case let r: utf8.invalid => { fail(); };
|
||||
case let r: rune => { if (r != 0x1F980u32: rune) { fail(); }; };
|
||||
};
|
||||
};
|
||||
|
||||
// ---- decoder.next: invalid inputs -------------------------------------
|
||||
// Cases mirror ref/hare/encoding/utf8/decode.ha:127-167 (surrogate /
|
||||
// overlong / out-of-range / bad-continuation).
|
||||
|
||||
@test fn decode_surrogate() void = {
|
||||
let src: [3]u8;
|
||||
src[0] = 0xEDu8; src[1] = 0xA0u8; src[2] = 0x80u8; // U+D800
|
||||
let d: utf8.decoder = utf8.decode(src[0:3]);
|
||||
match (utf8.next(&d)) {
|
||||
case let r: utf8.invalid => void;
|
||||
case let r: rune => { fail(); };
|
||||
case utf8.done => { fail(); };
|
||||
case utf8.more => { fail(); };
|
||||
};
|
||||
};
|
||||
|
||||
@test fn decode_overlong() void = {
|
||||
let src: [4]u8;
|
||||
src[0] = 0xF0u8; src[1] = 0x82u8; src[2] = 0x82u8; src[3] = 0xACu8;
|
||||
let d: utf8.decoder = utf8.decode(src[0:4]);
|
||||
match (utf8.next(&d)) {
|
||||
case let r: utf8.invalid => void;
|
||||
case let r: rune => { fail(); };
|
||||
case utf8.done => { fail(); };
|
||||
case utf8.more => { fail(); };
|
||||
};
|
||||
};
|
||||
|
||||
@test fn decode_out_of_range() void = {
|
||||
let src: [4]u8;
|
||||
src[0] = 0xF5u8; src[1] = 0x94u8; src[2] = 0x80u8; src[3] = 0x80u8;
|
||||
let d: utf8.decoder = utf8.decode(src[0:4]);
|
||||
match (utf8.next(&d)) {
|
||||
case let r: utf8.invalid => void;
|
||||
case let r: rune => { fail(); };
|
||||
case utf8.done => { fail(); };
|
||||
case utf8.more => { fail(); };
|
||||
};
|
||||
};
|
||||
|
||||
// ref/hare/encoding/utf8/decode.ha:141 — `[0xC2, 0xFF]`: legal 2-byte
|
||||
// lead followed by non-continuation. Pins next-table cell (state 1,
|
||||
// byte 0xFF) returning -1; distinct from overlong (which is filtered
|
||||
// in state 3/5/7 by lead-byte-aware sub-states).
|
||||
@test fn decode_bad_continuation() void = {
|
||||
let src: [2]u8;
|
||||
src[0] = 0xC2u8; src[1] = 0xFFu8;
|
||||
let d: utf8.decoder = utf8.decode(src[0:2]);
|
||||
match (utf8.next(&d)) {
|
||||
case let r: utf8.invalid => void;
|
||||
case let r: rune => { fail(); };
|
||||
case utf8.done => { fail(); };
|
||||
case utf8.more => { fail(); };
|
||||
};
|
||||
};
|
||||
|
||||
// ref/hare/encoding/utf8/decode.ha:151 — `[0xF4, 0x8F, 0xBF, 0xBF]` =
|
||||
// U+10FFFF, the largest legal codepoint. Pins the upper boundary;
|
||||
// pairs with the existing `0xF5…` out-of-range row.
|
||||
@test fn decode_max_in_range() void = {
|
||||
let src: [4]u8;
|
||||
src[0] = 0xF4u8; src[1] = 0x8Fu8; src[2] = 0xBFu8; src[3] = 0xBFu8;
|
||||
let d: utf8.decoder = utf8.decode(src[0:4]);
|
||||
match (utf8.next(&d)) {
|
||||
case let r: rune => { if (r != 0x10FFFFu32: rune) { fail(); }; };
|
||||
case let r: utf8.invalid => { fail(); };
|
||||
case utf8.done => { fail(); };
|
||||
case utf8.more => { fail(); };
|
||||
};
|
||||
};
|
||||
|
||||
// ---- decoder.next: truncated → more -----------------------------------
|
||||
|
||||
@test fn decode_truncated() void = {
|
||||
let src: [2]u8;
|
||||
src[0] = 0xE3u8; src[1] = 0x81u8; // missing 3rd byte
|
||||
let d: utf8.decoder = utf8.decode(src[0:2]);
|
||||
match (utf8.next(&d)) {
|
||||
case utf8.more => void;
|
||||
case let r: rune => { fail(); };
|
||||
case utf8.done => { fail(); };
|
||||
case let r: utf8.invalid => { fail(); };
|
||||
};
|
||||
};
|
||||
|
||||
// ---- decoder.next: done at EOI ----------------------------------------
|
||||
|
||||
@test fn decode_done() void = {
|
||||
let src: [1]u8;
|
||||
let d: utf8.decoder = utf8.decode(src[0:0]);
|
||||
match (utf8.next(&d)) {
|
||||
case utf8.done => void;
|
||||
case let r: rune => { fail(); };
|
||||
case utf8.more => { fail(); };
|
||||
case let r: utf8.invalid => { fail(); };
|
||||
};
|
||||
};
|
||||
|
||||
// ---- validate: well-formed mixed-width vs malformed -------------------
|
||||
// ref/hare/encoding/utf8/decode.ha:207. Mixed input mirrors Hare's
|
||||
// decode @test ('こんにちは' + NUL).
|
||||
|
||||
@test fn validate_mixed_ok() void = {
|
||||
let src: [16]u8;
|
||||
src[0] = 0xE3u8; src[1] = 0x81u8; src[2] = 0x93u8; // こ
|
||||
src[3] = 0xE3u8; src[4] = 0x82u8; src[5] = 0x93u8; // ん
|
||||
src[6] = 0xE3u8; src[7] = 0x81u8; src[8] = 0xABu8; // に
|
||||
src[9] = 0xE3u8; src[10] = 0x81u8; src[11] = 0xA1u8; // ち
|
||||
src[12] = 0xE3u8; src[13] = 0x81u8; src[14] = 0xAFu8; // は
|
||||
src[15] = 0u8;
|
||||
match (utf8.validate(src[0:16])) {
|
||||
case let e: utf8.invalid => { fail(); };
|
||||
case void => void;
|
||||
};
|
||||
};
|
||||
|
||||
@test fn validate_malformed() void = {
|
||||
let src: [3]u8;
|
||||
src[0] = 0xEDu8; src[1] = 0xA0u8; src[2] = 0x80u8; // surrogate
|
||||
match (utf8.validate(src[0:3])) {
|
||||
case let e: utf8.invalid => void;
|
||||
case void => { fail(); };
|
||||
};
|
||||
};
|
||||
|
||||
@test fn validate_empty_ok() void = {
|
||||
let src: [1]u8;
|
||||
match (utf8.validate(src[0:0])) {
|
||||
case let e: utf8.invalid => { fail(); };
|
||||
case void => void;
|
||||
};
|
||||
};
|
||||
|
||||
// ---- round-trip: encode → decode → equal rune --------------------------
|
||||
|
||||
@test fn roundtrip() void = {
|
||||
let runes: [4]u32;
|
||||
runes[0] = 0x41u32;
|
||||
runes[1] = 0xE9u32;
|
||||
runes[2] = 0x20ACu32;
|
||||
runes[3] = 0x1F980u32;
|
||||
let i: i32 = 0;
|
||||
for (i < 4) {
|
||||
let buf: [4]u8;
|
||||
let n: i32 = utf8.encoderune(buf[0:4], runes[i]: rune);
|
||||
let d: utf8.decoder = utf8.decode(buf[0:n]);
|
||||
match (utf8.next(&d)) {
|
||||
case let r: rune => {
|
||||
if ((r: u32) != runes[i]) { fail(); };
|
||||
};
|
||||
case utf8.done => { fail(); };
|
||||
case utf8.more => { fail(); };
|
||||
case let e: utf8.invalid => { fail(); };
|
||||
};
|
||||
i += 1;
|
||||
};
|
||||
};
|
||||
|
||||
export fn main() i32 = {
|
||||
signalled = 1; runesz_ranges();
|
||||
signalled = 2; utf8sz_classify();
|
||||
signalled = 3; encode_ascii();
|
||||
signalled = 4; encode_two_byte();
|
||||
signalled = 5; encode_three_byte();
|
||||
signalled = 6; encode_four_byte();
|
||||
signalled = 7; decode_one_byte();
|
||||
signalled = 8; decode_two_byte();
|
||||
signalled = 9; decode_three_byte();
|
||||
signalled = 10; decode_four_byte();
|
||||
signalled = 11; decode_surrogate();
|
||||
signalled = 12; decode_overlong();
|
||||
signalled = 13; decode_out_of_range();
|
||||
signalled = 14; decode_bad_continuation();
|
||||
signalled = 15; decode_max_in_range();
|
||||
signalled = 16; decode_truncated();
|
||||
signalled = 17; decode_done();
|
||||
signalled = 18; validate_mixed_ok();
|
||||
signalled = 19; validate_malformed();
|
||||
signalled = 20; validate_empty_ok();
|
||||
signalled = 21; roundtrip();
|
||||
return 0;
|
||||
};
|
||||
51
test/wcc/968_utf8_run.c
Normal file
51
test/wcc/968_utf8_run.c
Normal file
@@ -0,0 +1,51 @@
|
||||
/*
|
||||
* 968_utf8_run — execute the lib/encoding/utf8 @test fixture under the
|
||||
* C-side `ww run` driver and assert exit 0.
|
||||
*
|
||||
* Thin wrapper mirroring 979_hex_run / 977_time_run: utf8test.ww carries
|
||||
* its own `export fn main()` that drives the @test fns and signals which
|
||||
* case failed via the exit code (signalled + 10).
|
||||
*
|
||||
* Slot 968: stdlib block 970–989 was full at landing time.
|
||||
*/
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <unistd.h>
|
||||
#include <sys/wait.h>
|
||||
|
||||
static int
|
||||
runwait(const char *cmd)
|
||||
{
|
||||
int rc = system(cmd);
|
||||
if (rc == -1) return -1;
|
||||
if (WIFEXITED(rc)) return WEXITSTATUS(rc);
|
||||
return 1;
|
||||
}
|
||||
|
||||
int
|
||||
main(void)
|
||||
{
|
||||
const char *bin = getenv("BIN");
|
||||
if (!bin) bin = "out/bin";
|
||||
char absbin[1024];
|
||||
if (bin[0] != '/') {
|
||||
char cwd[1024];
|
||||
if (getcwd(cwd, sizeof cwd) == NULL) return 1;
|
||||
snprintf(absbin, sizeof absbin, "%s/%s", cwd, bin);
|
||||
bin = absbin;
|
||||
}
|
||||
char cwd[1024];
|
||||
if (getcwd(cwd, sizeof cwd) == NULL) return 1;
|
||||
|
||||
const char *src = "lib/encoding/utf8/utf8test.ww";
|
||||
char path[1024], cmd[2048];
|
||||
snprintf(path, sizeof path, "%s/%s", cwd, src);
|
||||
snprintf(cmd, sizeof cmd, "%s/ww run %s", bin, path);
|
||||
int rc = runwait(cmd);
|
||||
if (rc != 0) {
|
||||
fprintf(stderr, "utf8_run FAIL: %s exited %d\n", src, rc);
|
||||
return 1;
|
||||
}
|
||||
printf("utf8_run: %s ok\n", src);
|
||||
return 0;
|
||||
}
|
||||
Reference in New Issue
Block a user