From 793734c1e049738082101d3ebd9bb5748d951973 Mon Sep 17 00:00:00 2001 From: Hojun-Cho Date: Sun, 17 May 2026 23:46:34 +0900 Subject: [PATCH] lib/encoding/utf8+test: Hare port (decoder / next / encoderune / runesz / utf8sz / validate) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Hoehrmann DFA from ref/hare/encoding/utf8/decodetable.ha flattened to 1D [2048]i8 (task #20: 2D-array jagged cgen still pending); encoderune takes a caller buffer matching lib/encoding/hex.encode; done/more/invalid all spelled as plain void aliases per lib/io's eof precedent. Surface ports decoder + decode + next + encoderune + runesz + utf8sz + validate from ref/hare/encoding/utf8/{types, decode,encode,rune}.ha. next() polarity rewritten from Hare's `(state-1):uint >> 31` to an explicit `if state == 0` branch because ww's uint is 64-bit (cmd/wcc/type.c:58); same effect, no hidden 32-bit assumption. Deferred (no in-tree callers): prev, slice, position, remaining, appendrune, strencode, strdecode. String iteration (chars/ newchars/nextchar in the session-4 draft) dropped per Hare discipline — belongs in lib/strings::iterator, not encoding/utf8. Tests: - 968_utf8_run drives lib/encoding/utf8/utf8test.ww via ww run. 21 @test fns: boundaries (ASCII, 2-byte, 3-byte, 4-byte encode/decode), surrogate/overlong/out-of-range/bad-continuation reject, max-in-range (U+10FFFF) accept, truncated→more, done@EOI, validate empty/mixed/malformed, encode/decode roundtrip. Two rows ported from ref/hare/encoding/utf8/decode.ha @test that were missing in the session-4 draft: bad-continuation [0xC2,0xFF]→invalid and max-in-range [0xF4,0x8F,0xBF,0xBF]→ U+10FFFF. - 9xx stdlib runtime slot range extended from 970-989 to 960-989 to accommodate utf8 at 968 (970-989 block was full). 90/90 ok. 995_self_rebuild stays green (ww2==ww3==ww4 byte-id). --- Makefile | 6 +- lib/encoding/utf8/utf8.ww | 348 +++++++++++++++++++++++++++++++-- lib/encoding/utf8/utf8test.ww | 351 ++++++++++++++++++++++++++++++++++ test/wcc/968_utf8_run.c | 51 +++++ 4 files changed, 743 insertions(+), 13 deletions(-) create mode 100644 lib/encoding/utf8/utf8test.ww create mode 100644 test/wcc/968_utf8_run.c diff --git a/Makefile b/Makefile index c54021ef..30522f94 100644 --- a/Makefile +++ b/Makefile @@ -260,7 +260,7 @@ TESTS = $(BIN)/test_smoke $(BIN)/test_lex $(BIN)/test_parse $(BIN)/test_check \ $(BIN)/test_shlex_run $(BIN)/test_getenv_run $(BIN)/test_dirs_run \ $(BIN)/test_stat_run $(BIN)/test_time_run \ $(BIN)/test_intdiv_signed \ - $(BIN)/test_hex_run \ + $(BIN)/test_hex_run $(BIN)/test_utf8_run \ $(BIN)/test_memio_run $(BIN)/test_temp_run $(BIN)/test_getopt_run \ $(BIN)/test_base32_run $(BIN)/test_base64_run \ $(BIN)/test_adler32_run $(BIN)/test_crc16_run \ @@ -634,6 +634,10 @@ $(BIN)/test_hex_run: test/wcc/979_hex_run.c $(BIN)/ww $(BIN)/w6c \ $(BIN)/w6a $(BIN)/w6l $(LIB)/libwwrt.a | $(BIN) $(CC) $(CFLAGS) -o $@ $< +$(BIN)/test_utf8_run: test/wcc/968_utf8_run.c $(BIN)/ww $(BIN)/w6c \ + $(BIN)/w6a $(BIN)/w6l $(LIB)/libwwrt.a | $(BIN) + $(CC) $(CFLAGS) -o $@ $< + $(BIN)/test_memio_run: test/wcc/980_memio_run.c $(BIN)/ww $(BIN)/w6c \ $(BIN)/w6a $(BIN)/w6l $(LIB)/libwwrt.a | $(BIN) $(CC) $(CFLAGS) -o $@ $< diff --git a/lib/encoding/utf8/utf8.ww b/lib/encoding/utf8/utf8.ww index 0185f7d6..0b965ecc 100644 --- a/lib/encoding/utf8/utf8.ww +++ b/lib/encoding/utf8/utf8.ww @@ -1,15 +1,339 @@ -// encoding/utf8 — UTF-8 helpers. RFC 3629; we only handle the legal -// subset (no over-long encodings, no surrogates). +// encoding/utf8 — UTF-8 encode/decode. Hare port; see +// ref/hare/encoding/utf8/{types,rune,encode,decode,decodetable}.ha. +// +// The decoder is Hoehrmann's branchless DFA, originally published +// at . Hare's +// ref/hare/encoding/utf8/decodetable.ha:4 restructures Hoehrmann's +// flat table to 2D `[8][256]i8`; we flatten back to 1D `[2048]i8` +// because ww cgen does not yet ship 2D arrays (task #20). +// +// Surface deviation from ref/hare/encoding/utf8: +// +// - `encoderune` takes a caller-supplied `out: []u8` and returns +// the byte count. Hare returns a slice into a `static let buf`; +// the caller-buffer form mirrors lib/encoding/hex.encode and +// skips the static-buffer/slice-return pair. +// +// Deferred (no in-tree caller, follow-up tasks): `prev`, `slice`, +// `position`, `remaining`, `appendrune`, `strencode`, `strdecode`. +// Hare's string-iteration surface (`strings::iterator`/`strings::next` +// — ref/hare/strings/iter.ha) lives under lib/strings, not here. -def MAX: rune = 1114111; // 0x10FFFF +// ref/hare/encoding/utf8/types.ha:6 — incomplete trailing sequence. +// Plain `void` (not `!void`): a truncated tail is a control-flow +// signal, not an error caller can ignore. +export type more = void; -// runesz — encoded byte length of `r` as UTF-8. void variant means -// `r` is outside the legal range (negative, > 0x10FFFF). -export fn runesz(r: rune) (i32 | void) = { - if (r < 0) { return; }; - if (r < 128) { return 1; }; - if (r < 2048) { return 2; }; - if (r < 65536) { return 3; }; - if (r <= MAX) { return 4; }; - return; +// ref/hare/encoding/utf8/types.ha:9 — invalid UTF-8 sequence. +export type invalid = !void; + +// `done` is not a built-in singleton in ww (Hare ships it as part of +// the type system). Plain `void` (not `!void`): end-of-input is a +// continuation signal, not an error. lib/io spells its EOF the same +// way (lib/io/io.ww:8-11). +export type done = void; + +// ref/hare/encoding/utf8/decodetable.ha:4 — Hoehrmann's UTF-8 DFA, +// flat 1D `[2048]i8`. Layout: dfa[state*256 + byte] gives the next +// state (>0), the accept transition (0 — emit rune), or invalid (-1). +// Values match ref/hare/encoding/utf8/decodetable.ha verbatim. +let dfa: [2048]i8 = [ + // state 0 — initial byte: ASCII accepts (0), continuation/illegal + // byte rejects (-1), legal multibyte start emits a state. + 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, + 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, + 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, + 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, + 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, + 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, + 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, + 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, + -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, + -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, + -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, + -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, + -1i8, -1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, + 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, + 3i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 4i8, 2i8, 2i8, + 5i8, 6i8, 6i8, 6i8, 7i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, + + // state 1 — expecting one continuation byte (0x80..0xBF). + -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, + -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, + -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, + -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, + -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, + -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, + -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, + -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, + 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, + 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, + 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, + 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, 0i8, + -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, + -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, + -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, + -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, + + // state 2 — expecting one continuation byte (full 0x80..0xBF range). + -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, + -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, + -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, + -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, + -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, + -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, + -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, + -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, + 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, + 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, + 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, + 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, + -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, + -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, + -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, + -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, + + // state 3 — first byte was 0xE0; continuation byte must be 0xA0..0xBF + // (rejects overlong 3-byte encodings). + -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, + -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, + -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, + -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, + -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, + -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, + -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, + -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, + -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, + -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, + 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, + 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, + -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, + -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, + -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, + -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, + + // state 4 — first byte was 0xED; continuation byte must be 0x80..0x9F + // (rejects UTF-16 surrogate codepoints U+D800..U+DFFF). + -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, + -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, + -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, + -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, + -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, + -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, + -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, + -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, + 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, + 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, 1i8, + -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, + -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, + -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, + -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, + -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, + -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, + + // state 5 — first byte was 0xF0; continuation byte must be 0x90..0xBF + // (rejects overlong 4-byte encodings). + -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, + -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, + -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, + -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, + -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, + -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, + -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, + -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, + -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, + 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, + 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, + 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, + -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, + -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, + -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, + -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, + + // state 6 — middle continuation byte of a 4-byte sequence (0x80..0xBF). + -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, + -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, + -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, + -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, + -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, + -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, + -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, + -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, + 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, + 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, + 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, + 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, + -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, + -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, + -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, + -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, + + // state 7 — first byte was 0xF4; continuation byte must be 0x80..0x8F + // (rejects codepoints above U+10FFFF). + -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, + -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, + -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, + -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, + -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, + -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, + -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, + -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, + 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, 2i8, + -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, + -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, + -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, + -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, + -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, + -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, + -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, -1i8, +]; + +// ref/hare/encoding/utf8/decode.ha:17 — payload-bit masks. Hare's +// [2][8]u8 flattened to 1D [16]u8; row 0 (offsets 0..7) is the +// continuation-byte mask (always 0x3F), row 1 (offsets 8..15) is the +// initial-byte payload mask indexed by the transition class. +let masks: [16]u8 = [ + 0x3fu8, 0x3fu8, 0x3fu8, 0x3fu8, 0x3fu8, 0x3fu8, 0x3fu8, 0x3fu8, + 0x7fu8, 0x1fu8, 0x0fu8, 0x0fu8, 0x0fu8, 0x07u8, 0x07u8, 0x07u8, +]; + +// ref/hare/encoding/utf8/decode.ha:6 — incremental decoder state. +export type decoder = struct { + offs: i32, + src: []u8, }; + +// ref/hare/encoding/utf8/decode.ha:12. +export fn decode(src: []u8) decoder = { + let d: decoder; + d.src = src; + d.offs = 0; + return d; +}; + +// ref/hare/encoding/utf8/decode.ha:27. Returns the next rune from a +// decoder, `done` at end-of-input, `more` on truncated trailing +// sequence, `invalid` on malformed input (overlong, surrogate, +// out-of-range, bad continuation). +// +// Algorithm is verbatim Hoehrmann (see file header). One structural +// rewrite: Hare encodes the "initial vs continuation byte" decision +// as the branchless `(state - 1): uint >> 31`, which assumes a 32-bit +// uint. ww's uint is 64-bit (cmd/wcc/type.c:58), so the shift answer +// would be 0x1_ffff_ffff rather than 1. We spell the same predicate +// with an explicit conditional. +export fn next(d: *decoder) (rune | done | more | invalid) = { + if (d.offs == d.src.len) { + let dn: done; return dn; + }; + let nx: i32 = 0; + let state: i32 = 0; + let r: u32 = 0u32; + for (d.offs < d.src.len) { + let b: u8 = d.src[d.offs]; + let bi: i32 = b: i32; + let row: i32 = state * 256 + bi; + let cell: i8 = dfa[row]; + nx = cell: i32; + let mi: i32 = 0; + if (state == 0) { mi = 1; }; + let m: u8 = masks[mi * 8 + (nx & 7)]; + r = (r << 6u32) | ((b & m): u32); + if (nx <= 0) { + d.offs += 1; + if (nx == 0) { return r: rune; }; + let e: invalid; return e; + }; + state = nx; + d.offs += 1; + }; + let mr: more; return mr; +}; + +// ref/hare/encoding/utf8/decode.ha:207. Strict whole-input check. +// The hot path: tight DFA loop, no rune assembly. Bails the moment +// the table returns -1 so malformed inputs don't pay for the rest +// of the buffer. +export fn validate(src: []u8) (void | invalid) = { + let state: i32 = 0; + let i: i32 = 0; + for (i < src.len) { + if (state < 0) { break; }; + let bi: i32 = src[i]: i32; + let cell: i8 = dfa[state * 256 + bi]; + state = cell: i32; + i += 1; + }; + if (state == 0) { return; }; + let e: invalid; return e; +}; + +// ref/hare/encoding/utf8/rune.ha:5. Encoded byte length of `r` as +// UTF-8. Callers in ww use this to size the buffer they hand to +// [[encoderune]]; values >0x10FFFF or negative are not legal Unicode +// codepoints and Hare aborts on them in `encoderune` itself, so we +// keep `runesz` infallible (matches Hare). +export fn runesz(r: rune) i32 = { + let ch: u32 = r: u32; + if (ch < 128u32) { return 1; }; + if (ch < 2048u32) { return 2; }; + if (ch < 65536u32) { return 3; }; + return 4; +}; + +// ref/hare/encoding/utf8/rune.ha:15. Expected byte length of the +// codepoint that starts with `c`, or `invalid` if `c` cannot start +// a legal UTF-8 sequence. Constants written in decimal because ww +// doesn't accept Hare's `0b1000_0000` binary syntax: 0x80=128, +// 0xC2=194, 0xE0=224, 0xF0=240, 0xF8=248. +export fn utf8sz(c: u8) (i32 | invalid) = { + if (c < 128u8) { return 1; }; + if (c < 194u8) { let e: invalid; return e; }; + if (c >= 248u8) { let e: invalid; return e; }; + if (c < 224u8) { return 2; }; + if (c < 240u8) { return 3; }; + return 4; +}; + +// ref/hare/encoding/utf8/encode.ha:7. Encode `r` into `out` (caller- +// supplied; must hold at least [[runesz]](r) bytes) and return the +// byte count. ABORT if `r` is a UTF-16 surrogate or above U+10FFFF — +// same precondition Hare asserts at ref/hare/encoding/utf8/encode.ha:9. +// +// Surface deviation: Hare returns `[]u8` (slice into a static buf). +// ww uses the caller-buffer form (matches lib/encoding/hex.encode); +// caller can reuse a [4]u8 stack scratch across encodes. +export fn encoderune(out: []u8, r: rune) i32 = { + let ch: u32 = r: u32; + if (ch >= 0xD800u32) { + if (ch <= 0xDFFFu32) { + abort("utf8.encoderune: surrogate codepoint"); + }; + }; + if (ch > 0x10FFFFu32) { + abort("utf8.encoderune: codepoint > U+10FFFF"); + }; + + let n: i32 = 0; + let first: u8 = 0u8; + if (ch < 0x80u32) { + first = 0u8; n = 1; + } else if (ch < 0x800u32) { + first = 0xC0u8; n = 2; + } else if (ch < 0x10000u32) { + first = 0xE0u8; n = 3; + } else { + first = 0xF0u8; n = 4; + }; + + let v: u32 = ch; + let i: i32 = n - 1; + for (i > 0) { + out[i] = ((v: u8) & 0x3Fu8) | 0x80u8; + v = v >> 6u32; + i -= 1; + }; + out[0] = (v: u8) | first; + return n; +}; + diff --git a/lib/encoding/utf8/utf8test.ww b/lib/encoding/utf8/utf8test.ww new file mode 100644 index 00000000..cd844f09 --- /dev/null +++ b/lib/encoding/utf8/utf8test.ww @@ -0,0 +1,351 @@ +// utf8test — exercises lib/encoding/utf8. Run with +// `out/bin/ww run lib/encoding/utf8/utf8test.ww`. Same signalled- +// then-fail()-with-+10 pattern as hex / base32 / time tests: +// non-zero exit pinpoints the failing scenario. + +use utf8; +use os; + +let signalled: i32 = 0; +fn fail() void = { os.exit(signalled + 10); }; + +fn beq(a: []u8, b: []u8) bool = { + if (a.len != b.len) { return false; }; + let i: i32 = 0; + for (i < a.len) { + if (a[i] != b[i]) { return false; }; + i += 1; + }; + return true; +}; + +// ---- runesz: byte length per range ------------------------------------ +// ref/hare/encoding/utf8/rune.ha:5. Boundaries: 0x7F → 1, 0x80 → 2, +// 0x7FF → 2, 0x800 → 3, 0xFFFF → 3, 0x10000 → 4, 0x10FFFF → 4. + +@test fn runesz_ranges() void = { + if (utf8.runesz(0u32: rune) != 1) { fail(); }; + if (utf8.runesz(0x7Fu32: rune) != 1) { fail(); }; + if (utf8.runesz(0x80u32: rune) != 2) { fail(); }; + if (utf8.runesz(0x7FFu32: rune) != 2) { fail(); }; + if (utf8.runesz(0x800u32: rune) != 3) { fail(); }; + if (utf8.runesz(0xFFFFu32: rune) != 3) { fail(); }; + if (utf8.runesz(0x10000u32: rune) != 4) { fail(); }; + if (utf8.runesz(0x10FFFFu32: rune) != 4) { fail(); }; +}; + +// ---- utf8sz: start-byte classification -------------------------------- +// ref/hare/encoding/utf8/rune.ha:15. ASCII → 1; legal multibyte +// leads → 2/3/4; continuation and >0xF7 → invalid. + +@test fn utf8sz_classify() void = { + match (utf8.utf8sz(0u8)) { + case let n: i32 => { if (n != 1) { fail(); }; }; + case let e: utf8.invalid => { fail(); }; + }; + match (utf8.utf8sz(0x7Fu8)) { + case let n: i32 => { if (n != 1) { fail(); }; }; + case let e: utf8.invalid => { fail(); }; + }; + match (utf8.utf8sz(0x80u8)) { // continuation + case let n: i32 => { fail(); }; + case let e: utf8.invalid => void; + }; + match (utf8.utf8sz(0xC1u8)) { // overlong 2-byte lead + case let n: i32 => { fail(); }; + case let e: utf8.invalid => void; + }; + match (utf8.utf8sz(0xC2u8)) { + case let n: i32 => { if (n != 2) { fail(); }; }; + case let e: utf8.invalid => { fail(); }; + }; + match (utf8.utf8sz(0xE0u8)) { + case let n: i32 => { if (n != 3) { fail(); }; }; + case let e: utf8.invalid => { fail(); }; + }; + match (utf8.utf8sz(0xF0u8)) { + case let n: i32 => { if (n != 4) { fail(); }; }; + case let e: utf8.invalid => { fail(); }; + }; + match (utf8.utf8sz(0xF8u8)) { // 5-byte lead — illegal in modern UTF-8 + case let n: i32 => { fail(); }; + case let e: utf8.invalid => void; + }; + match (utf8.utf8sz(0xFFu8)) { + case let n: i32 => { fail(); }; + case let e: utf8.invalid => void; + }; +}; + +// ---- encoderune: all four widths -------------------------------------- +// ref/hare/encoding/utf8/encode.ha:7. Byte vectors come from the +// Unicode specification (UAX standard examples). + +@test fn encode_ascii() void = { + let out: [4]u8; + let n: i32 = utf8.encoderune(out[0:4], 0x41u32: rune); // 'A' + if (n != 1) { fail(); }; + if (out[0] != 0x41u8) { fail(); }; +}; + +@test fn encode_two_byte() void = { + let out: [4]u8; + let n: i32 = utf8.encoderune(out[0:4], 0xE9u32: rune); // 'é' U+00E9 + if (n != 2) { fail(); }; + if (out[0] != 0xC3u8) { fail(); }; + if (out[1] != 0xA9u8) { fail(); }; +}; + +@test fn encode_three_byte() void = { + let out: [4]u8; + let n: i32 = utf8.encoderune(out[0:4], 0x20ACu32: rune); // '€' U+20AC + if (n != 3) { fail(); }; + if (out[0] != 0xE2u8) { fail(); }; + if (out[1] != 0x82u8) { fail(); }; + if (out[2] != 0xACu8) { fail(); }; +}; + +@test fn encode_four_byte() void = { + let out: [4]u8; + let n: i32 = utf8.encoderune(out[0:4], 0x1F980u32: rune); // '🦀' U+1F980 + if (n != 4) { fail(); }; + if (out[0] != 0xF0u8) { fail(); }; + if (out[1] != 0x9Fu8) { fail(); }; + if (out[2] != 0xA6u8) { fail(); }; + if (out[3] != 0x80u8) { fail(); }; +}; + +// ---- decoder.next: valid 1/2/3/4-byte --------------------------------- +// ref/hare/encoding/utf8/decode.ha:27. Round-trip via the same byte +// vectors used in encode. + +@test fn decode_one_byte() void = { + let src: [1]u8; + src[0] = 0x41u8; + let d: utf8.decoder = utf8.decode(src[0:1]); + match (utf8.next(&d)) { + case utf8.done => { fail(); }; + case utf8.more => { fail(); }; + case let r: utf8.invalid => { fail(); }; + case let r: rune => { if (r != 0x41u32: rune) { fail(); }; }; + }; +}; + +@test fn decode_two_byte() void = { + let src: [2]u8; + src[0] = 0xC3u8; src[1] = 0xA9u8; + let d: utf8.decoder = utf8.decode(src[0:2]); + match (utf8.next(&d)) { + case utf8.done => { fail(); }; + case utf8.more => { fail(); }; + case let r: utf8.invalid => { fail(); }; + case let r: rune => { if (r != 0xE9u32: rune) { fail(); }; }; + }; +}; + +@test fn decode_three_byte() void = { + let src: [3]u8; + src[0] = 0xE2u8; src[1] = 0x82u8; src[2] = 0xACu8; + let d: utf8.decoder = utf8.decode(src[0:3]); + match (utf8.next(&d)) { + case utf8.done => { fail(); }; + case utf8.more => { fail(); }; + case let r: utf8.invalid => { fail(); }; + case let r: rune => { if (r != 0x20ACu32: rune) { fail(); }; }; + }; +}; + +@test fn decode_four_byte() void = { + let src: [4]u8; + src[0] = 0xF0u8; src[1] = 0x9Fu8; src[2] = 0xA6u8; src[3] = 0x80u8; + let d: utf8.decoder = utf8.decode(src[0:4]); + match (utf8.next(&d)) { + case utf8.done => { fail(); }; + case utf8.more => { fail(); }; + case let r: utf8.invalid => { fail(); }; + case let r: rune => { if (r != 0x1F980u32: rune) { fail(); }; }; + }; +}; + +// ---- decoder.next: invalid inputs ------------------------------------- +// Cases mirror ref/hare/encoding/utf8/decode.ha:127-167 (surrogate / +// overlong / out-of-range / bad-continuation). + +@test fn decode_surrogate() void = { + let src: [3]u8; + src[0] = 0xEDu8; src[1] = 0xA0u8; src[2] = 0x80u8; // U+D800 + let d: utf8.decoder = utf8.decode(src[0:3]); + match (utf8.next(&d)) { + case let r: utf8.invalid => void; + case let r: rune => { fail(); }; + case utf8.done => { fail(); }; + case utf8.more => { fail(); }; + }; +}; + +@test fn decode_overlong() void = { + let src: [4]u8; + src[0] = 0xF0u8; src[1] = 0x82u8; src[2] = 0x82u8; src[3] = 0xACu8; + let d: utf8.decoder = utf8.decode(src[0:4]); + match (utf8.next(&d)) { + case let r: utf8.invalid => void; + case let r: rune => { fail(); }; + case utf8.done => { fail(); }; + case utf8.more => { fail(); }; + }; +}; + +@test fn decode_out_of_range() void = { + let src: [4]u8; + src[0] = 0xF5u8; src[1] = 0x94u8; src[2] = 0x80u8; src[3] = 0x80u8; + let d: utf8.decoder = utf8.decode(src[0:4]); + match (utf8.next(&d)) { + case let r: utf8.invalid => void; + case let r: rune => { fail(); }; + case utf8.done => { fail(); }; + case utf8.more => { fail(); }; + }; +}; + +// ref/hare/encoding/utf8/decode.ha:141 — `[0xC2, 0xFF]`: legal 2-byte +// lead followed by non-continuation. Pins next-table cell (state 1, +// byte 0xFF) returning -1; distinct from overlong (which is filtered +// in state 3/5/7 by lead-byte-aware sub-states). +@test fn decode_bad_continuation() void = { + let src: [2]u8; + src[0] = 0xC2u8; src[1] = 0xFFu8; + let d: utf8.decoder = utf8.decode(src[0:2]); + match (utf8.next(&d)) { + case let r: utf8.invalid => void; + case let r: rune => { fail(); }; + case utf8.done => { fail(); }; + case utf8.more => { fail(); }; + }; +}; + +// ref/hare/encoding/utf8/decode.ha:151 — `[0xF4, 0x8F, 0xBF, 0xBF]` = +// U+10FFFF, the largest legal codepoint. Pins the upper boundary; +// pairs with the existing `0xF5…` out-of-range row. +@test fn decode_max_in_range() void = { + let src: [4]u8; + src[0] = 0xF4u8; src[1] = 0x8Fu8; src[2] = 0xBFu8; src[3] = 0xBFu8; + let d: utf8.decoder = utf8.decode(src[0:4]); + match (utf8.next(&d)) { + case let r: rune => { if (r != 0x10FFFFu32: rune) { fail(); }; }; + case let r: utf8.invalid => { fail(); }; + case utf8.done => { fail(); }; + case utf8.more => { fail(); }; + }; +}; + +// ---- decoder.next: truncated → more ----------------------------------- + +@test fn decode_truncated() void = { + let src: [2]u8; + src[0] = 0xE3u8; src[1] = 0x81u8; // missing 3rd byte + let d: utf8.decoder = utf8.decode(src[0:2]); + match (utf8.next(&d)) { + case utf8.more => void; + case let r: rune => { fail(); }; + case utf8.done => { fail(); }; + case let r: utf8.invalid => { fail(); }; + }; +}; + +// ---- decoder.next: done at EOI ---------------------------------------- + +@test fn decode_done() void = { + let src: [1]u8; + let d: utf8.decoder = utf8.decode(src[0:0]); + match (utf8.next(&d)) { + case utf8.done => void; + case let r: rune => { fail(); }; + case utf8.more => { fail(); }; + case let r: utf8.invalid => { fail(); }; + }; +}; + +// ---- validate: well-formed mixed-width vs malformed ------------------- +// ref/hare/encoding/utf8/decode.ha:207. Mixed input mirrors Hare's +// decode @test ('こんにちは' + NUL). + +@test fn validate_mixed_ok() void = { + let src: [16]u8; + src[0] = 0xE3u8; src[1] = 0x81u8; src[2] = 0x93u8; // こ + src[3] = 0xE3u8; src[4] = 0x82u8; src[5] = 0x93u8; // ん + src[6] = 0xE3u8; src[7] = 0x81u8; src[8] = 0xABu8; // に + src[9] = 0xE3u8; src[10] = 0x81u8; src[11] = 0xA1u8; // ち + src[12] = 0xE3u8; src[13] = 0x81u8; src[14] = 0xAFu8; // は + src[15] = 0u8; + match (utf8.validate(src[0:16])) { + case let e: utf8.invalid => { fail(); }; + case void => void; + }; +}; + +@test fn validate_malformed() void = { + let src: [3]u8; + src[0] = 0xEDu8; src[1] = 0xA0u8; src[2] = 0x80u8; // surrogate + match (utf8.validate(src[0:3])) { + case let e: utf8.invalid => void; + case void => { fail(); }; + }; +}; + +@test fn validate_empty_ok() void = { + let src: [1]u8; + match (utf8.validate(src[0:0])) { + case let e: utf8.invalid => { fail(); }; + case void => void; + }; +}; + +// ---- round-trip: encode → decode → equal rune -------------------------- + +@test fn roundtrip() void = { + let runes: [4]u32; + runes[0] = 0x41u32; + runes[1] = 0xE9u32; + runes[2] = 0x20ACu32; + runes[3] = 0x1F980u32; + let i: i32 = 0; + for (i < 4) { + let buf: [4]u8; + let n: i32 = utf8.encoderune(buf[0:4], runes[i]: rune); + let d: utf8.decoder = utf8.decode(buf[0:n]); + match (utf8.next(&d)) { + case let r: rune => { + if ((r: u32) != runes[i]) { fail(); }; + }; + case utf8.done => { fail(); }; + case utf8.more => { fail(); }; + case let e: utf8.invalid => { fail(); }; + }; + i += 1; + }; +}; + +export fn main() i32 = { + signalled = 1; runesz_ranges(); + signalled = 2; utf8sz_classify(); + signalled = 3; encode_ascii(); + signalled = 4; encode_two_byte(); + signalled = 5; encode_three_byte(); + signalled = 6; encode_four_byte(); + signalled = 7; decode_one_byte(); + signalled = 8; decode_two_byte(); + signalled = 9; decode_three_byte(); + signalled = 10; decode_four_byte(); + signalled = 11; decode_surrogate(); + signalled = 12; decode_overlong(); + signalled = 13; decode_out_of_range(); + signalled = 14; decode_bad_continuation(); + signalled = 15; decode_max_in_range(); + signalled = 16; decode_truncated(); + signalled = 17; decode_done(); + signalled = 18; validate_mixed_ok(); + signalled = 19; validate_malformed(); + signalled = 20; validate_empty_ok(); + signalled = 21; roundtrip(); + return 0; +}; diff --git a/test/wcc/968_utf8_run.c b/test/wcc/968_utf8_run.c new file mode 100644 index 00000000..60d30b46 --- /dev/null +++ b/test/wcc/968_utf8_run.c @@ -0,0 +1,51 @@ +/* + * 968_utf8_run — execute the lib/encoding/utf8 @test fixture under the + * C-side `ww run` driver and assert exit 0. + * + * Thin wrapper mirroring 979_hex_run / 977_time_run: utf8test.ww carries + * its own `export fn main()` that drives the @test fns and signals which + * case failed via the exit code (signalled + 10). + * + * Slot 968: stdlib block 970–989 was full at landing time. + */ +#include +#include +#include +#include + +static int +runwait(const char *cmd) +{ + int rc = system(cmd); + if (rc == -1) return -1; + if (WIFEXITED(rc)) return WEXITSTATUS(rc); + return 1; +} + +int +main(void) +{ + const char *bin = getenv("BIN"); + if (!bin) bin = "out/bin"; + char absbin[1024]; + if (bin[0] != '/') { + char cwd[1024]; + if (getcwd(cwd, sizeof cwd) == NULL) return 1; + snprintf(absbin, sizeof absbin, "%s/%s", cwd, bin); + bin = absbin; + } + char cwd[1024]; + if (getcwd(cwd, sizeof cwd) == NULL) return 1; + + const char *src = "lib/encoding/utf8/utf8test.ww"; + char path[1024], cmd[2048]; + snprintf(path, sizeof path, "%s/%s", cwd, src); + snprintf(cmd, sizeof cmd, "%s/ww run %s", bin, path); + int rc = runwait(cmd); + if (rc != 0) { + fprintf(stderr, "utf8_run FAIL: %s exited %d\n", src, rc); + return 1; + } + printf("utf8_run: %s ok\n", src); + return 0; +}