Files
ww/lib/encoding/utf8/decode_test.ww
Hojun-Cho aadc6618f0 lib: banner purge + WHY-only comment sweep (rule 8)
Every // ---- section banner dies (132 -> 0): names carry the WHAT.
Narration deleted (filename restatements, run-with lines, what-the-
next-line-does); every ref/hare cite, task cite, divergence, ABI/
layout contract, and ownership qualifier kept (borrowed-view lines
restored where the sweep over-cut). Comment-only proven: all 442
walk-workdir .s and 32 import-probe .s byte-identical before/after;
libbyteid 56-roster all-ID.
2026-08-08 21:10:18 +09:00

512 lines
16 KiB
Plaintext

// Vectors mirror ref/hare/encoding/utf8/decode.ha. A failing row aborts
// via the assert/abort builtin (task #5 @test conversion).
package utf8_test;
import bytes;
import encoding.utf8;
// ref/hare/encoding/utf8/decode.ha:27. Round-trip via the same byte
// vectors used in encode.
@test fn decode_one_byte() void = {
let src: [1]u8;
src[0] = 0x41u8;
let d: utf8.decoder = utf8.decode(src[0:1]);
match (utf8.next(&d)) {
case utf8.done => { abort(); };
case utf8.more => { abort(); };
case let r: utf8.invalid => { abort(); };
case let r: rune => { assert(!(r != 0x41u32: rune)); };
};
};
@test fn decode_two_byte() void = {
let src: [2]u8;
src[0] = 0xC3u8; src[1] = 0xA9u8;
let d: utf8.decoder = utf8.decode(src[0:2]);
match (utf8.next(&d)) {
case utf8.done => { abort(); };
case utf8.more => { abort(); };
case let r: utf8.invalid => { abort(); };
case let r: rune => { assert(!(r != 0xE9u32: rune)); };
};
};
@test fn decode_three_byte() void = {
let src: [3]u8;
src[0] = 0xE2u8; src[1] = 0x82u8; src[2] = 0xACu8;
let d: utf8.decoder = utf8.decode(src[0:3]);
match (utf8.next(&d)) {
case utf8.done => { abort(); };
case utf8.more => { abort(); };
case let r: utf8.invalid => { abort(); };
case let r: rune => { assert(!(r != 0x20ACu32: rune)); };
};
};
@test fn decode_four_byte() void = {
let src: [4]u8;
src[0] = 0xF0u8; src[1] = 0x9Fu8; src[2] = 0xA6u8; src[3] = 0x80u8;
let d: utf8.decoder = utf8.decode(src[0:4]);
match (utf8.next(&d)) {
case utf8.done => { abort(); };
case utf8.more => { abort(); };
case let r: utf8.invalid => { abort(); };
case let r: rune => { assert(!(r != 0x1F980u32: rune)); };
};
};
// Cases mirror ref/hare/encoding/utf8/decode.ha:127-167 (surrogate /
// overlong / out-of-range / bad-continuation).
@test fn decode_surrogate() void = {
let src: [3]u8;
src[0] = 0xEDu8; src[1] = 0xA0u8; src[2] = 0x80u8; // U+D800
let d: utf8.decoder = utf8.decode(src[0:3]);
match (utf8.next(&d)) {
case let r: utf8.invalid => void;
case let r: rune => { abort(); };
case utf8.done => { abort(); };
case utf8.more => { abort(); };
};
};
@test fn decode_overlong() void = {
let src: [4]u8;
src[0] = 0xF0u8; src[1] = 0x82u8; src[2] = 0x82u8; src[3] = 0xACu8;
let d: utf8.decoder = utf8.decode(src[0:4]);
match (utf8.next(&d)) {
case let r: utf8.invalid => void;
case let r: rune => { abort(); };
case utf8.done => { abort(); };
case utf8.more => { abort(); };
};
};
@test fn decode_out_of_range() void = {
let src: [4]u8;
src[0] = 0xF5u8; src[1] = 0x94u8; src[2] = 0x80u8; src[3] = 0x80u8;
let d: utf8.decoder = utf8.decode(src[0:4]);
match (utf8.next(&d)) {
case let r: utf8.invalid => void;
case let r: rune => { abort(); };
case utf8.done => { abort(); };
case utf8.more => { abort(); };
};
};
// ref/hare/encoding/utf8/decode.ha:141 — `[0xC2, 0xFF]`: legal 2-byte
// lead followed by non-continuation. Pins next-table cell (state 1,
// byte 0xFF) returning -1; distinct from overlong (which is filtered
// in state 3/5/7 by lead-byte-aware sub-states).
@test fn decode_bad_continuation() void = {
let src: [2]u8;
src[0] = 0xC2u8; src[1] = 0xFFu8;
let d: utf8.decoder = utf8.decode(src[0:2]);
match (utf8.next(&d)) {
case let r: utf8.invalid => void;
case let r: rune => { abort(); };
case utf8.done => { abort(); };
case utf8.more => { abort(); };
};
};
// ref/hare/encoding/utf8/decode.ha:151 — `[0xF4, 0x8F, 0xBF, 0xBF]` =
// U+10FFFF, the largest legal codepoint. Pins the upper boundary;
// pairs with the existing `0xF5…` out-of-range row.
@test fn decode_max_in_range() void = {
let src: [4]u8;
src[0] = 0xF4u8; src[1] = 0x8Fu8; src[2] = 0xBFu8; src[3] = 0xBFu8;
let d: utf8.decoder = utf8.decode(src[0:4]);
match (utf8.next(&d)) {
case let r: rune => { assert(!(r != 0x10FFFFu32: rune)); };
case let r: utf8.invalid => { abort(); };
case utf8.done => { abort(); };
case utf8.more => { abort(); };
};
};
@test fn decode_truncated() void = {
let src: [2]u8;
src[0] = 0xE3u8; src[1] = 0x81u8; // missing 3rd byte
let d: utf8.decoder = utf8.decode(src[0:2]);
match (utf8.next(&d)) {
case utf8.more => void;
case let r: rune => { abort(); };
case utf8.done => { abort(); };
case let r: utf8.invalid => { abort(); };
};
};
@test fn decode_done() void = {
let src: [1]u8;
let d: utf8.decoder = utf8.decode(src[0:0]);
match (utf8.next(&d)) {
case utf8.done => void;
case let r: rune => { abort(); };
case utf8.more => { abort(); };
case let r: utf8.invalid => { abort(); };
};
};
// ref/hare/encoding/utf8/decode.ha:207. Mixed input mirrors Hare's
// decode @test ('こんにちは' + NUL).
@test fn validate_mixed_ok() void = {
let src: [16]u8;
src[0] = 0xE3u8; src[1] = 0x81u8; src[2] = 0x93u8; // こ
src[3] = 0xE3u8; src[4] = 0x82u8; src[5] = 0x93u8; // ん
src[6] = 0xE3u8; src[7] = 0x81u8; src[8] = 0xABu8; // に
src[9] = 0xE3u8; src[10] = 0x81u8; src[11] = 0xA1u8; // ち
src[12] = 0xE3u8; src[13] = 0x81u8; src[14] = 0xAFu8; // は
src[15] = 0u8;
match (utf8.validate(src[0:16])) {
case let e: utf8.invalid => { abort(); };
case void => void;
};
};
@test fn validate_malformed() void = {
let src: [3]u8;
src[0] = 0xEDu8; src[1] = 0xA0u8; src[2] = 0x80u8; // surrogate
match (utf8.validate(src[0:3])) {
case let e: utf8.invalid => void;
case void => { abort(); };
};
};
@test fn validate_empty_ok() void = {
let src: [1]u8;
match (utf8.validate(src[0:0])) {
case let e: utf8.invalid => { abort(); };
case void => void;
};
};
// ref/hare/encoding/utf8/decode.ha:53-55. Cursor at offs=0 has no prior
// codepoint.
@test fn prev_done_at_start() void = {
let src: [4]u8;
src[0] = 0x41u8;
let d: utf8.decoder = utf8.decode(src[0:1]);
match (utf8.prev(&d)) {
case utf8.done => void;
case let r: rune => { abort(); };
case utf8.more => { abort(); };
case let e: utf8.invalid => { abort(); };
};
};
// ref/hare/encoding/utf8/decode.ha:56-71. Forward-decode one rune,
// reverse-decode back to the same rune.
@test fn prev_one_byte() void = {
let src: [1]u8;
src[0] = 0x41u8;
let d: utf8.decoder = utf8.decode(src[0:1]);
match (utf8.next(&d)) {
case let r: rune => void;
case => { abort(); };
};
match (utf8.prev(&d)) {
case let r: rune => { assert(!(r != 0x41u32: rune)); };
case => { abort(); };
};
};
@test fn prev_two_byte() void = {
let src: [2]u8;
src[0] = 0xC3u8; src[1] = 0xA9u8; // 'é'
let d: utf8.decoder = utf8.decode(src[0:2]);
match (utf8.next(&d)) {
case let r: rune => void;
case => { abort(); };
};
match (utf8.prev(&d)) {
case let r: rune => { assert(!(r != 0xE9u32: rune)); };
case => { abort(); };
};
};
@test fn prev_three_byte() void = {
let src: [3]u8;
src[0] = 0xE2u8; src[1] = 0x82u8; src[2] = 0xACu8; // '€'
let d: utf8.decoder = utf8.decode(src[0:3]);
match (utf8.next(&d)) {
case let r: rune => void;
case => { abort(); };
};
match (utf8.prev(&d)) {
case let r: rune => { assert(!(r != 0x20ACu32: rune)); };
case => { abort(); };
};
};
@test fn prev_four_byte() void = {
let src: [4]u8;
src[0] = 0xF0u8; src[1] = 0x9Fu8; src[2] = 0xA6u8; src[3] = 0x80u8; // '🦀'
let d: utf8.decoder = utf8.decode(src[0:4]);
match (utf8.next(&d)) {
case let r: rune => void;
case => { abort(); };
};
match (utf8.prev(&d)) {
case let r: rune => { assert(!(r != 0x1F980u32: rune)); };
case => { abort(); };
};
};
// ref/hare/encoding/utf8/decode.ha:85-111. After consuming all runes
// forward, prev walks back through them in reverse order; prev at the
// start-of-input then returns done.
@test fn prev_mixed_roundtrip() void = {
let src: [16]u8;
src[0] = 0xE3u8; src[1] = 0x81u8; src[2] = 0x93u8; // こ
src[3] = 0xE3u8; src[4] = 0x82u8; src[5] = 0x93u8; // ん
src[6] = 0xE3u8; src[7] = 0x81u8; src[8] = 0xABu8; // に
src[9] = 0xE3u8; src[10] = 0x81u8; src[11] = 0xA1u8; // ち
src[12] = 0xE3u8; src[13] = 0x81u8; src[14] = 0xAFu8; // は
src[15] = 0u8;
let d: utf8.decoder = utf8.decode(src[0:16]);
let fwd: [6]u32;
let i: i32 = 0;
for (i < 6) {
match (utf8.next(&d)) {
case let r: rune => { fwd[i] = r: u32; };
case => { abort(); };
};
i += 1;
};
assert(!(utf8.position(&d) != 16));
i = 0;
for (i < 6) {
match (utf8.prev(&d)) {
case let r: rune => {
assert(!((r: u32) != fwd[5 - i]));
};
case => { abort(); };
};
i += 1;
};
match (utf8.prev(&d)) {
case utf8.done => void;
case => { abort(); };
};
};
// ref/hare/encoding/utf8/decode.ha:113-150. Two-continuation walk
// reaches offs=0 without an initial byte → more; otherwise the forward
// re-decode catches the malformed run as invalid.
// ref/hare/encoding/utf8/decode.ha:117 — `[0xA0, 0xA1]` is all
// continuation bytes; walking back from offs=2 finds no initial byte
// within the 4-step bound nor before offs underflows, so the loop
// exits via the `>= 0` guard and returns `more`.
@test fn prev_continuation_only_more() void = {
let src: [2]u8;
src[0] = 0xA0u8; src[1] = 0xA1u8;
let d: utf8.decoder = utf8.decode(src[0:2]);
d.offs = 2;
match (utf8.prev(&d)) {
case utf8.more => void;
case let r: rune => { abort(); };
case utf8.done => { abort(); };
case let e: utf8.invalid => { abort(); };
};
};
// #70: after prev() returns `more` it leaves offs out of range (SIZE_MAX,
// via the unsigned underflow). A subsequent next() must see offs >= len
// and return `more` — NOT read one byte before the buffer and decode a
// garbage rune. The byte before the slice (buf[2] = 0x41 = 'A') is the
// planted sentinel the old i32 offs=-1 path returned as rune 65.
@test fn prev_more_then_next_no_oob() void = {
let buf: [4]u8;
buf[0] = 0x58u8; buf[1] = 0x59u8; buf[2] = 0x41u8; // 'A' sentinel
buf[3] = 0xA0u8; // lone continuation
let d: utf8.decoder = utf8.decode(buf[3:4]);
match (utf8.next(&d)) {
case let e: utf8.invalid => void;
case => { abort(); };
};
match (utf8.prev(&d)) {
case utf8.more => void;
case => { abort(); };
};
match (utf8.next(&d)) {
case utf8.more => void; // safe: not src[-1]'s 'A'
case let r: rune => { abort(); };
case utf8.done => { abort(); };
case let e: utf8.invalid => { abort(); };
};
};
@test fn prev_incomplete_invalid() void = {
let src: [2]u8;
src[0] = 0xE3u8; src[1] = 0x81u8;
let d: utf8.decoder = utf8.decode(src[0:2]);
d.offs = 2;
match (utf8.prev(&d)) {
case let e: utf8.invalid => void;
case let r: rune => { abort(); };
case utf8.done => { abort(); };
case utf8.more => { abort(); };
};
};
@test fn prev_surrogate_invalid() void = {
let src: [3]u8;
src[0] = 0xEDu8; src[1] = 0xA0u8; src[2] = 0x80u8;
let d: utf8.decoder = utf8.decode(src[0:3]);
d.offs = 3;
match (utf8.prev(&d)) {
case let e: utf8.invalid => void;
case let r: rune => { abort(); };
case utf8.done => { abort(); };
case utf8.more => { abort(); };
};
};
@test fn prev_overlong_invalid() void = {
let src: [4]u8;
src[0] = 0xF0u8; src[1] = 0x82u8; src[2] = 0x82u8; src[3] = 0xACu8;
let d: utf8.decoder = utf8.decode(src[0:4]);
d.offs = 4;
match (utf8.prev(&d)) {
case let e: utf8.invalid => void;
case let r: rune => { abort(); };
case utf8.done => { abort(); };
case utf8.more => { abort(); };
};
};
// ref/hare/encoding/utf8/decode.ha:146 — extra continuation byte after
// a complete 2-byte codepoint. prev from offs=3 finds 0xC2 (initial
// byte), forward-decodes back to offs=2, but n=3 ≠ 2 → invalid.
@test fn prev_extracont_invalid() void = {
let src: [3]u8;
src[0] = 0xC2u8; src[1] = 0xA3u8; src[2] = 0x95u8;
let d: utf8.decoder = utf8.decode(src[0:3]);
d.offs = 3;
match (utf8.prev(&d)) {
case let e: utf8.invalid => void;
case let r: rune => { abort(); };
case utf8.done => { abort(); };
case utf8.more => { abort(); };
};
};
// ref/hare/encoding/utf8/decode.ha:158-163 — max in-range codepoint
// reverse-decoded. Pins state-7 acceptance via prev: from offs=4 the
// walk skips 3 continuations, lands on 0xF4 (state-7 lead), and the
// forward re-decode reproduces U+10FFFF.
@test fn prev_max_in_range() void = {
let src: [4]u8;
src[0] = 0xF4u8; src[1] = 0x8Fu8; src[2] = 0xBFu8; src[3] = 0xBFu8;
let d: utf8.decoder = utf8.decode(src[0:4]);
d.offs = 4;
match (utf8.prev(&d)) {
case let r: rune => { assert(!(r != 0x10FFFFu32: rune)); };
case let e: utf8.invalid => { abort(); };
case utf8.done => { abort(); };
case utf8.more => { abort(); };
};
};
// ref/hare/encoding/utf8/decode.ha:166-169 — `[0xF5, 0x94, 0x80, 0x80]`:
// 0xF5 is not a legal initial byte (dfa state-0 cell is -1). prev from
// offs=4 walks 4 bytes without finding an initial byte, trips the
// `n - d.offs == 4` bound, and returns invalid. The only test that
// exercises this branch (prev_continuation_only_more underflows past
// 0 before reaching 4).
@test fn prev_min_out_of_range() void = {
let src: [4]u8;
src[0] = 0xF5u8; src[1] = 0x94u8; src[2] = 0x80u8; src[3] = 0x80u8;
let d: utf8.decoder = utf8.decode(src[0:4]);
d.offs = 4;
match (utf8.prev(&d)) {
case let e: utf8.invalid => void;
case let r: rune => { abort(); };
case utf8.done => { abort(); };
case utf8.more => { abort(); };
};
};
// ref/hare/encoding/utf8/decode.ha:172-198.
@test fn remaining_slice_position() void = {
let src: [16]u8;
src[0] = 0xE3u8; src[1] = 0x81u8; src[2] = 0x93u8; // こ
src[3] = 0xE3u8; src[4] = 0x82u8; src[5] = 0x93u8; // ん
src[6] = 0xE3u8; src[7] = 0x81u8; src[8] = 0xABu8; // に
src[9] = 0xE3u8; src[10] = 0x81u8; src[11] = 0xA1u8; // ち
src[12] = 0xE3u8; src[13] = 0x81u8; src[14] = 0xAFu8; // は
src[15] = 0u8;
// Two decode() calls instead of `let d2 = d1` per Hare style —
// let-copy on struct >8B is a Class A cstage+wwstage miscompile
// (task #32; rhs ident is dropped, slot zero-inits). Both calls
// take the same src[0:16] slice so d1.src.ptr == d2.src.ptr,
// matching Hare's `d2 = d1` for slice()'s same-source precondition.
let d1: utf8.decoder = utf8.decode(src[0:16]);
let d2: utf8.decoder = utf8.decode(src[0:16]);
assert(!(!bytes.equal(utf8.remaining(&d1), src[0:16])));
assert(!(utf8.slice(&d1, &d2).len != 0));
assert(!(utf8.slice(&d2, &d1).len != 0));
assert(!(utf8.position(&d1) != 0));
let i: i32 = 0;
for (i < 2) {
match (utf8.next(&d1)) { case let r: rune => void; case => { abort(); }; };
match (utf8.next(&d2)) { case let r: rune => void; case => { abort(); }; };
i += 1;
};
assert(!(utf8.position(&d1) != 6));
assert(!(!bytes.equal(utf8.remaining(&d1), src[6:16])));
assert(!(utf8.slice(&d1, &d2).len != 0));
i = 0;
for (i < 3) {
match (utf8.next(&d2)) { case let r: rune => void; case => { abort(); }; };
i += 1;
};
assert(!(utf8.position(&d2) != 15));
assert(!(!bytes.equal(utf8.remaining(&d2), src[15:16])));
assert(!(!bytes.equal(utf8.slice(&d1, &d2), src[6:15])));
i = 0;
for (i < 3) {
match (utf8.next(&d1)) { case let r: rune => void; case => { abort(); }; };
i += 1;
};
assert(!(utf8.slice(&d1, &d2).len != 0));
match (utf8.next(&d1)) { case let r: rune => void; case => { abort(); }; };
assert(!(utf8.remaining(&d1).len != 0));
};
@test fn roundtrip() void = {
let runes: [4]u32;
runes[0] = 0x41u32;
runes[1] = 0xE9u32;
runes[2] = 0x20ACu32;
runes[3] = 0x1F980u32;
let i: i32 = 0;
for (i < 4) {
let buf: [4]u8;
let n: i32 = utf8.encoderune(buf[0:4], runes[i]: rune);
let d: utf8.decoder = utf8.decode(buf[0:n]);
match (utf8.next(&d)) {
case let r: rune => {
assert(!((r: u32) != runes[i]));
};
case utf8.done => { abort(); };
case utf8.more => { abort(); };
case let e: utf8.invalid => { abort(); };
};
i += 1;
};
};