lib: split utf8 tests into rune/encode/decode/types per ref/hare/encoding/utf8
Pure move: utf8_test.ww dissolves along its banner seams into sibling
rune_test.ww (runesz/utf8sz), encode_test.ww (encoderune),
decode_test.ww (decoder next/prev/validate/remaining/slice/position
plus the encode->decode round-trip), and types_test.ww (strerror),
mirroring ref/hare/encoding/utf8/{rune,encode,decode,types}.ha
ownership (types.ha:12 owns strerror). Blocks are byte-identical;
only the banner lines are deleted. The streq helper moves with its
sole consumer, strerror_cases, into types_test.ww. The impl utf8.ww
is untouched (no banners).
Consumers: Makefile LIBRARY_TESTS replaces the utf8_test.ww entry
with the four new entries at the same position; the
test/byteid/libbyteid_test.ww roster row becomes four fx rows,
NENTEXPECT 52->55 (+3).
This commit is contained in:
@@ -1,120 +1,13 @@
|
||||
// utf8test — exercises lib/encoding/utf8. Run with
|
||||
// `out/bin/ww run lib/encoding/utf8/utf8test.ww`. A failing row aborts
|
||||
// via the assert/abort builtin (task #5 @test conversion).
|
||||
// decodetest — exercises the utf8 decoder: next/prev/validate/
|
||||
// remaining/slice/position, plus the encode→decode round-trip. A
|
||||
// failing row aborts via the assert/abort builtin (task #5 @test
|
||||
// conversion). Vectors mirror ref/hare/encoding/utf8/decode.ha.
|
||||
|
||||
package utf8_test;
|
||||
|
||||
import bytes;
|
||||
import encoding.utf8;
|
||||
|
||||
|
||||
fn streq(a: str, b: str) bool = {
|
||||
if (a.len != b.len) { return false; };
|
||||
let i: i32 = 0;
|
||||
for (i < a.len) {
|
||||
if (a[i] != b[i]) { return false; };
|
||||
i += 1;
|
||||
};
|
||||
return true;
|
||||
};
|
||||
|
||||
// ---- runesz: byte length per range ------------------------------------
|
||||
// ref/hare/encoding/utf8/rune.ha:5. Boundaries: 0x7F → 1, 0x80 → 2,
|
||||
// 0x7FF → 2, 0x800 → 3, 0xFFFF → 3, 0x10000 → 4, 0x10FFFF → 4.
|
||||
|
||||
@test fn runesz_ranges() void = {
|
||||
assert(!(utf8.runesz(0u32: rune) != 1));
|
||||
assert(!(utf8.runesz(0x7Fu32: rune) != 1));
|
||||
assert(!(utf8.runesz(0x80u32: rune) != 2));
|
||||
assert(!(utf8.runesz(0x7FFu32: rune) != 2));
|
||||
assert(!(utf8.runesz(0x800u32: rune) != 3));
|
||||
assert(!(utf8.runesz(0xFFFFu32: rune) != 3));
|
||||
assert(!(utf8.runesz(0x10000u32: rune) != 4));
|
||||
assert(!(utf8.runesz(0x10FFFFu32: rune) != 4));
|
||||
};
|
||||
|
||||
// ---- utf8sz: start-byte classification --------------------------------
|
||||
// ref/hare/encoding/utf8/rune.ha:15. ASCII → 1; legal multibyte
|
||||
// leads → 2/3/4; continuation and >0xF7 → invalid.
|
||||
|
||||
@test fn utf8sz_classify() void = {
|
||||
match (utf8.utf8sz(0u8)) {
|
||||
case let n: i32 => { assert(!(n != 1)); };
|
||||
case let e: utf8.invalid => { abort(); };
|
||||
};
|
||||
match (utf8.utf8sz(0x7Fu8)) {
|
||||
case let n: i32 => { assert(!(n != 1)); };
|
||||
case let e: utf8.invalid => { abort(); };
|
||||
};
|
||||
match (utf8.utf8sz(0x80u8)) { // continuation
|
||||
case let n: i32 => { abort(); };
|
||||
case let e: utf8.invalid => void;
|
||||
};
|
||||
match (utf8.utf8sz(0xC1u8)) { // overlong 2-byte lead
|
||||
case let n: i32 => { abort(); };
|
||||
case let e: utf8.invalid => void;
|
||||
};
|
||||
match (utf8.utf8sz(0xC2u8)) {
|
||||
case let n: i32 => { assert(!(n != 2)); };
|
||||
case let e: utf8.invalid => { abort(); };
|
||||
};
|
||||
match (utf8.utf8sz(0xE0u8)) {
|
||||
case let n: i32 => { assert(!(n != 3)); };
|
||||
case let e: utf8.invalid => { abort(); };
|
||||
};
|
||||
match (utf8.utf8sz(0xF0u8)) {
|
||||
case let n: i32 => { assert(!(n != 4)); };
|
||||
case let e: utf8.invalid => { abort(); };
|
||||
};
|
||||
match (utf8.utf8sz(0xF8u8)) { // 5-byte lead — illegal in modern UTF-8
|
||||
case let n: i32 => { abort(); };
|
||||
case let e: utf8.invalid => void;
|
||||
};
|
||||
match (utf8.utf8sz(0xFFu8)) {
|
||||
case let n: i32 => { abort(); };
|
||||
case let e: utf8.invalid => void;
|
||||
};
|
||||
};
|
||||
|
||||
// ---- encoderune: all four widths --------------------------------------
|
||||
// ref/hare/encoding/utf8/encode.ha:7. Byte vectors come from the
|
||||
// Unicode specification (UAX standard examples).
|
||||
|
||||
@test fn encode_ascii() void = {
|
||||
let out: [4]u8;
|
||||
let n: i32 = utf8.encoderune(out[0:4], 0x41u32: rune); // 'A'
|
||||
assert(!(n != 1));
|
||||
assert(!(out[0] != 0x41u8));
|
||||
};
|
||||
|
||||
@test fn encode_two_byte() void = {
|
||||
let out: [4]u8;
|
||||
let n: i32 = utf8.encoderune(out[0:4], 0xE9u32: rune); // 'é' U+00E9
|
||||
assert(!(n != 2));
|
||||
assert(!(out[0] != 0xC3u8));
|
||||
assert(!(out[1] != 0xA9u8));
|
||||
};
|
||||
|
||||
@test fn encode_three_byte() void = {
|
||||
let out: [4]u8;
|
||||
let n: i32 = utf8.encoderune(out[0:4], 0x20ACu32: rune); // '€' U+20AC
|
||||
assert(!(n != 3));
|
||||
assert(!(out[0] != 0xE2u8));
|
||||
assert(!(out[1] != 0x82u8));
|
||||
assert(!(out[2] != 0xACu8));
|
||||
};
|
||||
|
||||
@test fn encode_four_byte() void = {
|
||||
let out: [4]u8;
|
||||
let n: i32 = utf8.encoderune(out[0:4], 0x1F980u32: rune); // '🦀' U+1F980
|
||||
assert(!(n != 4));
|
||||
assert(!(out[0] != 0xF0u8));
|
||||
assert(!(out[1] != 0x9Fu8));
|
||||
assert(!(out[2] != 0xA6u8));
|
||||
assert(!(out[3] != 0x80u8));
|
||||
};
|
||||
|
||||
// ---- decoder.next: valid 1/2/3/4-byte ---------------------------------
|
||||
// ref/hare/encoding/utf8/decode.ha:27. Round-trip via the same byte
|
||||
// vectors used in encode.
|
||||
|
||||
@@ -166,7 +59,6 @@ fn streq(a: str, b: str) bool = {
|
||||
};
|
||||
};
|
||||
|
||||
// ---- decoder.next: invalid inputs -------------------------------------
|
||||
// Cases mirror ref/hare/encoding/utf8/decode.ha:127-167 (surrogate /
|
||||
// overlong / out-of-range / bad-continuation).
|
||||
|
||||
@@ -237,8 +129,6 @@ fn streq(a: str, b: str) bool = {
|
||||
};
|
||||
};
|
||||
|
||||
// ---- decoder.next: truncated → more -----------------------------------
|
||||
|
||||
@test fn decode_truncated() void = {
|
||||
let src: [2]u8;
|
||||
src[0] = 0xE3u8; src[1] = 0x81u8; // missing 3rd byte
|
||||
@@ -251,8 +141,6 @@ fn streq(a: str, b: str) bool = {
|
||||
};
|
||||
};
|
||||
|
||||
// ---- decoder.next: done at EOI ----------------------------------------
|
||||
|
||||
@test fn decode_done() void = {
|
||||
let src: [1]u8;
|
||||
let d: utf8.decoder = utf8.decode(src[0:0]);
|
||||
@@ -264,7 +152,6 @@ fn streq(a: str, b: str) bool = {
|
||||
};
|
||||
};
|
||||
|
||||
// ---- validate: well-formed mixed-width vs malformed -------------------
|
||||
// ref/hare/encoding/utf8/decode.ha:207. Mixed input mirrors Hare's
|
||||
// decode @test ('こんにちは' + NUL).
|
||||
|
||||
@@ -299,7 +186,6 @@ fn streq(a: str, b: str) bool = {
|
||||
};
|
||||
};
|
||||
|
||||
// ---- decoder.prev: done at start-of-input -----------------------------
|
||||
// ref/hare/encoding/utf8/decode.ha:53-55. Cursor at offs=0 has no prior
|
||||
// codepoint.
|
||||
|
||||
@@ -315,7 +201,6 @@ fn streq(a: str, b: str) bool = {
|
||||
};
|
||||
};
|
||||
|
||||
// ---- decoder.prev: each codepoint width round-trips -------------------
|
||||
// ref/hare/encoding/utf8/decode.ha:56-71. Forward-decode one rune,
|
||||
// reverse-decode back to the same rune.
|
||||
|
||||
@@ -375,7 +260,6 @@ fn streq(a: str, b: str) bool = {
|
||||
};
|
||||
};
|
||||
|
||||
// ---- decoder.prev: full forward + full reverse on mixed input ---------
|
||||
// ref/hare/encoding/utf8/decode.ha:85-111. After consuming all runes
|
||||
// forward, prev walks back through them in reverse order; prev at the
|
||||
// start-of-input then returns done.
|
||||
@@ -415,7 +299,6 @@ fn streq(a: str, b: str) bool = {
|
||||
};
|
||||
};
|
||||
|
||||
// ---- decoder.prev: invalid inputs -------------------------------------
|
||||
// ref/hare/encoding/utf8/decode.ha:113-150. Two-continuation walk
|
||||
// reaches offs=0 without an initial byte → more; otherwise the forward
|
||||
// re-decode catches the malformed run as invalid.
|
||||
@@ -554,7 +437,6 @@ fn streq(a: str, b: str) bool = {
|
||||
};
|
||||
};
|
||||
|
||||
// ---- remaining / slice / position -------------------------------------
|
||||
// ref/hare/encoding/utf8/decode.ha:172-198.
|
||||
|
||||
@test fn remaining_slice_position() void = {
|
||||
@@ -607,17 +489,6 @@ fn streq(a: str, b: str) bool = {
|
||||
assert(!(utf8.remaining(&d1).len != 0));
|
||||
};
|
||||
|
||||
// ---- strerror: constant rendering -------------------------------------
|
||||
// ref/hare/encoding/utf8/types.ha:12. `invalid` is payload-free; the
|
||||
// renderer collapses to a single fixture row.
|
||||
|
||||
@test fn strerror_cases() void = {
|
||||
let e: utf8.invalid;
|
||||
assert(!(!streq(utf8.strerror(e), "Invalid UTF-8")));
|
||||
};
|
||||
|
||||
// ---- round-trip: encode → decode → equal rune --------------------------
|
||||
|
||||
@test fn roundtrip() void = {
|
||||
let runes: [4]u32;
|
||||
runes[0] = 0x41u32;
|
||||
44
lib/encoding/utf8/encode_test.ww
Normal file
44
lib/encoding/utf8/encode_test.ww
Normal file
@@ -0,0 +1,44 @@
|
||||
// encodetest — exercises utf8.encoderune. A failing row aborts via
|
||||
// the assert/abort builtin (task #5 @test conversion).
|
||||
// Vectors mirror ref/hare/encoding/utf8/encode.ha.
|
||||
|
||||
package utf8_test;
|
||||
|
||||
import encoding.utf8;
|
||||
|
||||
// ref/hare/encoding/utf8/encode.ha:7. Byte vectors come from the
|
||||
// Unicode specification (UAX standard examples).
|
||||
|
||||
@test fn encode_ascii() void = {
|
||||
let out: [4]u8;
|
||||
let n: i32 = utf8.encoderune(out[0:4], 0x41u32: rune); // 'A'
|
||||
assert(!(n != 1));
|
||||
assert(!(out[0] != 0x41u8));
|
||||
};
|
||||
|
||||
@test fn encode_two_byte() void = {
|
||||
let out: [4]u8;
|
||||
let n: i32 = utf8.encoderune(out[0:4], 0xE9u32: rune); // 'é' U+00E9
|
||||
assert(!(n != 2));
|
||||
assert(!(out[0] != 0xC3u8));
|
||||
assert(!(out[1] != 0xA9u8));
|
||||
};
|
||||
|
||||
@test fn encode_three_byte() void = {
|
||||
let out: [4]u8;
|
||||
let n: i32 = utf8.encoderune(out[0:4], 0x20ACu32: rune); // '€' U+20AC
|
||||
assert(!(n != 3));
|
||||
assert(!(out[0] != 0xE2u8));
|
||||
assert(!(out[1] != 0x82u8));
|
||||
assert(!(out[2] != 0xACu8));
|
||||
};
|
||||
|
||||
@test fn encode_four_byte() void = {
|
||||
let out: [4]u8;
|
||||
let n: i32 = utf8.encoderune(out[0:4], 0x1F980u32: rune); // '🦀' U+1F980
|
||||
assert(!(n != 4));
|
||||
assert(!(out[0] != 0xF0u8));
|
||||
assert(!(out[1] != 0x9Fu8));
|
||||
assert(!(out[2] != 0xA6u8));
|
||||
assert(!(out[3] != 0x80u8));
|
||||
};
|
||||
63
lib/encoding/utf8/rune_test.ww
Normal file
63
lib/encoding/utf8/rune_test.ww
Normal file
@@ -0,0 +1,63 @@
|
||||
// runetest — exercises utf8.runesz/utf8sz. A failing row aborts via
|
||||
// the assert/abort builtin (task #5 @test conversion).
|
||||
// Vectors mirror ref/hare/encoding/utf8/rune.ha.
|
||||
|
||||
package utf8_test;
|
||||
|
||||
import encoding.utf8;
|
||||
|
||||
// ref/hare/encoding/utf8/rune.ha:5. Boundaries: 0x7F → 1, 0x80 → 2,
|
||||
// 0x7FF → 2, 0x800 → 3, 0xFFFF → 3, 0x10000 → 4, 0x10FFFF → 4.
|
||||
|
||||
@test fn runesz_ranges() void = {
|
||||
assert(!(utf8.runesz(0u32: rune) != 1));
|
||||
assert(!(utf8.runesz(0x7Fu32: rune) != 1));
|
||||
assert(!(utf8.runesz(0x80u32: rune) != 2));
|
||||
assert(!(utf8.runesz(0x7FFu32: rune) != 2));
|
||||
assert(!(utf8.runesz(0x800u32: rune) != 3));
|
||||
assert(!(utf8.runesz(0xFFFFu32: rune) != 3));
|
||||
assert(!(utf8.runesz(0x10000u32: rune) != 4));
|
||||
assert(!(utf8.runesz(0x10FFFFu32: rune) != 4));
|
||||
};
|
||||
|
||||
// ref/hare/encoding/utf8/rune.ha:15. ASCII → 1; legal multibyte
|
||||
// leads → 2/3/4; continuation and >0xF7 → invalid.
|
||||
|
||||
@test fn utf8sz_classify() void = {
|
||||
match (utf8.utf8sz(0u8)) {
|
||||
case let n: i32 => { assert(!(n != 1)); };
|
||||
case let e: utf8.invalid => { abort(); };
|
||||
};
|
||||
match (utf8.utf8sz(0x7Fu8)) {
|
||||
case let n: i32 => { assert(!(n != 1)); };
|
||||
case let e: utf8.invalid => { abort(); };
|
||||
};
|
||||
match (utf8.utf8sz(0x80u8)) { // continuation
|
||||
case let n: i32 => { abort(); };
|
||||
case let e: utf8.invalid => void;
|
||||
};
|
||||
match (utf8.utf8sz(0xC1u8)) { // overlong 2-byte lead
|
||||
case let n: i32 => { abort(); };
|
||||
case let e: utf8.invalid => void;
|
||||
};
|
||||
match (utf8.utf8sz(0xC2u8)) {
|
||||
case let n: i32 => { assert(!(n != 2)); };
|
||||
case let e: utf8.invalid => { abort(); };
|
||||
};
|
||||
match (utf8.utf8sz(0xE0u8)) {
|
||||
case let n: i32 => { assert(!(n != 3)); };
|
||||
case let e: utf8.invalid => { abort(); };
|
||||
};
|
||||
match (utf8.utf8sz(0xF0u8)) {
|
||||
case let n: i32 => { assert(!(n != 4)); };
|
||||
case let e: utf8.invalid => { abort(); };
|
||||
};
|
||||
match (utf8.utf8sz(0xF8u8)) { // 5-byte lead — illegal in modern UTF-8
|
||||
case let n: i32 => { abort(); };
|
||||
case let e: utf8.invalid => void;
|
||||
};
|
||||
match (utf8.utf8sz(0xFFu8)) {
|
||||
case let n: i32 => { abort(); };
|
||||
case let e: utf8.invalid => void;
|
||||
};
|
||||
};
|
||||
25
lib/encoding/utf8/types_test.ww
Normal file
25
lib/encoding/utf8/types_test.ww
Normal file
@@ -0,0 +1,25 @@
|
||||
// typestest — exercises utf8.strerror. A failing row aborts via the
|
||||
// assert/abort builtin (task #5 @test conversion).
|
||||
// Mirrors ref/hare/encoding/utf8/types.ha.
|
||||
|
||||
package utf8_test;
|
||||
|
||||
import encoding.utf8;
|
||||
|
||||
fn streq(a: str, b: str) bool = {
|
||||
if (a.len != b.len) { return false; };
|
||||
let i: i32 = 0;
|
||||
for (i < a.len) {
|
||||
if (a[i] != b[i]) { return false; };
|
||||
i += 1;
|
||||
};
|
||||
return true;
|
||||
};
|
||||
|
||||
// ref/hare/encoding/utf8/types.ha:12. `invalid` is payload-free; the
|
||||
// renderer collapses to a single fixture row.
|
||||
|
||||
@test fn strerror_cases() void = {
|
||||
let e: utf8.invalid;
|
||||
assert(!(!streq(utf8.strerror(e), "Invalid UTF-8")));
|
||||
};
|
||||
Reference in New Issue
Block a user