diff --git a/Makefile b/Makefile index 3d15566b..d1791f28 100644 --- a/Makefile +++ b/Makefile @@ -490,7 +490,9 @@ LIBRARY_TESTS = lib/errors/errno_test.ww lib/ascii/ascii_test.ww \ lib/strings/index_test.ww lib/strings/compare_test.ww \ lib/bytes/equal_test.ww lib/bytes/index_test.ww \ lib/bytes/contains_test.ww lib/bytes/tokenize_test.ww \ - lib/bytes/trim_test.ww lib/encoding/utf8/utf8_test.ww \ + lib/bytes/trim_test.ww lib/encoding/utf8/rune_test.ww \ + lib/encoding/utf8/encode_test.ww lib/encoding/utf8/decode_test.ww \ + lib/encoding/utf8/types_test.ww \ lib/bufio/bufio_test.ww lib/math/random/random_test.ww \ lib/math/checked/checked_test.ww lib/fmt/fmt_test.ww \ lib/log/log_test.ww lib/log/silent_test.ww lib/fnmatch/fnmatch_test.ww \ diff --git a/lib/encoding/utf8/utf8_test.ww b/lib/encoding/utf8/decode_test.ww similarity index 77% rename from lib/encoding/utf8/utf8_test.ww rename to lib/encoding/utf8/decode_test.ww index 3ed55885..9205bb10 100644 --- a/lib/encoding/utf8/utf8_test.ww +++ b/lib/encoding/utf8/decode_test.ww @@ -1,120 +1,13 @@ -// utf8test — exercises lib/encoding/utf8. Run with -// `out/bin/ww run lib/encoding/utf8/utf8test.ww`. A failing row aborts -// via the assert/abort builtin (task #5 @test conversion). +// decodetest — exercises the utf8 decoder: next/prev/validate/ +// remaining/slice/position, plus the encode→decode round-trip. A +// failing row aborts via the assert/abort builtin (task #5 @test +// conversion). Vectors mirror ref/hare/encoding/utf8/decode.ha. package utf8_test; import bytes; import encoding.utf8; - -fn streq(a: str, b: str) bool = { - if (a.len != b.len) { return false; }; - let i: i32 = 0; - for (i < a.len) { - if (a[i] != b[i]) { return false; }; - i += 1; - }; - return true; -}; - -// ---- runesz: byte length per range ------------------------------------ -// ref/hare/encoding/utf8/rune.ha:5. Boundaries: 0x7F → 1, 0x80 → 2, -// 0x7FF → 2, 0x800 → 3, 0xFFFF → 3, 0x10000 → 4, 0x10FFFF → 4. - -@test fn runesz_ranges() void = { - assert(!(utf8.runesz(0u32: rune) != 1)); - assert(!(utf8.runesz(0x7Fu32: rune) != 1)); - assert(!(utf8.runesz(0x80u32: rune) != 2)); - assert(!(utf8.runesz(0x7FFu32: rune) != 2)); - assert(!(utf8.runesz(0x800u32: rune) != 3)); - assert(!(utf8.runesz(0xFFFFu32: rune) != 3)); - assert(!(utf8.runesz(0x10000u32: rune) != 4)); - assert(!(utf8.runesz(0x10FFFFu32: rune) != 4)); -}; - -// ---- utf8sz: start-byte classification -------------------------------- -// ref/hare/encoding/utf8/rune.ha:15. ASCII → 1; legal multibyte -// leads → 2/3/4; continuation and >0xF7 → invalid. - -@test fn utf8sz_classify() void = { - match (utf8.utf8sz(0u8)) { - case let n: i32 => { assert(!(n != 1)); }; - case let e: utf8.invalid => { abort(); }; - }; - match (utf8.utf8sz(0x7Fu8)) { - case let n: i32 => { assert(!(n != 1)); }; - case let e: utf8.invalid => { abort(); }; - }; - match (utf8.utf8sz(0x80u8)) { // continuation - case let n: i32 => { abort(); }; - case let e: utf8.invalid => void; - }; - match (utf8.utf8sz(0xC1u8)) { // overlong 2-byte lead - case let n: i32 => { abort(); }; - case let e: utf8.invalid => void; - }; - match (utf8.utf8sz(0xC2u8)) { - case let n: i32 => { assert(!(n != 2)); }; - case let e: utf8.invalid => { abort(); }; - }; - match (utf8.utf8sz(0xE0u8)) { - case let n: i32 => { assert(!(n != 3)); }; - case let e: utf8.invalid => { abort(); }; - }; - match (utf8.utf8sz(0xF0u8)) { - case let n: i32 => { assert(!(n != 4)); }; - case let e: utf8.invalid => { abort(); }; - }; - match (utf8.utf8sz(0xF8u8)) { // 5-byte lead — illegal in modern UTF-8 - case let n: i32 => { abort(); }; - case let e: utf8.invalid => void; - }; - match (utf8.utf8sz(0xFFu8)) { - case let n: i32 => { abort(); }; - case let e: utf8.invalid => void; - }; -}; - -// ---- encoderune: all four widths -------------------------------------- -// ref/hare/encoding/utf8/encode.ha:7. Byte vectors come from the -// Unicode specification (UAX standard examples). - -@test fn encode_ascii() void = { - let out: [4]u8; - let n: i32 = utf8.encoderune(out[0:4], 0x41u32: rune); // 'A' - assert(!(n != 1)); - assert(!(out[0] != 0x41u8)); -}; - -@test fn encode_two_byte() void = { - let out: [4]u8; - let n: i32 = utf8.encoderune(out[0:4], 0xE9u32: rune); // 'é' U+00E9 - assert(!(n != 2)); - assert(!(out[0] != 0xC3u8)); - assert(!(out[1] != 0xA9u8)); -}; - -@test fn encode_three_byte() void = { - let out: [4]u8; - let n: i32 = utf8.encoderune(out[0:4], 0x20ACu32: rune); // '€' U+20AC - assert(!(n != 3)); - assert(!(out[0] != 0xE2u8)); - assert(!(out[1] != 0x82u8)); - assert(!(out[2] != 0xACu8)); -}; - -@test fn encode_four_byte() void = { - let out: [4]u8; - let n: i32 = utf8.encoderune(out[0:4], 0x1F980u32: rune); // '🦀' U+1F980 - assert(!(n != 4)); - assert(!(out[0] != 0xF0u8)); - assert(!(out[1] != 0x9Fu8)); - assert(!(out[2] != 0xA6u8)); - assert(!(out[3] != 0x80u8)); -}; - -// ---- decoder.next: valid 1/2/3/4-byte --------------------------------- // ref/hare/encoding/utf8/decode.ha:27. Round-trip via the same byte // vectors used in encode. @@ -166,7 +59,6 @@ fn streq(a: str, b: str) bool = { }; }; -// ---- decoder.next: invalid inputs ------------------------------------- // Cases mirror ref/hare/encoding/utf8/decode.ha:127-167 (surrogate / // overlong / out-of-range / bad-continuation). @@ -237,8 +129,6 @@ fn streq(a: str, b: str) bool = { }; }; -// ---- decoder.next: truncated → more ----------------------------------- - @test fn decode_truncated() void = { let src: [2]u8; src[0] = 0xE3u8; src[1] = 0x81u8; // missing 3rd byte @@ -251,8 +141,6 @@ fn streq(a: str, b: str) bool = { }; }; -// ---- decoder.next: done at EOI ---------------------------------------- - @test fn decode_done() void = { let src: [1]u8; let d: utf8.decoder = utf8.decode(src[0:0]); @@ -264,7 +152,6 @@ fn streq(a: str, b: str) bool = { }; }; -// ---- validate: well-formed mixed-width vs malformed ------------------- // ref/hare/encoding/utf8/decode.ha:207. Mixed input mirrors Hare's // decode @test ('こんにちは' + NUL). @@ -299,7 +186,6 @@ fn streq(a: str, b: str) bool = { }; }; -// ---- decoder.prev: done at start-of-input ----------------------------- // ref/hare/encoding/utf8/decode.ha:53-55. Cursor at offs=0 has no prior // codepoint. @@ -315,7 +201,6 @@ fn streq(a: str, b: str) bool = { }; }; -// ---- decoder.prev: each codepoint width round-trips ------------------- // ref/hare/encoding/utf8/decode.ha:56-71. Forward-decode one rune, // reverse-decode back to the same rune. @@ -375,7 +260,6 @@ fn streq(a: str, b: str) bool = { }; }; -// ---- decoder.prev: full forward + full reverse on mixed input --------- // ref/hare/encoding/utf8/decode.ha:85-111. After consuming all runes // forward, prev walks back through them in reverse order; prev at the // start-of-input then returns done. @@ -415,7 +299,6 @@ fn streq(a: str, b: str) bool = { }; }; -// ---- decoder.prev: invalid inputs ------------------------------------- // ref/hare/encoding/utf8/decode.ha:113-150. Two-continuation walk // reaches offs=0 without an initial byte → more; otherwise the forward // re-decode catches the malformed run as invalid. @@ -554,7 +437,6 @@ fn streq(a: str, b: str) bool = { }; }; -// ---- remaining / slice / position ------------------------------------- // ref/hare/encoding/utf8/decode.ha:172-198. @test fn remaining_slice_position() void = { @@ -607,17 +489,6 @@ fn streq(a: str, b: str) bool = { assert(!(utf8.remaining(&d1).len != 0)); }; -// ---- strerror: constant rendering ------------------------------------- -// ref/hare/encoding/utf8/types.ha:12. `invalid` is payload-free; the -// renderer collapses to a single fixture row. - -@test fn strerror_cases() void = { - let e: utf8.invalid; - assert(!(!streq(utf8.strerror(e), "Invalid UTF-8"))); -}; - -// ---- round-trip: encode → decode → equal rune -------------------------- - @test fn roundtrip() void = { let runes: [4]u32; runes[0] = 0x41u32; diff --git a/lib/encoding/utf8/encode_test.ww b/lib/encoding/utf8/encode_test.ww new file mode 100644 index 00000000..dd56fd53 --- /dev/null +++ b/lib/encoding/utf8/encode_test.ww @@ -0,0 +1,44 @@ +// encodetest — exercises utf8.encoderune. A failing row aborts via +// the assert/abort builtin (task #5 @test conversion). +// Vectors mirror ref/hare/encoding/utf8/encode.ha. + +package utf8_test; + +import encoding.utf8; + +// ref/hare/encoding/utf8/encode.ha:7. Byte vectors come from the +// Unicode specification (UAX standard examples). + +@test fn encode_ascii() void = { + let out: [4]u8; + let n: i32 = utf8.encoderune(out[0:4], 0x41u32: rune); // 'A' + assert(!(n != 1)); + assert(!(out[0] != 0x41u8)); +}; + +@test fn encode_two_byte() void = { + let out: [4]u8; + let n: i32 = utf8.encoderune(out[0:4], 0xE9u32: rune); // 'é' U+00E9 + assert(!(n != 2)); + assert(!(out[0] != 0xC3u8)); + assert(!(out[1] != 0xA9u8)); +}; + +@test fn encode_three_byte() void = { + let out: [4]u8; + let n: i32 = utf8.encoderune(out[0:4], 0x20ACu32: rune); // '€' U+20AC + assert(!(n != 3)); + assert(!(out[0] != 0xE2u8)); + assert(!(out[1] != 0x82u8)); + assert(!(out[2] != 0xACu8)); +}; + +@test fn encode_four_byte() void = { + let out: [4]u8; + let n: i32 = utf8.encoderune(out[0:4], 0x1F980u32: rune); // '🦀' U+1F980 + assert(!(n != 4)); + assert(!(out[0] != 0xF0u8)); + assert(!(out[1] != 0x9Fu8)); + assert(!(out[2] != 0xA6u8)); + assert(!(out[3] != 0x80u8)); +}; diff --git a/lib/encoding/utf8/rune_test.ww b/lib/encoding/utf8/rune_test.ww new file mode 100644 index 00000000..263a3c08 --- /dev/null +++ b/lib/encoding/utf8/rune_test.ww @@ -0,0 +1,63 @@ +// runetest — exercises utf8.runesz/utf8sz. A failing row aborts via +// the assert/abort builtin (task #5 @test conversion). +// Vectors mirror ref/hare/encoding/utf8/rune.ha. + +package utf8_test; + +import encoding.utf8; + +// ref/hare/encoding/utf8/rune.ha:5. Boundaries: 0x7F → 1, 0x80 → 2, +// 0x7FF → 2, 0x800 → 3, 0xFFFF → 3, 0x10000 → 4, 0x10FFFF → 4. + +@test fn runesz_ranges() void = { + assert(!(utf8.runesz(0u32: rune) != 1)); + assert(!(utf8.runesz(0x7Fu32: rune) != 1)); + assert(!(utf8.runesz(0x80u32: rune) != 2)); + assert(!(utf8.runesz(0x7FFu32: rune) != 2)); + assert(!(utf8.runesz(0x800u32: rune) != 3)); + assert(!(utf8.runesz(0xFFFFu32: rune) != 3)); + assert(!(utf8.runesz(0x10000u32: rune) != 4)); + assert(!(utf8.runesz(0x10FFFFu32: rune) != 4)); +}; + +// ref/hare/encoding/utf8/rune.ha:15. ASCII → 1; legal multibyte +// leads → 2/3/4; continuation and >0xF7 → invalid. + +@test fn utf8sz_classify() void = { + match (utf8.utf8sz(0u8)) { + case let n: i32 => { assert(!(n != 1)); }; + case let e: utf8.invalid => { abort(); }; + }; + match (utf8.utf8sz(0x7Fu8)) { + case let n: i32 => { assert(!(n != 1)); }; + case let e: utf8.invalid => { abort(); }; + }; + match (utf8.utf8sz(0x80u8)) { // continuation + case let n: i32 => { abort(); }; + case let e: utf8.invalid => void; + }; + match (utf8.utf8sz(0xC1u8)) { // overlong 2-byte lead + case let n: i32 => { abort(); }; + case let e: utf8.invalid => void; + }; + match (utf8.utf8sz(0xC2u8)) { + case let n: i32 => { assert(!(n != 2)); }; + case let e: utf8.invalid => { abort(); }; + }; + match (utf8.utf8sz(0xE0u8)) { + case let n: i32 => { assert(!(n != 3)); }; + case let e: utf8.invalid => { abort(); }; + }; + match (utf8.utf8sz(0xF0u8)) { + case let n: i32 => { assert(!(n != 4)); }; + case let e: utf8.invalid => { abort(); }; + }; + match (utf8.utf8sz(0xF8u8)) { // 5-byte lead — illegal in modern UTF-8 + case let n: i32 => { abort(); }; + case let e: utf8.invalid => void; + }; + match (utf8.utf8sz(0xFFu8)) { + case let n: i32 => { abort(); }; + case let e: utf8.invalid => void; + }; +}; diff --git a/lib/encoding/utf8/types_test.ww b/lib/encoding/utf8/types_test.ww new file mode 100644 index 00000000..40f7d65f --- /dev/null +++ b/lib/encoding/utf8/types_test.ww @@ -0,0 +1,25 @@ +// typestest — exercises utf8.strerror. A failing row aborts via the +// assert/abort builtin (task #5 @test conversion). +// Mirrors ref/hare/encoding/utf8/types.ha. + +package utf8_test; + +import encoding.utf8; + +fn streq(a: str, b: str) bool = { + if (a.len != b.len) { return false; }; + let i: i32 = 0; + for (i < a.len) { + if (a[i] != b[i]) { return false; }; + i += 1; + }; + return true; +}; + +// ref/hare/encoding/utf8/types.ha:12. `invalid` is payload-free; the +// renderer collapses to a single fixture row. + +@test fn strerror_cases() void = { + let e: utf8.invalid; + assert(!(!streq(utf8.strerror(e), "Invalid UTF-8"))); +}; diff --git a/test/byteid/libbyteid_test.ww b/test/byteid/libbyteid_test.ww index 9b10ed13..54557b7c 100644 --- a/test/byteid/libbyteid_test.ww +++ b/test/byteid/libbyteid_test.ww @@ -43,7 +43,7 @@ import time; def MID: i32 = 0; def MDIVERGE: i32 = 1; def MWWREJECT: i32 = 2; -def NENTEXPECT: i32 = 55; +def NENTEXPECT: i32 = 58; type ent = struct { fixture: str, // repo-relative .ww; "" -> probe entry @@ -83,7 +83,7 @@ fn pri(p: str, inc: str, sentinel: str, moddir: str) ent = { return e; }; -// The 55-unit roster. Graduation history lives in git (the retired C +// The 58-unit roster. Graduation history lives in git (the retired C // carrier's table comments); cites are kept only where a non-ID pin // would need them. fn corpus() []ent = { @@ -96,7 +96,10 @@ fn corpus() []ent = { append(es, fx("lib/dirs/dirs_test.ww")); append(es, fx("lib/encoding/base32/base32_test.ww")); append(es, fx("lib/encoding/hex/hex_test.ww")); - append(es, fx("lib/encoding/utf8/utf8_test.ww")); + append(es, fx("lib/encoding/utf8/rune_test.ww")); + append(es, fx("lib/encoding/utf8/encode_test.ww")); + append(es, fx("lib/encoding/utf8/decode_test.ww")); + append(es, fx("lib/encoding/utf8/types_test.ww")); append(es, fx("lib/getopt/getopt_test.ww")); append(es, fx("lib/hash/adler32/adler32_test.ww")); append(es, fx("lib/hash/crc16/crc16_test.ww"));