// utf8test — exercises lib/encoding/utf8. Run with // `out/bin/ww run lib/encoding/utf8/utf8test.ww`. Same signalled- // then-fail()-with-+10 pattern as hex / base32 / time tests: // non-zero exit pinpoints the failing scenario. package utf8; import utf8; import os; let signalled: i32 = 0; fn fail() void = { os.exit(signalled + 10); }; // ---- runesz: byte length per range ------------------------------------ // ref/hare/encoding/utf8/rune.ha:5. Boundaries: 0x7F → 1, 0x80 → 2, // 0x7FF → 2, 0x800 → 3, 0xFFFF → 3, 0x10000 → 4, 0x10FFFF → 4. @test fn runesz_ranges() void = { if (utf8.runesz(0u32: rune) != 1) { fail(); }; if (utf8.runesz(0x7Fu32: rune) != 1) { fail(); }; if (utf8.runesz(0x80u32: rune) != 2) { fail(); }; if (utf8.runesz(0x7FFu32: rune) != 2) { fail(); }; if (utf8.runesz(0x800u32: rune) != 3) { fail(); }; if (utf8.runesz(0xFFFFu32: rune) != 3) { fail(); }; if (utf8.runesz(0x10000u32: rune) != 4) { fail(); }; if (utf8.runesz(0x10FFFFu32: rune) != 4) { fail(); }; }; // ---- utf8sz: start-byte classification -------------------------------- // ref/hare/encoding/utf8/rune.ha:15. ASCII → 1; legal multibyte // leads → 2/3/4; continuation and >0xF7 → invalid. @test fn utf8sz_classify() void = { match (utf8.utf8sz(0u8)) { case let n: i32 => { if (n != 1) { fail(); }; }; case let e: utf8.invalid => { fail(); }; }; match (utf8.utf8sz(0x7Fu8)) { case let n: i32 => { if (n != 1) { fail(); }; }; case let e: utf8.invalid => { fail(); }; }; match (utf8.utf8sz(0x80u8)) { // continuation case let n: i32 => { fail(); }; case let e: utf8.invalid => void; }; match (utf8.utf8sz(0xC1u8)) { // overlong 2-byte lead case let n: i32 => { fail(); }; case let e: utf8.invalid => void; }; match (utf8.utf8sz(0xC2u8)) { case let n: i32 => { if (n != 2) { fail(); }; }; case let e: utf8.invalid => { fail(); }; }; match (utf8.utf8sz(0xE0u8)) { case let n: i32 => { if (n != 3) { fail(); }; }; case let e: utf8.invalid => { fail(); }; }; match (utf8.utf8sz(0xF0u8)) { case let n: i32 => { if (n != 4) { fail(); }; }; case let e: utf8.invalid => { fail(); }; }; match (utf8.utf8sz(0xF8u8)) { // 5-byte lead — illegal in modern UTF-8 case let n: i32 => { fail(); }; case let e: utf8.invalid => void; }; match (utf8.utf8sz(0xFFu8)) { case let n: i32 => { fail(); }; case let e: utf8.invalid => void; }; }; // ---- encoderune: all four widths -------------------------------------- // ref/hare/encoding/utf8/encode.ha:7. Byte vectors come from the // Unicode specification (UAX standard examples). @test fn encode_ascii() void = { let out: [4]u8; let n: i32 = utf8.encoderune(out[0:4], 0x41u32: rune); // 'A' if (n != 1) { fail(); }; if (out[0] != 0x41u8) { fail(); }; }; @test fn encode_two_byte() void = { let out: [4]u8; let n: i32 = utf8.encoderune(out[0:4], 0xE9u32: rune); // 'é' U+00E9 if (n != 2) { fail(); }; if (out[0] != 0xC3u8) { fail(); }; if (out[1] != 0xA9u8) { fail(); }; }; @test fn encode_three_byte() void = { let out: [4]u8; let n: i32 = utf8.encoderune(out[0:4], 0x20ACu32: rune); // '€' U+20AC if (n != 3) { fail(); }; if (out[0] != 0xE2u8) { fail(); }; if (out[1] != 0x82u8) { fail(); }; if (out[2] != 0xACu8) { fail(); }; }; @test fn encode_four_byte() void = { let out: [4]u8; let n: i32 = utf8.encoderune(out[0:4], 0x1F980u32: rune); // '🦀' U+1F980 if (n != 4) { fail(); }; if (out[0] != 0xF0u8) { fail(); }; if (out[1] != 0x9Fu8) { fail(); }; if (out[2] != 0xA6u8) { fail(); }; if (out[3] != 0x80u8) { fail(); }; }; // ---- decoder.next: valid 1/2/3/4-byte --------------------------------- // ref/hare/encoding/utf8/decode.ha:27. Round-trip via the same byte // vectors used in encode. @test fn decode_one_byte() void = { let src: [1]u8; src[0] = 0x41u8; let d: utf8.decoder = utf8.decode(src[0:1]); match (utf8.next(&d)) { case utf8.done => { fail(); }; case utf8.more => { fail(); }; case let r: utf8.invalid => { fail(); }; case let r: rune => { if (r != 0x41u32: rune) { fail(); }; }; }; }; @test fn decode_two_byte() void = { let src: [2]u8; src[0] = 0xC3u8; src[1] = 0xA9u8; let d: utf8.decoder = utf8.decode(src[0:2]); match (utf8.next(&d)) { case utf8.done => { fail(); }; case utf8.more => { fail(); }; case let r: utf8.invalid => { fail(); }; case let r: rune => { if (r != 0xE9u32: rune) { fail(); }; }; }; }; @test fn decode_three_byte() void = { let src: [3]u8; src[0] = 0xE2u8; src[1] = 0x82u8; src[2] = 0xACu8; let d: utf8.decoder = utf8.decode(src[0:3]); match (utf8.next(&d)) { case utf8.done => { fail(); }; case utf8.more => { fail(); }; case let r: utf8.invalid => { fail(); }; case let r: rune => { if (r != 0x20ACu32: rune) { fail(); }; }; }; }; @test fn decode_four_byte() void = { let src: [4]u8; src[0] = 0xF0u8; src[1] = 0x9Fu8; src[2] = 0xA6u8; src[3] = 0x80u8; let d: utf8.decoder = utf8.decode(src[0:4]); match (utf8.next(&d)) { case utf8.done => { fail(); }; case utf8.more => { fail(); }; case let r: utf8.invalid => { fail(); }; case let r: rune => { if (r != 0x1F980u32: rune) { fail(); }; }; }; }; // ---- decoder.next: invalid inputs ------------------------------------- // Cases mirror ref/hare/encoding/utf8/decode.ha:127-167 (surrogate / // overlong / out-of-range / bad-continuation). @test fn decode_surrogate() void = { let src: [3]u8; src[0] = 0xEDu8; src[1] = 0xA0u8; src[2] = 0x80u8; // U+D800 let d: utf8.decoder = utf8.decode(src[0:3]); match (utf8.next(&d)) { case let r: utf8.invalid => void; case let r: rune => { fail(); }; case utf8.done => { fail(); }; case utf8.more => { fail(); }; }; }; @test fn decode_overlong() void = { let src: [4]u8; src[0] = 0xF0u8; src[1] = 0x82u8; src[2] = 0x82u8; src[3] = 0xACu8; let d: utf8.decoder = utf8.decode(src[0:4]); match (utf8.next(&d)) { case let r: utf8.invalid => void; case let r: rune => { fail(); }; case utf8.done => { fail(); }; case utf8.more => { fail(); }; }; }; @test fn decode_out_of_range() void = { let src: [4]u8; src[0] = 0xF5u8; src[1] = 0x94u8; src[2] = 0x80u8; src[3] = 0x80u8; let d: utf8.decoder = utf8.decode(src[0:4]); match (utf8.next(&d)) { case let r: utf8.invalid => void; case let r: rune => { fail(); }; case utf8.done => { fail(); }; case utf8.more => { fail(); }; }; }; // ref/hare/encoding/utf8/decode.ha:141 — `[0xC2, 0xFF]`: legal 2-byte // lead followed by non-continuation. Pins next-table cell (state 1, // byte 0xFF) returning -1; distinct from overlong (which is filtered // in state 3/5/7 by lead-byte-aware sub-states). @test fn decode_bad_continuation() void = { let src: [2]u8; src[0] = 0xC2u8; src[1] = 0xFFu8; let d: utf8.decoder = utf8.decode(src[0:2]); match (utf8.next(&d)) { case let r: utf8.invalid => void; case let r: rune => { fail(); }; case utf8.done => { fail(); }; case utf8.more => { fail(); }; }; }; // ref/hare/encoding/utf8/decode.ha:151 — `[0xF4, 0x8F, 0xBF, 0xBF]` = // U+10FFFF, the largest legal codepoint. Pins the upper boundary; // pairs with the existing `0xF5…` out-of-range row. @test fn decode_max_in_range() void = { let src: [4]u8; src[0] = 0xF4u8; src[1] = 0x8Fu8; src[2] = 0xBFu8; src[3] = 0xBFu8; let d: utf8.decoder = utf8.decode(src[0:4]); match (utf8.next(&d)) { case let r: rune => { if (r != 0x10FFFFu32: rune) { fail(); }; }; case let r: utf8.invalid => { fail(); }; case utf8.done => { fail(); }; case utf8.more => { fail(); }; }; }; // ---- decoder.next: truncated → more ----------------------------------- @test fn decode_truncated() void = { let src: [2]u8; src[0] = 0xE3u8; src[1] = 0x81u8; // missing 3rd byte let d: utf8.decoder = utf8.decode(src[0:2]); match (utf8.next(&d)) { case utf8.more => void; case let r: rune => { fail(); }; case utf8.done => { fail(); }; case let r: utf8.invalid => { fail(); }; }; }; // ---- decoder.next: done at EOI ---------------------------------------- @test fn decode_done() void = { let src: [1]u8; let d: utf8.decoder = utf8.decode(src[0:0]); match (utf8.next(&d)) { case utf8.done => void; case let r: rune => { fail(); }; case utf8.more => { fail(); }; case let r: utf8.invalid => { fail(); }; }; }; // ---- validate: well-formed mixed-width vs malformed ------------------- // ref/hare/encoding/utf8/decode.ha:207. Mixed input mirrors Hare's // decode @test ('こんにちは' + NUL). @test fn validate_mixed_ok() void = { let src: [16]u8; src[0] = 0xE3u8; src[1] = 0x81u8; src[2] = 0x93u8; // こ src[3] = 0xE3u8; src[4] = 0x82u8; src[5] = 0x93u8; // ん src[6] = 0xE3u8; src[7] = 0x81u8; src[8] = 0xABu8; // に src[9] = 0xE3u8; src[10] = 0x81u8; src[11] = 0xA1u8; // ち src[12] = 0xE3u8; src[13] = 0x81u8; src[14] = 0xAFu8; // は src[15] = 0u8; match (utf8.validate(src[0:16])) { case let e: utf8.invalid => { fail(); }; case void => void; }; }; @test fn validate_malformed() void = { let src: [3]u8; src[0] = 0xEDu8; src[1] = 0xA0u8; src[2] = 0x80u8; // surrogate match (utf8.validate(src[0:3])) { case let e: utf8.invalid => void; case void => { fail(); }; }; }; @test fn validate_empty_ok() void = { let src: [1]u8; match (utf8.validate(src[0:0])) { case let e: utf8.invalid => { fail(); }; case void => void; }; }; // ---- round-trip: encode → decode → equal rune -------------------------- @test fn roundtrip() void = { let runes: [4]u32; runes[0] = 0x41u32; runes[1] = 0xE9u32; runes[2] = 0x20ACu32; runes[3] = 0x1F980u32; let i: i32 = 0; for (i < 4) { let buf: [4]u8; let n: i32 = utf8.encoderune(buf[0:4], runes[i]: rune); let d: utf8.decoder = utf8.decode(buf[0:n]); match (utf8.next(&d)) { case let r: rune => { if ((r: u32) != runes[i]) { fail(); }; }; case utf8.done => { fail(); }; case utf8.more => { fail(); }; case let e: utf8.invalid => { fail(); }; }; i += 1; }; }; export fn main() i32 = { signalled = 1; runesz_ranges(); signalled = 2; utf8sz_classify(); signalled = 3; encode_ascii(); signalled = 4; encode_two_byte(); signalled = 5; encode_three_byte(); signalled = 6; encode_four_byte(); signalled = 7; decode_one_byte(); signalled = 8; decode_two_byte(); signalled = 9; decode_three_byte(); signalled = 10; decode_four_byte(); signalled = 11; decode_surrogate(); signalled = 12; decode_overlong(); signalled = 13; decode_out_of_range(); signalled = 14; decode_bad_continuation(); signalled = 15; decode_max_in_range(); signalled = 16; decode_truncated(); signalled = 17; decode_done(); signalled = 18; validate_mixed_ok(); signalled = 19; validate_malformed(); signalled = 20; validate_empty_ok(); signalled = 21; roundtrip(); return 0; };