ww/lib/encoding/utf8/utf8test.ww

// utf8test — exercises lib/encoding/utf8. Run with
// `out/bin/ww run lib/encoding/utf8/utf8test.ww`. Same signalled-
// then-fail()-with-+10 pattern as hex / base32 / time tests:
// non-zero exit pinpoints the failing scenario.

package utf8;

import utf8;
import os;

let signalled: i32 = 0;
fn fail() void = { os.exit(signalled + 10); };

// ---- runesz: byte length per range ------------------------------------
// ref/hare/encoding/utf8/rune.ha:5. Boundaries: 0x7F → 1, 0x80 → 2,
// 0x7FF → 2, 0x800 → 3, 0xFFFF → 3, 0x10000 → 4, 0x10FFFF → 4.

@test fn runesz_ranges() void = {
	if (utf8.runesz(0u32: rune)        != 1) { fail(); };
	if (utf8.runesz(0x7Fu32: rune)     != 1) { fail(); };
	if (utf8.runesz(0x80u32: rune)     != 2) { fail(); };
	if (utf8.runesz(0x7FFu32: rune)    != 2) { fail(); };
	if (utf8.runesz(0x800u32: rune)    != 3) { fail(); };
	if (utf8.runesz(0xFFFFu32: rune)   != 3) { fail(); };
	if (utf8.runesz(0x10000u32: rune)  != 4) { fail(); };
	if (utf8.runesz(0x10FFFFu32: rune) != 4) { fail(); };
};

// ---- utf8sz: start-byte classification --------------------------------
// ref/hare/encoding/utf8/rune.ha:15. ASCII → 1; legal multibyte
// leads → 2/3/4; continuation and >0xF7 → invalid.

@test fn utf8sz_classify() void = {
	match (utf8.utf8sz(0u8)) {
	case let n: i32           => { if (n != 1) { fail(); }; };
	case let e: utf8.invalid  => { fail(); };
	};
	match (utf8.utf8sz(0x7Fu8)) {
	case let n: i32           => { if (n != 1) { fail(); }; };
	case let e: utf8.invalid  => { fail(); };
	};
	match (utf8.utf8sz(0x80u8)) {	// continuation
	case let n: i32           => { fail(); };
	case let e: utf8.invalid  => void;
	};
	match (utf8.utf8sz(0xC1u8)) {	// overlong 2-byte lead
	case let n: i32           => { fail(); };
	case let e: utf8.invalid  => void;
	};
	match (utf8.utf8sz(0xC2u8)) {
	case let n: i32           => { if (n != 2) { fail(); }; };
	case let e: utf8.invalid  => { fail(); };
	};
	match (utf8.utf8sz(0xE0u8)) {
	case let n: i32           => { if (n != 3) { fail(); }; };
	case let e: utf8.invalid  => { fail(); };
	};
	match (utf8.utf8sz(0xF0u8)) {
	case let n: i32           => { if (n != 4) { fail(); }; };
	case let e: utf8.invalid  => { fail(); };
	};
	match (utf8.utf8sz(0xF8u8)) {	// 5-byte lead — illegal in modern UTF-8
	case let n: i32           => { fail(); };
	case let e: utf8.invalid  => void;
	};
	match (utf8.utf8sz(0xFFu8)) {
	case let n: i32           => { fail(); };
	case let e: utf8.invalid  => void;
	};
};

// ---- encoderune: all four widths --------------------------------------
// ref/hare/encoding/utf8/encode.ha:7. Byte vectors come from the
// Unicode specification (UAX standard examples).

@test fn encode_ascii() void = {
	let out: [4]u8;
	let n: i32 = utf8.encoderune(out[0:4], 0x41u32: rune);	// 'A'
	if (n != 1) { fail(); };
	if (out[0] != 0x41u8) { fail(); };
};

@test fn encode_two_byte() void = {
	let out: [4]u8;
	let n: i32 = utf8.encoderune(out[0:4], 0xE9u32: rune);	// 'é' U+00E9
	if (n != 2) { fail(); };
	if (out[0] != 0xC3u8) { fail(); };
	if (out[1] != 0xA9u8) { fail(); };
};

@test fn encode_three_byte() void = {
	let out: [4]u8;
	let n: i32 = utf8.encoderune(out[0:4], 0x20ACu32: rune);	// '€' U+20AC
	if (n != 3) { fail(); };
	if (out[0] != 0xE2u8) { fail(); };
	if (out[1] != 0x82u8) { fail(); };
	if (out[2] != 0xACu8) { fail(); };
};

@test fn encode_four_byte() void = {
	let out: [4]u8;
	let n: i32 = utf8.encoderune(out[0:4], 0x1F980u32: rune);	// '🦀' U+1F980
	if (n != 4) { fail(); };
	if (out[0] != 0xF0u8) { fail(); };
	if (out[1] != 0x9Fu8) { fail(); };
	if (out[2] != 0xA6u8) { fail(); };
	if (out[3] != 0x80u8) { fail(); };
};

// ---- decoder.next: valid 1/2/3/4-byte ---------------------------------
// ref/hare/encoding/utf8/decode.ha:27. Round-trip via the same byte
// vectors used in encode.

@test fn decode_one_byte() void = {
	let src: [1]u8;
	src[0] = 0x41u8;
	let d: utf8.decoder = utf8.decode(src[0:1]);
	match (utf8.next(&d)) {
	case utf8.done           => { fail(); };
	case utf8.more           => { fail(); };
	case let r: utf8.invalid => { fail(); };
	case let r: rune         => { if (r != 0x41u32: rune) { fail(); }; };
	};
};

@test fn decode_two_byte() void = {
	let src: [2]u8;
	src[0] = 0xC3u8; src[1] = 0xA9u8;
	let d: utf8.decoder = utf8.decode(src[0:2]);
	match (utf8.next(&d)) {
	case utf8.done           => { fail(); };
	case utf8.more           => { fail(); };
	case let r: utf8.invalid => { fail(); };
	case let r: rune         => { if (r != 0xE9u32: rune) { fail(); }; };
	};
};

@test fn decode_three_byte() void = {
	let src: [3]u8;
	src[0] = 0xE2u8; src[1] = 0x82u8; src[2] = 0xACu8;
	let d: utf8.decoder = utf8.decode(src[0:3]);
	match (utf8.next(&d)) {
	case utf8.done           => { fail(); };
	case utf8.more           => { fail(); };
	case let r: utf8.invalid => { fail(); };
	case let r: rune         => { if (r != 0x20ACu32: rune) { fail(); }; };
	};
};

@test fn decode_four_byte() void = {
	let src: [4]u8;
	src[0] = 0xF0u8; src[1] = 0x9Fu8; src[2] = 0xA6u8; src[3] = 0x80u8;
	let d: utf8.decoder = utf8.decode(src[0:4]);
	match (utf8.next(&d)) {
	case utf8.done           => { fail(); };
	case utf8.more           => { fail(); };
	case let r: utf8.invalid => { fail(); };
	case let r: rune         => { if (r != 0x1F980u32: rune) { fail(); }; };
	};
};

// ---- decoder.next: invalid inputs -------------------------------------
// Cases mirror ref/hare/encoding/utf8/decode.ha:127-167 (surrogate /
// overlong / out-of-range / bad-continuation).

@test fn decode_surrogate() void = {
	let src: [3]u8;
	src[0] = 0xEDu8; src[1] = 0xA0u8; src[2] = 0x80u8;	// U+D800
	let d: utf8.decoder = utf8.decode(src[0:3]);
	match (utf8.next(&d)) {
	case let r: utf8.invalid => void;
	case let r: rune         => { fail(); };
	case utf8.done           => { fail(); };
	case utf8.more           => { fail(); };
	};
};

@test fn decode_overlong() void = {
	let src: [4]u8;
	src[0] = 0xF0u8; src[1] = 0x82u8; src[2] = 0x82u8; src[3] = 0xACu8;
	let d: utf8.decoder = utf8.decode(src[0:4]);
	match (utf8.next(&d)) {
	case let r: utf8.invalid => void;
	case let r: rune         => { fail(); };
	case utf8.done           => { fail(); };
	case utf8.more           => { fail(); };
	};
};

@test fn decode_out_of_range() void = {
	let src: [4]u8;
	src[0] = 0xF5u8; src[1] = 0x94u8; src[2] = 0x80u8; src[3] = 0x80u8;
	let d: utf8.decoder = utf8.decode(src[0:4]);
	match (utf8.next(&d)) {
	case let r: utf8.invalid => void;
	case let r: rune         => { fail(); };
	case utf8.done           => { fail(); };
	case utf8.more           => { fail(); };
	};
};

// ref/hare/encoding/utf8/decode.ha:141 — `[0xC2, 0xFF]`: legal 2-byte
// lead followed by non-continuation. Pins next-table cell (state 1,
// byte 0xFF) returning -1; distinct from overlong (which is filtered
// in state 3/5/7 by lead-byte-aware sub-states).
@test fn decode_bad_continuation() void = {
	let src: [2]u8;
	src[0] = 0xC2u8; src[1] = 0xFFu8;
	let d: utf8.decoder = utf8.decode(src[0:2]);
	match (utf8.next(&d)) {
	case let r: utf8.invalid => void;
	case let r: rune         => { fail(); };
	case utf8.done           => { fail(); };
	case utf8.more           => { fail(); };
	};
};

// ref/hare/encoding/utf8/decode.ha:151 — `[0xF4, 0x8F, 0xBF, 0xBF]` =
// U+10FFFF, the largest legal codepoint. Pins the upper boundary;
// pairs with the existing `0xF5…` out-of-range row.
@test fn decode_max_in_range() void = {
	let src: [4]u8;
	src[0] = 0xF4u8; src[1] = 0x8Fu8; src[2] = 0xBFu8; src[3] = 0xBFu8;
	let d: utf8.decoder = utf8.decode(src[0:4]);
	match (utf8.next(&d)) {
	case let r: rune         => { if (r != 0x10FFFFu32: rune) { fail(); }; };
	case let r: utf8.invalid => { fail(); };
	case utf8.done           => { fail(); };
	case utf8.more           => { fail(); };
	};
};

// ---- decoder.next: truncated → more -----------------------------------

@test fn decode_truncated() void = {
	let src: [2]u8;
	src[0] = 0xE3u8; src[1] = 0x81u8;	// missing 3rd byte
	let d: utf8.decoder = utf8.decode(src[0:2]);
	match (utf8.next(&d)) {
	case utf8.more           => void;
	case let r: rune         => { fail(); };
	case utf8.done           => { fail(); };
	case let r: utf8.invalid => { fail(); };
	};
};

// ---- decoder.next: done at EOI ----------------------------------------

@test fn decode_done() void = {
	let src: [1]u8;
	let d: utf8.decoder = utf8.decode(src[0:0]);
	match (utf8.next(&d)) {
	case utf8.done           => void;
	case let r: rune         => { fail(); };
	case utf8.more           => { fail(); };
	case let r: utf8.invalid => { fail(); };
	};
};

// ---- validate: well-formed mixed-width vs malformed -------------------
// ref/hare/encoding/utf8/decode.ha:207. Mixed input mirrors Hare's
// decode @test ('こんにちは' + NUL).

@test fn validate_mixed_ok() void = {
	let src: [16]u8;
	src[0]  = 0xE3u8; src[1]  = 0x81u8; src[2]  = 0x93u8;	// こ
	src[3]  = 0xE3u8; src[4]  = 0x82u8; src[5]  = 0x93u8;	// ん
	src[6]  = 0xE3u8; src[7]  = 0x81u8; src[8]  = 0xABu8;	// に
	src[9]  = 0xE3u8; src[10] = 0x81u8; src[11] = 0xA1u8;	// ち
	src[12] = 0xE3u8; src[13] = 0x81u8; src[14] = 0xAFu8;	// は
	src[15] = 0u8;
	match (utf8.validate(src[0:16])) {
	case let e: utf8.invalid => { fail(); };
	case void => void;
	};
};

@test fn validate_malformed() void = {
	let src: [3]u8;
	src[0] = 0xEDu8; src[1] = 0xA0u8; src[2] = 0x80u8;	// surrogate
	match (utf8.validate(src[0:3])) {
	case let e: utf8.invalid => void;
	case void => { fail(); };
	};
};

@test fn validate_empty_ok() void = {
	let src: [1]u8;
	match (utf8.validate(src[0:0])) {
	case let e: utf8.invalid => { fail(); };
	case void => void;
	};
};

// ---- round-trip: encode → decode → equal rune --------------------------

@test fn roundtrip() void = {
	let runes: [4]u32;
	runes[0] = 0x41u32;
	runes[1] = 0xE9u32;
	runes[2] = 0x20ACu32;
	runes[3] = 0x1F980u32;
	let i: i32 = 0;
	for (i < 4) {
		let buf: [4]u8;
		let n: i32 = utf8.encoderune(buf[0:4], runes[i]: rune);
		let d: utf8.decoder = utf8.decode(buf[0:n]);
		match (utf8.next(&d)) {
		case let r: rune         => {
			if ((r: u32) != runes[i]) { fail(); };
		};
		case utf8.done           => { fail(); };
		case utf8.more           => { fail(); };
		case let e: utf8.invalid => { fail(); };
		};
		i += 1;
	};
};

export fn main() i32 = {
	signalled = 1;  runesz_ranges();
	signalled = 2;  utf8sz_classify();
	signalled = 3;  encode_ascii();
	signalled = 4;  encode_two_byte();
	signalled = 5;  encode_three_byte();
	signalled = 6;  encode_four_byte();
	signalled = 7;  decode_one_byte();
	signalled = 8;  decode_two_byte();
	signalled = 9;  decode_three_byte();
	signalled = 10; decode_four_byte();
	signalled = 11; decode_surrogate();
	signalled = 12; decode_overlong();
	signalled = 13; decode_out_of_range();
	signalled = 14; decode_bad_continuation();
	signalled = 15; decode_max_in_range();
	signalled = 16; decode_truncated();
	signalled = 17; decode_done();
	signalled = 18; validate_mixed_ok();
	signalled = 19; validate_malformed();
	signalled = 20; validate_empty_ok();
	signalled = 21; roundtrip();
	return 0;
};