encoding/utf8: enforce scalar and buffer bounds

lib/encoding/utf8/decode_test.ww

@@ -5,6 +5,7 @@ package utf8_test;
 
 import bytes;
 import encoding.utf8;
+import test;
 
 // ref/hare/encoding/utf8/decode.ha:27. Round-trip via the same byte
 // vectors used in encode.
@@ -487,6 +488,27 @@ import encoding.utf8;
 	assert(!(utf8.remaining(&d1).len != 0));
 };
 
+// Pointer equality alone does not establish that two decoders have the same
+// source: differently bounded views over one allocation have different valid
+// offset ranges.
+@test fn slice_mismatched_source_length_aborts() void = {
+	test.expectabort();
+	let src: [4]u8;
+	let begin: utf8.decoder = utf8.decode(src[0:1]);
+	let end: utf8.decoder = utf8.decode(src[0:4]);
+	utf8.slice(&begin, &end);
+};
+
+@test fn slice_offset_past_source_aborts() void = {
+	test.expectabort();
+	let src: [1]u8;
+	let begin: utf8.decoder = utf8.decode(src[0:1]);
+	let end: utf8.decoder = utf8.decode(src[0:1]);
+	begin.offs = 2;
+	end.offs = 2;
+	utf8.slice(&begin, &end);
+};
+
 @test fn roundtrip() void = {
 	let runes: [4]u32;
 	runes[0] = 0x41u32;

lib/encoding/utf8/encode_test.ww

@@ -4,6 +4,7 @@
 package utf8_test;
 
 import encoding.utf8;
+import test;
 
 // ref/hare/encoding/utf8/encode.ha:7. Byte vectors come from the
 // Unicode specification (UAX standard examples).
@@ -41,3 +42,11 @@ import encoding.utf8;
 	assert(!(out[2] != 0xA6u8));
 	assert(!(out[3] != 0x80u8));
 };
+
+// The ww API writes into caller storage, unlike Hare's static-buffer return.
+// A short slice must fail before the first out-of-bounds store.
+@test fn encode_short_buffer_aborts() void = {
+	test.expectabort();
+	let out: [1]u8;
+	utf8.encoderune(out[0:1], 0xE9u32: rune);
+};

lib/encoding/utf8/rune_test.ww

@@ -20,7 +20,7 @@ import encoding.utf8;
 };
 
 // ref/hare/encoding/utf8/rune.ha:15. ASCII → 1; legal multibyte
-// leads → 2/3/4; continuation and >0xF7 → invalid.
+// leads → 2/3/4; continuation and >0xF4 → invalid.
 
 @test fn utf8sz_classify() void = {
 	match (utf8.utf8sz(0u8)) {
@@ -51,7 +51,19 @@ import encoding.utf8;
 	case let n: i32           => { assert(!(n != 4)); };
 	case let e: utf8.invalid  => { abort(); };
 	};
-	match (utf8.utf8sz(0xF8u8)) {	// 5-byte lead — illegal in modern UTF-8
+	match (utf8.utf8sz(0xF4u8)) {	// U+10FFFF may begin with F4
+	case let n: i32           => { assert(!(n != 4)); };
+	case let e: utf8.invalid  => { abort(); };
+	};
+	match (utf8.utf8sz(0xF5u8)) {	// above Unicode's scalar range
+	case let n: i32           => { abort(); };
+	case let e: utf8.invalid  => void;
+	};
+	match (utf8.utf8sz(0xF6u8)) {
+	case let n: i32           => { abort(); };
+	case let e: utf8.invalid  => void;
+	};
+	match (utf8.utf8sz(0xF7u8)) {
 	case let n: i32           => { abort(); };
 	case let e: utf8.invalid  => void;
 	};

lib/encoding/utf8/utf8.ww

@@ -300,11 +300,13 @@ export fn runesz(r: rune) i32 = {
 // codepoint that starts with `c`, or `invalid` if `c` cannot start
 // a legal UTF-8 sequence. Constants written in decimal because ww
 // doesn't accept Hare's `0b1000_0000` binary syntax: 0x80=128,
-// 0xC2=194, 0xE0=224, 0xF0=240, 0xF8=248.
+// 0xC2=194, 0xE0=224, 0xF0=240, 0xF5=245.
 export fn utf8sz(c: u8) (i32 | invalid) = {
 	if (c < 128u8) { return 1; };
 	if (c < 194u8) { let e: invalid; return e; };
-	if (c >= 248u8) { let e: invalid; return e; };
+	// F4 is the last legal four-byte lead: F4 8F BF BF encodes
+	// U+10FFFF. F5 and above cannot begin a Unicode scalar value.
+	if (c >= 245u8) { let e: invalid; return e; };
 	if (c < 224u8) { return 2; };
 	if (c < 240u8) { return 3; };
 	return 4;
@@ -340,6 +342,7 @@ export fn encoderune(out: []u8, r: rune) i32 = {
 	} else {
 		first = 0xF0u8; n = 4;
 	};
+	assert(out.len >= n, "utf8.encoderune: output buffer is too small");
 
 	let v: u32 = ch;
 	let i: i32 = n - 1;
@@ -431,12 +434,16 @@ export fn remaining(d: *decoder) []u8 = {
 };
 
 // ref/hare/encoding/utf8/decode.ha:80. Borrowed view of the bytes
-// between two decoders' positions. Precondition (Hare asserts both):
+// between two decoders' positions. The decoders must share one complete
-// the decoders share the same source, and `begin.offs <= end.offs`.
+// source view, both offsets must remain in that view, and begin must not
+// follow end.
 export fn slice(begin: *decoder, end: *decoder) []u8 = {
-	if (begin.src.ptr != end.src.ptr) {
+	if (begin.src.ptr != end.src.ptr || begin.src.len != end.src.len) {
 		abort("utf8.slice: decoders from different sources");
 	};
+	if (begin.offs > begin.src.len: size || end.offs > end.src.len: size) {
+		abort("utf8.slice: decoder offset past end of source");
+	};
 	if (begin.offs > end.offs) {
 		abort("utf8.slice: begin past end");
 	};