diff --git a/lib/encoding/utf8/decode_test.ww b/lib/encoding/utf8/decode_test.ww index 386413da..e2975cb1 100644 --- a/lib/encoding/utf8/decode_test.ww +++ b/lib/encoding/utf8/decode_test.ww @@ -5,6 +5,7 @@ package utf8_test; import bytes; import encoding.utf8; +import test; // ref/hare/encoding/utf8/decode.ha:27. Round-trip via the same byte // vectors used in encode. @@ -487,6 +488,27 @@ import encoding.utf8; assert(!(utf8.remaining(&d1).len != 0)); }; +// Pointer equality alone does not establish that two decoders have the same +// source: differently bounded views over one allocation have different valid +// offset ranges. +@test fn slice_mismatched_source_length_aborts() void = { + test.expectabort(); + let src: [4]u8; + let begin: utf8.decoder = utf8.decode(src[0:1]); + let end: utf8.decoder = utf8.decode(src[0:4]); + utf8.slice(&begin, &end); +}; + +@test fn slice_offset_past_source_aborts() void = { + test.expectabort(); + let src: [1]u8; + let begin: utf8.decoder = utf8.decode(src[0:1]); + let end: utf8.decoder = utf8.decode(src[0:1]); + begin.offs = 2; + end.offs = 2; + utf8.slice(&begin, &end); +}; + @test fn roundtrip() void = { let runes: [4]u32; runes[0] = 0x41u32; diff --git a/lib/encoding/utf8/encode_test.ww b/lib/encoding/utf8/encode_test.ww index c28a0c2b..ffdd0ecc 100644 --- a/lib/encoding/utf8/encode_test.ww +++ b/lib/encoding/utf8/encode_test.ww @@ -4,6 +4,7 @@ package utf8_test; import encoding.utf8; +import test; // ref/hare/encoding/utf8/encode.ha:7. Byte vectors come from the // Unicode specification (UAX standard examples). @@ -41,3 +42,11 @@ import encoding.utf8; assert(!(out[2] != 0xA6u8)); assert(!(out[3] != 0x80u8)); }; + +// The ww API writes into caller storage, unlike Hare's static-buffer return. +// A short slice must fail before the first out-of-bounds store. +@test fn encode_short_buffer_aborts() void = { + test.expectabort(); + let out: [1]u8; + utf8.encoderune(out[0:1], 0xE9u32: rune); +}; diff --git a/lib/encoding/utf8/rune_test.ww b/lib/encoding/utf8/rune_test.ww index 6733d707..35a85791 100644 --- a/lib/encoding/utf8/rune_test.ww +++ b/lib/encoding/utf8/rune_test.ww @@ -20,7 +20,7 @@ import encoding.utf8; }; // ref/hare/encoding/utf8/rune.ha:15. ASCII → 1; legal multibyte -// leads → 2/3/4; continuation and >0xF7 → invalid. +// leads → 2/3/4; continuation and >0xF4 → invalid. @test fn utf8sz_classify() void = { match (utf8.utf8sz(0u8)) { @@ -51,7 +51,19 @@ import encoding.utf8; case let n: i32 => { assert(!(n != 4)); }; case let e: utf8.invalid => { abort(); }; }; - match (utf8.utf8sz(0xF8u8)) { // 5-byte lead — illegal in modern UTF-8 + match (utf8.utf8sz(0xF4u8)) { // U+10FFFF may begin with F4 + case let n: i32 => { assert(!(n != 4)); }; + case let e: utf8.invalid => { abort(); }; + }; + match (utf8.utf8sz(0xF5u8)) { // above Unicode's scalar range + case let n: i32 => { abort(); }; + case let e: utf8.invalid => void; + }; + match (utf8.utf8sz(0xF6u8)) { + case let n: i32 => { abort(); }; + case let e: utf8.invalid => void; + }; + match (utf8.utf8sz(0xF7u8)) { case let n: i32 => { abort(); }; case let e: utf8.invalid => void; }; diff --git a/lib/encoding/utf8/utf8.ww b/lib/encoding/utf8/utf8.ww index d38ef090..2e346660 100644 --- a/lib/encoding/utf8/utf8.ww +++ b/lib/encoding/utf8/utf8.ww @@ -300,11 +300,13 @@ export fn runesz(r: rune) i32 = { // codepoint that starts with `c`, or `invalid` if `c` cannot start // a legal UTF-8 sequence. Constants written in decimal because ww // doesn't accept Hare's `0b1000_0000` binary syntax: 0x80=128, -// 0xC2=194, 0xE0=224, 0xF0=240, 0xF8=248. +// 0xC2=194, 0xE0=224, 0xF0=240, 0xF5=245. export fn utf8sz(c: u8) (i32 | invalid) = { if (c < 128u8) { return 1; }; if (c < 194u8) { let e: invalid; return e; }; - if (c >= 248u8) { let e: invalid; return e; }; + // F4 is the last legal four-byte lead: F4 8F BF BF encodes + // U+10FFFF. F5 and above cannot begin a Unicode scalar value. + if (c >= 245u8) { let e: invalid; return e; }; if (c < 224u8) { return 2; }; if (c < 240u8) { return 3; }; return 4; @@ -340,6 +342,7 @@ export fn encoderune(out: []u8, r: rune) i32 = { } else { first = 0xF0u8; n = 4; }; + assert(out.len >= n, "utf8.encoderune: output buffer is too small"); let v: u32 = ch; let i: i32 = n - 1; @@ -431,12 +434,16 @@ export fn remaining(d: *decoder) []u8 = { }; // ref/hare/encoding/utf8/decode.ha:80. Borrowed view of the bytes -// between two decoders' positions. Precondition (Hare asserts both): -// the decoders share the same source, and `begin.offs <= end.offs`. +// between two decoders' positions. The decoders must share one complete +// source view, both offsets must remain in that view, and begin must not +// follow end. export fn slice(begin: *decoder, end: *decoder) []u8 = { - if (begin.src.ptr != end.src.ptr) { + if (begin.src.ptr != end.src.ptr || begin.src.len != end.src.len) { abort("utf8.slice: decoders from different sources"); }; + if (begin.offs > begin.src.len: size || end.offs > end.src.len: size) { + abort("utf8.slice: decoder offset past end of source"); + }; if (begin.offs > end.offs) { abort("utf8.slice: begin past end"); };