Hojun-Cho/ww
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

ww source: reject malformed UTF-8

Browse Source
This commit is contained in:
Hojun-Cho
2026-08-22 03:14:53 +09:00
parent d61805262e
commit 4069eda942
10 changed files with 1146 additions and 30 deletions
Download Patch File Download Diff File Expand all files Collapse all files

103
lib/ww/syntax/lex.ww
View File

@@ -52,6 +52,7 @@ export type lex = struct {
col: i32,
errs: i32,
nulcount: u64,
utf8count: u64,
// a `//ww:module-reset` directive was seen in the last skipped run;
// lexnext emits TK_MODRESET before the next real token (#16 opt-B).
modreset: i32,
@@ -74,6 +75,54 @@ fn bomat(src: *u8, len: u64, off: u64) bool = {
&& src[i + 2] == 0xbfu8;
};
// Width of the valid UTF-8 scalar sequence beginning at off, or zero. This
// mirrors unicode/utf8.DecodeRune's validity partition without changing WW's
// established byte-oriented treatment of valid non-ASCII token text.
fn utf8seqwidth(src: *u8, len: u64, off: u64) i32 = {
if (off >= len) { return 0; };
let c: u8 = src[off];
if (c < 128u8) { return 1; };
if (c >= 194u8 && c <= 223u8 && len - off >= 2u64
&& src[off + 1u64] >= 128u8 && src[off + 1u64] <= 191u8) {
return 2;
};
if (len - off >= 3u64 && src[off + 2u64] >= 128u8
&& src[off + 2u64] <= 191u8) {
let c1: u8 = src[off + 1u64];
if ((c == 224u8 && c1 >= 160u8 && c1 <= 191u8)
|| (c >= 225u8 && c <= 236u8 && c1 >= 128u8 && c1 <= 191u8)
|| (c == 237u8 && c1 >= 128u8 && c1 <= 159u8)
|| (c >= 238u8 && c <= 239u8 && c1 >= 128u8 && c1 <= 191u8)) {
return 3;
};
};
if (len - off >= 4u64 && src[off + 2u64] >= 128u8
&& src[off + 2u64] <= 191u8 && src[off + 3u64] >= 128u8
&& src[off + 3u64] <= 191u8) {
let c1: u8 = src[off + 1u64];
if ((c == 240u8 && c1 >= 144u8 && c1 <= 191u8)
|| (c >= 241u8 && c <= 243u8 && c1 >= 128u8 && c1 <= 191u8)
|| (c == 244u8 && c1 >= 128u8 && c1 <= 143u8)) {
return 4;
};
};
return 0;
};
fn utf8bytevalid(src: *u8, len: u64, off: u64) bool = {
if (utf8seqwidth(src, len, off) != 0) { return true; };
let c: u8 = src[off];
if (c < 128u8 || c > 191u8) { return false; };
let back: u64 = 1u64;
for (back <= 3u64 && back <= off) {
if (utf8seqwidth(src, len, off - back) > (back: i32)) {
return true;
};
back += 1u64;
};
return false;
};
export fn lexinit(l: *lex, file: str, src: *u8, len: u64) void = {
l.file = file;
l.src = src;
@@ -83,6 +132,7 @@ export fn lexinit(l: *lex, file: str, src: *u8, len: u64) void = {
l.col = 1;
l.errs = 0;
l.nulcount = 0u64;
l.utf8count = 0u64;
l.modreset = 0;
l.modpathset = 0;
l.modresetpathset = 0;
@@ -111,15 +161,34 @@ fn lskipnul(l: *lex) void = {
};
};
// Return the raw offset of a logical byte lookahead. Raw NUL bytes do not
// occupy a slot in the token stream: Go's source.nextch diagnoses them and
// immediately resumes decoding at the following character.
// Go 1.26.5 syntax.source.nextch diagnoses and discards one raw byte for each
// utf8.DecodeRune RuneError of width one.
fn lskiputf8(l: *lex) void = {
for (l.lpos < l.srclen
&& srcb(l, l.lpos) >= 128
&& !utf8bytevalid(l.src, l.srclen, l.lpos)) {
let ep: pos;
ep.file = l.file;
ep.line = l.line;
ep.col = l.col;
l.lpos += 1u64;
l.col += 1;
errat(l, &ep, "invalid UTF-8 encoding");
l.utf8count += 1u64;
};
};
// Return the raw offset of a logical byte lookahead. Raw NUL and malformed
// UTF-8 bytes do not occupy a slot in the token stream: Go's source.nextch
// diagnoses them and immediately resumes at the following byte.
fn lrawoff(l: *lex, ahead0: u64) u64 = {
let p: u64 = l.lpos;
let ahead: u64 = ahead0;
for (true) {
for (p < l.srclen) {
if (srcb(l, p) != 0) { break; };
let b: i32 = srcb(l, p);
if (b != 0 && (b < 128
|| utf8bytevalid(l.src, l.srclen, p))) { break; };
p += 1u64;
};
if (ahead == 0u64 || p >= l.srclen) { return p; };
@@ -135,6 +204,10 @@ fn lpeek(l: *lex, ahead: u64) i32 = {
if (l.lpos >= l.srclen) { return -1; };
let c: i32 = srcb(l, l.lpos);
if (c == 0) { lskipnul(l); continue; };
if (c >= 128 && !utf8bytevalid(l.src, l.srclen, l.lpos)) {
lskiputf8(l);
continue;
};
if (ahead == 0u64) {
if (bomat(l.src, l.srclen, l.lpos)) { return 0xFEFF; };
return c;
@@ -151,6 +224,10 @@ fn lget(l: *lex) i32 = {
if (l.lpos >= l.srclen) { return -1; };
let c: i32 = srcb(l, l.lpos);
if (c == 0) { lskipnul(l); continue; };
if (c >= 128 && !utf8bytevalid(l.src, l.srclen, l.lpos)) {
lskiputf8(l);
continue;
};
if (bomat(l.src, l.srclen, l.lpos)) {
let bp: pos;
bp.file = l.file;
@@ -172,16 +249,17 @@ fn lget(l: *lex) i32 = {
};
};
// Copy one raw source span into token text while omitting diagnosed NUL
// bytes. This keeps keyword, numeric, suffix, and directive recovery on the
// same logical character stream as lpeek/lget.
// Copy one raw source span into token text while omitting diagnosed NUL and
// malformed UTF-8 bytes. This keeps keyword, numeric, suffix, and directive
// recovery on the same logical character stream as lpeek/lget.
fn lexspan(l: *lex, begin: u64, end: u64) str = {
let buf: []u8 = alloc([], end - begin + 1u64)!;
let i: u64 = begin;
let j: u64 = 0u64;
for (i < end) {
let b: i32 = srcb(l, i);
if (b != 0) {
if (b != 0 && (b < 128
|| utf8bytevalid(l.src, l.srclen, i))) {
buf[j] = b: u8;
j += 1u64;
};
@@ -221,6 +299,7 @@ fn errat(l: *lex, p: *pos, msg: str) void = {
fn linecomment(l: *lex) void = {
let begin: u64 = l.lpos;
let nulbegin: u64 = l.nulcount;
let utf8begin: u64 = l.utf8count;
for (true) {
let c: i32 = lpeek(l, 0u64);
if (c < 0 || c == '\n') { break; };
@@ -228,7 +307,7 @@ fn linecomment(l: *lex) void = {
};
let end: u64 = l.lpos;
let body: str;
if (l.nulcount == nulbegin) {
if (l.nulcount == nulbegin && l.utf8count == utf8begin) {
body.ptr = l.src + begin;
body.len = (end - begin): i32;
} else {
@@ -534,6 +613,7 @@ fn lexnum(l: *lex, start: *pos, out: *tok) void = {
out.col = start.col;
let begin: u64 = l.lpos;
let nulbegin: u64 = l.nulcount;
let utf8begin: u64 = l.utf8count;
let base: i32 = 10;
let isfloat: bool = false;
@@ -583,7 +663,7 @@ fn lexnum(l: *lex, start: *pos, out: *tok) void = {
};
let rawend: u64 = l.lpos;
if (l.nulcount == nulbegin) {
if (l.nulcount == nulbegin && l.utf8count == utf8begin) {
let view: str;
view.ptr = l.src + begin;
view.len = (rawend - begin): i32;
@@ -671,6 +751,7 @@ fn lexnum(l: *lex, start: *pos, out: *tok) void = {
fn lexident(l: *lex, start: *pos, out: *tok) void = {
let begin: u64 = l.lpos;
let nulbegin: u64 = l.nulcount;
let utf8begin: u64 = l.utf8count;
for (true) {
let c: i32 = lpeek(l, 0u64);
if (c < 0) { break; };
@@ -678,7 +759,7 @@ fn lexident(l: *lex, start: *pos, out: *tok) void = {
lget(l);
};
let text: str;
if (l.nulcount == nulbegin) {
if (l.nulcount == nulbegin && l.utf8count == utf8begin) {
let view: str;
view.ptr = l.src + begin;
view.len = (l.lpos - begin): i32;

135
lib/ww/syntax/tok_test.ww
View File

@@ -437,3 +437,138 @@ fn doublebomerror() bool = {
assert(bomerror("\"", "\""));
assert(bomerror("'", "'"));
};
fn utf8source(dst: *u8, before: str, middle: *u8, middlen: i32,
after: str) u64 = {
let n: i32 = 0;
let i: i32 = 0;
for (i < before.len) { dst[n] = before[i]; n += 1; i += 1; };
i = 0;
for (i < middlen) { dst[n] = middle[i]; n += 1; i += 1; };
i = 0;
for (i < after.len) { dst[n] = after[i]; n += 1; i += 1; };
return n: u64;
};
fn checkmalformedutf8(raw: *u8, rawlen: i32, errs: i32) void = {
let src: [32]u8;
let n: u64 = utf8source(src.ptr, "f", raw, rawlen, "n bar");
let l: syntax.lex;
syntax.lexinit(&l, "t", src.ptr, n);
let t: syntax.tok;
syntax.lexnext(&l, &t);
assert(t.kind == syntax.tkind.TK_FN && t.line == 1 && t.col == 1);
syntax.lexnext(&l, &t);
assert(t.kind == syntax.tkind.TK_IDENT && t.text == "bar");
assert(t.line == 1 && t.col == 4 + rawlen);
syntax.lexnext(&l, &t);
assert(t.kind == syntax.tkind.TK_EOF && l.errs == errs);
};
// Go 1.26.5 source.nextch reports and filters one byte for every
// DecodeRune width-one malformed result.
@test fn malformed_utf8_width_one_recovery() void = {
let lead: [1]u8 = [0xffu8];
let continuation: [1]u8 = [0x80u8];
let overlong: [2]u8 = [0xc0u8, 0x80u8];
let surrogate: [3]u8 = [0xedu8, 0xa0u8, 0x80u8];
let outofrange: [4]u8 = [0xf4u8, 0x90u8, 0x80u8, 0x80u8];
let truncated: [2]u8 = [0xe2u8, 0x82u8];
checkmalformedutf8(lead.ptr, 1, 1);
checkmalformedutf8(continuation.ptr, 1, 1);
checkmalformedutf8(overlong.ptr, 2, 2);
checkmalformedutf8(surrogate.ptr, 3, 3);
checkmalformedutf8(outofrange.ptr, 4, 4);
checkmalformedutf8(truncated.ptr, 2, 2);
};
@test fn valid_utf8_is_preserved() void = {
let raw: [12]u8 = [
0xc3u8, 0xa9u8,
0xeau8, 0xb0u8, 0x80u8,
0xefu8, 0xbfu8, 0xbdu8,
0xf0u8, 0x9fu8, 0x98u8, 0x80u8,
];
let src: [32]u8;
let n: u64 = utf8source(src.ptr, "\"", raw.ptr, 12, "\" fn");
let l: syntax.lex;
syntax.lexinit(&l, "t", src.ptr, n);
let t: syntax.tok;
syntax.lexnext(&l, &t);
assert(t.kind == syntax.tkind.TK_STR && t.text.len == 12);
let i: i32 = 0;
for (i < 12) { assert(t.text[i] == raw[i]); i += 1; };
syntax.lexnext(&l, &t);
assert(t.kind == syntax.tkind.TK_FN && t.line == 1 && t.col == 16);
syntax.lexnext(&l, &t);
assert(t.kind == syntax.tkind.TK_EOF && l.errs == 0);
n = utf8source(src.ptr, "// ", raw.ptr, 12, "\nfn");
syntax.lexinit(&l, "t", src.ptr, n);
syntax.lexnext(&l, &t);
assert(t.kind == syntax.tkind.TK_FN && t.line == 2 && t.col == 1);
syntax.lexnext(&l, &t);
assert(t.kind == syntax.tkind.TK_EOF && l.errs == 0);
};
@test fn malformed_utf8_token_recovery() void = {
let bad: [1]u8 = [0xffu8];
let src: [64]u8;
let l: syntax.lex;
let t: syntax.tok;
let n: u64 = utf8source(src.ptr, "1", bad.ptr, 1, "_0");
syntax.lexinit(&l, "t", src.ptr, n);
syntax.lexnext(&l, &t);
assert(t.kind == syntax.tkind.TK_INT && t.uval == 10u64 && l.errs == 1);
n = utf8source(src.ptr, "=", bad.ptr, 1, "=");
syntax.lexinit(&l, "t", src.ptr, n);
syntax.lexnext(&l, &t);
assert(t.kind == syntax.tkind.TK_EQ && l.errs == 1);
n = utf8source(src.ptr, "/", bad.ptr, 1, "/ comment\nfn");
syntax.lexinit(&l, "t", src.ptr, n);
syntax.lexnext(&l, &t);
assert(t.kind == syntax.tkind.TK_FN && l.errs == 1);
n = utf8source(src.ptr, "/* end *", bad.ptr, 1, "/ fn");
syntax.lexinit(&l, "t", src.ptr, n);
syntax.lexnext(&l, &t);
assert(t.kind == syntax.tkind.TK_FN && l.errs == 1);
n = utf8source(src.ptr, "\"a", bad.ptr, 1, "b\"");
syntax.lexinit(&l, "t", src.ptr, n);
syntax.lexnext(&l, &t);
assert(t.kind == syntax.tkind.TK_STR && t.text == "ab" && l.errs == 1);
n = utf8source(src.ptr, "\"\\", bad.ptr, 1, "n\"");
syntax.lexinit(&l, "t", src.ptr, n);
syntax.lexnext(&l, &t);
assert(t.kind == syntax.tkind.TK_STR && t.text.len == 1);
assert(t.text[0] == '\n' && l.errs == 1);
n = utf8source(src.ptr, "\"\\x0", bad.ptr, 1, "0\"");
syntax.lexinit(&l, "t", src.ptr, n);
syntax.lexnext(&l, &t);
assert(t.kind == syntax.tkind.TK_STR && t.text.len == 1);
assert(t.text[0] == 0u8 && l.errs == 1);
};
@test fn malformed_utf8_bom_nul_columns() void = {
let src: [12]u8 = [
0xefu8, 0xbbu8, 0xbfu8,
'f': u8, 0xffu8, 0u8, 'n': u8,
' ': u8, 'b': u8, 'a': u8, 'r': u8, 0u8,
];
let l: syntax.lex;
syntax.lexinit(&l, "t", src.ptr, 11u64);
let t: syntax.tok;
syntax.lexnext(&l, &t);
assert(t.kind == syntax.tkind.TK_FN && t.line == 1 && t.col == 4);
syntax.lexnext(&l, &t);
assert(t.kind == syntax.tkind.TK_IDENT && t.text == "bar");
assert(t.line == 1 && t.col == 9);
syntax.lexnext(&l, &t);
assert(t.kind == syntax.tkind.TK_EOF);
assert(l.errs == 2 && l.nulcount == 1u64);
};