Hojun-Cho/ww
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

ww source: reject malformed UTF-8

Browse Source
This commit is contained in:
Hojun-Cho
2026-08-22 03:14:53 +09:00
parent d61805262e
commit 4069eda942
10 changed files with 1146 additions and 30 deletions
Download Patch File Download Diff File Expand all files Collapse all files

120
test/wcc/100_lex.c
View File

@@ -240,6 +240,113 @@ static const struct {
{ "f\0\0n", sizeof "f\0\0n" - 1, "fn", 2 },
};
/* Go 1.26.5 source.nextch reports each DecodeRune width-one result and
* removes that byte before scanner recovery. */
static const struct {
const char *src;
size_t len;
const char *expect;
int errs;
} utf8rows[] = {
{ "f" "\xff" "n bar", sizeof "f" "\xff" "n bar" - 1,
"fn IDENT(bar)", 1 },
{ "f" "\x80" "n bar", sizeof "f" "\x80" "n bar" - 1,
"fn IDENT(bar)", 1 },
{ "f" "\xc0\x80" "n bar", sizeof "f" "\xc0\x80" "n bar" - 1,
"fn IDENT(bar)", 2 },
{ "f" "\xed\xa0\x80" "n bar",
sizeof "f" "\xed\xa0\x80" "n bar" - 1,
"fn IDENT(bar)", 3 },
{ "f" "\xf4\x90\x80\x80" "n bar",
sizeof "f" "\xf4\x90\x80\x80" "n bar" - 1,
"fn IDENT(bar)", 4 },
{ "f" "\xe2\x82" "n bar", sizeof "f" "\xe2\x82" "n bar" - 1,
"fn IDENT(bar)", 2 },
{ "1" "\xff" "_0", sizeof "1" "\xff" "_0" - 1,
"INT(10)", 1 },
{ "=" "\xff" "=", sizeof "=" "\xff" "=" - 1,
"==", 1 },
{ "/" "\xff" "/ comment\nfn",
sizeof "/" "\xff" "/ comment\nfn" - 1, "fn", 1 },
{ "/* end *" "\xff" "/ fn",
sizeof "/* end *" "\xff" "/ fn" - 1, "fn", 1 },
{ "\"a" "\xff" "b\"", sizeof "\"a" "\xff" "b\"" - 1,
"STR(ab)", 1 },
};
static int
runutf8valid(void)
{
static const char strsrc[] =
"\"" "\xc3\xa9" "\xea\xb0\x80" "\xef\xbf\xbd"
"\xf0\x9f\x98\x80" "\" fn";
static const char commentsrc[] =
"// " "\xc3\xa9" "\xea\xb0\x80" "\xef\xbf\xbd"
"\xf0\x9f\x98\x80" "\nfn";
static const unsigned char want[] = {
0xc3, 0xa9, 0xea, 0xb0, 0x80, 0xef, 0xbf, 0xbd,
0xf0, 0x9f, 0x98, 0x80,
};
Arena *a = newarena();
Lex l;
lexinit(&l, a, "<test>", strsrc, sizeof strsrc - 1);
Tok s = lexnext(&l);
Tok f = lexnext(&l);
int ok = s.kind == TK_STR && s.tlen == sizeof want
&& memcmp(s.text, want, sizeof want) == 0
&& f.kind == TK_FN && f.pos.col == 16 && l.errs == 0
&& lexnext(&l).kind == TK_EOF;
lexinit(&l, a, "<test>", commentsrc, sizeof commentsrc - 1);
f = lexnext(&l);
ok = ok && f.kind == TK_FN && f.pos.line == 2 && f.pos.col == 1
&& l.errs == 0 && lexnext(&l).kind == TK_EOF;
if (!ok)
fprintf(stderr, "valid UTF-8 preservation failed\n");
freearena(a);
return ok;
}
static int
runutf8cols(void)
{
static const char src[] =
"\xef\xbb\xbf" "f" "\xff" "\0" "n bar";
Arena *a = newarena();
Lex l;
lexinit(&l, a, "<test>", src, sizeof src - 1);
Tok f = lexnext(&l);
Tok b = lexnext(&l);
int ok = f.kind == TK_FN && f.pos.line == 1 && f.pos.col == 4
&& b.kind == TK_IDENT && b.pos.line == 1 && b.pos.col == 9
&& strcmp(b.text, "bar") == 0 && l.errs == 2
&& l.nulcount == 1 && lexnext(&l).kind == TK_EOF;
if (!ok)
fprintf(stderr, "UTF-8 raw-byte columns or BOM/NUL recovery failed\n");
freearena(a);
return ok;
}
static int
runutf8escape(void)
{
static const char line[] = "\"\\" "\xff" "n\"";
static const char hex[] = "\"\\x0" "\xff" "0\"";
Arena *a = newarena();
Lex l;
lexinit(&l, a, "<test>", line, sizeof line - 1);
Tok t = lexnext(&l);
int ok = t.kind == TK_STR && t.tlen == 1 && t.text[0] == '\n'
&& l.errs == 1 && lexnext(&l).kind == TK_EOF;
lexinit(&l, a, "<test>", hex, sizeof hex - 1);
t = lexnext(&l);
ok = ok && t.kind == TK_STR && t.tlen == 1 && t.text[0] == '\0'
&& l.errs == 1 && lexnext(&l).kind == TK_EOF;
if (!ok)
fprintf(stderr, "UTF-8 escape recovery failed\n");
freearena(a);
return ok;
}
static int
runescapenul(void)
{
@@ -333,10 +440,23 @@ main(void)
fail++;
}
}
for (size_t i = 0; i < sizeof utf8rows / sizeof utf8rows[0]; i++) {
if (!runrown(utf8rows[i].src, utf8rows[i].len,
utf8rows[i].expect, utf8rows[i].errs)) {
fprintf(stderr, "UTF-8 row %zu failed\n", i);
fail++;
}
}
if (!runescapenul())
fail++;
if (!runsuffixnul())
fail++;
if (!runutf8valid())
fail++;
if (!runutf8cols())
fail++;
if (!runutf8escape())
fail++;
if (!runlongdirective())
fail++;
if (fail) {