Hojun-Cho/ww
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

ww source: reject raw NUL characters

Browse Source
This commit is contained in:
Hojun-Cho
2026-08-21 23:03:14 +09:00
parent 9381f8fb8e
commit 4b9968e4e6
10 changed files with 1304 additions and 320 deletions
Download Patch File Download Diff File Expand all files Collapse all files

340
cmd/wcc/lex.c
View File

@@ -34,38 +34,91 @@ lexinit(Lex *l, Arena *a, const char *file, const char *src, u64 len)
}
}
static void
lskipnul(Lex *l)
{
while (l->pos < l->srclen && l->src[l->pos] == '\0') {
Pos p = { l->file, l->line, l->col };
l->pos++;
l->col++;
errorf(p, "invalid NUL character");
l->errs++;
l->nulcount++;
}
}
/* Return the raw offset of a logical byte lookahead. Raw NUL bytes do not
* occupy a slot in the token stream: Go's source.nextch diagnoses them and
* immediately resumes decoding at the following character. */
static u64
lrawoff(Lex *l, u64 ahead)
{
u64 p = l->pos;
for (;;) {
while (p < l->srclen && l->src[p] == '\0')
p++;
if (ahead == 0 || p >= l->srclen)
return p;
p++;
ahead--;
}
}
static int
lpeek(Lex *l, u64 ahead)
{
u64 p = l->pos + ahead;
if (p >= l->srclen)
return -1;
if (bomat(l->src, l->srclen, p))
return 0xfeff;
return (unsigned char)l->src[p];
/* Drain NUL at the current decoder position even when it is followed by
* EOF; otherwise a trailing NUL could disappear without a diagnostic. */
for (;;) {
if (l->pos >= l->srclen)
return -1;
int c = (unsigned char)l->src[l->pos];
if (c == 0) {
lskipnul(l);
continue;
}
if (ahead == 0) {
if (bomat(l->src, l->srclen, l->pos))
return 0xfeff;
return c;
}
u64 p = lrawoff(l, ahead);
if (p >= l->srclen)
return -1;
if (bomat(l->src, l->srclen, p))
return 0xfeff;
return (unsigned char)l->src[p];
}
}
static int
lget(Lex *l)
{
if (l->pos >= l->srclen)
return -1;
if (bomat(l->src, l->srclen, l->pos)) {
Pos p = { l->file, l->line, l->col };
l->pos += 3;
l->col += 3;
errorf(p, "invalid BOM in the middle of the file");
l->errs++;
return 0xfeff;
for (;;) {
if (l->pos >= l->srclen)
return -1;
int c = (unsigned char)l->src[l->pos];
if (c == 0) {
lskipnul(l);
continue;
}
if (bomat(l->src, l->srclen, l->pos)) {
Pos p = { l->file, l->line, l->col };
l->pos += 3;
l->col += 3;
errorf(p, "invalid BOM in the middle of the file");
l->errs++;
return 0xfeff;
}
l->pos++;
if (c == '\n') {
l->line++;
l->col = 1;
} else {
l->col++;
}
return c;
}
int c = (unsigned char)l->src[l->pos++];
if (c == '\n') {
l->line++;
l->col = 1;
} else {
l->col++;
}
return c;
}
static Pos
@@ -75,6 +128,22 @@ lpos(Lex *l)
return p;
}
/* Copy one raw source span into token text while omitting diagnosed NUL
* bytes. This keeps keyword, numeric, suffix, and directive recovery on the
* same logical character stream as lpeek/lget. */
static char *
lexspan(Lex *l, u64 begin, u64 end, u64 *len)
{
char *s = amalloc(l->a, end - begin + 1);
u64 j = 0;
for (u64 i = begin; i < end; i++)
if (l->src[i] != '\0')
s[j++] = l->src[i];
s[j] = '\0';
*len = j;
return s;
}
static int
isidstart(int c)
{
@@ -94,6 +163,70 @@ ishex(int c)
(c >= 'A' && c <= 'F');
}
/* Consume one // body, then classify the driver's internal module directive
* from its logical (NUL-filtered) text. Keeping a single moving lexer cursor
* makes even long generated module paths linear rather than repeatedly
* rescanning from the start of the comment. The trailing newline remains for
* the ordinary whitespace loop. */
static void
linecomment(Lex *l)
{
u64 begin = l->pos;
u64 nulbegin = l->nulcount;
int c;
while ((c = lpeek(l, 0)) >= 0 && c != '\n')
lget(l);
u64 end = l->pos;
u64 n;
const char *body;
if (l->nulcount == nulbegin) {
n = end - begin;
body = l->src + begin;
} else {
body = lexspan(l, begin, end, &n);
}
static const char pre[] = "ww:module";
const u64 plen = sizeof pre - 1;
if (n < plen || memcmp(body, pre, plen) != 0 || n == plen)
return;
u64 i = plen;
if (body[i] == '-') {
static const char rest[] = "-reset";
const u64 rlen = sizeof rest - 1;
if (n - i < rlen || memcmp(body + i, rest, rlen) != 0)
return;
i += rlen;
if (i == n) {
l->modreset = 1;
l->modpath = NULL; /* #9: reset supersedes a pending path */
return;
}
if (body[i] != ' ' && body[i] != '\t')
return;
while (i < n && (body[i] == ' ' || body[i] == '\t'))
i++;
u64 s = i;
while (i < n && body[i] != '\r' && body[i] != ' '
&& body[i] != '\t')
i++;
l->modreset = 1;
l->modpath = NULL; /* #9: see above */
if (i > s)
l->modresetpath = astrndup(l->a, body + s, i - s);
return;
}
if (body[i] != ' ' && body[i] != '\t')
return;
while (i < n && (body[i] == ' ' || body[i] == '\t'))
i++;
u64 s = i;
while (i < n && body[i] != '\r' && body[i] != ' ' && body[i] != '\t')
i++;
if (i > s)
l->modpath = astrndup(l->a, body + s, i - s);
}
static int
skipws(Lex *l)
{
@@ -107,78 +240,9 @@ skipws(Lex *l)
}
if (c == '/' && lpeek(l, 1) == '/') {
lget(l); lget(l);
/* #16 option-B: the driver emits `//ww:module-reset`
* before a package-less file's bytes; recognize the
* whole-line directive (without consuming differently)
* and flag it so lexnext emits TK_MODRESET. The body is
* then skipped like any comment. Mirrors the removed
* `// MODULE:` lexer directive. */
{
static const char pre[] = "ww:module";
size_t i = 0;
while (pre[i] && lpeek(l, i) == pre[i])
i++;
if (pre[i] == '\0') {
int nx = lpeek(l, i);
if (nx == '-') {
static const char rest[] = "-reset";
size_t j = 0;
while (rest[j] && lpeek(l, i + j) == rest[j])
j++;
if (rest[j] == '\0') {
int af = lpeek(l, i + j);
if (af == '\n' || af < 0) {
l->modreset = 1;
l->modpath = NULL; /* #9: a reset supersedes a path opened
* earlier in this skipws run (empty/
* export-less inlined module body) */
} else if (af == ' '
|| af == '\t') {
/* `//ww:module-reset <path>`
* — sep primary body tagged by
* its full dotted import path so
* definer == importer (#57). */
size_t k = i + j;
while (lpeek(l, k) == ' '
|| lpeek(l, k) == '\t')
k++;
size_t s = k;
int dch;
while ((dch = lpeek(l, k)) >= 0
&& dch != '\n'
&& dch != '\r'
&& dch != ' '
&& dch != '\t')
k++;
l->modreset = 1;
l->modpath = NULL; /* #9: see above — clear pending path */
if (k > s)
l->modresetpath =
astrndup(l->a,
l->src + l->pos + s,
k - s);
}
}
} else if (nx == ' ' || nx == '\t') {
/* `//ww:module <path>` — M1 import boundary. */
size_t k = i;
while (lpeek(l, k) == ' '
|| lpeek(l, k) == '\t')
k++;
size_t s = k;
int ch;
while ((ch = lpeek(l, k)) >= 0
&& ch != '\n' && ch != '\r'
&& ch != ' ' && ch != '\t')
k++;
if (k > s)
l->modpath = astrndup(l->a,
l->src + l->pos + s, k - s);
}
}
}
while ((c = lpeek(l, 0)) >= 0 && c != '\n')
lget(l);
/* #16 option-B: classify the driver's whole-line internal
* module boundary after consuming it once. */
linecomment(l);
continue;
}
if (c == '/' && lpeek(l, 1) == '*') {
@@ -318,11 +382,38 @@ escape(Lex *l, int *out)
return -1;
}
static const char *
lextypesuffix(Lex *l, u64 *len)
{
char got[4];
u64 n = 0;
while (n < sizeof got && isidcont(lpeek(l, n))) {
got[n] = (char)lpeek(l, n);
n++;
}
if (n == 0 || n > 3 || isidcont(lpeek(l, n)))
return NULL;
static const char *const names[] = {
"i8", "i16", "i32", "i64",
"u8", "u16", "u32", "u64",
"f32", "f64", NULL
};
for (int i = 0; names[i]; i++) {
u64 nl = strlen(names[i]);
if (nl == n && memcmp(names[i], got, n) == 0) {
*len = n;
return names[i];
}
}
return NULL;
}
static Tok
lexnum(Lex *l, Pos start)
{
Tok t = (Tok){ TK_INT, start, NULL, 0, {0}, TK_NONE };
u64 begin = l->pos;
u64 nulbegin = l->nulcount;
int base = 10;
int isfloat = 0;
int c = lpeek(l, 0);
@@ -361,8 +452,14 @@ lexnum(Lex *l, Pos start)
}
}
u64 n = l->pos - begin;
t.text = astrndup(l->a, l->src + begin, n);
u64 rawend = l->pos;
u64 n;
if (l->nulcount == nulbegin) {
n = rawend - begin;
t.text = astrndup(l->a, l->src + begin, n);
} else {
t.text = lexspan(l, begin, rawend, &n);
}
t.tlen = n;
if (isfloat) {
@@ -371,8 +468,8 @@ lexnum(Lex *l, Pos start)
char *clean = amalloc(l->a, n + 1);
u64 j = 0;
for (u64 i = 0; i < n; i++)
if (l->src[begin + i] != '_')
clean[j++] = l->src[begin + i];
if (t.text[i] != '_')
clean[j++] = t.text[i];
clean[j] = '\0';
errno = 0;
t.v.fval = strtod(clean, NULL);
@@ -381,7 +478,7 @@ lexnum(Lex *l, Pos start)
l->errs++;
}
} else {
const char *digs = l->src + begin;
const char *digs = t.text;
u64 dn = n;
if (base != 10) {
digs += 2;
@@ -398,28 +495,12 @@ lexnum(Lex *l, Pos start)
/* A typed suffix must be glued (no whitespace) to the digits. */
if (isidstart(lpeek(l, 0))) {
u64 sb = l->pos;
i32 sc = l->col;
while (isidcont(lpeek(l, 0))) lget(l);
u64 sl = l->pos - sb;
const char *names[] = {
"i8", "i16", "i32", "i64",
"u8", "u16", "u32", "u64",
"f32", "f64", NULL
};
const char *match = NULL;
for (int i = 0; names[i]; i++) {
u64 nl = strlen(names[i]);
if (nl == sl && memcmp(names[i], l->src + sb, nl) == 0) {
match = names[i];
break;
}
}
u64 sl;
const char *match = lextypesuffix(l, &sl);
if (match) {
t.tsuffix = astrndup(l->a, l->src + sb, sl);
} else {
l->pos = sb;
l->col = sc;
for (u64 i = 0; i < sl; i++)
lget(l);
t.tsuffix = astrndup(l->a, match, sl);
}
}
return t;
@@ -429,18 +510,27 @@ static Tok
lexident(Lex *l, Pos start)
{
u64 begin = l->pos;
u64 nulbegin = l->nulcount;
while (isidcont(lpeek(l, 0)))
lget(l);
u64 n = l->pos - begin;
const char *p = l->src + begin;
u64 n;
const char *p;
int filtered = l->nulcount != nulbegin;
if (!filtered) {
n = l->pos - begin;
p = l->src + begin;
} else {
p = lexspan(l, begin, l->pos, &n);
}
char *text = filtered ? (char *)p : astrndup(l->a, p, n);
/* bare '_' is the discard marker. `_x`, `_1` are normal idents. */
if (n == 1 && p[0] == '_') {
Tok t = (Tok){ TK_UNDER, start, astrndup(l->a, p, n), n, {0}, TK_NONE };
Tok t = (Tok){ TK_UNDER, start, text, n, {0}, TK_NONE };
return t;
}
Tkind k = kwlookup(p, n);
Tok t = (Tok){ k != TK_NONE ? k : TK_IDENT, start,
astrndup(l->a, p, n), n, {0}, TK_NONE };
text, n, {0}, TK_NONE };
return t;
}

1
cmd/wcc/ww.h
View File

@@ -202,6 +202,7 @@ struct Lex {
i32 col;
Arena *a; /* token-text arena */
int errs;
u64 nulcount; /* raw NUL bytes diagnosed by the source decoder */
int modreset; /* a `//ww:module-reset` directive was seen in
* the last skipped run; lexnext emits TK_MODRESET
* before the next real token. */