Hojun-Cho/ww
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

ww source: reject raw NUL characters

Browse Source
This commit is contained in:
Hojun-Cho
2026-08-21 23:03:14 +09:00
parent 9381f8fb8e
commit 4b9968e4e6
10 changed files with 1304 additions and 320 deletions
Download Patch File Download Diff File Expand all files Collapse all files

420
lib/ww/syntax/lex.ww
View File

@@ -51,6 +51,7 @@ export type lex = struct {
line: i32,
col: i32,
errs: i32,
nulcount: u64,
// a `//ww:module-reset` directive was seen in the last skipped run;
// lexnext emits TK_MODRESET before the next real token (#16 opt-B).
modreset: i32,
@@ -81,6 +82,7 @@ export fn lexinit(l: *lex, file: str, src: *u8, len: u64) void = {
l.line = 1;
l.col = 1;
l.errs = 0;
l.nulcount = 0u64;
l.modreset = 0;
l.modpathset = 0;
l.modresetpathset = 0;
@@ -95,34 +97,101 @@ fn srcb(l: *lex, off: u64) i32 = {
return b: i32;
};
fn lskipnul(l: *lex) void = {
for (l.lpos < l.srclen) {
if (srcb(l, l.lpos) != 0) { break; };
let np: pos;
np.file = l.file;
np.line = l.line;
np.col = l.col;
l.lpos += 1u64;
l.col += 1;
errat(l, &np, "invalid NUL character");
l.nulcount += 1u64;
};
};
// Return the raw offset of a logical byte lookahead. Raw NUL bytes do not
// occupy a slot in the token stream: Go's source.nextch diagnoses them and
// immediately resumes decoding at the following character.
fn lrawoff(l: *lex, ahead0: u64) u64 = {
let p: u64 = l.lpos;
let ahead: u64 = ahead0;
for (true) {
for (p < l.srclen) {
if (srcb(l, p) != 0) { break; };
p += 1u64;
};
if (ahead == 0u64 || p >= l.srclen) { return p; };
p += 1u64;
ahead -= 1u64;
};
};
fn lpeek(l: *lex, ahead: u64) i32 = {
let p: u64 = l.lpos + ahead;
if (p >= l.srclen) { return -1; };
if (bomat(l.src, l.srclen, p)) { return 0xFEFF; };
return srcb(l, p);
// Drain NUL at the current decoder position even when it is followed by
// EOF; otherwise a trailing NUL could disappear without a diagnostic.
for (true) {
if (l.lpos >= l.srclen) { return -1; };
let c: i32 = srcb(l, l.lpos);
if (c == 0) { lskipnul(l); continue; };
if (ahead == 0u64) {
if (bomat(l.src, l.srclen, l.lpos)) { return 0xFEFF; };
return c;
};
let p: u64 = lrawoff(l, ahead);
if (p >= l.srclen) { return -1; };
if (bomat(l.src, l.srclen, p)) { return 0xFEFF; };
return srcb(l, p);
};
};
fn lget(l: *lex) i32 = {
if (l.lpos >= l.srclen) { return -1; };
if (bomat(l.src, l.srclen, l.lpos)) {
let bp: pos;
bp.file = l.file;
bp.line = l.line;
bp.col = l.col;
l.lpos += 3u64;
l.col += 3;
errat(l, &bp, "invalid BOM in the middle of the file");
return 0xFEFF;
for (true) {
if (l.lpos >= l.srclen) { return -1; };
let c: i32 = srcb(l, l.lpos);
if (c == 0) { lskipnul(l); continue; };
if (bomat(l.src, l.srclen, l.lpos)) {
let bp: pos;
bp.file = l.file;
bp.line = l.line;
bp.col = l.col;
l.lpos += 3u64;
l.col += 3;
errat(l, &bp, "invalid BOM in the middle of the file");
return 0xFEFF;
};
l.lpos += 1u64;
if (c == '\n') {
l.line += 1;
l.col = 1;
} else {
l.col += 1;
};
return c;
};
let c: i32 = srcb(l, l.lpos);
l.lpos += 1u64;
if (c == '\n') {
l.line += 1;
l.col = 1;
} else {
l.col += 1;
};
// Copy one raw source span into token text while omitting diagnosed NUL
// bytes. This keeps keyword, numeric, suffix, and directive recovery on the
// same logical character stream as lpeek/lget.
fn lexspan(l: *lex, begin: u64, end: u64) str = {
let buf: []u8 = alloc([], end - begin + 1u64)!;
let i: u64 = begin;
let j: u64 = 0u64;
for (i < end) {
let b: i32 = srcb(l, i);
if (b != 0) {
buf[j] = b: u8;
j += 1u64;
};
i += 1u64;
};
return c;
buf[j] = 0u8;
let out: str;
out.ptr = buf.ptr;
out.len = j: i32;
return out;
};
fn curpos(l: *lex, out: *pos) void = {
@@ -146,6 +215,81 @@ fn errat(l: *lex, p: *pos, msg: str) void = {
l.errs += 1;
};
// Consume one // body, then classify the driver's internal module directive
// from its logical (NUL-filtered) text. A single moving lexer cursor keeps long
// generated module paths linear. The trailing newline remains for skipws.
fn linecomment(l: *lex) void = {
let begin: u64 = l.lpos;
let nulbegin: u64 = l.nulcount;
for (true) {
let c: i32 = lpeek(l, 0u64);
if (c < 0 || c == '\n') { break; };
lget(l);
};
let end: u64 = l.lpos;
let body: str;
if (l.nulcount == nulbegin) {
body.ptr = l.src + begin;
body.len = (end - begin): i32;
} else {
body = lexspan(l, begin, end);
};
let pre: str = "ww:module";
if (!strings.hasprefix(body, pre) || body.len == pre.len) { return; };
let i: i32 = pre.len;
if (body[i] == '-') {
let rest: str = "-reset";
if (body.len - i < rest.len) { return; };
let j: i32 = 0;
for (j < rest.len) {
if (body[i + j] != rest[j]) { return; };
j += 1;
};
i += rest.len;
if (i == body.len) {
l.modreset = 1;
l.modpathset = 0; // #9: reset supersedes a pending path
return;
};
if (body[i] != ' ' && body[i] != '\t') { return; };
for (i < body.len && (body[i] == ' ' || body[i] == '\t')) {
i += 1;
};
let s: i32 = i;
for (i < body.len && body[i] != '\r' && body[i] != ' '
&& body[i] != '\t') {
i += 1;
};
l.modreset = 1;
l.modpathset = 0; // #9: see above
if (i > s) {
let view: str;
view.ptr = body.ptr + (s: u64);
view.len = i - s;
l.modresetpath = strings.dup(view);
l.modresetpathset = 1;
};
return;
};
if (body[i] != ' ' && body[i] != '\t') { return; };
for (i < body.len && (body[i] == ' ' || body[i] == '\t')) {
i += 1;
};
let s: i32 = i;
for (i < body.len && body[i] != '\r' && body[i] != ' '
&& body[i] != '\t') {
i += 1;
};
if (i > s) {
let view: str;
view.ptr = body.ptr + (s: u64);
view.len = i - s;
l.modpath = strings.dup(view);
l.modpathset = 1;
};
};
fn skipws(l: *lex) bool = {
for (true) {
let c: i32 = lpeek(l, 0u64);
@@ -158,111 +302,9 @@ fn skipws(l: *lex) bool = {
let c2: i32 = lpeek(l, 1u64);
if (c2 == '/') {
lget(l); lget(l);
// #16 opt-B: recognize the driver's curmod-reset
// boundary directive `//ww:module-reset` (whole
// line) and flag it; lexnext emits TK_MODRESET.
// The body is then skipped like any comment.
// Mirrors cstage lex.c skipws. Compare via lpeek
// (no consume) so the skip loop below is unchanged.
let pre: str = "ww:module";
let di: i32 = 0;
let matched: bool = true;
for (di < pre.len) {
if (lpeek(l, di: u64) != pre[di]: i32) {
matched = false; break;
};
di += 1;
};
if (matched) {
let nx: i32 = lpeek(l, pre.len: u64);
if (nx == '-') {
let rest: str = "-reset";
let rj: i32 = 0;
let rm: bool = true;
for (rj < rest.len) {
if (lpeek(l, (pre.len + rj): u64)
!= rest[rj]: i32) {
rm = false; break;
};
rj += 1;
};
if (rm) {
let af: i32 = lpeek(l,
(pre.len + rest.len): u64);
if (af == '\n') { l.modreset = 1; l.modpathset = 0; } // #9: reset supersedes pending path (empty module body)
else { if (af < 0) { l.modreset = 1; l.modpathset = 0; } // #9: see above
else { if (af == ' ' || af == '\t') {
// `//ww:module-reset <path>` — sep
// primary body tagged by its full
// dotted import path (#57).
let k: u64 =
(pre.len + rest.len): u64;
for (true) {
let sc: i32 = lpeek(l, k);
if (sc == ' ' || sc == '\t') {
k += 1u64; continue;
};
break;
};
let s0: u64 = k;
for (true) {
let pc: i32 = lpeek(l, k);
if (pc < 0) { break; };
if (pc == '\n' || pc == '\r'
|| pc == ' '
|| pc == '\t') {
break;
};
k += 1u64;
};
l.modreset = 1;
l.modpathset = 0; // #9: see above — clear pending path
if (k > s0) {
let view: str;
view.ptr =
l.src + l.lpos + s0;
view.len = (k - s0): i32;
l.modresetpath =
strings.dup(view);
l.modresetpathset = 1;
};
}; }; };
};
} else { if (nx == ' ' || nx == '\t') {
// `//ww:module <path>` — M1 import boundary.
let k: u64 = pre.len: u64;
for (true) {
let sc: i32 = lpeek(l, k);
if (sc == ' ' || sc == '\t') {
k += 1u64; continue;
};
break;
};
let s0: u64 = k;
for (true) {
let pc: i32 = lpeek(l, k);
if (pc < 0) { break; };
if (pc == '\n' || pc == '\r'
|| pc == ' ' || pc == '\t') {
break;
};
k += 1u64;
};
if (k > s0) {
let view: str;
view.ptr = l.src + l.lpos + s0;
view.len = (k - s0): i32;
l.modpath = strings.dup(view);
l.modpathset = 1;
};
}; };
};
for (true) {
let cx: i32 = lpeek(l, 0u64);
if (cx < 0) { return false; };
if (cx == '\n') { break; };
lget(l);
};
// #16 opt-B: classify the driver's whole-line internal
// module boundary after consuming it once.
linecomment(l);
continue;
};
if (c2 == '*') {
@@ -395,6 +437,38 @@ fn escape(l: *lex, out: *i32) bool = {
return false;
};
fn lextypesuffix(l: *lex, out: *str) bool = {
let got: [4]u8;
let n: u64 = 0u64;
for (n < 4u64) {
let c: i32 = lpeek(l, n);
if (c < 0 || !isidpart(c: rune)) { break; };
got[n] = c: u8;
n += 1u64;
};
if (n == 0u64 || n > 3u64) { return false; };
let tail: i32 = lpeek(l, n);
if (tail >= 0) {
if (isidpart(tail: rune)) { return false; };
};
let names: []str = ["i8", "i16", "i32", "i64",
"u8", "u16", "u32", "u64", "f32", "f64"];
let i: i32 = 0;
for (i < names.len) {
if (names[i].len: u64 == n) {
let same: bool = true;
let j: i32 = 0;
for (j < names[i].len) {
if (names[i][j] != got[j]) { same = false; break; };
j += 1;
};
if (same) { *out = names[i]; return true; };
};
i += 1;
};
return false;
};
fn scandecimalrun(l: *lex) void = {
for (true) {
let c: i32 = lpeek(l, 0u64);
@@ -459,6 +533,7 @@ fn lexnum(l: *lex, start: *pos, out: *tok) void = {
out.line = start.line;
out.col = start.col;
let begin: u64 = l.lpos;
let nulbegin: u64 = l.nulcount;
let base: i32 = 10;
let isfloat: bool = false;
@@ -507,11 +582,16 @@ fn lexnum(l: *lex, start: *pos, out: *tok) void = {
};
};
let n: u64 = l.lpos - begin;
let view: str;
view.ptr = l.src + begin;
view.len = n: i32;
out.text = strings.dup(view);
let rawend: u64 = l.lpos;
if (l.nulcount == nulbegin) {
let view: str;
view.ptr = l.src + begin;
view.len = (rawend - begin): i32;
out.text = strings.dup(view);
} else {
out.text = lexspan(l, begin, rawend);
};
let n: u64 = out.text.len: u64;
if (isfloat) {
out.kind = tkind.TK_FLOAT;
@@ -522,7 +602,7 @@ fn lexnum(l: *lex, start: *pos, out: *tok) void = {
let i: u64 = 0u64;
let j: u64 = 0u64;
for (i < n) {
let b: u8 = l.src[begin + i];
let b: u8 = out.text[i];
if (b != '_') {
clean[j] = b;
j += 1u64;
@@ -561,7 +641,7 @@ fn lexnum(l: *lex, start: *pos, out: *tok) void = {
let pu: *u64 = (&fv): *u64;
out.uval = *pu;
} else {
let digs: *u8 = l.src + begin;
let digs: *u8 = out.text.ptr;
let dn: u64 = n;
if (base != 10) {
digs = digs + 2u64;
@@ -578,49 +658,11 @@ fn lexnum(l: *lex, start: *pos, out: *tok) void = {
let pc: i32 = lpeek(l, 0u64);
if (pc >= 0) {
if (isidstart(pc: rune)) {
let sb: u64 = l.lpos;
let sc: i32 = l.col;
for (true) {
let cc: i32 = lpeek(l, 0u64);
if (cc < 0) { break; };
if (!isidpart(cc: rune)) { break; };
lget(l);
};
let sl: u64 = l.lpos - sb;
let p: *u8 = l.src + sb;
let isok: bool = false;
if (sl == 2u64) {
if (p[0] == 'i') {
if (p[1] == '8') { isok = true; }; // i8
};
if (p[0] == 'u') {
if (p[1] == '8') { isok = true; }; // u8
};
};
if (sl == 3u64) {
if (p[0] == 'i') {
if (p[1] == '1') { if (p[2] == '6') { isok = true; }; }; // i16
if (p[1] == '3') { if (p[2] == '2') { isok = true; }; }; // i32
if (p[1] == '6') { if (p[2] == '4') { isok = true; }; }; // i64
};
if (p[0] == 'u') {
if (p[1] == '1') { if (p[2] == '6') { isok = true; }; };
if (p[1] == '3') { if (p[2] == '2') { isok = true; }; };
if (p[1] == '6') { if (p[2] == '4') { isok = true; }; };
};
if (p[0] == 'f') {
if (p[1] == '3') { if (p[2] == '2') { isok = true; }; }; // f32
if (p[1] == '6') { if (p[2] == '4') { isok = true; }; }; // f64
};
};
if (isok) {
let view: str;
view.ptr = p;
view.len = sl: i32;
out.tsuffix = strings.dup(view);
} else {
l.lpos = sb;
l.col = sc;
let suffix: str;
if (lextypesuffix(l, &suffix)) {
let i: i32 = 0;
for (i < suffix.len) { lget(l); i += 1; };
out.tsuffix = strings.dup(suffix);
};
};
};
@@ -628,14 +670,24 @@ fn lexnum(l: *lex, start: *pos, out: *tok) void = {
fn lexident(l: *lex, start: *pos, out: *tok) void = {
let begin: u64 = l.lpos;
let nulbegin: u64 = l.nulcount;
for (true) {
let c: i32 = lpeek(l, 0u64);
if (c < 0) { break; };
if (!isidpart(c: rune)) { break; };
lget(l);
};
let n: u64 = l.lpos - begin;
let p: *u8 = l.src + begin;
let text: str;
if (l.nulcount == nulbegin) {
let view: str;
view.ptr = l.src + begin;
view.len = (l.lpos - begin): i32;
text = strings.dup(view);
} else {
text = lexspan(l, begin, l.lpos);
};
let n: u64 = text.len: u64;
let p: *u8 = text.ptr;
out.file = start.file;
out.line = start.line;
out.col = start.col;
@@ -643,10 +695,7 @@ fn lexident(l: *lex, start: *pos, out: *tok) void = {
if (n == 1u64) {
if (p[0] == '_') {
out.kind = tkind.TK_UNDER;
let view: str;
view.ptr = p;
view.len = n: i32;
out.text = strings.dup(view);
out.text = text;
return;
};
};
@@ -656,10 +705,7 @@ fn lexident(l: *lex, start: *pos, out: *tok) void = {
} else {
out.kind = tkind.TK_IDENT;
};
let view: str;
view.ptr = p;
view.len = n: i32;
out.text = strings.dup(view);
out.text = text;
};
fn lexstr(l: *lex, start: *pos, out: *tok) void = {

24
lib/ww/syntax/tok_test.ww
View File

@@ -339,6 +339,30 @@ fn checkfloat(src: str, want: u64) void = {
assert(!(t.text != "bar"));
};
@test fn long_module_directive_is_linear() void = {
let pre: str = "//ww:module ";
let tail: str = "\nfn";
let pathlen: i32 = 16384;
let total: i32 = pre.len + pathlen + tail.len;
let src: []u8 = alloc([], total: u64)!;
src.len = total;
let i: i32 = 0;
for (i < pre.len) { src[i] = pre[i]; i += 1; };
for (i < pre.len + pathlen) { src[i] = 'a': u8; i += 1; };
let j: i32 = 0;
for (j < tail.len) { src[i] = tail[j]; i += 1; j += 1; };
let l: syntax.lex;
syntax.lexinit(&l, "t", src.ptr, src.len: u64);
let t: syntax.tok;
syntax.lexnext(&l, &t);
assert(t.kind == syntax.tkind.TK_MODPATH);
assert(t.text.len == pathlen);
assert(t.text[0] == 'a' && t.text[pathlen - 1] == 'a');
syntax.lexnext(&l, &t);
assert(t.kind == syntax.tkind.TK_FN);
assert(l.errs == 0);
};
fn bomsource(dst: *u8, before: str, after: str) u64 = {
let n: u64 = 0u64;
let i: i32 = 0;