Hojun-Cho/ww
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

lib/ww/lex: kwlookup if-ladder -> [N]str parallel-table linear scan (kwtab, rides #18/#8)

Browse Source 
Replace kwlookup's 30-arm streqn if-ladder with two parallel module-level
tables — kwnames: [30]str + kwkinds: [30]tkind — scanned linearly via
strings.compare, and delete the hand-rolled streqn. Rides #18 (module-
level [N]str static-init + relocations) for the kwnames data and #8
([N]enum element sizing) for the kwkinds[i] read, which is itself the
construct that surfaced the #8 elemsizeofc/array-init-store miscompile.

Two parallel arrays rather than a [N]kwent array-of-struct: a str inside
an aggregate element is the filed #18 follow-up. Mirrors the C twin
cmd/wcc/tok.c kwlookup (N=30, linear, no hash). Source re-applied on top
of the #8 cgen fix and regenerated fresh: the wwstage-compiled toktest
now runs correctly (exit 0, no segfault) where pre-#8 it smashed the
frame on the local [N]tkind init store.
This commit is contained in:
Hojun-Cho
2026-06-03 14:50:18 +09:00
parent d39691a3d7
commit 5c1e9c0bfb
4 changed files with 178 additions and 160 deletions
Download Patch File Download Diff File Expand all files Collapse all files

79
lib/ww/lex/tok.ww
View File

@@ -13,6 +13,7 @@ package lex;
import os;
import strconv;
import strings;
// ---- tkind ------------------------------------------------------------
// Mirror of the C `Tkind` enum in cmd/wcc/ww.h. Numeric values are
@@ -149,50 +150,46 @@ type tok = struct {
// ---- keyword lookup ---------------------------------------------------
fn streqn(a: *u8, b: str, n: i32) bool = {
if (b.len != n) { return false; };
let i: i32 = 0;
for (i < n) {
if (a[i] != b[i]) { return false; };
i += 1;
};
return true;
};
// keep alphabetised, so kwlookup is easy to read — mirrors the C twin
// cmd/wcc/tok.c:18-47. Two parallel arrays, not a [N]kwent array-of-
// struct: a str *inside* an aggregate element is the filed #18 follow-up
// (str-in-aggregate). `let`, not `def`: #18's module-level [N]str static
// init is scoped to the DATAW (`let`) directive (A_DATAR needs a DATAW
// holder); `def [N]str` is the same filed follow-up. kwkinds is a plain
// [N]tkind enum byte-array (pre-#18 path). Explicit [30], NOT [_]:
// [_] static-init silently miscompiles to a zero-length array in-tree
// (probe at cc69daf — len() returns 0, no diagnostic); filed bug.
let kwnames: [30]str = [
"as", "break", "case", "chan", "const", "continue", "def", "defer",
"else", "enum", "export", "false", "fn", "for", "if", "is",
"import", "let", "match", "nil", "package", "proc", "return",
"static", "struct", "switch", "true", "type", "void", "yield",
];
let kwkinds: [30]tkind = [
tkind.TK_AS, tkind.TK_BREAK, tkind.TK_CASE, tkind.TK_CHAN,
tkind.TK_CONST, tkind.TK_CONTINUE, tkind.TK_DEF, tkind.TK_DEFER,
tkind.TK_ELSE, tkind.TK_ENUM, tkind.TK_EXPORT, tkind.TK_FALSE,
tkind.TK_FN, tkind.TK_FOR, tkind.TK_IF, tkind.TK_IS,
tkind.TK_USE, tkind.TK_LET, tkind.TK_MATCH, tkind.TK_NIL,
tkind.TK_MODULE, tkind.TK_PROC, tkind.TK_RETURN, tkind.TK_STATIC,
tkind.TK_STRUCT, tkind.TK_SWITCH, tkind.TK_TRUE, tkind.TK_TYPE,
tkind.TK_VOID, tkind.TK_YIELD,
];
// kwlookup — returns the matching TK_* keyword kind for a byte run,
// or tkind.TK_NONE if it's an ordinary identifier. Linear search over a
// small alphabetised list, matching cmd/wcc/tok.c.
// or tkind.TK_NONE if it's an ordinary identifier. Linear scan over the
// table, matching cmd/wcc/tok.c:kwlookup (N=30, no hash).
export fn kwlookup(p: *u8, n: i32) tkind = {
if (streqn(p, "as", n)) { return tkind.TK_AS; };
if (streqn(p, "break", n)) { return tkind.TK_BREAK; };
if (streqn(p, "case", n)) { return tkind.TK_CASE; };
if (streqn(p, "chan", n)) { return tkind.TK_CHAN; };
if (streqn(p, "const", n)) { return tkind.TK_CONST; };
if (streqn(p, "continue", n)) { return tkind.TK_CONTINUE; };
if (streqn(p, "def", n)) { return tkind.TK_DEF; };
if (streqn(p, "defer", n)) { return tkind.TK_DEFER; };
if (streqn(p, "else", n)) { return tkind.TK_ELSE; };
if (streqn(p, "enum", n)) { return tkind.TK_ENUM; };
if (streqn(p, "export", n)) { return tkind.TK_EXPORT; };
if (streqn(p, "false", n)) { return tkind.TK_FALSE; };
if (streqn(p, "fn", n)) { return tkind.TK_FN; };
if (streqn(p, "for", n)) { return tkind.TK_FOR; };
if (streqn(p, "if", n)) { return tkind.TK_IF; };
if (streqn(p, "is", n)) { return tkind.TK_IS; };
if (streqn(p, "let", n)) { return tkind.TK_LET; };
if (streqn(p, "import", n)) { return tkind.TK_USE; };
if (streqn(p, "match", n)) { return tkind.TK_MATCH; };
if (streqn(p, "nil", n)) { return tkind.TK_NIL; };
if (streqn(p, "package", n)) { return tkind.TK_MODULE; };
if (streqn(p, "proc", n)) { return tkind.TK_PROC; };
if (streqn(p, "return", n)) { return tkind.TK_RETURN; };
if (streqn(p, "static", n)) { return tkind.TK_STATIC; };
if (streqn(p, "struct", n)) { return tkind.TK_STRUCT; };
if (streqn(p, "switch", n)) { return tkind.TK_SWITCH; };
if (streqn(p, "true", n)) { return tkind.TK_TRUE; };
if (streqn(p, "type", n)) { return tkind.TK_TYPE; };
if (streqn(p, "void", n)) { return tkind.TK_VOID; };
if (streqn(p, "yield", n)) { return tkind.TK_YIELD; };
let cand: str;
cand.ptr = p;
cand.len = n;
let i: i32 = 0;
for (i < len(kwnames)) {
if (strings.compare(cand, kwnames[i]) == 0) {
return kwkinds[i];
};
i += 1;
};
return tkind.TK_NONE;
};

101
lib/ww/lex/toktest.ww
View File

@@ -5,7 +5,8 @@
//
// tokname is checked against every tkind value (the full ladder the
// switch replaced, plus the unknown-kind fallback); kwlookup is
// checked against every keyword it recognises plus a non-keyword.
// checked against every keyword it recognises plus non-keyword and
// near-miss (prefix/superstring/exact-width) identifiers.
// tokprint is driven over a temp file: it pins the kind-dispatch
// switch (STR/IDENT/ERR vs INT/RUNE vs the value-less default) and,
// through the STR text, fputq's full escape switch (\\, ", \n, \t,
@@ -138,44 +139,70 @@ fn checkkw(s: str, want: tkind) void = {
if (kwlookup(s.ptr, s.len) != want) { fail(); };
};
// Every keyword kwlookup recognises (mirrors the streqn ladder), plus
// non-keywords that must fall through to TK_NONE.
// Table-driven (parallel-array idiom; tuple rows blocked by #111). The
// kw rows pin all 30 keywords kwlookup recognises, 1:1 with the
// kwnames/kwkinds table (and cmd/wcc/tok.c:18-47), incl. the two remaps
// import->TK_USE and package->TK_MODULE. The nk rows pin the
// fall-through to TK_NONE. signalled = base + row keeps a failure
// pinpointable. Explicit dims, NOT [_]: [_] static-init silently
// miscompiles to a zero-length array in-tree (probe, cc69daf), which
// would void the loop body — the very hole a table test must not have.
@test fn kwlookup_cases() void = {
signalled = 200; checkkw("as", tkind.TK_AS);
signalled = 201; checkkw("break", tkind.TK_BREAK);
signalled = 202; checkkw("case", tkind.TK_CASE);
signalled = 203; checkkw("chan", tkind.TK_CHAN);
signalled = 204; checkkw("const", tkind.TK_CONST);
signalled = 205; checkkw("continue", tkind.TK_CONTINUE);
signalled = 206; checkkw("def", tkind.TK_DEF);
signalled = 207; checkkw("defer", tkind.TK_DEFER);
signalled = 208; checkkw("else", tkind.TK_ELSE);
signalled = 209; checkkw("enum", tkind.TK_ENUM);
signalled = 210; checkkw("export", tkind.TK_EXPORT);
signalled = 211; checkkw("false", tkind.TK_FALSE);
signalled = 212; checkkw("fn", tkind.TK_FN);
signalled = 213; checkkw("for", tkind.TK_FOR);
signalled = 214; checkkw("if", tkind.TK_IF);
signalled = 215; checkkw("is", tkind.TK_IS);
signalled = 216; checkkw("let", tkind.TK_LET);
signalled = 217; checkkw("import", tkind.TK_USE);
signalled = 218; checkkw("match", tkind.TK_MATCH);
signalled = 219; checkkw("nil", tkind.TK_NIL);
signalled = 220; checkkw("package", tkind.TK_MODULE);
signalled = 221; checkkw("proc", tkind.TK_PROC);
signalled = 222; checkkw("return", tkind.TK_RETURN);
signalled = 223; checkkw("static", tkind.TK_STATIC);
signalled = 224; checkkw("struct", tkind.TK_STRUCT);
signalled = 225; checkkw("switch", tkind.TK_SWITCH);
signalled = 226; checkkw("true", tkind.TK_TRUE);
signalled = 227; checkkw("type", tkind.TK_TYPE);
signalled = 228; checkkw("void", tkind.TK_VOID);
signalled = 229; checkkw("yield", tkind.TK_YIELD);
let kwin: [30]str = [
"as", "break", "case", "chan", "const", "continue", "def", "defer",
"else", "enum", "export", "false", "fn", "for", "if", "is",
"import", "let", "match", "nil", "package", "proc", "return",
"static", "struct", "switch", "true", "type", "void", "yield",
];
let kwexp: [30]tkind = [
tkind.TK_AS, tkind.TK_BREAK, tkind.TK_CASE, tkind.TK_CHAN,
tkind.TK_CONST, tkind.TK_CONTINUE, tkind.TK_DEF, tkind.TK_DEFER,
tkind.TK_ELSE, tkind.TK_ENUM, tkind.TK_EXPORT, tkind.TK_FALSE,
tkind.TK_FN, tkind.TK_FOR, tkind.TK_IF, tkind.TK_IS,
tkind.TK_USE, tkind.TK_LET, tkind.TK_MATCH, tkind.TK_NIL,
tkind.TK_MODULE, tkind.TK_PROC, tkind.TK_RETURN, tkind.TK_STATIC,
tkind.TK_STRUCT, tkind.TK_SWITCH, tkind.TK_TRUE, tkind.TK_TYPE,
tkind.TK_VOID, tkind.TK_YIELD,
];
let i: i32 = 0;
for (i < len(kwin)) {
signalled = 200 + i;
checkkw(kwin[i], kwexp[i]);
i += 1;
};
// Non-keywords fall through to TK_NONE — an ordinary identifier and
// a near-miss prefix of a real keyword.
signalled = 230; checkkw("xyzzy", tkind.TK_NONE);
signalled = 231; checkkw("fns", tkind.TK_NONE);
// Non-keywords that must fall through to TK_NONE. Rows exercise the
// length-guard + bytewise reject inside strings.compare: keyword
// SUPERSTRINGS (longer, shared prefix), proper PREFIXES of a keyword
// (shorter, shared leading bytes), exact-length non-keywords at the
// 2/3/4-byte keyword widths, and Hare bmap "keywords" (alloc/len/
// size/append/assert) that are deliberately NOT ww lexer keywords.
let nk: [18]str = [
"xyzzy", // ordinary identifier
"fns", // superstring of "fn"
"ifx", // superstring of "if"
"iffy", // superstring of "if"
"form", // superstring of "for"
"fora", // superstring of "for"
"asx", // superstring of "as"
"i", // proper prefix of if / is / import
"co", // proper prefix of const / continue
"swit", // proper prefix of switch
"xx", // len-2 non-kw (as/fn/if/is width)
"zzz", // len-3 non-kw (def/for/nil width)
"abcd", // len-4 non-kw (case/enum/true/type width)
"alloc", // Hare bmap keyword, NOT a ww lexer keyword
"len", // Hare bmap keyword, NOT a ww lexer keyword
"size", // Hare bmap keyword, NOT a ww lexer keyword
"append", // Hare bmap keyword, NOT a ww lexer keyword
"assert", // Hare bmap keyword, NOT a ww lexer keyword
];
let j: i32 = 0;
for (j < len(nk)) {
signalled = 230 + j;
checkkw(nk[j], tkind.TK_NONE);
j += 1;
};
};
// checkprint — tokprint `t` to a freshly-rewound fd, read the bytes

79
selfhost/cmd/w6c/main.combined.ww
View File

@@ -6386,6 +6386,7 @@ package lex;
import os;
import strconv;
import strings;
// ---- tkind ------------------------------------------------------------
// Mirror of the C `Tkind` enum in cmd/wcc/ww.h. Numeric values are
@@ -6522,50 +6523,46 @@ type tok = struct {
// ---- keyword lookup ---------------------------------------------------
fn streqn(a: *u8, b: str, n: i32) bool = {
if (b.len != n) { return false; };
let i: i32 = 0;
for (i < n) {
if (a[i] != b[i]) { return false; };
i += 1;
};
return true;
};
// keep alphabetised, so kwlookup is easy to read — mirrors the C twin
// cmd/wcc/tok.c:18-47. Two parallel arrays, not a [N]kwent array-of-
// struct: a str *inside* an aggregate element is the filed #18 follow-up
// (str-in-aggregate). `let`, not `def`: #18's module-level [N]str static
// init is scoped to the DATAW (`let`) directive (A_DATAR needs a DATAW
// holder); `def [N]str` is the same filed follow-up. kwkinds is a plain
// [N]tkind enum byte-array (pre-#18 path). Explicit [30], NOT [_]:
// [_] static-init silently miscompiles to a zero-length array in-tree
// (probe at cc69daf — len() returns 0, no diagnostic); filed bug.
let kwnames: [30]str = [
"as", "break", "case", "chan", "const", "continue", "def", "defer",
"else", "enum", "export", "false", "fn", "for", "if", "is",
"import", "let", "match", "nil", "package", "proc", "return",
"static", "struct", "switch", "true", "type", "void", "yield",
];
let kwkinds: [30]tkind = [
tkind.TK_AS, tkind.TK_BREAK, tkind.TK_CASE, tkind.TK_CHAN,
tkind.TK_CONST, tkind.TK_CONTINUE, tkind.TK_DEF, tkind.TK_DEFER,
tkind.TK_ELSE, tkind.TK_ENUM, tkind.TK_EXPORT, tkind.TK_FALSE,
tkind.TK_FN, tkind.TK_FOR, tkind.TK_IF, tkind.TK_IS,
tkind.TK_USE, tkind.TK_LET, tkind.TK_MATCH, tkind.TK_NIL,
tkind.TK_MODULE, tkind.TK_PROC, tkind.TK_RETURN, tkind.TK_STATIC,
tkind.TK_STRUCT, tkind.TK_SWITCH, tkind.TK_TRUE, tkind.TK_TYPE,
tkind.TK_VOID, tkind.TK_YIELD,
];
// kwlookup — returns the matching TK_* keyword kind for a byte run,
// or tkind.TK_NONE if it's an ordinary identifier. Linear search over a
// small alphabetised list, matching cmd/wcc/tok.c.
// or tkind.TK_NONE if it's an ordinary identifier. Linear scan over the
// table, matching cmd/wcc/tok.c:kwlookup (N=30, no hash).
export fn kwlookup(p: *u8, n: i32) tkind = {
if (streqn(p, "as", n)) { return tkind.TK_AS; };
if (streqn(p, "break", n)) { return tkind.TK_BREAK; };
if (streqn(p, "case", n)) { return tkind.TK_CASE; };
if (streqn(p, "chan", n)) { return tkind.TK_CHAN; };
if (streqn(p, "const", n)) { return tkind.TK_CONST; };
if (streqn(p, "continue", n)) { return tkind.TK_CONTINUE; };
if (streqn(p, "def", n)) { return tkind.TK_DEF; };
if (streqn(p, "defer", n)) { return tkind.TK_DEFER; };
if (streqn(p, "else", n)) { return tkind.TK_ELSE; };
if (streqn(p, "enum", n)) { return tkind.TK_ENUM; };
if (streqn(p, "export", n)) { return tkind.TK_EXPORT; };
if (streqn(p, "false", n)) { return tkind.TK_FALSE; };
if (streqn(p, "fn", n)) { return tkind.TK_FN; };
if (streqn(p, "for", n)) { return tkind.TK_FOR; };
if (streqn(p, "if", n)) { return tkind.TK_IF; };
if (streqn(p, "is", n)) { return tkind.TK_IS; };
if (streqn(p, "let", n)) { return tkind.TK_LET; };
if (streqn(p, "import", n)) { return tkind.TK_USE; };
if (streqn(p, "match", n)) { return tkind.TK_MATCH; };
if (streqn(p, "nil", n)) { return tkind.TK_NIL; };
if (streqn(p, "package", n)) { return tkind.TK_MODULE; };
if (streqn(p, "proc", n)) { return tkind.TK_PROC; };
if (streqn(p, "return", n)) { return tkind.TK_RETURN; };
if (streqn(p, "static", n)) { return tkind.TK_STATIC; };
if (streqn(p, "struct", n)) { return tkind.TK_STRUCT; };
if (streqn(p, "switch", n)) { return tkind.TK_SWITCH; };
if (streqn(p, "true", n)) { return tkind.TK_TRUE; };
if (streqn(p, "type", n)) { return tkind.TK_TYPE; };
if (streqn(p, "void", n)) { return tkind.TK_VOID; };
if (streqn(p, "yield", n)) { return tkind.TK_YIELD; };
let cand: str;
cand.ptr = p;
cand.len = n;
let i: i32 = 0;
for (i < len(kwnames)) {
if (strings.compare(cand, kwnames[i]) == 0) {
return kwkinds[i];
};
i += 1;
};
return tkind.TK_NONE;
};

79
selfhost/cmd/wwdump/main.combined.ww
View File

@@ -6386,6 +6386,7 @@ package lex;
import os;
import strconv;
import strings;
// ---- tkind ------------------------------------------------------------
// Mirror of the C `Tkind` enum in cmd/wcc/ww.h. Numeric values are
@@ -6522,50 +6523,46 @@ type tok = struct {
// ---- keyword lookup ---------------------------------------------------
fn streqn(a: *u8, b: str, n: i32) bool = {
if (b.len != n) { return false; };
let i: i32 = 0;
for (i < n) {
if (a[i] != b[i]) { return false; };
i += 1;
};
return true;
};
// keep alphabetised, so kwlookup is easy to read — mirrors the C twin
// cmd/wcc/tok.c:18-47. Two parallel arrays, not a [N]kwent array-of-
// struct: a str *inside* an aggregate element is the filed #18 follow-up
// (str-in-aggregate). `let`, not `def`: #18's module-level [N]str static
// init is scoped to the DATAW (`let`) directive (A_DATAR needs a DATAW
// holder); `def [N]str` is the same filed follow-up. kwkinds is a plain
// [N]tkind enum byte-array (pre-#18 path). Explicit [30], NOT [_]:
// [_] static-init silently miscompiles to a zero-length array in-tree
// (probe at cc69daf — len() returns 0, no diagnostic); filed bug.
let kwnames: [30]str = [
"as", "break", "case", "chan", "const", "continue", "def", "defer",
"else", "enum", "export", "false", "fn", "for", "if", "is",
"import", "let", "match", "nil", "package", "proc", "return",
"static", "struct", "switch", "true", "type", "void", "yield",
];
let kwkinds: [30]tkind = [
tkind.TK_AS, tkind.TK_BREAK, tkind.TK_CASE, tkind.TK_CHAN,
tkind.TK_CONST, tkind.TK_CONTINUE, tkind.TK_DEF, tkind.TK_DEFER,
tkind.TK_ELSE, tkind.TK_ENUM, tkind.TK_EXPORT, tkind.TK_FALSE,
tkind.TK_FN, tkind.TK_FOR, tkind.TK_IF, tkind.TK_IS,
tkind.TK_USE, tkind.TK_LET, tkind.TK_MATCH, tkind.TK_NIL,
tkind.TK_MODULE, tkind.TK_PROC, tkind.TK_RETURN, tkind.TK_STATIC,
tkind.TK_STRUCT, tkind.TK_SWITCH, tkind.TK_TRUE, tkind.TK_TYPE,
tkind.TK_VOID, tkind.TK_YIELD,
];
// kwlookup — returns the matching TK_* keyword kind for a byte run,
// or tkind.TK_NONE if it's an ordinary identifier. Linear search over a
// small alphabetised list, matching cmd/wcc/tok.c.
// or tkind.TK_NONE if it's an ordinary identifier. Linear scan over the
// table, matching cmd/wcc/tok.c:kwlookup (N=30, no hash).
export fn kwlookup(p: *u8, n: i32) tkind = {
if (streqn(p, "as", n)) { return tkind.TK_AS; };
if (streqn(p, "break", n)) { return tkind.TK_BREAK; };
if (streqn(p, "case", n)) { return tkind.TK_CASE; };
if (streqn(p, "chan", n)) { return tkind.TK_CHAN; };
if (streqn(p, "const", n)) { return tkind.TK_CONST; };
if (streqn(p, "continue", n)) { return tkind.TK_CONTINUE; };
if (streqn(p, "def", n)) { return tkind.TK_DEF; };
if (streqn(p, "defer", n)) { return tkind.TK_DEFER; };
if (streqn(p, "else", n)) { return tkind.TK_ELSE; };
if (streqn(p, "enum", n)) { return tkind.TK_ENUM; };
if (streqn(p, "export", n)) { return tkind.TK_EXPORT; };
if (streqn(p, "false", n)) { return tkind.TK_FALSE; };
if (streqn(p, "fn", n)) { return tkind.TK_FN; };
if (streqn(p, "for", n)) { return tkind.TK_FOR; };
if (streqn(p, "if", n)) { return tkind.TK_IF; };
if (streqn(p, "is", n)) { return tkind.TK_IS; };
if (streqn(p, "let", n)) { return tkind.TK_LET; };
if (streqn(p, "import", n)) { return tkind.TK_USE; };
if (streqn(p, "match", n)) { return tkind.TK_MATCH; };
if (streqn(p, "nil", n)) { return tkind.TK_NIL; };
if (streqn(p, "package", n)) { return tkind.TK_MODULE; };
if (streqn(p, "proc", n)) { return tkind.TK_PROC; };
if (streqn(p, "return", n)) { return tkind.TK_RETURN; };
if (streqn(p, "static", n)) { return tkind.TK_STATIC; };
if (streqn(p, "struct", n)) { return tkind.TK_STRUCT; };
if (streqn(p, "switch", n)) { return tkind.TK_SWITCH; };
if (streqn(p, "true", n)) { return tkind.TK_TRUE; };
if (streqn(p, "type", n)) { return tkind.TK_TYPE; };
if (streqn(p, "void", n)) { return tkind.TK_VOID; };
if (streqn(p, "yield", n)) { return tkind.TK_YIELD; };
let cand: str;
cand.ptr = p;
cand.len = n;
let i: i32 = 0;
for (i < len(kwnames)) {
if (strings.compare(cand, kwnames[i]) == 0) {
return kwkinds[i];
};
i += 1;
};
return tkind.TK_NONE;
};