diff --git a/lib/ww/lex/tok.ww b/lib/ww/lex/tok.ww index e921b002..5ef72be2 100644 --- a/lib/ww/lex/tok.ww +++ b/lib/ww/lex/tok.ww @@ -13,6 +13,7 @@ package lex; import os; import strconv; +import strings; // ---- tkind ------------------------------------------------------------ // Mirror of the C `Tkind` enum in cmd/wcc/ww.h. Numeric values are @@ -149,50 +150,46 @@ type tok = struct { // ---- keyword lookup --------------------------------------------------- -fn streqn(a: *u8, b: str, n: i32) bool = { - if (b.len != n) { return false; }; - let i: i32 = 0; - for (i < n) { - if (a[i] != b[i]) { return false; }; - i += 1; - }; - return true; -}; +// keep alphabetised, so kwlookup is easy to read — mirrors the C twin +// cmd/wcc/tok.c:18-47. Two parallel arrays, not a [N]kwent array-of- +// struct: a str *inside* an aggregate element is the filed #18 follow-up +// (str-in-aggregate). `let`, not `def`: #18's module-level [N]str static +// init is scoped to the DATAW (`let`) directive (A_DATAR needs a DATAW +// holder); `def [N]str` is the same filed follow-up. kwkinds is a plain +// [N]tkind enum byte-array (pre-#18 path). Explicit [30], NOT [_]: +// [_] static-init silently miscompiles to a zero-length array in-tree +// (probe at cc69daf — len() returns 0, no diagnostic); filed bug. +let kwnames: [30]str = [ + "as", "break", "case", "chan", "const", "continue", "def", "defer", + "else", "enum", "export", "false", "fn", "for", "if", "is", + "import", "let", "match", "nil", "package", "proc", "return", + "static", "struct", "switch", "true", "type", "void", "yield", +]; +let kwkinds: [30]tkind = [ + tkind.TK_AS, tkind.TK_BREAK, tkind.TK_CASE, tkind.TK_CHAN, + tkind.TK_CONST, tkind.TK_CONTINUE, tkind.TK_DEF, tkind.TK_DEFER, + tkind.TK_ELSE, tkind.TK_ENUM, tkind.TK_EXPORT, tkind.TK_FALSE, + tkind.TK_FN, tkind.TK_FOR, tkind.TK_IF, tkind.TK_IS, + tkind.TK_USE, tkind.TK_LET, tkind.TK_MATCH, tkind.TK_NIL, + tkind.TK_MODULE, tkind.TK_PROC, tkind.TK_RETURN, tkind.TK_STATIC, + tkind.TK_STRUCT, tkind.TK_SWITCH, tkind.TK_TRUE, tkind.TK_TYPE, + tkind.TK_VOID, tkind.TK_YIELD, +]; // kwlookup — returns the matching TK_* keyword kind for a byte run, -// or tkind.TK_NONE if it's an ordinary identifier. Linear search over a -// small alphabetised list, matching cmd/wcc/tok.c. +// or tkind.TK_NONE if it's an ordinary identifier. Linear scan over the +// table, matching cmd/wcc/tok.c:kwlookup (N=30, no hash). export fn kwlookup(p: *u8, n: i32) tkind = { - if (streqn(p, "as", n)) { return tkind.TK_AS; }; - if (streqn(p, "break", n)) { return tkind.TK_BREAK; }; - if (streqn(p, "case", n)) { return tkind.TK_CASE; }; - if (streqn(p, "chan", n)) { return tkind.TK_CHAN; }; - if (streqn(p, "const", n)) { return tkind.TK_CONST; }; - if (streqn(p, "continue", n)) { return tkind.TK_CONTINUE; }; - if (streqn(p, "def", n)) { return tkind.TK_DEF; }; - if (streqn(p, "defer", n)) { return tkind.TK_DEFER; }; - if (streqn(p, "else", n)) { return tkind.TK_ELSE; }; - if (streqn(p, "enum", n)) { return tkind.TK_ENUM; }; - if (streqn(p, "export", n)) { return tkind.TK_EXPORT; }; - if (streqn(p, "false", n)) { return tkind.TK_FALSE; }; - if (streqn(p, "fn", n)) { return tkind.TK_FN; }; - if (streqn(p, "for", n)) { return tkind.TK_FOR; }; - if (streqn(p, "if", n)) { return tkind.TK_IF; }; - if (streqn(p, "is", n)) { return tkind.TK_IS; }; - if (streqn(p, "let", n)) { return tkind.TK_LET; }; - if (streqn(p, "import", n)) { return tkind.TK_USE; }; - if (streqn(p, "match", n)) { return tkind.TK_MATCH; }; - if (streqn(p, "nil", n)) { return tkind.TK_NIL; }; - if (streqn(p, "package", n)) { return tkind.TK_MODULE; }; - if (streqn(p, "proc", n)) { return tkind.TK_PROC; }; - if (streqn(p, "return", n)) { return tkind.TK_RETURN; }; - if (streqn(p, "static", n)) { return tkind.TK_STATIC; }; - if (streqn(p, "struct", n)) { return tkind.TK_STRUCT; }; - if (streqn(p, "switch", n)) { return tkind.TK_SWITCH; }; - if (streqn(p, "true", n)) { return tkind.TK_TRUE; }; - if (streqn(p, "type", n)) { return tkind.TK_TYPE; }; - if (streqn(p, "void", n)) { return tkind.TK_VOID; }; - if (streqn(p, "yield", n)) { return tkind.TK_YIELD; }; + let cand: str; + cand.ptr = p; + cand.len = n; + let i: i32 = 0; + for (i < len(kwnames)) { + if (strings.compare(cand, kwnames[i]) == 0) { + return kwkinds[i]; + }; + i += 1; + }; return tkind.TK_NONE; }; diff --git a/lib/ww/lex/toktest.ww b/lib/ww/lex/toktest.ww index 95d551b7..5dd81e1e 100644 --- a/lib/ww/lex/toktest.ww +++ b/lib/ww/lex/toktest.ww @@ -5,7 +5,8 @@ // // tokname is checked against every tkind value (the full ladder the // switch replaced, plus the unknown-kind fallback); kwlookup is -// checked against every keyword it recognises plus a non-keyword. +// checked against every keyword it recognises plus non-keyword and +// near-miss (prefix/superstring/exact-width) identifiers. // tokprint is driven over a temp file: it pins the kind-dispatch // switch (STR/IDENT/ERR vs INT/RUNE vs the value-less default) and, // through the STR text, fputq's full escape switch (\\, ", \n, \t, @@ -138,44 +139,70 @@ fn checkkw(s: str, want: tkind) void = { if (kwlookup(s.ptr, s.len) != want) { fail(); }; }; -// Every keyword kwlookup recognises (mirrors the streqn ladder), plus -// non-keywords that must fall through to TK_NONE. +// Table-driven (parallel-array idiom; tuple rows blocked by #111). The +// kw rows pin all 30 keywords kwlookup recognises, 1:1 with the +// kwnames/kwkinds table (and cmd/wcc/tok.c:18-47), incl. the two remaps +// import->TK_USE and package->TK_MODULE. The nk rows pin the +// fall-through to TK_NONE. signalled = base + row keeps a failure +// pinpointable. Explicit dims, NOT [_]: [_] static-init silently +// miscompiles to a zero-length array in-tree (probe, cc69daf), which +// would void the loop body — the very hole a table test must not have. @test fn kwlookup_cases() void = { - signalled = 200; checkkw("as", tkind.TK_AS); - signalled = 201; checkkw("break", tkind.TK_BREAK); - signalled = 202; checkkw("case", tkind.TK_CASE); - signalled = 203; checkkw("chan", tkind.TK_CHAN); - signalled = 204; checkkw("const", tkind.TK_CONST); - signalled = 205; checkkw("continue", tkind.TK_CONTINUE); - signalled = 206; checkkw("def", tkind.TK_DEF); - signalled = 207; checkkw("defer", tkind.TK_DEFER); - signalled = 208; checkkw("else", tkind.TK_ELSE); - signalled = 209; checkkw("enum", tkind.TK_ENUM); - signalled = 210; checkkw("export", tkind.TK_EXPORT); - signalled = 211; checkkw("false", tkind.TK_FALSE); - signalled = 212; checkkw("fn", tkind.TK_FN); - signalled = 213; checkkw("for", tkind.TK_FOR); - signalled = 214; checkkw("if", tkind.TK_IF); - signalled = 215; checkkw("is", tkind.TK_IS); - signalled = 216; checkkw("let", tkind.TK_LET); - signalled = 217; checkkw("import", tkind.TK_USE); - signalled = 218; checkkw("match", tkind.TK_MATCH); - signalled = 219; checkkw("nil", tkind.TK_NIL); - signalled = 220; checkkw("package", tkind.TK_MODULE); - signalled = 221; checkkw("proc", tkind.TK_PROC); - signalled = 222; checkkw("return", tkind.TK_RETURN); - signalled = 223; checkkw("static", tkind.TK_STATIC); - signalled = 224; checkkw("struct", tkind.TK_STRUCT); - signalled = 225; checkkw("switch", tkind.TK_SWITCH); - signalled = 226; checkkw("true", tkind.TK_TRUE); - signalled = 227; checkkw("type", tkind.TK_TYPE); - signalled = 228; checkkw("void", tkind.TK_VOID); - signalled = 229; checkkw("yield", tkind.TK_YIELD); + let kwin: [30]str = [ + "as", "break", "case", "chan", "const", "continue", "def", "defer", + "else", "enum", "export", "false", "fn", "for", "if", "is", + "import", "let", "match", "nil", "package", "proc", "return", + "static", "struct", "switch", "true", "type", "void", "yield", + ]; + let kwexp: [30]tkind = [ + tkind.TK_AS, tkind.TK_BREAK, tkind.TK_CASE, tkind.TK_CHAN, + tkind.TK_CONST, tkind.TK_CONTINUE, tkind.TK_DEF, tkind.TK_DEFER, + tkind.TK_ELSE, tkind.TK_ENUM, tkind.TK_EXPORT, tkind.TK_FALSE, + tkind.TK_FN, tkind.TK_FOR, tkind.TK_IF, tkind.TK_IS, + tkind.TK_USE, tkind.TK_LET, tkind.TK_MATCH, tkind.TK_NIL, + tkind.TK_MODULE, tkind.TK_PROC, tkind.TK_RETURN, tkind.TK_STATIC, + tkind.TK_STRUCT, tkind.TK_SWITCH, tkind.TK_TRUE, tkind.TK_TYPE, + tkind.TK_VOID, tkind.TK_YIELD, + ]; + let i: i32 = 0; + for (i < len(kwin)) { + signalled = 200 + i; + checkkw(kwin[i], kwexp[i]); + i += 1; + }; - // Non-keywords fall through to TK_NONE — an ordinary identifier and - // a near-miss prefix of a real keyword. - signalled = 230; checkkw("xyzzy", tkind.TK_NONE); - signalled = 231; checkkw("fns", tkind.TK_NONE); + // Non-keywords that must fall through to TK_NONE. Rows exercise the + // length-guard + bytewise reject inside strings.compare: keyword + // SUPERSTRINGS (longer, shared prefix), proper PREFIXES of a keyword + // (shorter, shared leading bytes), exact-length non-keywords at the + // 2/3/4-byte keyword widths, and Hare bmap "keywords" (alloc/len/ + // size/append/assert) that are deliberately NOT ww lexer keywords. + let nk: [18]str = [ + "xyzzy", // ordinary identifier + "fns", // superstring of "fn" + "ifx", // superstring of "if" + "iffy", // superstring of "if" + "form", // superstring of "for" + "fora", // superstring of "for" + "asx", // superstring of "as" + "i", // proper prefix of if / is / import + "co", // proper prefix of const / continue + "swit", // proper prefix of switch + "xx", // len-2 non-kw (as/fn/if/is width) + "zzz", // len-3 non-kw (def/for/nil width) + "abcd", // len-4 non-kw (case/enum/true/type width) + "alloc", // Hare bmap keyword, NOT a ww lexer keyword + "len", // Hare bmap keyword, NOT a ww lexer keyword + "size", // Hare bmap keyword, NOT a ww lexer keyword + "append", // Hare bmap keyword, NOT a ww lexer keyword + "assert", // Hare bmap keyword, NOT a ww lexer keyword + ]; + let j: i32 = 0; + for (j < len(nk)) { + signalled = 230 + j; + checkkw(nk[j], tkind.TK_NONE); + j += 1; + }; }; // checkprint — tokprint `t` to a freshly-rewound fd, read the bytes diff --git a/selfhost/cmd/w6c/main.combined.ww b/selfhost/cmd/w6c/main.combined.ww index cf1da7f2..e6f3b870 100644 --- a/selfhost/cmd/w6c/main.combined.ww +++ b/selfhost/cmd/w6c/main.combined.ww @@ -6386,6 +6386,7 @@ package lex; import os; import strconv; +import strings; // ---- tkind ------------------------------------------------------------ // Mirror of the C `Tkind` enum in cmd/wcc/ww.h. Numeric values are @@ -6522,50 +6523,46 @@ type tok = struct { // ---- keyword lookup --------------------------------------------------- -fn streqn(a: *u8, b: str, n: i32) bool = { - if (b.len != n) { return false; }; - let i: i32 = 0; - for (i < n) { - if (a[i] != b[i]) { return false; }; - i += 1; - }; - return true; -}; +// keep alphabetised, so kwlookup is easy to read — mirrors the C twin +// cmd/wcc/tok.c:18-47. Two parallel arrays, not a [N]kwent array-of- +// struct: a str *inside* an aggregate element is the filed #18 follow-up +// (str-in-aggregate). `let`, not `def`: #18's module-level [N]str static +// init is scoped to the DATAW (`let`) directive (A_DATAR needs a DATAW +// holder); `def [N]str` is the same filed follow-up. kwkinds is a plain +// [N]tkind enum byte-array (pre-#18 path). Explicit [30], NOT [_]: +// [_] static-init silently miscompiles to a zero-length array in-tree +// (probe at cc69daf — len() returns 0, no diagnostic); filed bug. +let kwnames: [30]str = [ + "as", "break", "case", "chan", "const", "continue", "def", "defer", + "else", "enum", "export", "false", "fn", "for", "if", "is", + "import", "let", "match", "nil", "package", "proc", "return", + "static", "struct", "switch", "true", "type", "void", "yield", +]; +let kwkinds: [30]tkind = [ + tkind.TK_AS, tkind.TK_BREAK, tkind.TK_CASE, tkind.TK_CHAN, + tkind.TK_CONST, tkind.TK_CONTINUE, tkind.TK_DEF, tkind.TK_DEFER, + tkind.TK_ELSE, tkind.TK_ENUM, tkind.TK_EXPORT, tkind.TK_FALSE, + tkind.TK_FN, tkind.TK_FOR, tkind.TK_IF, tkind.TK_IS, + tkind.TK_USE, tkind.TK_LET, tkind.TK_MATCH, tkind.TK_NIL, + tkind.TK_MODULE, tkind.TK_PROC, tkind.TK_RETURN, tkind.TK_STATIC, + tkind.TK_STRUCT, tkind.TK_SWITCH, tkind.TK_TRUE, tkind.TK_TYPE, + tkind.TK_VOID, tkind.TK_YIELD, +]; // kwlookup — returns the matching TK_* keyword kind for a byte run, -// or tkind.TK_NONE if it's an ordinary identifier. Linear search over a -// small alphabetised list, matching cmd/wcc/tok.c. +// or tkind.TK_NONE if it's an ordinary identifier. Linear scan over the +// table, matching cmd/wcc/tok.c:kwlookup (N=30, no hash). export fn kwlookup(p: *u8, n: i32) tkind = { - if (streqn(p, "as", n)) { return tkind.TK_AS; }; - if (streqn(p, "break", n)) { return tkind.TK_BREAK; }; - if (streqn(p, "case", n)) { return tkind.TK_CASE; }; - if (streqn(p, "chan", n)) { return tkind.TK_CHAN; }; - if (streqn(p, "const", n)) { return tkind.TK_CONST; }; - if (streqn(p, "continue", n)) { return tkind.TK_CONTINUE; }; - if (streqn(p, "def", n)) { return tkind.TK_DEF; }; - if (streqn(p, "defer", n)) { return tkind.TK_DEFER; }; - if (streqn(p, "else", n)) { return tkind.TK_ELSE; }; - if (streqn(p, "enum", n)) { return tkind.TK_ENUM; }; - if (streqn(p, "export", n)) { return tkind.TK_EXPORT; }; - if (streqn(p, "false", n)) { return tkind.TK_FALSE; }; - if (streqn(p, "fn", n)) { return tkind.TK_FN; }; - if (streqn(p, "for", n)) { return tkind.TK_FOR; }; - if (streqn(p, "if", n)) { return tkind.TK_IF; }; - if (streqn(p, "is", n)) { return tkind.TK_IS; }; - if (streqn(p, "let", n)) { return tkind.TK_LET; }; - if (streqn(p, "import", n)) { return tkind.TK_USE; }; - if (streqn(p, "match", n)) { return tkind.TK_MATCH; }; - if (streqn(p, "nil", n)) { return tkind.TK_NIL; }; - if (streqn(p, "package", n)) { return tkind.TK_MODULE; }; - if (streqn(p, "proc", n)) { return tkind.TK_PROC; }; - if (streqn(p, "return", n)) { return tkind.TK_RETURN; }; - if (streqn(p, "static", n)) { return tkind.TK_STATIC; }; - if (streqn(p, "struct", n)) { return tkind.TK_STRUCT; }; - if (streqn(p, "switch", n)) { return tkind.TK_SWITCH; }; - if (streqn(p, "true", n)) { return tkind.TK_TRUE; }; - if (streqn(p, "type", n)) { return tkind.TK_TYPE; }; - if (streqn(p, "void", n)) { return tkind.TK_VOID; }; - if (streqn(p, "yield", n)) { return tkind.TK_YIELD; }; + let cand: str; + cand.ptr = p; + cand.len = n; + let i: i32 = 0; + for (i < len(kwnames)) { + if (strings.compare(cand, kwnames[i]) == 0) { + return kwkinds[i]; + }; + i += 1; + }; return tkind.TK_NONE; }; diff --git a/selfhost/cmd/wwdump/main.combined.ww b/selfhost/cmd/wwdump/main.combined.ww index 8d3a153f..4c0e363c 100644 --- a/selfhost/cmd/wwdump/main.combined.ww +++ b/selfhost/cmd/wwdump/main.combined.ww @@ -6386,6 +6386,7 @@ package lex; import os; import strconv; +import strings; // ---- tkind ------------------------------------------------------------ // Mirror of the C `Tkind` enum in cmd/wcc/ww.h. Numeric values are @@ -6522,50 +6523,46 @@ type tok = struct { // ---- keyword lookup --------------------------------------------------- -fn streqn(a: *u8, b: str, n: i32) bool = { - if (b.len != n) { return false; }; - let i: i32 = 0; - for (i < n) { - if (a[i] != b[i]) { return false; }; - i += 1; - }; - return true; -}; +// keep alphabetised, so kwlookup is easy to read — mirrors the C twin +// cmd/wcc/tok.c:18-47. Two parallel arrays, not a [N]kwent array-of- +// struct: a str *inside* an aggregate element is the filed #18 follow-up +// (str-in-aggregate). `let`, not `def`: #18's module-level [N]str static +// init is scoped to the DATAW (`let`) directive (A_DATAR needs a DATAW +// holder); `def [N]str` is the same filed follow-up. kwkinds is a plain +// [N]tkind enum byte-array (pre-#18 path). Explicit [30], NOT [_]: +// [_] static-init silently miscompiles to a zero-length array in-tree +// (probe at cc69daf — len() returns 0, no diagnostic); filed bug. +let kwnames: [30]str = [ + "as", "break", "case", "chan", "const", "continue", "def", "defer", + "else", "enum", "export", "false", "fn", "for", "if", "is", + "import", "let", "match", "nil", "package", "proc", "return", + "static", "struct", "switch", "true", "type", "void", "yield", +]; +let kwkinds: [30]tkind = [ + tkind.TK_AS, tkind.TK_BREAK, tkind.TK_CASE, tkind.TK_CHAN, + tkind.TK_CONST, tkind.TK_CONTINUE, tkind.TK_DEF, tkind.TK_DEFER, + tkind.TK_ELSE, tkind.TK_ENUM, tkind.TK_EXPORT, tkind.TK_FALSE, + tkind.TK_FN, tkind.TK_FOR, tkind.TK_IF, tkind.TK_IS, + tkind.TK_USE, tkind.TK_LET, tkind.TK_MATCH, tkind.TK_NIL, + tkind.TK_MODULE, tkind.TK_PROC, tkind.TK_RETURN, tkind.TK_STATIC, + tkind.TK_STRUCT, tkind.TK_SWITCH, tkind.TK_TRUE, tkind.TK_TYPE, + tkind.TK_VOID, tkind.TK_YIELD, +]; // kwlookup — returns the matching TK_* keyword kind for a byte run, -// or tkind.TK_NONE if it's an ordinary identifier. Linear search over a -// small alphabetised list, matching cmd/wcc/tok.c. +// or tkind.TK_NONE if it's an ordinary identifier. Linear scan over the +// table, matching cmd/wcc/tok.c:kwlookup (N=30, no hash). export fn kwlookup(p: *u8, n: i32) tkind = { - if (streqn(p, "as", n)) { return tkind.TK_AS; }; - if (streqn(p, "break", n)) { return tkind.TK_BREAK; }; - if (streqn(p, "case", n)) { return tkind.TK_CASE; }; - if (streqn(p, "chan", n)) { return tkind.TK_CHAN; }; - if (streqn(p, "const", n)) { return tkind.TK_CONST; }; - if (streqn(p, "continue", n)) { return tkind.TK_CONTINUE; }; - if (streqn(p, "def", n)) { return tkind.TK_DEF; }; - if (streqn(p, "defer", n)) { return tkind.TK_DEFER; }; - if (streqn(p, "else", n)) { return tkind.TK_ELSE; }; - if (streqn(p, "enum", n)) { return tkind.TK_ENUM; }; - if (streqn(p, "export", n)) { return tkind.TK_EXPORT; }; - if (streqn(p, "false", n)) { return tkind.TK_FALSE; }; - if (streqn(p, "fn", n)) { return tkind.TK_FN; }; - if (streqn(p, "for", n)) { return tkind.TK_FOR; }; - if (streqn(p, "if", n)) { return tkind.TK_IF; }; - if (streqn(p, "is", n)) { return tkind.TK_IS; }; - if (streqn(p, "let", n)) { return tkind.TK_LET; }; - if (streqn(p, "import", n)) { return tkind.TK_USE; }; - if (streqn(p, "match", n)) { return tkind.TK_MATCH; }; - if (streqn(p, "nil", n)) { return tkind.TK_NIL; }; - if (streqn(p, "package", n)) { return tkind.TK_MODULE; }; - if (streqn(p, "proc", n)) { return tkind.TK_PROC; }; - if (streqn(p, "return", n)) { return tkind.TK_RETURN; }; - if (streqn(p, "static", n)) { return tkind.TK_STATIC; }; - if (streqn(p, "struct", n)) { return tkind.TK_STRUCT; }; - if (streqn(p, "switch", n)) { return tkind.TK_SWITCH; }; - if (streqn(p, "true", n)) { return tkind.TK_TRUE; }; - if (streqn(p, "type", n)) { return tkind.TK_TYPE; }; - if (streqn(p, "void", n)) { return tkind.TK_VOID; }; - if (streqn(p, "yield", n)) { return tkind.TK_YIELD; }; + let cand: str; + cand.ptr = p; + cand.len = n; + let i: i32 = 0; + for (i < len(kwnames)) { + if (strings.compare(cand, kwnames[i]) == 0) { + return kwkinds[i]; + }; + i += 1; + }; return tkind.TK_NONE; };