Hojun-Cho
5c1e9c0bfb
Replace kwlookup's 30-arm streqn if-ladder with two parallel module-level tables — kwnames: [30]str + kwkinds: [30]tkind — scanned linearly via strings.compare, and delete the hand-rolled streqn. Rides #18 (module- level [N]str static-init + relocations) for the kwnames data and #8 ([N]enum element sizing) for the kwkinds[i] read, which is itself the construct that surfaced the #8 elemsizeofc/array-init-store miscompile. Two parallel arrays rather than a [N]kwent array-of-struct: a str inside an aggregate element is the filed #18 follow-up. Mirrors the C twin cmd/wcc/tok.c kwlookup (N=30, linear, no hash). Source re-applied on top of the #8 cgen fix and regenerated fresh: the wwstage-compiled toktest now runs correctly (exit 0, no segfault) where pre-#8 it smashed the frame on the local [N]tkind init store.
281 lines
11 KiB
Plaintext
281 lines
11 KiB
Plaintext
// toktest — functional-equivalence pin for [[tokname]], [[kwlookup]]
|
|
// (struct fold S1) and [[tokprint]]/fputq (struct fold S9, the
|
|
// if-ladder → switch folds in tok.ww).
|
|
// Run with `ww run -I lib/ww/lex lib/ww/lex/toktest.ww`.
|
|
//
|
|
// tokname is checked against every tkind value (the full ladder the
|
|
// switch replaced, plus the unknown-kind fallback); kwlookup is
|
|
// checked against every keyword it recognises plus non-keyword and
|
|
// near-miss (prefix/superstring/exact-width) identifiers.
|
|
// tokprint is driven over a temp file: it pins the kind-dispatch
|
|
// switch (STR/IDENT/ERR vs INT/RUNE vs the value-less default) and,
|
|
// through the STR text, fputq's full escape switch (\\, ", \n, \t,
|
|
// \r, the c<0x20 and c==0x7f \xNN arms, and the printable tail) —
|
|
// branches the 990_selfhost corpus does not exercise (source tokens
|
|
// hold raw `\`+`n`, never a literal control byte).
|
|
// A non-zero exit pinpoints the failing row (signalled + 10), same
|
|
// convention as asciitest. `package main` + bare `import tok/lex`
|
|
// mirrors wwdump (the only other external lex consumer).
|
|
|
|
package main;
|
|
|
|
import os;
|
|
import tok;
|
|
import lex;
|
|
|
|
let signalled: i32 = 0;
|
|
fn fail() void = { os.exit(signalled + 10); };
|
|
|
|
fn checkname(k: tkind, want: str) void = {
|
|
if (tokname(k) != want) { fail(); };
|
|
};
|
|
|
|
// One row per tkind value — same string the if-ladder returned. The
|
|
// final row pins the unknown-kind fallback ("<?>").
|
|
@test fn tokname_cases() void = {
|
|
signalled = 100; checkname(tkind.TK_NONE, "<none>");
|
|
signalled = 101; checkname(tkind.TK_EOF, "EOF");
|
|
signalled = 102; checkname(tkind.TK_ERR, "ERR");
|
|
signalled = 103; checkname(tkind.TK_IDENT, "IDENT");
|
|
signalled = 104; checkname(tkind.TK_INT, "INT");
|
|
signalled = 105; checkname(tkind.TK_FLOAT, "FLOAT");
|
|
signalled = 106; checkname(tkind.TK_RUNE, "RUNE");
|
|
signalled = 107; checkname(tkind.TK_STR, "STR");
|
|
|
|
signalled = 108; checkname(tkind.TK_FN, "fn");
|
|
signalled = 109; checkname(tkind.TK_LET, "let");
|
|
signalled = 110; checkname(tkind.TK_DEF, "def");
|
|
signalled = 111; checkname(tkind.TK_IF, "if");
|
|
signalled = 112; checkname(tkind.TK_ELSE, "else");
|
|
signalled = 113; checkname(tkind.TK_FOR, "for");
|
|
signalled = 114; checkname(tkind.TK_SWITCH, "switch");
|
|
signalled = 115; checkname(tkind.TK_CASE, "case");
|
|
signalled = 116; checkname(tkind.TK_RETURN, "return");
|
|
signalled = 117; checkname(tkind.TK_USE, "import");
|
|
signalled = 118; checkname(tkind.TK_TYPE, "type");
|
|
signalled = 119; checkname(tkind.TK_STRUCT, "struct");
|
|
signalled = 120; checkname(tkind.TK_DEFER, "defer");
|
|
signalled = 121; checkname(tkind.TK_BREAK, "break");
|
|
signalled = 122; checkname(tkind.TK_CONTINUE, "continue");
|
|
signalled = 123; checkname(tkind.TK_EXPORT, "export");
|
|
signalled = 124; checkname(tkind.TK_PROC, "proc");
|
|
signalled = 125; checkname(tkind.TK_CHAN, "chan");
|
|
signalled = 126; checkname(tkind.TK_NIL, "nil");
|
|
signalled = 127; checkname(tkind.TK_TRUE, "true");
|
|
signalled = 128; checkname(tkind.TK_FALSE, "false");
|
|
signalled = 129; checkname(tkind.TK_AS, "as");
|
|
signalled = 130; checkname(tkind.TK_IS, "is");
|
|
signalled = 131; checkname(tkind.TK_VOID, "void");
|
|
signalled = 132; checkname(tkind.TK_YIELD, "yield");
|
|
signalled = 133; checkname(tkind.TK_STATIC, "static");
|
|
signalled = 134; checkname(tkind.TK_MATCH, "match");
|
|
signalled = 135; checkname(tkind.TK_CONST, "const");
|
|
signalled = 136; checkname(tkind.TK_UNDER, "_");
|
|
signalled = 137; checkname(tkind.TK_ENUM, "enum");
|
|
signalled = 138; checkname(tkind.TK_MODULE, "package");
|
|
|
|
signalled = 139; checkname(tkind.TK_LPAREN, "(");
|
|
signalled = 140; checkname(tkind.TK_RPAREN, ")");
|
|
signalled = 141; checkname(tkind.TK_LBRACE, "{");
|
|
signalled = 142; checkname(tkind.TK_RBRACE, "}");
|
|
signalled = 143; checkname(tkind.TK_LBRACK, "[");
|
|
signalled = 144; checkname(tkind.TK_RBRACK, "]");
|
|
signalled = 145; checkname(tkind.TK_COMMA, ",");
|
|
signalled = 146; checkname(tkind.TK_SEMI, ";");
|
|
signalled = 147; checkname(tkind.TK_COLON, ":");
|
|
signalled = 148; checkname(tkind.TK_DOT, ".");
|
|
signalled = 149; checkname(tkind.TK_ELLIPSIS, "...");
|
|
signalled = 150; checkname(tkind.TK_DOTDOT, "..");
|
|
signalled = 151; checkname(tkind.TK_AT, "@");
|
|
signalled = 152; checkname(tkind.TK_QUESTION, "?");
|
|
|
|
signalled = 153; checkname(tkind.TK_ASSIGN, "=");
|
|
signalled = 154; checkname(tkind.TK_PLUSEQ, "+=");
|
|
signalled = 155; checkname(tkind.TK_MINUSEQ, "-=");
|
|
signalled = 156; checkname(tkind.TK_STAREQ, "*=");
|
|
signalled = 157; checkname(tkind.TK_SLASHEQ, "/=");
|
|
signalled = 158; checkname(tkind.TK_PERCENTEQ, "%=");
|
|
signalled = 159; checkname(tkind.TK_AMPEQ, "&=");
|
|
signalled = 160; checkname(tkind.TK_PIPEEQ, "|=");
|
|
signalled = 161; checkname(tkind.TK_CARETEQ, "^=");
|
|
signalled = 162; checkname(tkind.TK_LSHIFTEQ, "<<=");
|
|
signalled = 163; checkname(tkind.TK_RSHIFTEQ, ">>=");
|
|
|
|
signalled = 164; checkname(tkind.TK_PLUS, "+");
|
|
signalled = 165; checkname(tkind.TK_MINUS, "-");
|
|
signalled = 166; checkname(tkind.TK_STAR, "*");
|
|
signalled = 167; checkname(tkind.TK_SLASH, "/");
|
|
signalled = 168; checkname(tkind.TK_PERCENT, "%");
|
|
signalled = 169; checkname(tkind.TK_AMP, "&");
|
|
signalled = 170; checkname(tkind.TK_PIPE, "|");
|
|
signalled = 171; checkname(tkind.TK_CARET, "^");
|
|
signalled = 172; checkname(tkind.TK_TILDE, "~");
|
|
signalled = 173; checkname(tkind.TK_LSHIFT, "<<");
|
|
signalled = 174; checkname(tkind.TK_RSHIFT, ">>");
|
|
|
|
signalled = 175; checkname(tkind.TK_EQ, "==");
|
|
signalled = 176; checkname(tkind.TK_NEQ, "!=");
|
|
signalled = 177; checkname(tkind.TK_LT, "<");
|
|
signalled = 178; checkname(tkind.TK_LE, "<=");
|
|
signalled = 179; checkname(tkind.TK_GT, ">");
|
|
signalled = 180; checkname(tkind.TK_GE, ">=");
|
|
|
|
signalled = 181; checkname(tkind.TK_AND, "&&");
|
|
signalled = 182; checkname(tkind.TK_OR, "||");
|
|
signalled = 183; checkname(tkind.TK_NOT, "!");
|
|
|
|
signalled = 184; checkname(tkind.TK_LARROW, "<-");
|
|
signalled = 185; checkname(tkind.TK_ARROW, "->");
|
|
signalled = 186; checkname(tkind.TK_FATARROW, "=>");
|
|
|
|
signalled = 187; checkname(tkind.TK_LAST, "<last>");
|
|
|
|
// Unknown kind → the post-switch fallback. TK_LAST is the highest
|
|
// named value (87); 88 is out of band, exercising the "<?>" tail.
|
|
signalled = 188; checkname(88: tkind, "<?>");
|
|
};
|
|
|
|
fn checkkw(s: str, want: tkind) void = {
|
|
if (kwlookup(s.ptr, s.len) != want) { fail(); };
|
|
};
|
|
|
|
// Table-driven (parallel-array idiom; tuple rows blocked by #111). The
|
|
// kw rows pin all 30 keywords kwlookup recognises, 1:1 with the
|
|
// kwnames/kwkinds table (and cmd/wcc/tok.c:18-47), incl. the two remaps
|
|
// import->TK_USE and package->TK_MODULE. The nk rows pin the
|
|
// fall-through to TK_NONE. signalled = base + row keeps a failure
|
|
// pinpointable. Explicit dims, NOT [_]: [_] static-init silently
|
|
// miscompiles to a zero-length array in-tree (probe, cc69daf), which
|
|
// would void the loop body — the very hole a table test must not have.
|
|
@test fn kwlookup_cases() void = {
|
|
let kwin: [30]str = [
|
|
"as", "break", "case", "chan", "const", "continue", "def", "defer",
|
|
"else", "enum", "export", "false", "fn", "for", "if", "is",
|
|
"import", "let", "match", "nil", "package", "proc", "return",
|
|
"static", "struct", "switch", "true", "type", "void", "yield",
|
|
];
|
|
let kwexp: [30]tkind = [
|
|
tkind.TK_AS, tkind.TK_BREAK, tkind.TK_CASE, tkind.TK_CHAN,
|
|
tkind.TK_CONST, tkind.TK_CONTINUE, tkind.TK_DEF, tkind.TK_DEFER,
|
|
tkind.TK_ELSE, tkind.TK_ENUM, tkind.TK_EXPORT, tkind.TK_FALSE,
|
|
tkind.TK_FN, tkind.TK_FOR, tkind.TK_IF, tkind.TK_IS,
|
|
tkind.TK_USE, tkind.TK_LET, tkind.TK_MATCH, tkind.TK_NIL,
|
|
tkind.TK_MODULE, tkind.TK_PROC, tkind.TK_RETURN, tkind.TK_STATIC,
|
|
tkind.TK_STRUCT, tkind.TK_SWITCH, tkind.TK_TRUE, tkind.TK_TYPE,
|
|
tkind.TK_VOID, tkind.TK_YIELD,
|
|
];
|
|
let i: i32 = 0;
|
|
for (i < len(kwin)) {
|
|
signalled = 200 + i;
|
|
checkkw(kwin[i], kwexp[i]);
|
|
i += 1;
|
|
};
|
|
|
|
// Non-keywords that must fall through to TK_NONE. Rows exercise the
|
|
// length-guard + bytewise reject inside strings.compare: keyword
|
|
// SUPERSTRINGS (longer, shared prefix), proper PREFIXES of a keyword
|
|
// (shorter, shared leading bytes), exact-length non-keywords at the
|
|
// 2/3/4-byte keyword widths, and Hare bmap "keywords" (alloc/len/
|
|
// size/append/assert) that are deliberately NOT ww lexer keywords.
|
|
let nk: [18]str = [
|
|
"xyzzy", // ordinary identifier
|
|
"fns", // superstring of "fn"
|
|
"ifx", // superstring of "if"
|
|
"iffy", // superstring of "if"
|
|
"form", // superstring of "for"
|
|
"fora", // superstring of "for"
|
|
"asx", // superstring of "as"
|
|
"i", // proper prefix of if / is / import
|
|
"co", // proper prefix of const / continue
|
|
"swit", // proper prefix of switch
|
|
"xx", // len-2 non-kw (as/fn/if/is width)
|
|
"zzz", // len-3 non-kw (def/for/nil width)
|
|
"abcd", // len-4 non-kw (case/enum/true/type width)
|
|
"alloc", // Hare bmap keyword, NOT a ww lexer keyword
|
|
"len", // Hare bmap keyword, NOT a ww lexer keyword
|
|
"size", // Hare bmap keyword, NOT a ww lexer keyword
|
|
"append", // Hare bmap keyword, NOT a ww lexer keyword
|
|
"assert", // Hare bmap keyword, NOT a ww lexer keyword
|
|
];
|
|
let j: i32 = 0;
|
|
for (j < len(nk)) {
|
|
signalled = 230 + j;
|
|
checkkw(nk[j], tkind.TK_NONE);
|
|
j += 1;
|
|
};
|
|
};
|
|
|
|
// checkprint — tokprint `t` to a freshly-rewound fd, read the bytes
|
|
// back, and assert they equal `want`. The fd is RDWR; we lseek to 0
|
|
// before each write so earlier (possibly longer) content past want.len
|
|
// is irrelevant — only want.len bytes from offset 0 are compared.
|
|
fn checkprint(fd: i32, t: *tok, want: str) void = {
|
|
if (os.lseek(fd, 0i64, os.whence.SET) != 0i64) { fail(); };
|
|
tokprint(fd, t);
|
|
if (os.lseek(fd, 0i64, os.whence.SET) != 0i64) { fail(); };
|
|
let rbuf: [256]u8;
|
|
let z: i32 = 0;
|
|
for (z < 256) { rbuf[z] = 0u8; z += 1; };
|
|
let rd: i64 = os.read(fd, &rbuf[0], want.len: u64);
|
|
if (rd != want.len: i64) { fail(); };
|
|
let j: i32 = 0;
|
|
for (j < want.len) {
|
|
if (rbuf[j] != want.ptr[j]) { fail(); };
|
|
j += 1;
|
|
};
|
|
};
|
|
|
|
// One token per fputq/tokprint dispatch branch. The STR text packs
|
|
// every fputq escape: \\ " \n \t \r, a c<0x20 byte (0x01), c==0x7f,
|
|
// and a printable ('A'). `want` spells the exact emitted line.
|
|
@test fn tokprint_cases() void = {
|
|
let flags: os.flag = os.flag.RDWR | os.flag.CREATE | os.flag.TRUNC;
|
|
let fd: i32 = os.open("/tmp/ww_s9_tok.tmp", flags, 384i32);
|
|
if (fd < 0) { fail(); };
|
|
|
|
let t: tok;
|
|
t.file = "t";
|
|
t.line = 1;
|
|
t.col = 1;
|
|
|
|
signalled = 300;
|
|
t.kind = tkind.TK_STR;
|
|
t.text = "\\\"\n\t\r\x01\x7fA";
|
|
checkprint(fd, &t, "t:1:1 STR \"\\\\\\\"\\n\\t\\r\\x01\\x7fA\"\n");
|
|
|
|
signalled = 301;
|
|
t.kind = tkind.TK_IDENT;
|
|
t.text = "name";
|
|
checkprint(fd, &t, "t:1:1 IDENT \"name\"\n");
|
|
|
|
signalled = 302;
|
|
t.kind = tkind.TK_ERR;
|
|
t.text = "oops";
|
|
checkprint(fd, &t, "t:1:1 ERR \"oops\"\n");
|
|
|
|
signalled = 303;
|
|
t.kind = tkind.TK_INT;
|
|
t.uval = 42u64;
|
|
checkprint(fd, &t, "t:1:1 INT 42\n");
|
|
|
|
signalled = 304;
|
|
t.kind = tkind.TK_RUNE;
|
|
t.uval = 65u64;
|
|
checkprint(fd, &t, "t:1:1 RUNE 65\n");
|
|
|
|
// Default arm: a kind outside the switch appends no value.
|
|
signalled = 305;
|
|
t.kind = tkind.TK_NONE;
|
|
checkprint(fd, &t, "t:1:1 <none>\n");
|
|
|
|
if (os.close(fd) != 0) { fail(); };
|
|
if (os.remove("/tmp/ww_s9_tok.tmp") != 0) { fail(); };
|
|
};
|
|
|
|
export fn main() i32 = {
|
|
signalled = 1; tokname_cases();
|
|
signalled = 2; kwlookup_cases();
|
|
signalled = 3; tokprint_cases();
|
|
return 0;
|
|
};
|