// selfhost/cmd/wcc/tok.ww — port of cmd/wcc/tok.c plus the Tkind / // Tok / Pos shapes from cmd/wcc/ww.h. // // Token kind values must stay numerically equal to the C side: the // 990_selfhost test diffs ww-side wwdump output against C-side // wwdump output, byte-for-byte. Reordering this list shifts the // integers and breaks the diff. // // Bottom of file: tokprint, which emits one token per line in a // format identical to cmd/wcc/tok.c:tokprint(). use os; use strconv; // ---- Tkind ------------------------------------------------------------ // Mirror of the C enum in cmd/wcc/ww.h. Don't reorder. def TK_NONE: i32 = 0; def TK_EOF: i32 = 1; def TK_ERR: i32 = 2; def TK_IDENT: i32 = 3; def TK_INT: i32 = 4; def TK_FLOAT: i32 = 5; def TK_RUNE: i32 = 6; def TK_STR: i32 = 7; def TK_FN: i32 = 8; def TK_LET: i32 = 9; def TK_DEF: i32 = 10; def TK_IF: i32 = 11; def TK_ELSE: i32 = 12; def TK_FOR: i32 = 13; def TK_SWITCH: i32 = 14; def TK_CASE: i32 = 15; def TK_RETURN: i32 = 16; def TK_USE: i32 = 17; def TK_TYPE: i32 = 18; def TK_STRUCT: i32 = 19; def TK_DEFER: i32 = 20; def TK_BREAK: i32 = 21; def TK_CONTINUE: i32 = 22; def TK_EXPORT: i32 = 23; def TK_PROC: i32 = 24; def TK_CHAN: i32 = 25; def TK_NIL: i32 = 26; def TK_TRUE: i32 = 27; def TK_FALSE: i32 = 28; def TK_AS: i32 = 29; def TK_STATIC: i32 = 30; def TK_MATCH: i32 = 31; def TK_LPAREN: i32 = 32; def TK_RPAREN: i32 = 33; def TK_LBRACE: i32 = 34; def TK_RBRACE: i32 = 35; def TK_LBRACK: i32 = 36; def TK_RBRACK: i32 = 37; def TK_COMMA: i32 = 38; def TK_SEMI: i32 = 39; def TK_COLON: i32 = 40; def TK_DOT: i32 = 41; def TK_ELLIPSIS: i32 = 42; def TK_DOTDOT: i32 = 43; def TK_AT: i32 = 44; def TK_QUESTION: i32 = 45; def TK_ASSIGN: i32 = 46; def TK_PLUSEQ: i32 = 47; def TK_MINUSEQ: i32 = 48; def TK_STAREQ: i32 = 49; def TK_SLASHEQ: i32 = 50; def TK_PERCENTEQ: i32 = 51; def TK_AMPEQ: i32 = 52; def TK_PIPEEQ: i32 = 53; def TK_CARETEQ: i32 = 54; def TK_LSHIFTEQ: i32 = 55; def TK_RSHIFTEQ: i32 = 56; def TK_PLUS: i32 = 57; def TK_MINUS: i32 = 58; def TK_STAR: i32 = 59; def TK_SLASH: i32 = 60; def TK_PERCENT: i32 = 61; def TK_AMP: i32 = 62; def TK_PIPE: i32 = 63; def TK_CARET: i32 = 64; def TK_TILDE: i32 = 65; def TK_LSHIFT: i32 = 66; def TK_RSHIFT: i32 = 67; def TK_EQ: i32 = 68; def TK_NEQ: i32 = 69; def TK_LT: i32 = 70; def TK_LE: i32 = 71; def TK_GT: i32 = 72; def TK_GE: i32 = 73; def TK_AND: i32 = 74; def TK_OR: i32 = 75; def TK_NOT: i32 = 76; def TK_LARROW: i32 = 77; def TK_ARROW: i32 = 78; def TK_FATARROW: i32 = 79; def TK_LAST: i32 = 80; // ---- Pos / Tok -------------------------------------------------------- // // `pos` is used at error-reporting boundaries; we always pass it via // *pos so the value never gets struct-copied (w6c can't yet copy a // 24-byte struct). // // `tok` is flat — file/line/col live directly on the token rather than // nested inside a `pos` field. Same reason: nested struct field // assignment isn't supported, and flat primitives are. type pos = struct { file: str, line: i32, col: i32, }; type tok = struct { kind: i32, file: str, // path of the source the token came from line: i32, col: i32, text: str, // arena-owned token text (TK_IDENT, TK_STR, TK_ERR) uval: u64, // TK_INT, TK_RUNE fval: f64, // TK_FLOAT tsuffix: str, // typed numeric literal suffix or empty }; // ---- keyword lookup --------------------------------------------------- fn streq_n(a: *u8, b: str, n: i32) bool = { if (b.len != n) { return false; }; let i: i32 = 0; for (i < n) { if (a[i] != b[i]) { return false; }; i += 1; }; return true; }; // kwlookup — returns the matching TK_* keyword kind for a byte run, // or TK_NONE if it's an ordinary identifier. Linear search over a // small alphabetised list, matching cmd/wcc/tok.c. export fn kwlookup(p: *u8, n: i32) i32 = { if (streq_n(p, "as", n)) { return TK_AS; }; if (streq_n(p, "break", n)) { return TK_BREAK; }; if (streq_n(p, "case", n)) { return TK_CASE; }; if (streq_n(p, "chan", n)) { return TK_CHAN; }; if (streq_n(p, "continue", n)) { return TK_CONTINUE; }; if (streq_n(p, "def", n)) { return TK_DEF; }; if (streq_n(p, "defer", n)) { return TK_DEFER; }; if (streq_n(p, "else", n)) { return TK_ELSE; }; if (streq_n(p, "export", n)) { return TK_EXPORT; }; if (streq_n(p, "false", n)) { return TK_FALSE; }; if (streq_n(p, "fn", n)) { return TK_FN; }; if (streq_n(p, "for", n)) { return TK_FOR; }; if (streq_n(p, "if", n)) { return TK_IF; }; if (streq_n(p, "let", n)) { return TK_LET; }; if (streq_n(p, "match", n)) { return TK_MATCH; }; if (streq_n(p, "nil", n)) { return TK_NIL; }; if (streq_n(p, "proc", n)) { return TK_PROC; }; if (streq_n(p, "return", n)) { return TK_RETURN; }; if (streq_n(p, "static", n)) { return TK_STATIC; }; if (streq_n(p, "struct", n)) { return TK_STRUCT; }; if (streq_n(p, "switch", n)) { return TK_SWITCH; }; if (streq_n(p, "true", n)) { return TK_TRUE; }; if (streq_n(p, "type", n)) { return TK_TYPE; }; if (streq_n(p, "use", n)) { return TK_USE; }; return TK_NONE; }; // ---- tokname ---------------------------------------------------------- // // Returns the canonical printable spelling for a token kind. Matches // the C tokname()'s output exactly so wwdump output diffs cleanly. export fn tokname(k: i32) str = { if (k == TK_NONE) { return ""; }; if (k == TK_EOF) { return "EOF"; }; if (k == TK_ERR) { return "ERR"; }; if (k == TK_IDENT) { return "IDENT"; }; if (k == TK_INT) { return "INT"; }; if (k == TK_FLOAT) { return "FLOAT"; }; if (k == TK_RUNE) { return "RUNE"; }; if (k == TK_STR) { return "STR"; }; if (k == TK_FN) { return "fn"; }; if (k == TK_LET) { return "let"; }; if (k == TK_DEF) { return "def"; }; if (k == TK_IF) { return "if"; }; if (k == TK_ELSE) { return "else"; }; if (k == TK_FOR) { return "for"; }; if (k == TK_SWITCH) { return "switch"; }; if (k == TK_CASE) { return "case"; }; if (k == TK_RETURN) { return "return"; }; if (k == TK_USE) { return "use"; }; if (k == TK_TYPE) { return "type"; }; if (k == TK_STRUCT) { return "struct"; }; if (k == TK_DEFER) { return "defer"; }; if (k == TK_BREAK) { return "break"; }; if (k == TK_CONTINUE) { return "continue"; }; if (k == TK_EXPORT) { return "export"; }; if (k == TK_PROC) { return "proc"; }; if (k == TK_CHAN) { return "chan"; }; if (k == TK_NIL) { return "nil"; }; if (k == TK_TRUE) { return "true"; }; if (k == TK_FALSE) { return "false"; }; if (k == TK_AS) { return "as"; }; if (k == TK_STATIC) { return "static"; }; if (k == TK_MATCH) { return "match"; }; if (k == TK_LPAREN) { return "("; }; if (k == TK_RPAREN) { return ")"; }; if (k == TK_LBRACE) { return "{"; }; if (k == TK_RBRACE) { return "}"; }; if (k == TK_LBRACK) { return "["; }; if (k == TK_RBRACK) { return "]"; }; if (k == TK_COMMA) { return ","; }; if (k == TK_SEMI) { return ";"; }; if (k == TK_COLON) { return ":"; }; if (k == TK_DOT) { return "."; }; if (k == TK_ELLIPSIS) { return "..."; }; if (k == TK_DOTDOT) { return ".."; }; if (k == TK_AT) { return "@"; }; if (k == TK_QUESTION) { return "?"; }; if (k == TK_ASSIGN) { return "="; }; if (k == TK_PLUSEQ) { return "+="; }; if (k == TK_MINUSEQ) { return "-="; }; if (k == TK_STAREQ) { return "*="; }; if (k == TK_SLASHEQ) { return "/="; }; if (k == TK_PERCENTEQ) { return "%="; }; if (k == TK_AMPEQ) { return "&="; }; if (k == TK_PIPEEQ) { return "|="; }; if (k == TK_CARETEQ) { return "^="; }; if (k == TK_LSHIFTEQ) { return "<<="; }; if (k == TK_RSHIFTEQ) { return ">>="; }; if (k == TK_PLUS) { return "+"; }; if (k == TK_MINUS) { return "-"; }; if (k == TK_STAR) { return "*"; }; if (k == TK_SLASH) { return "/"; }; if (k == TK_PERCENT) { return "%"; }; if (k == TK_AMP) { return "&"; }; if (k == TK_PIPE) { return "|"; }; if (k == TK_CARET) { return "^"; }; if (k == TK_TILDE) { return "~"; }; if (k == TK_LSHIFT) { return "<<"; }; if (k == TK_RSHIFT) { return ">>"; }; if (k == TK_EQ) { return "=="; }; if (k == TK_NEQ) { return "!="; }; if (k == TK_LT) { return "<"; }; if (k == TK_LE) { return "<="; }; if (k == TK_GT) { return ">"; }; if (k == TK_GE) { return ">="; }; if (k == TK_AND) { return "&&"; }; if (k == TK_OR) { return "||"; }; if (k == TK_NOT) { return "!"; }; if (k == TK_LARROW) { return "<-"; }; if (k == TK_ARROW) { return "->"; }; if (k == TK_FATARROW) { return "=>"; }; if (k == TK_LAST) { return ""; }; return ""; }; // ---- writer for tokprint ---------------------------------------------- // // fputq mirrors cmd/wcc/tok.c:fputq — quote the string with C-style // escapes for \, ", \n, \t, \r and \xNN for other non-printables. fn fputc_byte(fd: i32, b: u8) void = { let buf: [1]u8; buf[0] = b; os.write(fd, buf.ptr, 1u64); }; fn fputs_str(fd: i32, s: str) void = { os.write(fd, s.ptr, s.len: u64); }; fn hexchar(n: u8) u8 = { if (n < 10u8) { return n + 48u8; }; // '0'..'9' return (n - 10u8) + 97u8; // 'a'..'f' }; fn fputhex2(fd: i32, b: u8) void = { let out: [4]u8; out[0] = 92u8; // '\\' out[1] = 120u8; // 'x' out[2] = hexchar(b >> 4u8); out[3] = hexchar(b & 15u8); os.write(fd, out.ptr, 4u64); }; fn fputq(fd: i32, p: *u8, n: i32) void = { fputc_byte(fd, 34u8); // '"' let i: i32 = 0; for (i < n) { let c: u8 = p[i]; if (c == 92u8) { // '\\' fputs_str(fd, "\\\\"); } else { if (c == 34u8) { // '"' fputs_str(fd, "\\\""); } else { if (c == 10u8) { // '\n' fputs_str(fd, "\\n"); } else { if (c == 9u8) { // '\t' fputs_str(fd, "\\t"); } else { if (c == 13u8) { // '\r' fputs_str(fd, "\\r"); } else { if (c < 32u8) { fputhex2(fd, c); } else { if (c == 127u8) { fputhex2(fd, c); } else { fputc_byte(fd, c); }; }; }; }; }; }; }; i += 1; }; fputc_byte(fd, 34u8); }; // tokprint — write one token line to fd. Format must match // cmd/wcc/tok.c:tokprint() byte-for-byte: that's the diff anchor. // ":: [ ]\n" // // Takes `t` by pointer because w6c can't yet pass a >16-byte struct // by value; the C version takes Tok by value. export fn tokprint(fd: i32, t: *tok) void = { // Chained-dot field reads (`t.x.y`) on str sub-fields aren't yet // reduced by w6c — `t.x.y` returns the whole str. Lift the str // fields into locals so we can use the str pseudo-field path. let tfile: str = t.file; let ttext: str = t.text; if (tfile.len > 0) { fputs_str(fd, tfile); } else { fputs_str(fd, ""); }; fputc_byte(fd, 58u8); // ':' let buf: [32]u8; let n: i32 = strconv.i64toa(buf[0:32], t.line: i64); os.write(fd, buf.ptr, n: u64); fputc_byte(fd, 58u8); n = strconv.i64toa(buf[0:32], t.col: i64); os.write(fd, buf.ptr, n: u64); fputc_byte(fd, 32u8); // ' ' fputs_str(fd, tokname(t.kind)); if (t.kind == TK_IDENT) { fputc_byte(fd, 32u8); fputq(fd, ttext.ptr, ttext.len); } else { if (t.kind == TK_STR) { fputc_byte(fd, 32u8); fputq(fd, ttext.ptr, ttext.len); } else { if (t.kind == TK_ERR) { fputc_byte(fd, 32u8); fputq(fd, ttext.ptr, ttext.len); } else { if (t.kind == TK_INT) { fputc_byte(fd, 32u8); n = strconv.u64toa(buf[0:32], t.uval); os.write(fd, buf.ptr, n: u64); } else { if (t.kind == TK_RUNE) { fputc_byte(fd, 32u8); n = strconv.u64toa(buf[0:32], t.uval); os.write(fd, buf.ptr, n: u64); };};};};}; // TK_FLOAT is intentionally not handled here — %g formatting // won't byte-match across implementations. Diff fixtures must // be float-free until we implement a stable float formatter. fputc_byte(fd, 10u8); // '\n' };