Hojun-Cho
be8a662f15
i64tos / u64tos / f64tos return a fresh owned str (caller frees via os.free) instead of writing into a caller-supplied [N]u8. Adds typed variants (i32tos / i16tos / i8tos and u32 / u16 / u8) and the missing base parameter on stoi64 / stou64 + typed parse wrappers. Base values are exported as plain-i32 `def`s (strconv.DEC, strconv.HEX_UPPER, ...) rather than a `base` enum: cross-module `strconv.base.DEC` chains miscompile in the cstage cgen — it emits a memory load through `base(SB)` rather than inlining the constant. The Sdef path resolves correctly, so callers say `strconv.DEC` and both cgens lower to an immediate. Also renames strings.byteindex / rbyteindex to strings.indexbyte / rindexbyte, matching bytes.indexbyte and reserving the Hare name `byteindex` for the future `(str | rune)`-needle shape. fmt drops printint / printlnint / fprintint — those were stand-ins for variadic `fmt::println(42)`; with the owned-str graduation the substitute is one call: `fmt.println(strconv.i64tos(42, strconv.DEC))`. strerror is sketched in a comment but not shipped — match arms over the wider `error = !(invalid | overflow)` union still expose a cstage-vs-wwstage spill divergence.
417 lines
13 KiB
Plaintext
417 lines
13 KiB
Plaintext
// lib/ww/lex/tok.ww — port of cmd/wcc/tok.c plus the Tkind /
|
|
// Tok / Pos shapes from cmd/wcc/ww.h.
|
|
//
|
|
// Token kind values must stay numerically equal to the C side: the
|
|
// 990_selfhost test diffs ww-side wwdump output against C-side
|
|
// wwdump output, byte-for-byte. Reordering this list shifts the
|
|
// integers and breaks the diff.
|
|
//
|
|
// Bottom of file: tokprint, which emits one token per line in a
|
|
// format identical to cmd/wcc/tok.c:tokprint().
|
|
|
|
use os;
|
|
use strconv;
|
|
|
|
// ---- tkind ------------------------------------------------------------
|
|
// Mirror of the C `Tkind` enum in cmd/wcc/ww.h. Numeric values are
|
|
// explicit and must stay in sync — the 990_selfhost test diffs wwdump
|
|
// output against the C side, byte for byte.
|
|
|
|
type tkind = enum i32 {
|
|
TK_NONE = 0,
|
|
TK_EOF = 1,
|
|
TK_ERR = 2,
|
|
TK_IDENT = 3,
|
|
TK_INT = 4,
|
|
TK_FLOAT = 5,
|
|
TK_RUNE = 6,
|
|
TK_STR = 7,
|
|
|
|
TK_FN = 8,
|
|
TK_LET = 9,
|
|
TK_DEF = 10,
|
|
TK_IF = 11,
|
|
TK_ELSE = 12,
|
|
TK_FOR = 13,
|
|
TK_SWITCH = 14,
|
|
TK_CASE = 15,
|
|
TK_RETURN = 16,
|
|
TK_USE = 17,
|
|
TK_TYPE = 18,
|
|
TK_STRUCT = 19,
|
|
TK_DEFER = 20,
|
|
TK_BREAK = 21,
|
|
TK_CONTINUE = 22,
|
|
TK_EXPORT = 23,
|
|
TK_PROC = 24,
|
|
TK_CHAN = 25,
|
|
TK_NIL = 26,
|
|
TK_TRUE = 27,
|
|
TK_FALSE = 28,
|
|
TK_AS = 29,
|
|
TK_STATIC = 30,
|
|
TK_MATCH = 31,
|
|
TK_CONST = 32,
|
|
TK_UNDER = 33,
|
|
|
|
TK_LPAREN = 34,
|
|
TK_RPAREN = 35,
|
|
TK_LBRACE = 36,
|
|
TK_RBRACE = 37,
|
|
TK_LBRACK = 38,
|
|
TK_RBRACK = 39,
|
|
TK_COMMA = 40,
|
|
TK_SEMI = 41,
|
|
TK_COLON = 42,
|
|
TK_DOT = 43,
|
|
TK_ELLIPSIS = 44,
|
|
TK_DOTDOT = 45,
|
|
TK_AT = 46,
|
|
TK_QUESTION = 47,
|
|
|
|
TK_ASSIGN = 48,
|
|
TK_PLUSEQ = 49,
|
|
TK_MINUSEQ = 50,
|
|
TK_STAREQ = 51,
|
|
TK_SLASHEQ = 52,
|
|
TK_PERCENTEQ = 53,
|
|
TK_AMPEQ = 54,
|
|
TK_PIPEEQ = 55,
|
|
TK_CARETEQ = 56,
|
|
TK_LSHIFTEQ = 57,
|
|
TK_RSHIFTEQ = 58,
|
|
|
|
TK_PLUS = 59,
|
|
TK_MINUS = 60,
|
|
TK_STAR = 61,
|
|
TK_SLASH = 62,
|
|
TK_PERCENT = 63,
|
|
TK_AMP = 64,
|
|
TK_PIPE = 65,
|
|
TK_CARET = 66,
|
|
TK_TILDE = 67,
|
|
TK_LSHIFT = 68,
|
|
TK_RSHIFT = 69,
|
|
|
|
TK_EQ = 70,
|
|
TK_NEQ = 71,
|
|
TK_LT = 72,
|
|
TK_LE = 73,
|
|
TK_GT = 74,
|
|
TK_GE = 75,
|
|
|
|
TK_AND = 76,
|
|
TK_OR = 77,
|
|
TK_NOT = 78,
|
|
|
|
TK_LARROW = 79,
|
|
TK_ARROW = 80,
|
|
TK_FATARROW = 81,
|
|
|
|
// Tail-appended values — keeps every prior TK_* numeric value
|
|
// stable for the 990_selfhost byte-diff against the C side.
|
|
TK_IS = 82,
|
|
TK_VOID = 83,
|
|
TK_YIELD = 84,
|
|
TK_ENUM = 85,
|
|
TK_LAST = 86,
|
|
};
|
|
|
|
// ---- Pos / Tok --------------------------------------------------------
|
|
//
|
|
// `pos` is used at error-reporting boundaries; we always pass it via
|
|
// *pos so the value never gets struct-copied (w6c can't yet copy a
|
|
// 24-byte struct).
|
|
//
|
|
// `tok` is flat — file/line/col live directly on the token rather than
|
|
// nested inside a `pos` field. Same reason: nested struct field
|
|
// assignment isn't supported, and flat primitives are.
|
|
|
|
type pos = struct {
|
|
file: str,
|
|
line: i32,
|
|
col: i32,
|
|
};
|
|
|
|
type tok = struct {
|
|
kind: tkind,
|
|
file: str, // path of the source the token came from
|
|
line: i32,
|
|
col: i32,
|
|
text: str, // arena-owned token text (tkind.TK_IDENT, tkind.TK_STR, tkind.TK_ERR)
|
|
uval: u64, // tkind.TK_INT, tkind.TK_RUNE
|
|
fval: f64, // tkind.TK_FLOAT
|
|
tsuffix: str, // typed numeric literal suffix or empty
|
|
};
|
|
|
|
// ---- keyword lookup ---------------------------------------------------
|
|
|
|
fn streqn(a: *u8, b: str, n: i32) bool = {
|
|
if (b.len != n) { return false; };
|
|
let i: i32 = 0;
|
|
for (i < n) {
|
|
if (a[i] != b[i]) { return false; };
|
|
i += 1;
|
|
};
|
|
return true;
|
|
};
|
|
|
|
// kwlookup — returns the matching TK_* keyword kind for a byte run,
|
|
// or tkind.TK_NONE if it's an ordinary identifier. Linear search over a
|
|
// small alphabetised list, matching cmd/wcc/tok.c.
|
|
export fn kwlookup(p: *u8, n: i32) tkind = {
|
|
if (streqn(p, "as", n)) { return tkind.TK_AS; };
|
|
if (streqn(p, "break", n)) { return tkind.TK_BREAK; };
|
|
if (streqn(p, "case", n)) { return tkind.TK_CASE; };
|
|
if (streqn(p, "chan", n)) { return tkind.TK_CHAN; };
|
|
if (streqn(p, "const", n)) { return tkind.TK_CONST; };
|
|
if (streqn(p, "continue", n)) { return tkind.TK_CONTINUE; };
|
|
if (streqn(p, "def", n)) { return tkind.TK_DEF; };
|
|
if (streqn(p, "defer", n)) { return tkind.TK_DEFER; };
|
|
if (streqn(p, "else", n)) { return tkind.TK_ELSE; };
|
|
if (streqn(p, "enum", n)) { return tkind.TK_ENUM; };
|
|
if (streqn(p, "export", n)) { return tkind.TK_EXPORT; };
|
|
if (streqn(p, "false", n)) { return tkind.TK_FALSE; };
|
|
if (streqn(p, "fn", n)) { return tkind.TK_FN; };
|
|
if (streqn(p, "for", n)) { return tkind.TK_FOR; };
|
|
if (streqn(p, "if", n)) { return tkind.TK_IF; };
|
|
if (streqn(p, "is", n)) { return tkind.TK_IS; };
|
|
if (streqn(p, "let", n)) { return tkind.TK_LET; };
|
|
if (streqn(p, "match", n)) { return tkind.TK_MATCH; };
|
|
if (streqn(p, "nil", n)) { return tkind.TK_NIL; };
|
|
if (streqn(p, "proc", n)) { return tkind.TK_PROC; };
|
|
if (streqn(p, "return", n)) { return tkind.TK_RETURN; };
|
|
if (streqn(p, "static", n)) { return tkind.TK_STATIC; };
|
|
if (streqn(p, "struct", n)) { return tkind.TK_STRUCT; };
|
|
if (streqn(p, "switch", n)) { return tkind.TK_SWITCH; };
|
|
if (streqn(p, "true", n)) { return tkind.TK_TRUE; };
|
|
if (streqn(p, "type", n)) { return tkind.TK_TYPE; };
|
|
if (streqn(p, "use", n)) { return tkind.TK_USE; };
|
|
if (streqn(p, "void", n)) { return tkind.TK_VOID; };
|
|
if (streqn(p, "yield", n)) { return tkind.TK_YIELD; };
|
|
return tkind.TK_NONE;
|
|
};
|
|
|
|
// ---- tokname ----------------------------------------------------------
|
|
//
|
|
// Returns the canonical printable spelling for a token kind. Matches
|
|
// the C tokname()'s output exactly so wwdump output diffs cleanly.
|
|
|
|
export fn tokname(k: tkind) str = {
|
|
if (k == tkind.TK_NONE) { return "<none>"; };
|
|
if (k == tkind.TK_EOF) { return "EOF"; };
|
|
if (k == tkind.TK_ERR) { return "ERR"; };
|
|
if (k == tkind.TK_IDENT) { return "IDENT"; };
|
|
if (k == tkind.TK_INT) { return "INT"; };
|
|
if (k == tkind.TK_FLOAT) { return "FLOAT"; };
|
|
if (k == tkind.TK_RUNE) { return "RUNE"; };
|
|
if (k == tkind.TK_STR) { return "STR"; };
|
|
|
|
if (k == tkind.TK_FN) { return "fn"; };
|
|
if (k == tkind.TK_LET) { return "let"; };
|
|
if (k == tkind.TK_DEF) { return "def"; };
|
|
if (k == tkind.TK_IF) { return "if"; };
|
|
if (k == tkind.TK_ELSE) { return "else"; };
|
|
if (k == tkind.TK_FOR) { return "for"; };
|
|
if (k == tkind.TK_SWITCH) { return "switch"; };
|
|
if (k == tkind.TK_CASE) { return "case"; };
|
|
if (k == tkind.TK_RETURN) { return "return"; };
|
|
if (k == tkind.TK_USE) { return "use"; };
|
|
if (k == tkind.TK_TYPE) { return "type"; };
|
|
if (k == tkind.TK_STRUCT) { return "struct"; };
|
|
if (k == tkind.TK_DEFER) { return "defer"; };
|
|
if (k == tkind.TK_BREAK) { return "break"; };
|
|
if (k == tkind.TK_CONTINUE) { return "continue"; };
|
|
if (k == tkind.TK_EXPORT) { return "export"; };
|
|
if (k == tkind.TK_PROC) { return "proc"; };
|
|
if (k == tkind.TK_CHAN) { return "chan"; };
|
|
if (k == tkind.TK_NIL) { return "nil"; };
|
|
if (k == tkind.TK_TRUE) { return "true"; };
|
|
if (k == tkind.TK_FALSE) { return "false"; };
|
|
if (k == tkind.TK_AS) { return "as"; };
|
|
if (k == tkind.TK_IS) { return "is"; };
|
|
if (k == tkind.TK_VOID) { return "void"; };
|
|
if (k == tkind.TK_YIELD) { return "yield"; };
|
|
if (k == tkind.TK_STATIC) { return "static"; };
|
|
if (k == tkind.TK_MATCH) { return "match"; };
|
|
if (k == tkind.TK_CONST) { return "const"; };
|
|
if (k == tkind.TK_UNDER) { return "_"; };
|
|
if (k == tkind.TK_ENUM) { return "enum"; };
|
|
|
|
if (k == tkind.TK_LPAREN) { return "("; };
|
|
if (k == tkind.TK_RPAREN) { return ")"; };
|
|
if (k == tkind.TK_LBRACE) { return "{"; };
|
|
if (k == tkind.TK_RBRACE) { return "}"; };
|
|
if (k == tkind.TK_LBRACK) { return "["; };
|
|
if (k == tkind.TK_RBRACK) { return "]"; };
|
|
if (k == tkind.TK_COMMA) { return ","; };
|
|
if (k == tkind.TK_SEMI) { return ";"; };
|
|
if (k == tkind.TK_COLON) { return ":"; };
|
|
if (k == tkind.TK_DOT) { return "."; };
|
|
if (k == tkind.TK_ELLIPSIS) { return "..."; };
|
|
if (k == tkind.TK_DOTDOT) { return ".."; };
|
|
if (k == tkind.TK_AT) { return "@"; };
|
|
if (k == tkind.TK_QUESTION) { return "?"; };
|
|
|
|
if (k == tkind.TK_ASSIGN) { return "="; };
|
|
if (k == tkind.TK_PLUSEQ) { return "+="; };
|
|
if (k == tkind.TK_MINUSEQ) { return "-="; };
|
|
if (k == tkind.TK_STAREQ) { return "*="; };
|
|
if (k == tkind.TK_SLASHEQ) { return "/="; };
|
|
if (k == tkind.TK_PERCENTEQ) { return "%="; };
|
|
if (k == tkind.TK_AMPEQ) { return "&="; };
|
|
if (k == tkind.TK_PIPEEQ) { return "|="; };
|
|
if (k == tkind.TK_CARETEQ) { return "^="; };
|
|
if (k == tkind.TK_LSHIFTEQ) { return "<<="; };
|
|
if (k == tkind.TK_RSHIFTEQ) { return ">>="; };
|
|
|
|
if (k == tkind.TK_PLUS) { return "+"; };
|
|
if (k == tkind.TK_MINUS) { return "-"; };
|
|
if (k == tkind.TK_STAR) { return "*"; };
|
|
if (k == tkind.TK_SLASH) { return "/"; };
|
|
if (k == tkind.TK_PERCENT) { return "%"; };
|
|
if (k == tkind.TK_AMP) { return "&"; };
|
|
if (k == tkind.TK_PIPE) { return "|"; };
|
|
if (k == tkind.TK_CARET) { return "^"; };
|
|
if (k == tkind.TK_TILDE) { return "~"; };
|
|
if (k == tkind.TK_LSHIFT) { return "<<"; };
|
|
if (k == tkind.TK_RSHIFT) { return ">>"; };
|
|
|
|
if (k == tkind.TK_EQ) { return "=="; };
|
|
if (k == tkind.TK_NEQ) { return "!="; };
|
|
if (k == tkind.TK_LT) { return "<"; };
|
|
if (k == tkind.TK_LE) { return "<="; };
|
|
if (k == tkind.TK_GT) { return ">"; };
|
|
if (k == tkind.TK_GE) { return ">="; };
|
|
|
|
if (k == tkind.TK_AND) { return "&&"; };
|
|
if (k == tkind.TK_OR) { return "||"; };
|
|
if (k == tkind.TK_NOT) { return "!"; };
|
|
|
|
if (k == tkind.TK_LARROW) { return "<-"; };
|
|
if (k == tkind.TK_ARROW) { return "->"; };
|
|
if (k == tkind.TK_FATARROW) { return "=>"; };
|
|
|
|
if (k == tkind.TK_LAST) { return "<last>"; };
|
|
return "<?>";
|
|
};
|
|
|
|
// ---- writer for tokprint ----------------------------------------------
|
|
//
|
|
// fputq mirrors cmd/wcc/tok.c:fputq — quote the string with C-style
|
|
// escapes for \, ", \n, \t, \r and \xNN for other non-printables.
|
|
|
|
fn fputcbyte(fd: i32, b: u8) void = {
|
|
let buf: [1]u8;
|
|
buf[0] = b;
|
|
os.write(fd, buf.ptr, 1u64);
|
|
};
|
|
|
|
fn fputsstr(fd: i32, s: str) void = {
|
|
os.write(fd, s.ptr, s.len: u64);
|
|
};
|
|
|
|
fn hexchar(n: u8) u8 = {
|
|
if (n < 10u8) { return n + 48u8; }; // '0'..'9'
|
|
return (n - 10u8) + 97u8; // 'a'..'f'
|
|
};
|
|
|
|
fn fputhex2(fd: i32, b: u8) void = {
|
|
let out: [4]u8;
|
|
out[0] = 92u8; // '\\'
|
|
out[1] = 120u8; // 'x'
|
|
out[2] = hexchar(b >> 4u8);
|
|
out[3] = hexchar(b & 15u8);
|
|
os.write(fd, out.ptr, 4u64);
|
|
};
|
|
|
|
fn fputq(fd: i32, p: *u8, n: i32) void = {
|
|
fputcbyte(fd, 34u8); // '"'
|
|
let i: i32 = 0;
|
|
for (i < n) {
|
|
let c: u8 = p[i];
|
|
if (c == 92u8) { // '\\'
|
|
fputsstr(fd, "\\\\");
|
|
} else {
|
|
if (c == 34u8) { // '"'
|
|
fputsstr(fd, "\\\"");
|
|
} else {
|
|
if (c == 10u8) { // '\n'
|
|
fputsstr(fd, "\\n");
|
|
} else {
|
|
if (c == 9u8) { // '\t'
|
|
fputsstr(fd, "\\t");
|
|
} else {
|
|
if (c == 13u8) { // '\r'
|
|
fputsstr(fd, "\\r");
|
|
} else {
|
|
if (c < 32u8) {
|
|
fputhex2(fd, c);
|
|
} else {
|
|
if (c == 127u8) {
|
|
fputhex2(fd, c);
|
|
} else {
|
|
fputcbyte(fd, c);
|
|
};
|
|
};
|
|
};
|
|
};
|
|
};
|
|
};
|
|
};
|
|
i += 1;
|
|
};
|
|
fputcbyte(fd, 34u8);
|
|
};
|
|
|
|
// tokprint — write one token line to fd. Format must match
|
|
// cmd/wcc/tok.c:tokprint() byte-for-byte: that's the diff anchor.
|
|
// "<file>:<line>:<col> <kindname>[ <value>]\n"
|
|
//
|
|
// Takes `t` by pointer because w6c can't yet pass a >16-byte struct
|
|
// by value; the C version takes Tok by value.
|
|
export fn tokprint(fd: i32, t: *tok) void = {
|
|
// Chained-dot field reads (`t.x.y`) on str sub-fields aren't yet
|
|
// reduced by w6c — `t.x.y` returns the whole str. Lift the str
|
|
// fields into locals so we can use the str pseudo-field path.
|
|
let tfile: str = t.file;
|
|
let ttext: str = t.text;
|
|
if (tfile.len > 0) {
|
|
fputsstr(fd, tfile);
|
|
} else {
|
|
fputsstr(fd, "<none>");
|
|
};
|
|
fputcbyte(fd, 58u8); // ':'
|
|
let ls: str = strconv.i64tos(t.line: i64, strconv.DEC);
|
|
os.write(fd, ls.ptr, ls.len: u64);
|
|
fputcbyte(fd, 58u8);
|
|
let cs: str = strconv.i64tos(t.col: i64, strconv.DEC);
|
|
os.write(fd, cs.ptr, cs.len: u64);
|
|
fputcbyte(fd, 32u8); // ' '
|
|
fputsstr(fd, tokname(t.kind));
|
|
|
|
if (t.kind == tkind.TK_IDENT) {
|
|
fputcbyte(fd, 32u8);
|
|
fputq(fd, ttext.ptr, ttext.len);
|
|
} else { if (t.kind == tkind.TK_STR) {
|
|
fputcbyte(fd, 32u8);
|
|
fputq(fd, ttext.ptr, ttext.len);
|
|
} else { if (t.kind == tkind.TK_ERR) {
|
|
fputcbyte(fd, 32u8);
|
|
fputq(fd, ttext.ptr, ttext.len);
|
|
} else { if (t.kind == tkind.TK_INT) {
|
|
fputcbyte(fd, 32u8);
|
|
let us: str = strconv.u64tos(t.uval, strconv.DEC);
|
|
os.write(fd, us.ptr, us.len: u64);
|
|
} else { if (t.kind == tkind.TK_RUNE) {
|
|
fputcbyte(fd, 32u8);
|
|
let us: str = strconv.u64tos(t.uval, strconv.DEC);
|
|
os.write(fd, us.ptr, us.len: u64);
|
|
};};};};};
|
|
// tkind.TK_FLOAT is intentionally not handled here — %g formatting
|
|
// won't byte-match across implementations. Diff fixtures must
|
|
// be float-free until we implement a stable float formatter.
|
|
|
|
fputcbyte(fd, 10u8); // '\n'
|
|
};
|