914 lines
24 KiB
Plaintext
914 lines
24 KiB
Plaintext
// Port of cmd/wcc/lex.c.
|
|
//
|
|
// The DFA, the helpers, and the order of decisions all mirror the C
|
|
// version exactly; any divergence surfaces as a cs/ww byte split in
|
|
// the compiler-output identity gates and is a port bug.
|
|
//
|
|
// Calling-convention note: w6c can't yet pass or return structs >16
|
|
// bytes by value, so `tok` and `pos` are passed by pointer (out
|
|
// params). The C version passes `Tok` by value; we differ here only
|
|
// in shape, not in observable behaviour. Token kind values stay
|
|
// numerically identical.
|
|
|
|
package syntax;
|
|
|
|
import os;
|
|
import ascii;
|
|
import strings;
|
|
import strconv;
|
|
import encoding.utf8;
|
|
|
|
// Lexer-local because the "alpha or '_' / alnum or '_'" set isn't part
|
|
// of Hare's ascii::; ascii::isalpha + the '_' check live here instead.
|
|
fn isidstart(c: rune) bool = {
|
|
if (ascii.isalpha(c)) { return true; };
|
|
if (c == '_') { return true; };
|
|
return false;
|
|
};
|
|
|
|
fn isidpart(c: rune) bool = {
|
|
if (ascii.isalnum(c)) { return true; };
|
|
if (c == '_') { return true; };
|
|
return false;
|
|
};
|
|
|
|
fn hexval(c: rune) (i32 | void) = {
|
|
if (ascii.isdigit(c)) { return (c - '0'): i32; };
|
|
if (c >= 'A') {
|
|
if (c <= 'F') { return ((c - 'A') + 10): i32; };
|
|
};
|
|
if (c >= 'a') {
|
|
if (c <= 'f') { return ((c - 'a') + 10): i32; };
|
|
};
|
|
return;
|
|
};
|
|
|
|
export type lex = struct {
|
|
file: str,
|
|
src: *u8, // raw bytes; not necessarily NUL-terminated
|
|
srclen: u64,
|
|
lpos: u64,
|
|
line: i32,
|
|
col: i32,
|
|
errs: i32,
|
|
// a `//ww:module-reset` directive was seen in the last skipped run;
|
|
// lexnext emits TK_MODRESET before the next real token (#16 opt-B).
|
|
modreset: i32,
|
|
// a `//ww:module <path>` directive was seen in the last skipped run;
|
|
// lexnext emits TK_MODPATH carrying this dotted path (M1 #22).
|
|
modpathset: i32,
|
|
modpath: str,
|
|
// a `//ww:module-reset <path>` directive was seen; the next TK_MODRESET
|
|
// carries this dotted path so the sep primary body mangles on the path,
|
|
// not its leaf clause (#57).
|
|
modresetpathset: i32,
|
|
modresetpath: str,
|
|
};
|
|
|
|
export fn lexinit(l: *lex, file: str, src: *u8, len: u64) void = {
|
|
l.file = file;
|
|
l.src = src;
|
|
l.srclen = len;
|
|
l.lpos = 0u64;
|
|
l.line = 1;
|
|
l.col = 1;
|
|
l.errs = 0;
|
|
l.modreset = 0;
|
|
l.modpathset = 0;
|
|
l.modresetpathset = 0;
|
|
};
|
|
|
|
fn srcb(l: *lex, off: u64) i32 = {
|
|
let i: i32 = off: i32;
|
|
let b: u8 = l.src[i];
|
|
return b: i32;
|
|
};
|
|
|
|
fn lpeek(l: *lex, ahead: u64) i32 = {
|
|
let p: u64 = l.lpos + ahead;
|
|
if (p >= l.srclen) { return -1; };
|
|
return srcb(l, p);
|
|
};
|
|
|
|
fn lget(l: *lex) i32 = {
|
|
if (l.lpos >= l.srclen) { return -1; };
|
|
let c: i32 = srcb(l, l.lpos);
|
|
l.lpos += 1u64;
|
|
if (c == '\n') {
|
|
l.line += 1;
|
|
l.col = 1;
|
|
} else {
|
|
l.col += 1;
|
|
};
|
|
return c;
|
|
};
|
|
|
|
fn curpos(l: *lex, out: *pos) void = {
|
|
out.file = l.file;
|
|
out.line = l.line;
|
|
out.col = l.col;
|
|
};
|
|
|
|
fn errat(l: *lex, p: *pos, msg: str) void = {
|
|
let pf: str = p.file;
|
|
os.write(2, pf.ptr, pf.len: u64);
|
|
os.write(2, ":".ptr, 1u64);
|
|
let ls: str = strconv.u64tos(p.line: u64, strconv.base.DEC);
|
|
os.write(2, ls.ptr, ls.len: u64);
|
|
os.write(2, ":".ptr, 1u64);
|
|
let cs: str = strconv.u64tos(p.col: u64, strconv.base.DEC);
|
|
os.write(2, cs.ptr, cs.len: u64);
|
|
os.write(2, ": error: ".ptr, 9u64);
|
|
os.write(2, msg.ptr, msg.len: u64);
|
|
os.write(2, "\n".ptr, 1u64);
|
|
l.errs += 1;
|
|
};
|
|
|
|
fn skipws(l: *lex) bool = {
|
|
for (true) {
|
|
let c: i32 = lpeek(l, 0u64);
|
|
if (c < 0) { return false; };
|
|
if (c == ' ') { lget(l); continue; };
|
|
if (c == '\t') { lget(l); continue; };
|
|
if (c == '\r') { lget(l); continue; };
|
|
if (c == '\n') { lget(l); continue; };
|
|
if (c == '/') {
|
|
let c2: i32 = lpeek(l, 1u64);
|
|
if (c2 == '/') {
|
|
lget(l); lget(l);
|
|
// #16 opt-B: recognize the driver's curmod-reset
|
|
// boundary directive `//ww:module-reset` (whole
|
|
// line) and flag it; lexnext emits TK_MODRESET.
|
|
// The body is then skipped like any comment.
|
|
// Mirrors cstage lex.c skipws. Compare via lpeek
|
|
// (no consume) so the skip loop below is unchanged.
|
|
let pre: str = "ww:module";
|
|
let di: i32 = 0;
|
|
let matched: bool = true;
|
|
for (di < pre.len) {
|
|
if (lpeek(l, di: u64) != pre[di]: i32) {
|
|
matched = false; break;
|
|
};
|
|
di += 1;
|
|
};
|
|
if (matched) {
|
|
let nx: i32 = lpeek(l, pre.len: u64);
|
|
if (nx == '-') {
|
|
let rest: str = "-reset";
|
|
let rj: i32 = 0;
|
|
let rm: bool = true;
|
|
for (rj < rest.len) {
|
|
if (lpeek(l, (pre.len + rj): u64)
|
|
!= rest[rj]: i32) {
|
|
rm = false; break;
|
|
};
|
|
rj += 1;
|
|
};
|
|
if (rm) {
|
|
let af: i32 = lpeek(l,
|
|
(pre.len + rest.len): u64);
|
|
if (af == '\n') { l.modreset = 1; l.modpathset = 0; } // #9: reset supersedes pending path (empty module body)
|
|
else { if (af < 0) { l.modreset = 1; l.modpathset = 0; } // #9: see above
|
|
else { if (af == ' ' || af == '\t') {
|
|
// `//ww:module-reset <path>` — sep
|
|
// primary body tagged by its full
|
|
// dotted import path (#57).
|
|
let k: u64 =
|
|
(pre.len + rest.len): u64;
|
|
for (true) {
|
|
let sc: i32 = lpeek(l, k);
|
|
if (sc == ' ' || sc == '\t') {
|
|
k += 1u64; continue;
|
|
};
|
|
break;
|
|
};
|
|
let s0: u64 = k;
|
|
for (true) {
|
|
let pc: i32 = lpeek(l, k);
|
|
if (pc < 0) { break; };
|
|
if (pc == '\n' || pc == '\r'
|
|
|| pc == ' '
|
|
|| pc == '\t') {
|
|
break;
|
|
};
|
|
k += 1u64;
|
|
};
|
|
l.modreset = 1;
|
|
l.modpathset = 0; // #9: see above — clear pending path
|
|
if (k > s0) {
|
|
let view: str;
|
|
view.ptr =
|
|
l.src + l.lpos + s0;
|
|
view.len = (k - s0): i32;
|
|
l.modresetpath =
|
|
strings.dup(view);
|
|
l.modresetpathset = 1;
|
|
};
|
|
}; }; };
|
|
};
|
|
} else { if (nx == ' ' || nx == '\t') {
|
|
// `//ww:module <path>` — M1 import boundary.
|
|
let k: u64 = pre.len: u64;
|
|
for (true) {
|
|
let sc: i32 = lpeek(l, k);
|
|
if (sc == ' ' || sc == '\t') {
|
|
k += 1u64; continue;
|
|
};
|
|
break;
|
|
};
|
|
let s0: u64 = k;
|
|
for (true) {
|
|
let pc: i32 = lpeek(l, k);
|
|
if (pc < 0) { break; };
|
|
if (pc == '\n' || pc == '\r'
|
|
|| pc == ' ' || pc == '\t') {
|
|
break;
|
|
};
|
|
k += 1u64;
|
|
};
|
|
if (k > s0) {
|
|
let view: str;
|
|
view.ptr = l.src + l.lpos + s0;
|
|
view.len = (k - s0): i32;
|
|
l.modpath = strings.dup(view);
|
|
l.modpathset = 1;
|
|
};
|
|
}; };
|
|
};
|
|
for (true) {
|
|
let cx: i32 = lpeek(l, 0u64);
|
|
if (cx < 0) { return false; };
|
|
if (cx == '\n') { break; };
|
|
lget(l);
|
|
};
|
|
continue;
|
|
};
|
|
if (c2 == '*') {
|
|
lget(l); lget(l);
|
|
let prev: i32 = -1;
|
|
for (true) {
|
|
let x: i32 = lget(l);
|
|
if (x < 0) {
|
|
let cp: pos;
|
|
curpos(l, &cp);
|
|
errat(l, &cp, "unterminated /* comment");
|
|
return false;
|
|
};
|
|
if (prev == '*') {
|
|
if (x == '/') { break; };
|
|
};
|
|
prev = x;
|
|
};
|
|
continue;
|
|
};
|
|
};
|
|
return true;
|
|
};
|
|
return false;
|
|
};
|
|
|
|
fn parseint(p: *u8, n: u64, base: i32, ok: *bool) u64 = {
|
|
let v: u64 = 0u64;
|
|
let b: u64 = base: u64;
|
|
let cutoff: u64 = ~0u64 / b;
|
|
let cutlim: u64 = ~0u64 % b;
|
|
let got: bool = false;
|
|
let i: u64 = 0u64;
|
|
for (i < n) {
|
|
let ix: i32 = i: i32;
|
|
let c: u8 = p[ix];
|
|
if (c == '_') {
|
|
i += 1u64;
|
|
continue;
|
|
};
|
|
let d: i32 = -1;
|
|
if (c >= 48u8) {
|
|
if (c <= 57u8) { d = (c - 48u8): i32; };
|
|
};
|
|
if (d < 0) {
|
|
if (c >= 97u8) {
|
|
if (c <= 102u8) { d = ((c - 97u8) + 10u8): i32; };
|
|
};
|
|
};
|
|
if (d < 0) {
|
|
if (c >= 65u8) {
|
|
if (c <= 70u8) { d = ((c - 65u8) + 10u8): i32; };
|
|
};
|
|
};
|
|
if (d < 0) { *ok = false; return 0u64; };
|
|
if (d >= base) { *ok = false; return 0u64; };
|
|
if (v > cutoff || (v == cutoff && (d: u64) > cutlim)) {
|
|
*ok = false;
|
|
return 0u64;
|
|
};
|
|
v = v * b + (d: u64);
|
|
got = true;
|
|
i += 1u64;
|
|
};
|
|
*ok = got;
|
|
return v;
|
|
};
|
|
|
|
// lexunicode — shared escape decoder for \xHH (n=2), \uHHHH (n=4),
|
|
// \UHHHHHHHH (n=8). All three yield a codepoint, not a raw byte —
|
|
// mirrors ref/hare/hare/lex/lex.ha:347 fn lex_unicode. Error strings
|
|
// copied verbatim from that reference for diagnostic fidelity (#50).
|
|
fn lexunicode(l: *lex, n: i32, out: *i32) bool = {
|
|
// u32 (not i32): an 8-digit \U with bit 31 set would go negative
|
|
// in i32 and slip past the `> 0x10FFFF` range check — cstage uses
|
|
// u32 here, so i32 would diverge (rule 10).
|
|
let u: u32 = 0u32;
|
|
let i: i32 = 0;
|
|
for (i < n) {
|
|
let c: i32 = lget(l);
|
|
if (c < 0) {
|
|
let cp: pos; curpos(l, &cp);
|
|
errat(l, &cp, "unexpected EOF scanning for escape");
|
|
return false;
|
|
};
|
|
if (!ascii.isxdigit(c: rune)) {
|
|
let cp: pos; curpos(l, &cp);
|
|
errat(l, &cp, "unexpected rune scanning for escape");
|
|
return false;
|
|
};
|
|
let d: i32 = hexval(c: rune)!;
|
|
u = (u << 4u32) | (d: u32);
|
|
i += 1;
|
|
};
|
|
if (u > 0x10FFFFu32) {
|
|
let cp: pos; curpos(l, &cp);
|
|
errat(l, &cp, "invalid unicode codepoint in escape");
|
|
return false;
|
|
};
|
|
if (u >= 0xD800u32) {
|
|
if (u < 0xE000u32) {
|
|
let cp: pos; curpos(l, &cp);
|
|
errat(l, &cp, "invalid unicode codepoint in escape");
|
|
return false;
|
|
};
|
|
};
|
|
*out = u: i32;
|
|
return true;
|
|
};
|
|
|
|
fn escape(l: *lex, out: *i32) bool = {
|
|
let c: i32 = lget(l);
|
|
if (c < 0) { return false; };
|
|
if (c == 'n') { *out = '\n'; return true; };
|
|
if (c == 't') { *out = '\t'; return true; };
|
|
if (c == 'r') { *out = '\r'; return true; };
|
|
if (c == '\\') { *out = '\\'; return true; };
|
|
if (c == '\'') { *out = '\''; return true; };
|
|
if (c == '"') { *out = '"'; return true; };
|
|
if (c == '0') { *out = '\0'; return true; };
|
|
if (c == 'a') { *out = '\a'; return true; };
|
|
if (c == 'b') { *out = '\b'; return true; };
|
|
if (c == 'f') { *out = '\f'; return true; };
|
|
if (c == 'v') { *out = '\v'; return true; };
|
|
if (c == 'x') { return lexunicode(l, 2, out); };
|
|
if (c == 'u') { return lexunicode(l, 4, out); };
|
|
if (c == 'U') { return lexunicode(l, 8, out); };
|
|
let cp: pos; curpos(l, &cp);
|
|
errat(l, &cp, "bad escape");
|
|
return false;
|
|
};
|
|
|
|
fn scandecimalrun(l: *lex) void = {
|
|
for (true) {
|
|
let c: i32 = lpeek(l, 0u64);
|
|
if (c < 0) { break; };
|
|
if (!ascii.isdigit(c: rune)) {
|
|
if (c != '_') { break; };
|
|
};
|
|
lget(l);
|
|
};
|
|
};
|
|
|
|
fn scanhexrun(l: *lex) void = {
|
|
for (true) {
|
|
let c: i32 = lpeek(l, 0u64);
|
|
if (c < 0) { break; };
|
|
if (!ascii.isxdigit(c: rune)) {
|
|
if (c != '_') { break; };
|
|
};
|
|
lget(l);
|
|
};
|
|
};
|
|
|
|
fn scanbinrun(l: *lex) void = {
|
|
for (true) {
|
|
let c: i32 = lpeek(l, 0u64);
|
|
if (c == '0') { lget(l); continue; };
|
|
if (c == '1') { lget(l); continue; };
|
|
if (c == '_') { lget(l); continue; };
|
|
break;
|
|
};
|
|
};
|
|
|
|
fn scanoctrun(l: *lex) void = {
|
|
for (true) {
|
|
let c: i32 = lpeek(l, 0u64);
|
|
if (c < '0') { break; };
|
|
if (c > '7') {
|
|
if (c != '_') { break; };
|
|
};
|
|
lget(l);
|
|
};
|
|
};
|
|
|
|
fn scanexp(l: *lex) void = {
|
|
let e: i32 = lpeek(l, 0u64);
|
|
if (e != 'e') { if (e != 'E') { return; }; };
|
|
lget(l);
|
|
let s: i32 = lpeek(l, 0u64);
|
|
if (s == '+') { lget(l); }
|
|
else { if (s == '-') { lget(l); }; };
|
|
for (true) {
|
|
let c: i32 = lpeek(l, 0u64);
|
|
if (c < 0) { break; };
|
|
if (!ascii.isdigit(c: rune)) { break; };
|
|
lget(l);
|
|
};
|
|
};
|
|
|
|
fn lexnum(l: *lex, start: *pos, out: *tok) void = {
|
|
out.kind = tkind.TK_INT;
|
|
out.file = start.file;
|
|
out.line = start.line;
|
|
out.col = start.col;
|
|
let begin: u64 = l.lpos;
|
|
let base: i32 = 10;
|
|
let isfloat: bool = false;
|
|
|
|
let c0: i32 = lpeek(l, 0u64);
|
|
let c1: i32 = lpeek(l, 1u64);
|
|
|
|
if (c0 == '0') {
|
|
if (c1 == 'x') {
|
|
lget(l); lget(l); base = 16; scanhexrun(l);
|
|
} else { if (c1 == 'X') {
|
|
lget(l); lget(l); base = 16; scanhexrun(l);
|
|
} else { if (c1 == 'b') {
|
|
lget(l); lget(l); base = 2; scanbinrun(l);
|
|
} else { if (c1 == 'B') {
|
|
lget(l); lget(l); base = 2; scanbinrun(l);
|
|
} else { if (c1 == 'o') {
|
|
lget(l); lget(l); base = 8; scanoctrun(l);
|
|
} else { if (c1 == 'O') {
|
|
lget(l); lget(l); base = 8; scanoctrun(l);
|
|
} else {
|
|
scandecimalrun(l);
|
|
if (lpeek(l, 0u64) == '.') {
|
|
let after: i32 = lpeek(l, 1u64);
|
|
if (after >= '0') {
|
|
if (after <= '9') {
|
|
isfloat = true;
|
|
lget(l);
|
|
scandecimalrun(l);
|
|
scanexp(l);
|
|
};
|
|
};
|
|
};
|
|
};};};};};};
|
|
} else {
|
|
scandecimalrun(l);
|
|
if (lpeek(l, 0u64) == '.') {
|
|
let after: i32 = lpeek(l, 1u64);
|
|
if (after >= '0') {
|
|
if (after <= '9') {
|
|
isfloat = true;
|
|
lget(l);
|
|
scandecimalrun(l);
|
|
scanexp(l);
|
|
};
|
|
};
|
|
};
|
|
};
|
|
|
|
let n: u64 = l.lpos - begin;
|
|
let view: str;
|
|
view.ptr = l.src + begin;
|
|
view.len = n: i32;
|
|
out.text = strings.dup(view);
|
|
|
|
if (isfloat) {
|
|
out.kind = tkind.TK_FLOAT;
|
|
// Strip underscores from the digits (Hare allows 1_000.5)
|
|
// before parsing — match what cmd/wcc/lex.c does with
|
|
// strtod over a cleaned buffer.
|
|
let clean: []u8 = alloc([], n + 1u64)!;
|
|
let i: u64 = 0u64;
|
|
let j: u64 = 0u64;
|
|
for (i < n) {
|
|
let b: u8 = l.src[begin + i];
|
|
if (b != '_') {
|
|
clean[j] = b;
|
|
j += 1u64;
|
|
};
|
|
i += 1u64;
|
|
};
|
|
clean[j] = 0u8;
|
|
let cleanv: str;
|
|
cleanv.ptr = clean.ptr;
|
|
cleanv.len = j: i32;
|
|
// strconv's correctly-rounded decimal engine — cstage folds
|
|
// via strtod, and a leaner pow-10 fold here was 1-2 ULP off
|
|
// on long-mantissa/extreme literals (cs≠ww DATA bits, #62).
|
|
// `0: f64` cast, not a 0.0 literal: 990's wwdump diff relies
|
|
// on this file tokenising identically through C and ww, and
|
|
// the C dumper %g-formats TK_FLOAT.fval while the ww dumper
|
|
// skips it.
|
|
// Retained divergence (task #21): SUBNORMAL literals are
|
|
// accepted here correctly-rounded (Hare stof semantics)
|
|
// but rejected by cstage (glibc strtod flags partial
|
|
// underflow with ERANGE).
|
|
let fv: f64 = 0: f64;
|
|
match (strconv.stof64(cleanv, strconv.base.DEC)) {
|
|
case let v: f64 => { fv = v; };
|
|
case let e: strconv.invalid => {
|
|
errat(l, start, "bad float literal");
|
|
};
|
|
case let e: strconv.overflow => {
|
|
errat(l, start, "bad float literal");
|
|
};
|
|
};
|
|
out.fval = fv;
|
|
// Stash the IEEE bits in uval — cgen consumers read floats
|
|
// as integers (n.uval) to avoid an SSE round-trip when
|
|
// materialising the constant.
|
|
let pu: *u64 = (&fv): *u64;
|
|
out.uval = *pu;
|
|
} else {
|
|
let digs: *u8 = l.src + begin;
|
|
let dn: u64 = n;
|
|
if (base != 10) {
|
|
digs = digs + 2u64;
|
|
dn -= 2u64;
|
|
};
|
|
let ok: bool = false;
|
|
out.uval = parseint(digs, dn, base, &ok);
|
|
if (!ok) {
|
|
errat(l, start, "bad integer literal");
|
|
out.kind = tkind.TK_ERR;
|
|
};
|
|
};
|
|
|
|
let pc: i32 = lpeek(l, 0u64);
|
|
if (pc >= 0) {
|
|
if (isidstart(pc: rune)) {
|
|
let sb: u64 = l.lpos;
|
|
let sc: i32 = l.col;
|
|
for (true) {
|
|
let cc: i32 = lpeek(l, 0u64);
|
|
if (cc < 0) { break; };
|
|
if (!isidpart(cc: rune)) { break; };
|
|
lget(l);
|
|
};
|
|
let sl: u64 = l.lpos - sb;
|
|
let p: *u8 = l.src + sb;
|
|
let isok: bool = false;
|
|
if (sl == 2u64) {
|
|
if (p[0] == 'i') {
|
|
if (p[1] == '8') { isok = true; }; // i8
|
|
};
|
|
if (p[0] == 'u') {
|
|
if (p[1] == '8') { isok = true; }; // u8
|
|
};
|
|
};
|
|
if (sl == 3u64) {
|
|
if (p[0] == 'i') {
|
|
if (p[1] == '1') { if (p[2] == '6') { isok = true; }; }; // i16
|
|
if (p[1] == '3') { if (p[2] == '2') { isok = true; }; }; // i32
|
|
if (p[1] == '6') { if (p[2] == '4') { isok = true; }; }; // i64
|
|
};
|
|
if (p[0] == 'u') {
|
|
if (p[1] == '1') { if (p[2] == '6') { isok = true; }; };
|
|
if (p[1] == '3') { if (p[2] == '2') { isok = true; }; };
|
|
if (p[1] == '6') { if (p[2] == '4') { isok = true; }; };
|
|
};
|
|
if (p[0] == 'f') {
|
|
if (p[1] == '3') { if (p[2] == '2') { isok = true; }; }; // f32
|
|
if (p[1] == '6') { if (p[2] == '4') { isok = true; }; }; // f64
|
|
};
|
|
};
|
|
if (isok) {
|
|
let view: str;
|
|
view.ptr = p;
|
|
view.len = sl: i32;
|
|
out.tsuffix = strings.dup(view);
|
|
} else {
|
|
l.lpos = sb;
|
|
l.col = sc;
|
|
};
|
|
};
|
|
};
|
|
};
|
|
|
|
fn lexident(l: *lex, start: *pos, out: *tok) void = {
|
|
let begin: u64 = l.lpos;
|
|
for (true) {
|
|
let c: i32 = lpeek(l, 0u64);
|
|
if (c < 0) { break; };
|
|
if (!isidpart(c: rune)) { break; };
|
|
lget(l);
|
|
};
|
|
let n: u64 = l.lpos - begin;
|
|
let p: *u8 = l.src + begin;
|
|
out.file = start.file;
|
|
out.line = start.line;
|
|
out.col = start.col;
|
|
// Bare '_' is the discard marker. `_x`, `_1` are normal idents.
|
|
if (n == 1u64) {
|
|
if (p[0] == '_') {
|
|
out.kind = tkind.TK_UNDER;
|
|
let view: str;
|
|
view.ptr = p;
|
|
view.len = n: i32;
|
|
out.text = strings.dup(view);
|
|
return;
|
|
};
|
|
};
|
|
let k: tkind = kwlookup(p, n: i32);
|
|
if (k != tkind.TK_NONE) {
|
|
out.kind = k;
|
|
} else {
|
|
out.kind = tkind.TK_IDENT;
|
|
};
|
|
let view: str;
|
|
view.ptr = p;
|
|
view.len = n: i32;
|
|
out.text = strings.dup(view);
|
|
};
|
|
|
|
fn lexstr(l: *lex, start: *pos, out: *tok) void = {
|
|
let cap: u64 = 32u64;
|
|
let nb: u64 = 0u64;
|
|
let buf: []u8 = alloc([], cap)!;
|
|
for (true) {
|
|
let c: i32 = lpeek(l, 0u64);
|
|
if (c < 0) {
|
|
errat(l, start, "unterminated string");
|
|
out.kind = tkind.TK_ERR;
|
|
out.file = start.file;
|
|
out.line = start.line;
|
|
out.col = start.col;
|
|
let view: str;
|
|
view.ptr = "".ptr;
|
|
view.len = 0;
|
|
out.text = strings.dup(view);
|
|
return;
|
|
};
|
|
if (c == '"') { lget(l); break; };
|
|
// Escape-decoded values are codepoints and UTF-8-encode into
|
|
// 1-4 bytes (mirrors Hare's memio::appendrune in lex_string,
|
|
// ref/hare/hare/lex/lex.ha:431). Raw source bytes are already
|
|
// UTF-8 and pass through unchanged — re-encoding them would
|
|
// double-encode the >0x7F continuation bytes.
|
|
let enc: [4]u8;
|
|
let el: i32 = 1;
|
|
if (c == '\\') {
|
|
let ch: i32 = 0;
|
|
lget(l);
|
|
if (!escape(l, &ch)) { ch = 0; };
|
|
el = utf8.encoderune(enc, ch: rune);
|
|
} else {
|
|
enc[0] = lget(l): u8;
|
|
el = 1;
|
|
};
|
|
if (nb + (el: u64) >= cap) {
|
|
let ncap: u64 = cap * 2u64;
|
|
for (nb + (el: u64) >= ncap) { ncap = ncap * 2u64; };
|
|
let nb2: []u8 = alloc([], ncap)!;
|
|
let i: u64 = 0u64;
|
|
for (i < nb) {
|
|
let ix: i32 = i: i32;
|
|
nb2[ix] = buf[ix];
|
|
i += 1u64;
|
|
};
|
|
buf = nb2;
|
|
cap = ncap;
|
|
};
|
|
let k: i32 = 0;
|
|
for (k < el) {
|
|
let nbi: i32 = nb: i32;
|
|
buf[nbi] = enc[k];
|
|
nb += 1u64;
|
|
k += 1;
|
|
};
|
|
};
|
|
out.kind = tkind.TK_STR;
|
|
out.file = start.file;
|
|
out.line = start.line;
|
|
out.col = start.col;
|
|
let s: str;
|
|
s.ptr = buf.ptr;
|
|
s.len = nb: i32;
|
|
out.text = s;
|
|
};
|
|
|
|
fn lexrune(l: *lex, start: *pos, out: *tok) void = {
|
|
let c: i32 = lpeek(l, 0u64);
|
|
if (c < 0) {
|
|
errat(l, start, "unterminated rune");
|
|
out.kind = tkind.TK_ERR;
|
|
out.file = start.file;
|
|
out.line = start.line;
|
|
out.col = start.col;
|
|
let view: str;
|
|
view.ptr = "".ptr;
|
|
view.len = 0;
|
|
out.text = strings.dup(view);
|
|
return;
|
|
};
|
|
let ch: i32 = 0;
|
|
if (c == '\\') {
|
|
lget(l);
|
|
if (!escape(l, &ch)) { ch = 0; };
|
|
} else {
|
|
ch = lget(l);
|
|
};
|
|
if (lpeek(l, 0u64) != '\'') {
|
|
errat(l, start, "rune literal missing closing '");
|
|
out.kind = tkind.TK_ERR;
|
|
out.file = start.file;
|
|
out.line = start.line;
|
|
out.col = start.col;
|
|
let view: str;
|
|
view.ptr = "".ptr;
|
|
view.len = 0;
|
|
out.text = strings.dup(view);
|
|
return;
|
|
};
|
|
lget(l);
|
|
out.kind = tkind.TK_RUNE;
|
|
out.file = start.file;
|
|
out.line = start.line;
|
|
out.col = start.col;
|
|
out.uval = ch: u64;
|
|
};
|
|
|
|
fn emitsimple(start: *pos, k: tkind, out: *tok) void = {
|
|
out.kind = k;
|
|
out.file = start.file;
|
|
out.line = start.line;
|
|
out.col = start.col;
|
|
};
|
|
|
|
fn setposfrom(out: *tok, p: *pos) void = {
|
|
out.file = p.file;
|
|
out.line = p.line;
|
|
out.col = p.col;
|
|
};
|
|
|
|
export fn lexnext(l: *lex, out: *tok) void = {
|
|
// Reset the out token so callers can rely on stale fields being
|
|
// cleared (they only inspect kind, pos, text, uval, fval, tsuffix
|
|
// per kind).
|
|
out.kind = tkind.TK_NONE;
|
|
out.uval = 0u64;
|
|
// out.fval starts cleared by the caller's stack-local init (lex.ww
|
|
// allocates the tok with `let t: tok;` which zeroes). We avoid
|
|
// writing a 0.0 literal here so this file itself stays float-free
|
|
// and the C/ww wwdump diff over it is byte-identical.
|
|
let empty: str;
|
|
empty.ptr = nil;
|
|
empty.len = 0;
|
|
out.text = empty;
|
|
out.tsuffix = empty;
|
|
|
|
let more: bool = skipws(l);
|
|
let start: pos; curpos(l, &start);
|
|
// A `//ww:module-reset` seen in the skipped run surfaces as its own
|
|
// token before the next real one (#16 opt-B boundary reset).
|
|
if (l.modreset != 0) {
|
|
l.modreset = 0;
|
|
emitsimple(&start, tkind.TK_MODRESET, out);
|
|
// path-carrying reset → text=path (#57); bare reset → text empty
|
|
if (l.modresetpathset != 0) {
|
|
l.modresetpathset = 0;
|
|
out.text = l.modresetpath;
|
|
};
|
|
return;
|
|
};
|
|
if (l.modpathset != 0) {
|
|
l.modpathset = 0;
|
|
emitsimple(&start, tkind.TK_MODPATH, out);
|
|
out.text = l.modpath;
|
|
return;
|
|
};
|
|
if (!more) {
|
|
emitsimple(&start, tkind.TK_EOF, out);
|
|
return;
|
|
};
|
|
let c: i32 = lpeek(l, 0u64);
|
|
|
|
if (c >= 0) {
|
|
if (isidstart(c: rune)) { lexident(l, &start, out); return; };
|
|
if (ascii.isdigit(c: rune)) { lexnum(l, &start, out); return; };
|
|
};
|
|
|
|
if (c == '"') { lget(l); lexstr(l, &start, out); return; };
|
|
if (c == '\'') { lget(l); lexrune(l, &start, out); return; };
|
|
|
|
lget(l);
|
|
|
|
if (c == '(') { emitsimple(&start, tkind.TK_LPAREN, out); return; };
|
|
if (c == ')') { emitsimple(&start, tkind.TK_RPAREN, out); return; };
|
|
if (c == '{') { emitsimple(&start, tkind.TK_LBRACE, out); return; };
|
|
if (c == '}') { emitsimple(&start, tkind.TK_RBRACE, out); return; };
|
|
if (c == '[') { emitsimple(&start, tkind.TK_LBRACK, out); return; };
|
|
if (c == ']') { emitsimple(&start, tkind.TK_RBRACK, out); return; };
|
|
if (c == ',') { emitsimple(&start, tkind.TK_COMMA, out); return; };
|
|
if (c == ';') { emitsimple(&start, tkind.TK_SEMI, out); return; };
|
|
if (c == ':') { emitsimple(&start, tkind.TK_COLON, out); return; };
|
|
if (c == '@') { emitsimple(&start, tkind.TK_AT, out); return; };
|
|
if (c == '?') { emitsimple(&start, tkind.TK_QUESTION, out); return; };
|
|
if (c == '~') { emitsimple(&start, tkind.TK_TILDE, out); return; };
|
|
|
|
if (c == '.') {
|
|
if (lpeek(l, 0u64) == '.') {
|
|
if (lpeek(l, 1u64) == '.') {
|
|
lget(l); lget(l);
|
|
emitsimple(&start, tkind.TK_ELLIPSIS, out); return;
|
|
};
|
|
lget(l);
|
|
emitsimple(&start, tkind.TK_DOTDOT, out); return;
|
|
};
|
|
emitsimple(&start, tkind.TK_DOT, out); return;
|
|
};
|
|
|
|
if (c == '+') {
|
|
if (lpeek(l, 0u64) == '=') { lget(l); emitsimple(&start, tkind.TK_PLUSEQ, out); return; };
|
|
emitsimple(&start, tkind.TK_PLUS, out); return;
|
|
};
|
|
if (c == '-') {
|
|
if (lpeek(l, 0u64) == '=') { lget(l); emitsimple(&start, tkind.TK_MINUSEQ, out); return; };
|
|
if (lpeek(l, 0u64) == '>') { lget(l); emitsimple(&start, tkind.TK_ARROW, out); return; };
|
|
emitsimple(&start, tkind.TK_MINUS, out); return;
|
|
};
|
|
if (c == '*') {
|
|
if (lpeek(l, 0u64) == '=') { lget(l); emitsimple(&start, tkind.TK_STAREQ, out); return; };
|
|
emitsimple(&start, tkind.TK_STAR, out); return;
|
|
};
|
|
if (c == '/') {
|
|
if (lpeek(l, 0u64) == '=') { lget(l); emitsimple(&start, tkind.TK_SLASHEQ, out); return; };
|
|
emitsimple(&start, tkind.TK_SLASH, out); return;
|
|
};
|
|
if (c == '%') {
|
|
if (lpeek(l, 0u64) == '=') { lget(l); emitsimple(&start, tkind.TK_PERCENTEQ, out); return; };
|
|
emitsimple(&start, tkind.TK_PERCENT, out); return;
|
|
};
|
|
if (c == '&') {
|
|
if (lpeek(l, 0u64) == '&') { lget(l); emitsimple(&start, tkind.TK_AND, out); return; };
|
|
if (lpeek(l, 0u64) == '=') { lget(l); emitsimple(&start, tkind.TK_AMPEQ, out); return; };
|
|
emitsimple(&start, tkind.TK_AMP, out); return;
|
|
};
|
|
if (c == '|') {
|
|
if (lpeek(l, 0u64) == '|') { lget(l); emitsimple(&start, tkind.TK_OR, out); return; };
|
|
if (lpeek(l, 0u64) == '=') { lget(l); emitsimple(&start, tkind.TK_PIPEEQ, out); return; };
|
|
emitsimple(&start, tkind.TK_PIPE, out); return;
|
|
};
|
|
if (c == '^') {
|
|
if (lpeek(l, 0u64) == '=') { lget(l); emitsimple(&start, tkind.TK_CARETEQ, out); return; };
|
|
emitsimple(&start, tkind.TK_CARET, out); return;
|
|
};
|
|
if (c == '=') {
|
|
if (lpeek(l, 0u64) == '=') { lget(l); emitsimple(&start, tkind.TK_EQ, out); return; };
|
|
if (lpeek(l, 0u64) == '>') { lget(l); emitsimple(&start, tkind.TK_FATARROW, out); return; };
|
|
emitsimple(&start, tkind.TK_ASSIGN, out); return;
|
|
};
|
|
if (c == '!') {
|
|
if (lpeek(l, 0u64) == '=') { lget(l); emitsimple(&start, tkind.TK_NEQ, out); return; };
|
|
emitsimple(&start, tkind.TK_NOT, out); return;
|
|
};
|
|
if (c == '<') {
|
|
if (lpeek(l, 0u64) == '<') {
|
|
lget(l);
|
|
if (lpeek(l, 0u64) == '=') { lget(l); emitsimple(&start, tkind.TK_LSHIFTEQ, out); return; };
|
|
emitsimple(&start, tkind.TK_LSHIFT, out); return;
|
|
};
|
|
if (lpeek(l, 0u64) == '=') { lget(l); emitsimple(&start, tkind.TK_LE, out); return; };
|
|
if (lpeek(l, 0u64) == '-') { lget(l); emitsimple(&start, tkind.TK_LARROW, out); return; };
|
|
emitsimple(&start, tkind.TK_LT, out); return;
|
|
};
|
|
if (c == '>') {
|
|
if (lpeek(l, 0u64) == '>') {
|
|
lget(l);
|
|
if (lpeek(l, 0u64) == '=') { lget(l); emitsimple(&start, tkind.TK_RSHIFTEQ, out); return; };
|
|
emitsimple(&start, tkind.TK_RSHIFT, out); return;
|
|
};
|
|
if (lpeek(l, 0u64) == '=') { lget(l); emitsimple(&start, tkind.TK_GE, out); return; };
|
|
emitsimple(&start, tkind.TK_GT, out); return;
|
|
};
|
|
|
|
errat(l, &start, "unexpected character");
|
|
out.kind = tkind.TK_ERR;
|
|
setposfrom(out, &start);
|
|
let one: [1]u8;
|
|
one[0] = c: u8;
|
|
let view: str;
|
|
view.ptr = one.ptr;
|
|
view.len = 1;
|
|
out.text = strings.dup(view);
|
|
};
|