ww/selfhost/cmd/wcc/lex.ww

// selfhost/cmd/wcc/lex.ww — port of cmd/wcc/lex.c.
//
// The DFA, the helpers, and the order of decisions all mirror the C
// version exactly. The 990_selfhost test diffs the resulting token
// stream against the C-side wwdump byte-for-byte; any divergence is
// a port bug.
//
// Calling-convention note: w6c can't yet pass or return structs >16
// bytes by value, so `tok` and `pos` are passed by pointer (out
// params). The C version passes `Tok` by value; we differ here only
// in shape, not in observable behaviour. Token kind values stay
// numerically identical.

use os;
use ascii;
use mem;
use tok;

type lex = struct {
	file:   str,
	src:    *u8,	// raw bytes; not necessarily NUL-terminated
	srclen: u64,
	lpos:   u64,
	line:   i32,
	col:    i32,
	a:      *arena,
	errs:   i32,
};

export fn lexinit(l: *lex, a: *arena, file: str, src: *u8, len: u64) void = {
	l.file = file;
	l.src = src;
	l.srclen = len;
	l.lpos = 0u64;
	l.line = 1;
	l.col = 1;
	l.a = a;
	l.errs = 0;
};

// srcb — byte at offset; helper that lifts the cast out of indexing.
fn srcb(l: *lex, off: u64) i32 = {
	let i: i32 = off: i32;
	let b: u8 = l.src[i];
	return b: i32;
};

fn lpeek(l: *lex, ahead: u64) i32 = {
	let p: u64 = l.lpos + ahead;
	if (p >= l.srclen) { return -1; };
	return srcb(l, p);
};

fn lget(l: *lex) i32 = {
	if (l.lpos >= l.srclen) { return -1; };
	let c: i32 = srcb(l, l.lpos);
	l.lpos += 1u64;
	if (c == 10) {	// '\n'
		l.line += 1;
		l.col = 1;
	} else {
		l.col += 1;
	};
	return c;
};

fn curpos(l: *lex, out: *pos) void = {
	out.file = l.file;
	out.line = l.line;
	out.col = l.col;
};

// putuint — write `v` (signed, but always non-negative here) to fd 2
// in decimal. Standalone so errat doesn't drag in fmt and create a
// dependency cycle with strconv.
fn putuint(fd: i32, v: i32) void = {
	let tmp: [16]u8;
	let i: i32 = 0;
	let n: i32 = v;
	for (n > 0) {
		tmp[i] = ((n % 10) + 48): u8;
		n = n / 10;
		i += 1;
	};
	if (i == 0) { tmp[0] = 48u8; i = 1; };
	let buf: [16]u8;
	let m: i32 = 0;
	for (i > 0) { i -= 1; buf[m] = tmp[i]; m += 1; };
	os.write(fd, buf.ptr, m: u64);
};

fn errat(l: *lex, p: *pos, msg: str) void = {
	let pf: str = p.file;
	os.write(2, pf.ptr, pf.len: u64);
	os.write(2, ":".ptr, 1u64);
	putuint(2, p.line);
	os.write(2, ":".ptr, 1u64);
	putuint(2, p.col);
	os.write(2, ": error: ".ptr, 9u64);
	os.write(2, msg.ptr, msg.len: u64);
	os.write(2, "\n".ptr, 1u64);
	l.errs += 1;
};

fn skipws(l: *lex) bool = {
	for (true) {
		let c: i32 = lpeek(l, 0u64);
		if (c < 0) { return false; };
		if (c == 32) { lget(l); continue; };
		if (c == 9)  { lget(l); continue; };
		if (c == 13) { lget(l); continue; };
		if (c == 10) { lget(l); continue; };
		if (c == 47) {	// '/'
			let c2: i32 = lpeek(l, 1u64);
			if (c2 == 47) {
				for (true) {
					let cx: i32 = lpeek(l, 0u64);
					if (cx < 0) { return false; };
					if (cx == 10) { break; };
					lget(l);
				};
				continue;
			};
			if (c2 == 42) {	// '*'
				lget(l); lget(l);
				let prev: i32 = -1;
				for (true) {
					let x: i32 = lget(l);
					if (x < 0) {
						let cp: pos;
						curpos(l, &cp);
						errat(l, &cp, "unterminated /* comment");
						return false;
					};
					if (prev == 42) {
						if (x == 47) { break; };
					};
					prev = x;
				};
				continue;
			};
		};
		return true;
	};
	return false;
};

fn parseint(p: *u8, n: u64, base: i32, ok: *bool) u64 = {
	let v: u64 = 0u64;
	let got: bool = false;
	let i: u64 = 0u64;
	for (i < n) {
		let ix: i32 = i: i32;
		let c: u8 = p[ix];
		if (c == 95u8) {	// '_'
			i += 1u64;
			continue;
		};
		let d: i32 = -1;
		if (c >= 48u8) {
			if (c <= 57u8) { d = (c - 48u8): i32; };
		};
		if (d < 0) {
			if (c >= 97u8) {
				if (c <= 102u8) { d = ((c - 97u8) + 10u8): i32; };
			};
		};
		if (d < 0) {
			if (c >= 65u8) {
				if (c <= 70u8) { d = ((c - 65u8) + 10u8): i32; };
			};
		};
		if (d < 0) { *ok = false; return 0u64; };
		if (d >= base) { *ok = false; return 0u64; };
		v = v * (base: u64) + (d: u64);
		got = true;
		i += 1u64;
	};
	*ok = got;
	return v;
};

fn escape(l: *lex, out: *i32) bool = {
	let c: i32 = lget(l);
	if (c < 0) { return false; };
	if (c == 110) { *out = 10; return true; };
	if (c == 116) { *out = 9;  return true; };
	if (c == 114) { *out = 13; return true; };
	if (c == 92)  { *out = 92; return true; };
	if (c == 39)  { *out = 39; return true; };
	if (c == 34)  { *out = 34; return true; };
	if (c == 48)  { *out = 0;  return true; };
	if (c == 97)  { *out = 7;  return true; };
	if (c == 98)  { *out = 8;  return true; };
	if (c == 102) { *out = 12; return true; };
	if (c == 118) { *out = 11; return true; };
	if (c == 120) {
		let hi: i32 = lget(l);
		let lo: i32 = lget(l);
		if (hi < 0) { return false; };
		if (lo < 0) { return false; };
		if (!ascii.ishex(hi: u8)) {
			let cp: pos; curpos(l, &cp);
			errat(l, &cp, "bad \\x escape");
			return false;
		};
		if (!ascii.ishex(lo: u8)) {
			let cp: pos; curpos(l, &cp);
			errat(l, &cp, "bad \\x escape");
			return false;
		};
		let h: i32 = ascii.digitval(hi: u8);
		let lv: i32 = ascii.digitval(lo: u8);
		*out = (h << 4) | lv;
		return true;
	};
	let cp: pos; curpos(l, &cp);
	errat(l, &cp, "bad escape");
	return false;
};

// scandecimalrun — consume a run of decimal digits and underscores.
fn scandecimalrun(l: *lex) void = {
	for (true) {
		let c: i32 = lpeek(l, 0u64);
		if (c < 0) { break; };
		if (!ascii.isdigit(c: u8)) {
			if (c != 95) { break; };
		};
		lget(l);
	};
};

fn scanhexrun(l: *lex) void = {
	for (true) {
		let c: i32 = lpeek(l, 0u64);
		if (c < 0) { break; };
		if (!ascii.ishex(c: u8)) {
			if (c != 95) { break; };
		};
		lget(l);
	};
};

fn scanbinrun(l: *lex) void = {
	for (true) {
		let c: i32 = lpeek(l, 0u64);
		if (c == 48) { lget(l); continue; };
		if (c == 49) { lget(l); continue; };
		if (c == 95) { lget(l); continue; };
		break;
	};
};

fn scanoctrun(l: *lex) void = {
	for (true) {
		let c: i32 = lpeek(l, 0u64);
		if (c < 48) { break; };
		if (c > 55) {
			if (c != 95) { break; };
		};
		lget(l);
	};
};

// scanexp — consume the [eE][+-]?[0-9]+ tail of a float, if present.
fn scanexp(l: *lex) void = {
	let e: i32 = lpeek(l, 0u64);
	if (e != 101) { if (e != 69) { return; }; };	// 'e' or 'E'
	lget(l);
	let s: i32 = lpeek(l, 0u64);
	if (s == 43) { lget(l); }
	else { if (s == 45) { lget(l); }; };
	for (true) {
		let c: i32 = lpeek(l, 0u64);
		if (c < 0) { break; };
		if (!ascii.isdigit(c: u8)) { break; };
		lget(l);
	};
};

fn lexnum(l: *lex, start: *pos, out: *tok) void = {
	out.kind = TK_INT;
	out.file = start.file;
	out.line = start.line;
	out.col  = start.col;
	let begin: u64 = l.lpos;
	let base: i32 = 10;
	let isfloat: bool = false;

	let c0: i32 = lpeek(l, 0u64);
	let c1: i32 = lpeek(l, 1u64);

	if (c0 == 48) {	// '0'
		if (c1 == 120) {	// 'x'
			lget(l); lget(l); base = 16; scanhexrun(l);
		} else { if (c1 == 88) {	// 'X'
			lget(l); lget(l); base = 16; scanhexrun(l);
		} else { if (c1 == 98) {	// 'b'
			lget(l); lget(l); base = 2; scanbinrun(l);
		} else { if (c1 == 66) {	// 'B'
			lget(l); lget(l); base = 2; scanbinrun(l);
		} else { if (c1 == 111) {	// 'o'
			lget(l); lget(l); base = 8; scanoctrun(l);
		} else { if (c1 == 79) {	// 'O'
			lget(l); lget(l); base = 8; scanoctrun(l);
		} else {
			scandecimalrun(l);
			if (lpeek(l, 0u64) == 46) {
				let after: i32 = lpeek(l, 1u64);
				if (after >= 48) {
					if (after <= 57) {
						isfloat = true;
						lget(l);
						scandecimalrun(l);
						scanexp(l);
					};
				};
			};
		};};};};};};
	} else {
		scandecimalrun(l);
		if (lpeek(l, 0u64) == 46) {
			let after: i32 = lpeek(l, 1u64);
			if (after >= 48) {
				if (after <= 57) {
					isfloat = true;
					lget(l);
					scandecimalrun(l);
					scanexp(l);
				};
			};
		};
	};

	let n: u64 = l.lpos - begin;
	out.text = astrndup(l.a, l.src + begin, n);

	if (isfloat) {
		// out.fval is already 0 from the top-of-lexnext clear.
		// We don't strtod the literal yet — the diff fixtures we
		// care about are float-free; any TK_FLOAT seen in source
		// gets a placeholder value until we wire a real parser.
		out.kind = TK_FLOAT;
	} else {
		let digs: *u8 = l.src + begin;
		let dn: u64 = n;
		if (base != 10) {
			digs = digs + 2u64;
			dn -= 2u64;
		};
		let ok: bool = false;
		out.uval = parseint(digs, dn, base, &ok);
		if (!ok) {
			errat(l, start, "bad integer literal");
			out.kind = TK_ERR;
		};
	};

	let pc: i32 = lpeek(l, 0u64);
	if (pc >= 0) {
		if (ascii.isidstart(pc: u8)) {
			let sb: u64 = l.lpos;
			for (true) {
				let cc: i32 = lpeek(l, 0u64);
				if (cc < 0) { break; };
				if (!ascii.isidpart(cc: u8)) { break; };
				lget(l);
			};
			let sl: u64 = l.lpos - sb;
			let p: *u8 = l.src + sb;
			let isok: bool = false;
			if (sl == 2u64) {
				if (p[0] == 105u8) {
					if (p[1] == 56u8) { isok = true; };	// i8
				};
				if (p[0] == 117u8) {
					if (p[1] == 56u8) { isok = true; };	// u8
				};
			};
			if (sl == 3u64) {
				if (p[0] == 105u8) {
					if (p[1] == 49u8) { if (p[2] == 54u8) { isok = true; }; };	// i16
					if (p[1] == 51u8) { if (p[2] == 50u8) { isok = true; }; };	// i32
					if (p[1] == 54u8) { if (p[2] == 52u8) { isok = true; }; };	// i64
				};
				if (p[0] == 117u8) {
					if (p[1] == 49u8) { if (p[2] == 54u8) { isok = true; }; };
					if (p[1] == 51u8) { if (p[2] == 50u8) { isok = true; }; };
					if (p[1] == 54u8) { if (p[2] == 52u8) { isok = true; }; };
				};
				if (p[0] == 102u8) {
					if (p[1] == 51u8) { if (p[2] == 50u8) { isok = true; }; };	// f32
					if (p[1] == 54u8) { if (p[2] == 52u8) { isok = true; }; };	// f64
				};
			};
			if (isok) {
				out.tsuffix = astrndup(l.a, p, sl);
			} else {
				l.lpos = sb;
			};
		};
	};
};

fn lexident(l: *lex, start: *pos, out: *tok) void = {
	let begin: u64 = l.lpos;
	for (true) {
		let c: i32 = lpeek(l, 0u64);
		if (c < 0) { break; };
		if (!ascii.isidpart(c: u8)) { break; };
		lget(l);
	};
	let n: u64 = l.lpos - begin;
	let p: *u8 = l.src + begin;
	let k: i32 = kwlookup(p, n: i32);
	out.file = start.file;
	out.line = start.line;
	out.col  = start.col;
	if (k != TK_NONE) {
		out.kind = k;
	} else {
		out.kind = TK_IDENT;
	};
	out.text = astrndup(l.a, p, n);
};

fn lexstr(l: *lex, start: *pos, out: *tok) void = {
	let cap: u64 = 32u64;
	let nb:  u64 = 0u64;
	let buf: *u8 = amalloc(l.a, cap): *u8;
	for (true) {
		let c: i32 = lpeek(l, 0u64);
		if (c < 0) {
			errat(l, start, "unterminated string");
			out.kind = TK_ERR;
			out.file = start.file;
	out.line = start.line;
	out.col  = start.col;
			out.text = astrndup(l.a, "".ptr, 0u64);
			return;
		};
		if (c == 34) { lget(l); break; };
		let ch: i32 = 0;
		if (c == 92) {
			lget(l);
			if (!escape(l, &ch)) { ch = 0; };
		} else {
			ch = lget(l);
		};
		if (nb + 1u64 >= cap) {
			let ncap: u64 = cap * 2u64;
			let nb2: *u8 = amalloc(l.a, ncap): *u8;
			let i: u64 = 0u64;
			for (i < nb) {
				let ix: i32 = i: i32;
				nb2[ix] = buf[ix];
				i += 1u64;
			};
			buf = nb2;
			cap = ncap;
		};
		let nbi: i32 = nb: i32;
		buf[nbi] = ch: u8;
		nb += 1u64;
	};
	out.kind = TK_STR;
	out.file = start.file;
	out.line = start.line;
	out.col  = start.col;
	let s: str;
	s.ptr = buf;
	s.len = nb: i32;
	out.text = s;
};

fn lexrune(l: *lex, start: *pos, out: *tok) void = {
	let c: i32 = lpeek(l, 0u64);
	if (c < 0) {
		errat(l, start, "unterminated rune");
		out.kind = TK_ERR;
		out.file = start.file;
	out.line = start.line;
	out.col  = start.col;
		out.text = astrndup(l.a, "".ptr, 0u64);
		return;
	};
	let ch: i32 = 0;
	if (c == 92) {
		lget(l);
		if (!escape(l, &ch)) { ch = 0; };
	} else {
		ch = lget(l);
	};
	if (lpeek(l, 0u64) != 39) {
		errat(l, start, "rune literal missing closing '");
		out.kind = TK_ERR;
		out.file = start.file;
	out.line = start.line;
	out.col  = start.col;
		out.text = astrndup(l.a, "".ptr, 0u64);
		return;
	};
	lget(l);
	out.kind = TK_RUNE;
	out.file = start.file;
	out.line = start.line;
	out.col  = start.col;
	out.uval = ch: u64;
};

fn emitsimple(start: *pos, k: i32, out: *tok) void = {
	out.kind = k;
	out.file = start.file;
	out.line = start.line;
	out.col  = start.col;
};

// setposfrom — copy file/line/col from a *pos into a tok. Used by
// the err-token path where we already have a pos.
fn setposfrom(out: *tok, p: *pos) void = {
	out.file = p.file;
	out.line = p.line;
	out.col  = p.col;
};

export fn lexnext(l: *lex, out: *tok) void = {
	// Reset the out token so callers can rely on stale fields being
	// cleared (they only inspect kind, pos, text, uval, fval, tsuffix
	// per kind).
	out.kind = TK_NONE;
	out.uval = 0u64;
	// out.fval starts cleared by the caller's stack-local init (lex.ww
	// allocates the tok with `let t: tok;` which zeroes). We avoid
	// writing a 0.0 literal here so this file itself stays float-free
	// and the C/ww wwdump diff over it is byte-identical.
	let empty: str;
	empty.ptr = nil;
	empty.len = 0;
	out.text = empty;
	out.tsuffix = empty;

	if (!skipws(l)) {
		let p: pos; curpos(l, &p);
		emitsimple(&p, TK_EOF, out);
		return;
	};
	let start: pos; curpos(l, &start);
	let c: i32 = lpeek(l, 0u64);

	if (c >= 0) {
		if (ascii.isidstart(c: u8)) { lexident(l, &start, out); return; };
		if (ascii.isdigit(c: u8))   { lexnum(l, &start, out); return; };
	};

	if (c == 34) { lget(l); lexstr(l, &start, out); return; };
	if (c == 39) { lget(l); lexrune(l, &start, out); return; };

	lget(l);

	if (c == 40)  { emitsimple(&start, TK_LPAREN, out); return; };
	if (c == 41)  { emitsimple(&start, TK_RPAREN, out); return; };
	if (c == 123) { emitsimple(&start, TK_LBRACE, out); return; };
	if (c == 125) { emitsimple(&start, TK_RBRACE, out); return; };
	if (c == 91)  { emitsimple(&start, TK_LBRACK, out); return; };
	if (c == 93)  { emitsimple(&start, TK_RBRACK, out); return; };
	if (c == 44)  { emitsimple(&start, TK_COMMA, out); return; };
	if (c == 59)  { emitsimple(&start, TK_SEMI, out); return; };
	if (c == 58)  { emitsimple(&start, TK_COLON, out); return; };
	if (c == 64)  { emitsimple(&start, TK_AT, out); return; };
	if (c == 63)  { emitsimple(&start, TK_QUESTION, out); return; };
	if (c == 126) { emitsimple(&start, TK_TILDE, out); return; };

	if (c == 46) {	// '.'
		if (lpeek(l, 0u64) == 46) {
			if (lpeek(l, 1u64) == 46) {
				lget(l); lget(l);
				emitsimple(&start, TK_ELLIPSIS, out); return;
			};
			lget(l);
			emitsimple(&start, TK_DOTDOT, out); return;
		};
		emitsimple(&start, TK_DOT, out); return;
	};

	if (c == 43) {
		if (lpeek(l, 0u64) == 61) { lget(l); emitsimple(&start, TK_PLUSEQ, out); return; };
		emitsimple(&start, TK_PLUS, out); return;
	};
	if (c == 45) {
		if (lpeek(l, 0u64) == 61) { lget(l); emitsimple(&start, TK_MINUSEQ, out); return; };
		if (lpeek(l, 0u64) == 62) { lget(l); emitsimple(&start, TK_ARROW, out); return; };
		emitsimple(&start, TK_MINUS, out); return;
	};
	if (c == 42) {
		if (lpeek(l, 0u64) == 61) { lget(l); emitsimple(&start, TK_STAREQ, out); return; };
		emitsimple(&start, TK_STAR, out); return;
	};
	if (c == 47) {
		if (lpeek(l, 0u64) == 61) { lget(l); emitsimple(&start, TK_SLASHEQ, out); return; };
		emitsimple(&start, TK_SLASH, out); return;
	};
	if (c == 37) {
		if (lpeek(l, 0u64) == 61) { lget(l); emitsimple(&start, TK_PERCENTEQ, out); return; };
		emitsimple(&start, TK_PERCENT, out); return;
	};
	if (c == 38) {
		if (lpeek(l, 0u64) == 38) { lget(l); emitsimple(&start, TK_AND, out); return; };
		if (lpeek(l, 0u64) == 61) { lget(l); emitsimple(&start, TK_AMPEQ, out); return; };
		emitsimple(&start, TK_AMP, out); return;
	};
	if (c == 124) {
		if (lpeek(l, 0u64) == 124) { lget(l); emitsimple(&start, TK_OR, out); return; };
		if (lpeek(l, 0u64) == 61)  { lget(l); emitsimple(&start, TK_PIPEEQ, out); return; };
		emitsimple(&start, TK_PIPE, out); return;
	};
	if (c == 94) {
		if (lpeek(l, 0u64) == 61) { lget(l); emitsimple(&start, TK_CARETEQ, out); return; };
		emitsimple(&start, TK_CARET, out); return;
	};
	if (c == 61) {
		if (lpeek(l, 0u64) == 61) { lget(l); emitsimple(&start, TK_EQ, out); return; };
		if (lpeek(l, 0u64) == 62) { lget(l); emitsimple(&start, TK_FATARROW, out); return; };
		emitsimple(&start, TK_ASSIGN, out); return;
	};
	if (c == 33) {
		if (lpeek(l, 0u64) == 61) { lget(l); emitsimple(&start, TK_NEQ, out); return; };
		emitsimple(&start, TK_NOT, out); return;
	};
	if (c == 60) {
		if (lpeek(l, 0u64) == 60) {
			lget(l);
			if (lpeek(l, 0u64) == 61) { lget(l); emitsimple(&start, TK_LSHIFTEQ, out); return; };
			emitsimple(&start, TK_LSHIFT, out); return;
		};
		if (lpeek(l, 0u64) == 61) { lget(l); emitsimple(&start, TK_LE, out); return; };
		if (lpeek(l, 0u64) == 45) { lget(l); emitsimple(&start, TK_LARROW, out); return; };
		emitsimple(&start, TK_LT, out); return;
	};
	if (c == 62) {
		if (lpeek(l, 0u64) == 62) {
			lget(l);
			if (lpeek(l, 0u64) == 61) { lget(l); emitsimple(&start, TK_RSHIFTEQ, out); return; };
			emitsimple(&start, TK_RSHIFT, out); return;
		};
		if (lpeek(l, 0u64) == 61) { lget(l); emitsimple(&start, TK_GE, out); return; };
		emitsimple(&start, TK_GT, out); return;
	};

	errat(l, &start, "unexpected character");
	out.kind = TK_ERR;
	setposfrom(out, &start);
	let one: [1]u8;
	one[0] = c: u8;
	out.text = astrndup(l.a, one.ptr, 1u64);
};