ww/lib/ww/lex/lex.ww

// lib/ww/lex/lex.ww — port of cmd/wcc/lex.c.
//
// The DFA, the helpers, and the order of decisions all mirror the C
// version exactly. The 990_selfhost test diffs the resulting token
// stream against the C-side wwdump byte-for-byte; any divergence is
// a port bug.
//
// Calling-convention note: w6c can't yet pass or return structs >16
// bytes by value, so `tok` and `pos` are passed by pointer (out
// params). The C version passes `Tok` by value; we differ here only
// in shape, not in observable behaviour. Token kind values stay
// numerically identical.

package lex;

// Sibling import (tok) auto-resolves via task #22 dir-enum when
// callers `import lex;` (which dir-enums lib/ww/lex/).
import os;
import ascii;
import mem;
import strings;

// isidstart / isidpart — identifier classification. Lexer-local
// because the "alpha or '_' / alnum or '_'" set isn't part of Hare's
// ascii::; ascii::isalpha + the '_' check live here instead.
fn isidstart(c: rune) bool = {
	if (ascii.isalpha(c)) { return true; };
	if (c == 95) { return true; };	// '_'
	return false;
};

fn isidpart(c: rune) bool = {
	if (ascii.isalnum(c)) { return true; };
	if (c == 95) { return true; };
	return false;
};

// hexval — value of `c` as a hex digit (0..15) or void if not a hex
// digit. Used by string-literal `\xHH` escapes.
fn hexval(c: rune) (i32 | void) = {
	if (ascii.isdigit(c)) { return (c - 48): i32; };
	if (c >= 65) {
		if (c <= 70) { return ((c - 65) + 10): i32; };	// 'A'..'F'
	};
	if (c >= 97) {
		if (c <= 102) { return ((c - 97) + 10): i32; };	// 'a'..'f'
	};
	return;
};

type lex = struct {
	file:   str,
	src:    *u8,	// raw bytes; not necessarily NUL-terminated
	srclen: u64,
	lpos:   u64,
	line:   i32,
	col:    i32,
	a:      *arena,
	errs:   i32,
};

export fn lexinit(l: *lex, a: *arena, file: str, src: *u8, len: u64) void = {
	l.file = file;
	l.src = src;
	l.srclen = len;
	l.lpos = 0u64;
	l.line = 1;
	l.col = 1;
	l.a = a;
	l.errs = 0;
};

// srcb — byte at offset; helper that lifts the cast out of indexing.
fn srcb(l: *lex, off: u64) i32 = {
	let i: i32 = off: i32;
	let b: u8 = l.src[i];
	return b: i32;
};

fn lpeek(l: *lex, ahead: u64) i32 = {
	let p: u64 = l.lpos + ahead;
	if (p >= l.srclen) { return -1; };
	return srcb(l, p);
};

fn lget(l: *lex) i32 = {
	if (l.lpos >= l.srclen) { return -1; };
	let c: i32 = srcb(l, l.lpos);
	l.lpos += 1u64;
	if (c == 10) {	// '\n'
		l.line += 1;
		l.col = 1;
	} else {
		l.col += 1;
	};
	return c;
};

fn curpos(l: *lex, out: *pos) void = {
	out.file = l.file;
	out.line = l.line;
	out.col = l.col;
};

// putuint — write `v` (signed, but always non-negative here) to fd 2
// in decimal. Standalone so errat doesn't drag in fmt and create a
// dependency cycle with strconv.
fn putuint(fd: i32, v: i32) void = {
	let tmp: [16]u8;
	let i: i32 = 0;
	let n: i32 = v;
	for (n > 0) {
		tmp[i] = ((n % 10) + 48): u8;
		n = n / 10;
		i += 1;
	};
	if (i == 0) { tmp[0] = 48u8; i = 1; };
	let buf: [16]u8;
	let m: i32 = 0;
	for (i > 0) { i -= 1; buf[m] = tmp[i]; m += 1; };
	os.write(fd, buf.ptr, m: u64);
};

fn errat(l: *lex, p: *pos, msg: str) void = {
	let pf: str = p.file;
	os.write(2, pf.ptr, pf.len: u64);
	os.write(2, ":".ptr, 1u64);
	putuint(2, p.line);
	os.write(2, ":".ptr, 1u64);
	putuint(2, p.col);
	os.write(2, ": error: ".ptr, 9u64);
	os.write(2, msg.ptr, msg.len: u64);
	os.write(2, "\n".ptr, 1u64);
	l.errs += 1;
};

fn skipws(l: *lex) bool = {
	for (true) {
		let c: i32 = lpeek(l, 0u64);
		if (c < 0) { return false; };
		if (c == 32) { lget(l); continue; };
		if (c == 9)  { lget(l); continue; };
		if (c == 13) { lget(l); continue; };
		if (c == 10) { lget(l); continue; };
		if (c == 47) {	// '/'
			let c2: i32 = lpeek(l, 1u64);
			if (c2 == 47) {
				lget(l); lget(l);	// consume '//'
				for (true) {
					let cx: i32 = lpeek(l, 0u64);
					if (cx < 0) { return false; };
					if (cx == 10) { break; };
					lget(l);
				};
				continue;
			};
			if (c2 == 42) {	// '*'
				lget(l); lget(l);
				let prev: i32 = -1;
				for (true) {
					let x: i32 = lget(l);
					if (x < 0) {
						let cp: pos;
						curpos(l, &cp);
						errat(l, &cp, "unterminated /* comment");
						return false;
					};
					if (prev == 42) {
						if (x == 47) { break; };
					};
					prev = x;
				};
				continue;
			};
		};
		return true;
	};
	return false;
};

fn parseint(p: *u8, n: u64, base: i32, ok: *bool) u64 = {
	let v: u64 = 0u64;
	let got: bool = false;
	let i: u64 = 0u64;
	for (i < n) {
		let ix: i32 = i: i32;
		let c: u8 = p[ix];
		if (c == 95u8) {	// '_'
			i += 1u64;
			continue;
		};
		let d: i32 = -1;
		if (c >= 48u8) {
			if (c <= 57u8) { d = (c - 48u8): i32; };
		};
		if (d < 0) {
			if (c >= 97u8) {
				if (c <= 102u8) { d = ((c - 97u8) + 10u8): i32; };
			};
		};
		if (d < 0) {
			if (c >= 65u8) {
				if (c <= 70u8) { d = ((c - 65u8) + 10u8): i32; };
			};
		};
		if (d < 0) { *ok = false; return 0u64; };
		if (d >= base) { *ok = false; return 0u64; };
		v = v * (base: u64) + (d: u64);
		got = true;
		i += 1u64;
	};
	*ok = got;
	return v;
};

fn escape(l: *lex, out: *i32) bool = {
	let c: i32 = lget(l);
	if (c < 0) { return false; };
	if (c == 110) { *out = 10; return true; };
	if (c == 116) { *out = 9;  return true; };
	if (c == 114) { *out = 13; return true; };
	if (c == 92)  { *out = 92; return true; };
	if (c == 39)  { *out = 39; return true; };
	if (c == 34)  { *out = 34; return true; };
	if (c == 48)  { *out = 0;  return true; };
	if (c == 97)  { *out = 7;  return true; };
	if (c == 98)  { *out = 8;  return true; };
	if (c == 102) { *out = 12; return true; };
	if (c == 118) { *out = 11; return true; };
	if (c == 120) {
		let hi: i32 = lget(l);
		let lo: i32 = lget(l);
		if (hi < 0) { return false; };
		if (lo < 0) { return false; };
		if (!ascii.isxdigit(hi: rune)) {
			let cp: pos; curpos(l, &cp);
			errat(l, &cp, "bad \\x escape");
			return false;
		};
		if (!ascii.isxdigit(lo: rune)) {
			let cp: pos; curpos(l, &cp);
			errat(l, &cp, "bad \\x escape");
			return false;
		};
		// Hex digits already validated by isxdigit above — `!`
		// (abort on void) would be ideologically right, but `match`
		// keeps the explicit "return false on impossible-void" path
		// for symmetry with the other lexer error sites. Use `!`
		// once we have a panic-with-position helper.
		let h: i32 = hexval(hi: rune)!;
		let lv: i32 = hexval(lo: rune)!;
		*out = (h << 4) | lv;
		return true;
	};
	let cp: pos; curpos(l, &cp);
	errat(l, &cp, "bad escape");
	return false;
};

// scandecimalrun — consume a run of decimal digits and underscores.
fn scandecimalrun(l: *lex) void = {
	for (true) {
		let c: i32 = lpeek(l, 0u64);
		if (c < 0) { break; };
		if (!ascii.isdigit(c: rune)) {
			if (c != 95) { break; };
		};
		lget(l);
	};
};

fn scanhexrun(l: *lex) void = {
	for (true) {
		let c: i32 = lpeek(l, 0u64);
		if (c < 0) { break; };
		if (!ascii.isxdigit(c: rune)) {
			if (c != 95) { break; };
		};
		lget(l);
	};
};

fn scanbinrun(l: *lex) void = {
	for (true) {
		let c: i32 = lpeek(l, 0u64);
		if (c == 48) { lget(l); continue; };
		if (c == 49) { lget(l); continue; };
		if (c == 95) { lget(l); continue; };
		break;
	};
};

fn scanoctrun(l: *lex) void = {
	for (true) {
		let c: i32 = lpeek(l, 0u64);
		if (c < 48) { break; };
		if (c > 55) {
			if (c != 95) { break; };
		};
		lget(l);
	};
};

// scanexp — consume the [eE][+-]?[0-9]+ tail of a float, if present.
fn scanexp(l: *lex) void = {
	let e: i32 = lpeek(l, 0u64);
	if (e != 101) { if (e != 69) { return; }; };	// 'e' or 'E'
	lget(l);
	let s: i32 = lpeek(l, 0u64);
	if (s == 43) { lget(l); }
	else { if (s == 45) { lget(l); }; };
	for (true) {
		let c: i32 = lpeek(l, 0u64);
		if (c < 0) { break; };
		if (!ascii.isdigit(c: rune)) { break; };
		lget(l);
	};
};

// parsef64 — minimal decimal-float parser. Reads digits[.digits][eE[+-]digits]
// from the first `n` bytes of `s` (no leading sign — the lexer emits
// the unary minus as a separate token). The result rounds to the
// nearest f64 only via the trailing pow-10 multiply; this matches
// `strtod` to 1 ULP on typical literals and is good enough for the
// wwstage's own use (no float literals appear in the bootstrap
// source). Anything past `n` or non-digit is silently ignored.
fn parsef64(s: *u8, n: u64) f64 = {
	let i: u64 = 0u64;
	let intp: i64 = 0i64;
	for (i < n) {
		let b: u8 = s[i];
		if (b < 48u8) { break; };
		if (b > 57u8) { break; };
		intp = intp * 10i64 + (b - 48u8): i64;
		i += 1u64;
	};
	let frac: i64 = 0i64;
	let fscale: i64 = 1i64;
	if (i < n) {
		if (s[i] == 46u8) {	// '.'
			i += 1u64;
			for (i < n) {
				let b: u8 = s[i];
				if (b < 48u8) { break; };
				if (b > 57u8) { break; };
				frac = frac * 10i64 + (b - 48u8): i64;
				fscale = fscale * 10i64;
				i += 1u64;
			};
		};
	};
	let exp: i32 = 0;
	let expneg: bool = false;
	if (i < n) {
		let e: u8 = s[i];
		if (e == 101u8 || e == 69u8) {	// 'e' / 'E'
			i += 1u64;
			if (i < n) {
				if (s[i] == 45u8) {	// '-'
					expneg = true;
					i += 1u64;
				} else { if (s[i] == 43u8) {	// '+'
					i += 1u64;
				};};
			};
			for (i < n) {
				let b: u8 = s[i];
				if (b < 48u8) { break; };
				if (b > 57u8) { break; };
				exp = exp * 10 + (b - 48u8): i32;
				i += 1u64;
			};
		};
	};
	let result: f64 = intp: f64;
	if (frac != 0i64) {
		result = result + (frac: f64) / (fscale: f64);
	};
	if (exp != 0) {
		// Use int-to-float casts so this file stays free of float
		// literals — 990's wwdump diff relies on lib/ww/lex/lex.ww
		// tokenising identically through C and ww, and the C dumper
		// %g-formats TK_FLOAT.fval while the ww dumper currently
		// skips it. Hiding the constants behind casts keeps both
		// sides emitting `FLOAT` with no payload.
		let factor: f64 = 1: f64;
		let ten: f64 = 10: f64;
		let k: i32 = 0;
		for (k < exp) { factor = factor * ten; k += 1; };
		if (expneg) { result = result / factor; }
		else        { result = result * factor; };
	};
	return result;
};

fn lexnum(l: *lex, start: *pos, out: *tok) void = {
	out.kind = tkind.TK_INT;
	out.file = start.file;
	out.line = start.line;
	out.col  = start.col;
	let begin: u64 = l.lpos;
	let base: i32 = 10;
	let isfloat: bool = false;

	let c0: i32 = lpeek(l, 0u64);
	let c1: i32 = lpeek(l, 1u64);

	if (c0 == 48) {	// '0'
		if (c1 == 120) {	// 'x'
			lget(l); lget(l); base = 16; scanhexrun(l);
		} else { if (c1 == 88) {	// 'X'
			lget(l); lget(l); base = 16; scanhexrun(l);
		} else { if (c1 == 98) {	// 'b'
			lget(l); lget(l); base = 2; scanbinrun(l);
		} else { if (c1 == 66) {	// 'B'
			lget(l); lget(l); base = 2; scanbinrun(l);
		} else { if (c1 == 111) {	// 'o'
			lget(l); lget(l); base = 8; scanoctrun(l);
		} else { if (c1 == 79) {	// 'O'
			lget(l); lget(l); base = 8; scanoctrun(l);
		} else {
			scandecimalrun(l);
			if (lpeek(l, 0u64) == 46) {
				let after: i32 = lpeek(l, 1u64);
				if (after >= 48) {
					if (after <= 57) {
						isfloat = true;
						lget(l);
						scandecimalrun(l);
						scanexp(l);
					};
				};
			};
		};};};};};};
	} else {
		scandecimalrun(l);
		if (lpeek(l, 0u64) == 46) {
			let after: i32 = lpeek(l, 1u64);
			if (after >= 48) {
				if (after <= 57) {
					isfloat = true;
					lget(l);
					scandecimalrun(l);
					scanexp(l);
				};
			};
		};
	};

	let n: u64 = l.lpos - begin;
	let view: str;
	view.ptr = l.src + begin;
	view.len = n: i32;
	out.text = strings.dup(view);

	if (isfloat) {
		out.kind = tkind.TK_FLOAT;
		// Strip underscores from the digits (Hare allows 1_000.5)
		// before parsing — match what cmd/wcc/lex.c does with
		// strtod over a cleaned buffer.
		let clean: []u8 = alloc([], n + 1u64)!;
		let i: u64 = 0u64;
		let j: u64 = 0u64;
		for (i < n) {
			let b: u8 = l.src[begin + i];
			if (b != 95u8) {	// '_'
				clean[j] = b;
				j += 1u64;
			};
			i += 1u64;
		};
		clean[j] = 0u8;
		let fv: f64 = parsef64(clean.ptr, j);
		out.fval = fv;
		// Stash the IEEE bits in uval — cgen consumers read floats
		// as integers (n.uval) to avoid an SSE round-trip when
		// materialising the constant.
		let pu: *u64 = (&fv): *u64;
		out.uval = *pu;
	} else {
		let digs: *u8 = l.src + begin;
		let dn: u64 = n;
		if (base != 10) {
			digs = digs + 2u64;
			dn -= 2u64;
		};
		let ok: bool = false;
		out.uval = parseint(digs, dn, base, &ok);
		if (!ok) {
			errat(l, start, "bad integer literal");
			out.kind = tkind.TK_ERR;
		};
	};

	let pc: i32 = lpeek(l, 0u64);
	if (pc >= 0) {
		if (isidstart(pc: rune)) {
			let sb: u64 = l.lpos;
			for (true) {
				let cc: i32 = lpeek(l, 0u64);
				if (cc < 0) { break; };
				if (!isidpart(cc: rune)) { break; };
				lget(l);
			};
			let sl: u64 = l.lpos - sb;
			let p: *u8 = l.src + sb;
			let isok: bool = false;
			if (sl == 2u64) {
				if (p[0] == 105u8) {
					if (p[1] == 56u8) { isok = true; };	// i8
				};
				if (p[0] == 117u8) {
					if (p[1] == 56u8) { isok = true; };	// u8
				};
			};
			if (sl == 3u64) {
				if (p[0] == 105u8) {
					if (p[1] == 49u8) { if (p[2] == 54u8) { isok = true; }; };	// i16
					if (p[1] == 51u8) { if (p[2] == 50u8) { isok = true; }; };	// i32
					if (p[1] == 54u8) { if (p[2] == 52u8) { isok = true; }; };	// i64
				};
				if (p[0] == 117u8) {
					if (p[1] == 49u8) { if (p[2] == 54u8) { isok = true; }; };
					if (p[1] == 51u8) { if (p[2] == 50u8) { isok = true; }; };
					if (p[1] == 54u8) { if (p[2] == 52u8) { isok = true; }; };
				};
				if (p[0] == 102u8) {
					if (p[1] == 51u8) { if (p[2] == 50u8) { isok = true; }; };	// f32
					if (p[1] == 54u8) { if (p[2] == 52u8) { isok = true; }; };	// f64
				};
			};
			if (isok) {
				let view: str;
				view.ptr = p;
				view.len = sl: i32;
				out.tsuffix = strings.dup(view);
			} else {
				l.lpos = sb;
			};
		};
	};
};

fn lexident(l: *lex, start: *pos, out: *tok) void = {
	let begin: u64 = l.lpos;
	for (true) {
		let c: i32 = lpeek(l, 0u64);
		if (c < 0) { break; };
		if (!isidpart(c: rune)) { break; };
		lget(l);
	};
	let n: u64 = l.lpos - begin;
	let p: *u8 = l.src + begin;
	out.file = start.file;
	out.line = start.line;
	out.col  = start.col;
	// Bare '_' is the discard marker. `_x`, `_1` are normal idents.
	if (n == 1u64) {
		if (p[0] == 95u8) {
			out.kind = tkind.TK_UNDER;
			let view: str;
			view.ptr = p;
			view.len = n: i32;
			out.text = strings.dup(view);
			return;
		};
	};
	let k: tkind = kwlookup(p, n: i32);
	if (k != tkind.TK_NONE) {
		out.kind = k;
	} else {
		out.kind = tkind.TK_IDENT;
	};
	let view: str;
	view.ptr = p;
	view.len = n: i32;
	out.text = strings.dup(view);
};

fn lexstr(l: *lex, start: *pos, out: *tok) void = {
	let cap: u64 = 32u64;
	let nb:  u64 = 0u64;
	let buf: []u8 = alloc([], cap)!;
	for (true) {
		let c: i32 = lpeek(l, 0u64);
		if (c < 0) {
			errat(l, start, "unterminated string");
			out.kind = tkind.TK_ERR;
			out.file = start.file;
	out.line = start.line;
	out.col  = start.col;
			let view: str;
			view.ptr = "".ptr;
			view.len = 0;
			out.text = strings.dup(view);
			return;
		};
		if (c == 34) { lget(l); break; };
		let ch: i32 = 0;
		if (c == 92) {
			lget(l);
			if (!escape(l, &ch)) { ch = 0; };
		} else {
			ch = lget(l);
		};
		if (nb + 1u64 >= cap) {
			let ncap: u64 = cap * 2u64;
			let nb2: []u8 = alloc([], ncap)!;
			let i: u64 = 0u64;
			for (i < nb) {
				let ix: i32 = i: i32;
				nb2[ix] = buf[ix];
				i += 1u64;
			};
			buf = nb2;
			cap = ncap;
		};
		let nbi: i32 = nb: i32;
		buf[nbi] = ch: u8;
		nb += 1u64;
	};
	out.kind = tkind.TK_STR;
	out.file = start.file;
	out.line = start.line;
	out.col  = start.col;
	let s: str;
	s.ptr = buf.ptr;
	s.len = nb: i32;
	out.text = s;
};

fn lexrune(l: *lex, start: *pos, out: *tok) void = {
	let c: i32 = lpeek(l, 0u64);
	if (c < 0) {
		errat(l, start, "unterminated rune");
		out.kind = tkind.TK_ERR;
		out.file = start.file;
	out.line = start.line;
	out.col  = start.col;
		let view: str;
		view.ptr = "".ptr;
		view.len = 0;
		out.text = strings.dup(view);
		return;
	};
	let ch: i32 = 0;
	if (c == 92) {
		lget(l);
		if (!escape(l, &ch)) { ch = 0; };
	} else {
		ch = lget(l);
	};
	if (lpeek(l, 0u64) != 39) {
		errat(l, start, "rune literal missing closing '");
		out.kind = tkind.TK_ERR;
		out.file = start.file;
	out.line = start.line;
	out.col  = start.col;
		let view: str;
		view.ptr = "".ptr;
		view.len = 0;
		out.text = strings.dup(view);
		return;
	};
	lget(l);
	out.kind = tkind.TK_RUNE;
	out.file = start.file;
	out.line = start.line;
	out.col  = start.col;
	out.uval = ch: u64;
};

fn emitsimple(start: *pos, k: tkind, out: *tok) void = {
	out.kind = k;
	out.file = start.file;
	out.line = start.line;
	out.col  = start.col;
};

// setposfrom — copy file/line/col from a *pos into a tok. Used by
// the err-token path where we already have a pos.
fn setposfrom(out: *tok, p: *pos) void = {
	out.file = p.file;
	out.line = p.line;
	out.col  = p.col;
};

export fn lexnext(l: *lex, out: *tok) void = {
	// Reset the out token so callers can rely on stale fields being
	// cleared (they only inspect kind, pos, text, uval, fval, tsuffix
	// per kind).
	out.kind = tkind.TK_NONE;
	out.uval = 0u64;
	// out.fval starts cleared by the caller's stack-local init (lex.ww
	// allocates the tok with `let t: tok;` which zeroes). We avoid
	// writing a 0.0 literal here so this file itself stays float-free
	// and the C/ww wwdump diff over it is byte-identical.
	let empty: str;
	empty.ptr = nil;
	empty.len = 0;
	out.text = empty;
	out.tsuffix = empty;

	if (!skipws(l)) {
		let p: pos; curpos(l, &p);
		emitsimple(&p, tkind.TK_EOF, out);
		return;
	};
	let start: pos; curpos(l, &start);
	let c: i32 = lpeek(l, 0u64);

	if (c >= 0) {
		if (isidstart(c: rune)) { lexident(l, &start, out); return; };
		if (ascii.isdigit(c: rune))   { lexnum(l, &start, out); return; };
	};

	if (c == 34) { lget(l); lexstr(l, &start, out); return; };
	if (c == 39) { lget(l); lexrune(l, &start, out); return; };

	lget(l);

	if (c == 40)  { emitsimple(&start, tkind.TK_LPAREN, out); return; };
	if (c == 41)  { emitsimple(&start, tkind.TK_RPAREN, out); return; };
	if (c == 123) { emitsimple(&start, tkind.TK_LBRACE, out); return; };
	if (c == 125) { emitsimple(&start, tkind.TK_RBRACE, out); return; };
	if (c == 91)  { emitsimple(&start, tkind.TK_LBRACK, out); return; };
	if (c == 93)  { emitsimple(&start, tkind.TK_RBRACK, out); return; };
	if (c == 44)  { emitsimple(&start, tkind.TK_COMMA, out); return; };
	if (c == 59)  { emitsimple(&start, tkind.TK_SEMI, out); return; };
	if (c == 58)  { emitsimple(&start, tkind.TK_COLON, out); return; };
	if (c == 64)  { emitsimple(&start, tkind.TK_AT, out); return; };
	if (c == 63)  { emitsimple(&start, tkind.TK_QUESTION, out); return; };
	if (c == 126) { emitsimple(&start, tkind.TK_TILDE, out); return; };

	if (c == 46) {	// '.'
		if (lpeek(l, 0u64) == 46) {
			if (lpeek(l, 1u64) == 46) {
				lget(l); lget(l);
				emitsimple(&start, tkind.TK_ELLIPSIS, out); return;
			};
			lget(l);
			emitsimple(&start, tkind.TK_DOTDOT, out); return;
		};
		emitsimple(&start, tkind.TK_DOT, out); return;
	};

	if (c == 43) {
		if (lpeek(l, 0u64) == 61) { lget(l); emitsimple(&start, tkind.TK_PLUSEQ, out); return; };
		emitsimple(&start, tkind.TK_PLUS, out); return;
	};
	if (c == 45) {
		if (lpeek(l, 0u64) == 61) { lget(l); emitsimple(&start, tkind.TK_MINUSEQ, out); return; };
		if (lpeek(l, 0u64) == 62) { lget(l); emitsimple(&start, tkind.TK_ARROW, out); return; };
		emitsimple(&start, tkind.TK_MINUS, out); return;
	};
	if (c == 42) {
		if (lpeek(l, 0u64) == 61) { lget(l); emitsimple(&start, tkind.TK_STAREQ, out); return; };
		emitsimple(&start, tkind.TK_STAR, out); return;
	};
	if (c == 47) {
		if (lpeek(l, 0u64) == 61) { lget(l); emitsimple(&start, tkind.TK_SLASHEQ, out); return; };
		emitsimple(&start, tkind.TK_SLASH, out); return;
	};
	if (c == 37) {
		if (lpeek(l, 0u64) == 61) { lget(l); emitsimple(&start, tkind.TK_PERCENTEQ, out); return; };
		emitsimple(&start, tkind.TK_PERCENT, out); return;
	};
	if (c == 38) {
		if (lpeek(l, 0u64) == 38) { lget(l); emitsimple(&start, tkind.TK_AND, out); return; };
		if (lpeek(l, 0u64) == 61) { lget(l); emitsimple(&start, tkind.TK_AMPEQ, out); return; };
		emitsimple(&start, tkind.TK_AMP, out); return;
	};
	if (c == 124) {
		if (lpeek(l, 0u64) == 124) { lget(l); emitsimple(&start, tkind.TK_OR, out); return; };
		if (lpeek(l, 0u64) == 61)  { lget(l); emitsimple(&start, tkind.TK_PIPEEQ, out); return; };
		emitsimple(&start, tkind.TK_PIPE, out); return;
	};
	if (c == 94) {
		if (lpeek(l, 0u64) == 61) { lget(l); emitsimple(&start, tkind.TK_CARETEQ, out); return; };
		emitsimple(&start, tkind.TK_CARET, out); return;
	};
	if (c == 61) {
		if (lpeek(l, 0u64) == 61) { lget(l); emitsimple(&start, tkind.TK_EQ, out); return; };
		if (lpeek(l, 0u64) == 62) { lget(l); emitsimple(&start, tkind.TK_FATARROW, out); return; };
		emitsimple(&start, tkind.TK_ASSIGN, out); return;
	};
	if (c == 33) {
		if (lpeek(l, 0u64) == 61) { lget(l); emitsimple(&start, tkind.TK_NEQ, out); return; };
		emitsimple(&start, tkind.TK_NOT, out); return;
	};
	if (c == 60) {
		if (lpeek(l, 0u64) == 60) {
			lget(l);
			if (lpeek(l, 0u64) == 61) { lget(l); emitsimple(&start, tkind.TK_LSHIFTEQ, out); return; };
			emitsimple(&start, tkind.TK_LSHIFT, out); return;
		};
		if (lpeek(l, 0u64) == 61) { lget(l); emitsimple(&start, tkind.TK_LE, out); return; };
		if (lpeek(l, 0u64) == 45) { lget(l); emitsimple(&start, tkind.TK_LARROW, out); return; };
		emitsimple(&start, tkind.TK_LT, out); return;
	};
	if (c == 62) {
		if (lpeek(l, 0u64) == 62) {
			lget(l);
			if (lpeek(l, 0u64) == 61) { lget(l); emitsimple(&start, tkind.TK_RSHIFTEQ, out); return; };
			emitsimple(&start, tkind.TK_RSHIFT, out); return;
		};
		if (lpeek(l, 0u64) == 61) { lget(l); emitsimple(&start, tkind.TK_GE, out); return; };
		emitsimple(&start, tkind.TK_GT, out); return;
	};

	errat(l, &start, "unexpected character");
	out.kind = tkind.TK_ERR;
	setposfrom(out, &start);
	let one: [1]u8;
	one[0] = c: u8;
	let view: str;
	view.ptr = one.ptr;
	view.len = 1;
	out.text = strings.dup(view);
};