ww/lib/ww/syntax/tok_test.ww

// toktest — functional-equivalence pin for [[tokname]], [[kwlookup]]
// (struct fold S1) and [[tokprint]]/fputq (struct fold S9, the
// if-ladder → switch folds in tok.ww).
// Run with `ww run -I lib/ww lib/ww/syntax/toktest.ww`.
//
// tokname is checked against every tkind value (the full ladder the
// switch replaced, plus the unknown-kind fallback); kwlookup is
// checked against every keyword it recognises plus non-keyword and
// near-miss (prefix/superstring/exact-width) identifiers.
// tokprint is driven over a temp file: it pins the kind-dispatch
// switch (STR/IDENT/ERR vs INT/RUNE vs the value-less default) and,
// through the STR text, fputq's full escape switch (\\, ", \n, \t,
// \r, the c<0x20 and c==0x7f \xNN arms, and the printable tail) —
// branches ordinary source tokens do not exercise (they hold raw
// `\`+`n`, never a literal control byte).
// A failing row aborts via the assert/abort builtin (task #5 @test
// conversion); per-row exit-code pinpoint is intentionally dropped (the
// abort reports the file, not the row; drew-t2-conversion-spec sec.5).
// `package main` + bare `import tok/lex` mirrors wwdump (the only other
// external lex consumer).

package syntax_test;

import os;
import syntax;
import temp;


fn checktokname(k: tkind, want: str) void = {
	assert(!(tokname(k) != want));
};

// One row per tkind value — same string the if-ladder returned. The
// final row pins the unknown-kind fallback ("<?>").
@test fn tokname_cases() void = {
	checktokname(tkind.TK_NONE, "<none>");
	checktokname(tkind.TK_EOF, "EOF");
	checktokname(tkind.TK_ERR, "ERR");
	checktokname(tkind.TK_IDENT, "IDENT");
	checktokname(tkind.TK_INT, "INT");
	checktokname(tkind.TK_FLOAT, "FLOAT");
	checktokname(tkind.TK_RUNE, "RUNE");
	checktokname(tkind.TK_STR, "STR");

	checktokname(tkind.TK_FN, "fn");
	checktokname(tkind.TK_LET, "let");
	checktokname(tkind.TK_DEF, "def");
	checktokname(tkind.TK_IF, "if");
	checktokname(tkind.TK_ELSE, "else");
	checktokname(tkind.TK_FOR, "for");
	checktokname(tkind.TK_SWITCH, "switch");
	checktokname(tkind.TK_CASE, "case");
	checktokname(tkind.TK_RETURN, "return");
	checktokname(tkind.TK_USE, "import");
	checktokname(tkind.TK_TYPE, "type");
	checktokname(tkind.TK_STRUCT, "struct");
	checktokname(tkind.TK_DEFER, "defer");
	checktokname(tkind.TK_BREAK, "break");
	checktokname(tkind.TK_CONTINUE, "continue");
	checktokname(tkind.TK_EXPORT, "export");
	checktokname(tkind.TK_PROC, "proc");
	checktokname(tkind.TK_CHAN, "chan");
	checktokname(tkind.TK_NIL, "nil");
	checktokname(tkind.TK_TRUE, "true");
	checktokname(tkind.TK_FALSE, "false");
	checktokname(tkind.TK_AS, "as");
	checktokname(tkind.TK_IS, "is");
	checktokname(tkind.TK_VOID, "void");
	checktokname(tkind.TK_YIELD, "yield");
	checktokname(tkind.TK_STATIC, "static");
	checktokname(tkind.TK_MATCH, "match");
	checktokname(tkind.TK_CONST, "const");
	checktokname(tkind.TK_UNDER, "_");
	checktokname(tkind.TK_ENUM, "enum");
	checktokname(tkind.TK_MODULE, "package");

	checktokname(tkind.TK_LPAREN, "(");
	checktokname(tkind.TK_RPAREN, ")");
	checktokname(tkind.TK_LBRACE, "{");
	checktokname(tkind.TK_RBRACE, "}");
	checktokname(tkind.TK_LBRACK, "[");
	checktokname(tkind.TK_RBRACK, "]");
	checktokname(tkind.TK_COMMA, ",");
	checktokname(tkind.TK_SEMI, ";");
	checktokname(tkind.TK_COLON, ":");
	checktokname(tkind.TK_DOT, ".");
	checktokname(tkind.TK_ELLIPSIS, "...");
	checktokname(tkind.TK_DOTDOT, "..");
	checktokname(tkind.TK_AT, "@");
	checktokname(tkind.TK_QUESTION, "?");

	checktokname(tkind.TK_ASSIGN, "=");
	checktokname(tkind.TK_PLUSEQ, "+=");
	checktokname(tkind.TK_MINUSEQ, "-=");
	checktokname(tkind.TK_STAREQ, "*=");
	checktokname(tkind.TK_SLASHEQ, "/=");
	checktokname(tkind.TK_PERCENTEQ, "%=");
	checktokname(tkind.TK_AMPEQ, "&=");
	checktokname(tkind.TK_PIPEEQ, "|=");
	checktokname(tkind.TK_CARETEQ, "^=");
	checktokname(tkind.TK_LSHIFTEQ, "<<=");
	checktokname(tkind.TK_RSHIFTEQ, ">>=");

	checktokname(tkind.TK_PLUS, "+");
	checktokname(tkind.TK_MINUS, "-");
	checktokname(tkind.TK_STAR, "*");
	checktokname(tkind.TK_SLASH, "/");
	checktokname(tkind.TK_PERCENT, "%");
	checktokname(tkind.TK_AMP, "&");
	checktokname(tkind.TK_PIPE, "|");
	checktokname(tkind.TK_CARET, "^");
	checktokname(tkind.TK_TILDE, "~");
	checktokname(tkind.TK_LSHIFT, "<<");
	checktokname(tkind.TK_RSHIFT, ">>");

	checktokname(tkind.TK_EQ, "==");
	checktokname(tkind.TK_NEQ, "!=");
	checktokname(tkind.TK_LT, "<");
	checktokname(tkind.TK_LE, "<=");
	checktokname(tkind.TK_GT, ">");
	checktokname(tkind.TK_GE, ">=");

	checktokname(tkind.TK_AND, "&&");
	checktokname(tkind.TK_OR, "||");
	checktokname(tkind.TK_NOT, "!");

	checktokname(tkind.TK_LARROW, "<-");
	checktokname(tkind.TK_ARROW, "->");
	checktokname(tkind.TK_FATARROW, "=>");

	checktokname(tkind.TK_MODRESET, "//ww:module-reset");
	checktokname(tkind.TK_MODPATH, "//ww:module");
	checktokname(tkind.TK_LAST, "<last>");

	// Unknown kind → the post-switch fallback. TK_LAST is the highest
	// named value (89, after TK_MODPATH=88 landed); 90 is out of band,
	// exercising the "<?>" tail.
	checktokname(90: tkind, "<?>");
};

fn checkkw(s: str, want: tkind) void = {
	assert(!(kwlookup(s.ptr, s.len) != want));
};

// Table-driven (parallel-array idiom; tuple rows blocked by #111). The
// kw rows pin all 30 keywords kwlookup recognises, 1:1 with the
// kwnames/kwkinds table (and cmd/wcc/tok.c:18-47), incl. the two remaps
// import->TK_USE and package->TK_MODULE. The nk rows pin the
// fall-through to TK_NONE. Explicit dims, NOT [_]: [_] static-init silently
// miscompiles to a zero-length array in-tree (probe, cc69daf), which
// would void the loop body — the very hole a table test must not have.
@test fn kwlookup_cases() void = {
	let kwin: [30]str = [
		"as", "break", "case", "chan", "const", "continue", "def", "defer",
		"else", "enum", "export", "false", "fn", "for", "if", "is",
		"import", "let", "match", "nil", "package", "proc", "return",
		"static", "struct", "switch", "true", "type", "void", "yield",
	];
	let kwexp: [30]tkind = [
		tkind.TK_AS, tkind.TK_BREAK, tkind.TK_CASE, tkind.TK_CHAN,
		tkind.TK_CONST, tkind.TK_CONTINUE, tkind.TK_DEF, tkind.TK_DEFER,
		tkind.TK_ELSE, tkind.TK_ENUM, tkind.TK_EXPORT, tkind.TK_FALSE,
		tkind.TK_FN, tkind.TK_FOR, tkind.TK_IF, tkind.TK_IS,
		tkind.TK_USE, tkind.TK_LET, tkind.TK_MATCH, tkind.TK_NIL,
		tkind.TK_MODULE, tkind.TK_PROC, tkind.TK_RETURN, tkind.TK_STATIC,
		tkind.TK_STRUCT, tkind.TK_SWITCH, tkind.TK_TRUE, tkind.TK_TYPE,
		tkind.TK_VOID, tkind.TK_YIELD,
	];
	let i: i32 = 0;
	for (i < len(kwin)) {
		checkkw(kwin[i], kwexp[i]);
		i += 1;
	};

	// Non-keywords that must fall through to TK_NONE. Rows exercise the
	// length-guard + bytewise reject inside strings.compare: keyword
	// SUPERSTRINGS (longer, shared prefix), proper PREFIXES of a keyword
	// (shorter, shared leading bytes), exact-length non-keywords at the
	// 2/3/4-byte keyword widths, and Hare bmap "keywords" (alloc/len/
	// size/append/assert) that are deliberately NOT ww lexer keywords.
	let nk: [18]str = [
		"xyzzy",	// ordinary identifier
		"fns",		// superstring of "fn"
		"ifx",		// superstring of "if"
		"iffy",		// superstring of "if"
		"form",		// superstring of "for"
		"fora",		// superstring of "for"
		"asx",		// superstring of "as"
		"i",		// proper prefix of if / is / import
		"co",		// proper prefix of const / continue
		"swit",		// proper prefix of switch
		"xx",		// len-2 non-kw (as/fn/if/is width)
		"zzz",		// len-3 non-kw (def/for/nil width)
		"abcd",		// len-4 non-kw (case/enum/true/type width)
		"alloc",	// Hare bmap keyword, NOT a ww lexer keyword
		"len",		// Hare bmap keyword, NOT a ww lexer keyword
		"size",		// Hare bmap keyword, NOT a ww lexer keyword
		"append",	// Hare bmap keyword, NOT a ww lexer keyword
		"assert",	// Hare bmap keyword, NOT a ww lexer keyword
	];
	let j: i32 = 0;
	for (j < len(nk)) {
		checkkw(nk[j], tkind.TK_NONE);
		j += 1;
	};
};

// checkprint — tokprint `t` to a freshly-rewound fd, read the bytes
// back, and assert they equal `want`. The fd is RDWR; we lseek to 0
// before each write so earlier (possibly longer) content past want.len
// is irrelevant — only want.len bytes from offset 0 are compared.
fn checkprint(fd: i32, t: *tok, want: str) void = {
	assert(!(os.lseek(fd, 0i64, os.whence.SET) != 0i64));
	tokprint(fd, t);
	assert(!(os.lseek(fd, 0i64, os.whence.SET) != 0i64));
	let rbuf: [256]u8;
	let z: i32 = 0;
	for (z < 256) { rbuf[z] = 0u8; z += 1; };
	let rd: i64 = os.read(fd, &rbuf[0], want.len: u64);
	assert(!(rd != want.len: i64));
	let j: i32 = 0;
	for (j < want.len) {
		assert(!(rbuf[j] != want.ptr[j]));
		j += 1;
	};
};

// One token per fputq/tokprint dispatch branch. The STR text packs
// every fputq escape: \\ " \n \t \r, a c<0x20 byte (0x01), c==0x7f,
// and a printable ('A'). `want` spells the exact emitted line.
@test fn tokprint_cases() void = {
	let fd: i32 = 0;
	let path: str;
	match (temp.named(&fd, &path, "/tmp", temp.mode.RDWR, 384i32)) {
	case void => {};
	case let e: os.oserror => abort();
	};
	assert(!(fd < 0));

	let t: tok;
	t.file = "t";
	t.line = 1;
	t.col = 1;

	t.kind = tkind.TK_STR;
	t.text = "\\\"\n\t\r\x01\x7fA";
	checkprint(fd, &t, "t:1:1 STR \"\\\\\\\"\\n\\t\\r\\x01\\x7fA\"\n");

	t.kind = tkind.TK_IDENT;
	t.text = "name";
	checkprint(fd, &t, "t:1:1 IDENT \"name\"\n");

	t.kind = tkind.TK_ERR;
	t.text = "oops";
	checkprint(fd, &t, "t:1:1 ERR \"oops\"\n");

	t.kind = tkind.TK_INT;
	t.uval = 42u64;
	checkprint(fd, &t, "t:1:1 INT 42\n");

	t.kind = tkind.TK_RUNE;
	t.uval = 65u64;
	checkprint(fd, &t, "t:1:1 RUNE 65\n");

	// Default arm: a kind outside the switch appends no value.
	t.kind = tkind.TK_NONE;
	checkprint(fd, &t, "t:1:1 <none>\n");

	assert(!(os.close(fd) != 0));
	assert(!(os.remove(path) != 0));
};

fn checkfloat(src: str, want: u64) void = {
	let l: lex;
	lexinit(&l, "t", src.ptr, src.len: u64);
	let t: tok;
	lexnext(&l, &t);
	assert(!(t.kind != tkind.TK_FLOAT));
	assert(!(t.uval != want));
};

// #62 pin: lexnum's float fold routes through strconv.stof64 and must
// produce the IEEE-754 correctly-rounded bits cstage gets from strtod
// — any rounding slip is a cs≠ww DATA divergence. Vectors pinned
// against a C strtod oracle. Rows cover the classes the retired
// pow-10 fold got wrong: >19-digit mantissas (its i64 accumulator
// overflowed), DBL_MIN/DBL_MAX extremes, and the decimal-fraction
// 1-ULP double-rounding cases; plus halfway-to-even, exponent forms,
// the underscore strip, the 53-digit exact-halfway pair at the 2^-53
// boundary (tie rounds to even, tie+1 rounds up — also the only
// >19-digit FRACTION rows), and an exact power of two.
@test fn floatfold_cases() void = {
	checkfloat("1.0000000000000002", 0x3FF0000000000001u64);
	checkfloat("9007199254740993.0", 0x4340000000000000u64);
	checkfloat("1.2345e67", 0x4DDD4E421712C0B7u64);
	checkfloat("0.1", 0x3FB999999999999Au64);
	checkfloat("1.1", 0x3FF199999999999Au64);
	checkfloat("123456789012345678901234567890.0", 0x45F8EE90FF6C373Eu64);
	checkfloat("2.2250738585072014e-308", 0x0010000000000000u64);
	checkfloat("0.3", 0x3FD3333333333333u64);
	checkfloat("3.141592653589793", 0x400921FB54442D18u64);
	checkfloat("1.7976931348623157e308", 0x7FEFFFFFFFFFFFFFu64);
	checkfloat("1.7976931348623158e308", 0x7FEFFFFFFFFFFFFFu64);
	checkfloat("7.2057594037927933e16", 0x4370000000000000u64);
	checkfloat("1000000000000000000000.0", 0x444B1AE4D6E2EF50u64);
	checkfloat("1_000.5", 0x408F440000000000u64);
	checkfloat("1.00000000000000011102230246251565404236316680908203125",
		0x3FF0000000000000u64);
	checkfloat("1.00000000000000011102230246251565404236316680908203126",
		0x3FF0000000000001u64);
	checkfloat("4503599627370497.5", 0x4330000000000002u64);
	checkfloat("0.5", 0x3FE0000000000000u64);
	checkfloat("1.0e308", 0x7FE1CCF385EBC8A0u64);
	checkfloat("2.225073858507202e-308", 0x0010000000000001u64);
};