ww/test/wcc/100_lex.c

/*
 * 100_lex — table-driven lexer tests.
 *
 * Each row is a (src, expected) pair. The expected string is the
 * concatenation of token names, space-separated. For literals we
 * also encode the value: e.g. INT(42), STR("hi"), IDENT(foo).
 *
 * EOF is implicit: the harness checks that lexnext returns TK_EOF
 * after the last expected token.
 */
#include "ww.h"
#include <string.h>
#include <stdlib.h>
#include <stdio.h>

static char *
toklit(Arena *a, Tok t)
{
	switch (t.kind) {
	case TK_IDENT: return aprintf(a, "IDENT(%s)", t.text);
	case TK_INT:   return aprintf(a, "INT(%llu)", (unsigned long long)t.v.uval);
	case TK_FLOAT: return aprintf(a, "FLOAT(%g)", t.v.fval);
	case TK_RUNE:  return aprintf(a, "RUNE(%llu)", (unsigned long long)t.v.uval);
	case TK_STR:   return aprintf(a, "STR(%s)", t.text);
	case TK_ERR:   return aprintf(a, "ERR(%s)", t.text);
	default:       return (char *)tokname(t.kind);
	}
}

static int
runrow(const char *src, const char *expect)
{
	Arena *a = newarena();
	Lex l;
	lexinit(&l, a, "<test>", src, strlen(src));

	char *got = amalloc(a, 1);
	got[0] = '\0';
	u64 cap = 1, n = 0;

	for (;;) {
		Tok t = lexnext(&l);
		if (t.kind == TK_EOF)
			break;
		const char *piece = toklit(a, t);
		u64 plen = strlen(piece);
		u64 need = n + plen + 2;
		if (need >= cap) {
			u64 nc = need * 2;
			char *nb = amalloc(a, nc);
			memcpy(nb, got, n);
			got = nb;
			cap = nc;
		}
		if (n) got[n++] = ' ';
		memcpy(got + n, piece, plen);
		n += plen;
		got[n] = '\0';
	}

	int ok = strcmp(got, expect) == 0;
	if (!ok) {
		fprintf(stderr, "lex mismatch:\n  src:    %s\n"
		    "  want:   %s\n  got:    %s\n", src, expect, got);
	}
	freearena(a);
	return ok;
}

struct row { const char *src, *expect; };

static const struct row rows[] = {
	{ "",                "" },
	{ "  \t\n  ",        "" },
	{ "// comment\n",    "" },
	{ "/* a /b/ c */",   "" },

	/* identifiers + keywords */
	{ "foo",             "IDENT(foo)" },
	{ "fn",              "fn" },
	{ "fn main",         "fn IDENT(main)" },
	{ "let x: i32 = 0;", "let IDENT(x) : IDENT(i32) = INT(0) ;" },
	{ "export fn",       "export fn" },
	{ "if else for switch case return import type struct defer break continue proc chan nil true false package",
	  "if else for switch case return import type struct defer break continue proc chan nil true false package" },

	/* numbers */
	{ "0",               "INT(0)" },
	{ "42",              "INT(42)" },
	{ "1_000_000",       "INT(1000000)" },
	{ "0xff",            "INT(255)" },
	{ "0xDE_AD_BE_EF",   "INT(3735928559)" },
	{ "0b1010",          "INT(10)" },
	{ "0o777",           "INT(511)" },
	{ "3.14",            "FLOAT(3.14)" },
	{ "1.5e3",           "FLOAT(1500)" },

	/* strings & runes */
	{ "\"hello\"",       "STR(hello)" },
	{ "\"a\\nb\"",       "STR(a\nb)" },
	{ "'A'",             "RUNE(65)" },
	{ "'\\n'",           "RUNE(10)" },
	{ "'\\x7f'",         "RUNE(127)" },

	/* unicode escapes — \u (4 hex) / \U (8 hex) share the \x codepoint
	 * path (ref/hare/hare/lex/lex.ha:347 lex_unicode). Rune literals
	 * carry the raw codepoint; string literals UTF-8-encode it. */
	{ "'\\u00e9'",       "RUNE(233)" },
	{ "'\\u20ac'",       "RUNE(8364)" },
	{ "'\\U0001F600'",   "RUNE(128512)" },
	{ "\"\\u00e9\"",     "STR(\xc3\xa9)" },
	{ "\"\\u20ac\"",     "STR(\xe2\x82\xac)" },
	{ "\"\\U0001F600\"", "STR(\xf0\x9f\x98\x80)" },
	{ "\"caf\\u00e9\"",  "STR(caf\xc3\xa9)" },
	/* \x now also yields a codepoint that UTF-8-encodes in strings:
	 * \xe9 -> U+00E9 -> 0xC3 0xA9 (matches Hare's appendrune). */
	{ "'\\xe9'",         "RUNE(233)" },
	{ "\"\\xe9\"",       "STR(\xc3\xa9)" },
	/* range boundaries: max valid codepoint, and the two edges that
	 * straddle the UTF-16 surrogate gap (0xD7FF ok / 0xE000 ok). */
	{ "'\\U0010FFFF'",   "RUNE(1114111)" },
	{ "'\\uD7FF'",       "RUNE(55295)" },
	{ "'\\uE000'",       "RUNE(57344)" },

	/* operators & punct */
	{ "+ - * / % == != < > <= >= && || !",
	  "+ - * / % == != < > <= >= && || !" },
	{ "= += -= *= /= %= &= |= ^= <<= >>=",
	  "= += -= *= /= %= &= |= ^= <<= >>=" },
	{ "<< >> & | ^ ~ ?",
	  "<< >> & | ^ ~ ?" },
	{ "( ) { } [ ] , ; : . ... @",
	  "( ) { } [ ] , ; : . ... @" },
	{ "<- ->",
	  "<- ->" },
	{ "@symbol(\"malloc\")",
	  "@ IDENT(symbol) ( STR(malloc) )" },

	/* mixed */
	{ "fn add(a: i32, b: i32) i32 = { return a + b; };",
	  "fn IDENT(add) ( IDENT(a) : IDENT(i32) , IDENT(b) : IDENT(i32) ) IDENT(i32) = { return IDENT(a) + IDENT(b) ; } ;" },
};

/* Escape error cases. A bad escape sets l.errs (the string/rune still
 * lexes with the offending codepoint zeroed), so detection is via the
 * error counter, not the token stream. The verbatim Hare messages live
 * at lexunicode in cmd/wcc/lex.c. */
static int
runerr(const char *src)
{
	Arena *a = newarena();
	Lex l;
	lexinit(&l, a, "<test>", src, strlen(src));
	for (;;) {
		Tok t = lexnext(&l);
		if (t.kind == TK_EOF)
			break;
	}
	int ok = l.errs > 0;
	if (!ok)
		fprintf(stderr, "expected escape error, none raised:\n  src: %s\n",
		    src);
	freearena(a);
	return ok;
}

static const char *const errrows[] = {
	"'\\uZ'",		/* unexpected rune scanning for escape */
	"\"\\u00g0\"",		/* non-hex digit inside a string escape */
	"'\\u00",		/* unexpected EOF scanning for escape */
	"'\\UFFFFFFFF'",	/* codepoint > 0x10FFFF (high bit set) */
	"'\\U00110000'",	/* exactly one past U+10FFFF */
	"'\\uD800'",		/* bottom of the UTF-16 surrogate range */
	"'\\uDFFF'",		/* top of the UTF-16 surrogate range */
};

int
main(void)
{
	int fail = 0;
	for (size_t i = 0; i < sizeof rows / sizeof rows[0]; i++) {
		if (!runrow(rows[i].src, rows[i].expect)) {
			fprintf(stderr, "row %zu failed\n", i);
			fail++;
		}
	}
	for (size_t i = 0; i < sizeof errrows / sizeof errrows[0]; i++) {
		if (!runerr(errrows[i])) {
			fprintf(stderr, "errrow %zu failed\n", i);
			fail++;
		}
	}
	if (fail) {
		fprintf(stderr, "%d/%zu lex tests failed\n", fail,
		    sizeof rows / sizeof rows[0]);
		return 1;
	}
	printf("lex: %zu/%zu ok\n", sizeof rows / sizeof rows[0],
	    sizeof rows / sizeof rows[0]);
	return 0;
}