ww/test/wwc/100_lex.c

/*
 * 100_lex — table-driven lexer tests.
 *
 * Each row is a (src, expected) pair. The expected string is the
 * concatenation of token names, space-separated. For literals we
 * also encode the value: e.g. INT(42), STR("hi"), IDENT(foo).
 *
 * EOF is implicit: the harness checks that lexnext returns TK_EOF
 * after the last expected token.
 */
#include "ww.h"
#include <string.h>
#include <stdlib.h>
#include <stdio.h>

static char *
toklit(Arena *a, Tok t)
{
	switch (t.kind) {
	case TK_IDENT: return aprintf(a, "IDENT(%s)", t.text);
	case TK_INT:   return aprintf(a, "INT(%llu)", (unsigned long long)t.v.uval);
	case TK_FLOAT: return aprintf(a, "FLOAT(%g)", t.v.fval);
	case TK_RUNE:  return aprintf(a, "RUNE(%llu)", (unsigned long long)t.v.uval);
	case TK_STR:   return aprintf(a, "STR(%s)", t.text);
	case TK_ERR:   return aprintf(a, "ERR(%s)", t.text);
	default:       return (char *)tokname(t.kind);
	}
}

static int
runrow(const char *src, const char *expect)
{
	Arena *a = newarena();
	Lex l;
	lexinit(&l, a, "<test>", src, strlen(src));

	char *got = amalloc(a, 1);
	got[0] = '\0';
	u64 cap = 1, n = 0;

	for (;;) {
		Tok t = lexnext(&l);
		if (t.kind == TK_EOF)
			break;
		const char *piece = toklit(a, t);
		u64 plen = strlen(piece);
		u64 need = n + plen + 2;
		if (need >= cap) {
			u64 nc = need * 2;
			char *nb = amalloc(a, nc);
			memcpy(nb, got, n);
			got = nb;
			cap = nc;
		}
		if (n) got[n++] = ' ';
		memcpy(got + n, piece, plen);
		n += plen;
		got[n] = '\0';
	}

	int ok = strcmp(got, expect) == 0;
	if (!ok) {
		fprintf(stderr, "lex mismatch:\n  src:    %s\n"
		    "  want:   %s\n  got:    %s\n", src, expect, got);
	}
	freearena(a);
	return ok;
}

struct row { const char *src, *expect; };

static const struct row rows[] = {
	{ "",                "" },
	{ "  \t\n  ",        "" },
	{ "// comment\n",    "" },
	{ "/* a /b/ c */",   "" },

	/* identifiers + keywords */
	{ "foo",             "IDENT(foo)" },
	{ "fn",              "fn" },
	{ "fn main",         "fn IDENT(main)" },
	{ "let x: i32 = 0;", "let IDENT(x) : IDENT(i32) = INT(0) ;" },
	{ "export fn",       "export fn" },
	{ "if else for switch case return use type struct defer break continue proc chan nil true false",
	  "if else for switch case return use type struct defer break continue proc chan nil true false" },

	/* numbers */
	{ "0",               "INT(0)" },
	{ "42",              "INT(42)" },
	{ "1_000_000",       "INT(1000000)" },
	{ "0xff",            "INT(255)" },
	{ "0xDE_AD_BE_EF",   "INT(3735928559)" },
	{ "0b1010",          "INT(10)" },
	{ "0o777",           "INT(511)" },
	{ "3.14",            "FLOAT(3.14)" },
	{ "1.5e3",           "FLOAT(1500)" },

	/* strings & runes */
	{ "\"hello\"",       "STR(hello)" },
	{ "\"a\\nb\"",       "STR(a\nb)" },
	{ "'A'",             "RUNE(65)" },
	{ "'\\n'",           "RUNE(10)" },
	{ "'\\x7f'",         "RUNE(127)" },

	/* operators & punct */
	{ "+ - * / % == != < > <= >= && || !",
	  "+ - * / % == != < > <= >= && || !" },
	{ "= += -= *= /= %= &= |= ^= <<= >>=",
	  "= += -= *= /= %= &= |= ^= <<= >>=" },
	{ "<< >> & | ^ ~ ?",
	  "<< >> & | ^ ~ ?" },
	{ "( ) { } [ ] , ; : . ... @",
	  "( ) { } [ ] , ; : . ... @" },
	{ "<- ->",
	  "<- ->" },
	{ "@symbol(\"malloc\")",
	  "@ IDENT(symbol) ( STR(malloc) )" },

	/* mixed */
	{ "fn add(a: i32, b: i32) i32 = { return a + b; };",
	  "fn IDENT(add) ( IDENT(a) : IDENT(i32) , IDENT(b) : IDENT(i32) ) IDENT(i32) = { return IDENT(a) + IDENT(b) ; } ;" },
};

int
main(void)
{
	int fail = 0;
	for (size_t i = 0; i < sizeof rows / sizeof rows[0]; i++) {
		if (!runrow(rows[i].src, rows[i].expect)) {
			fprintf(stderr, "row %zu failed\n", i);
			fail++;
		}
	}
	if (fail) {
		fprintf(stderr, "%d/%zu lex tests failed\n", fail,
		    sizeof rows / sizeof rows[0]);
		return 1;
	}
	printf("lex: %zu/%zu ok\n", sizeof rows / sizeof rows[0],
	    sizeof rows / sizeof rows[0]);
	return 0;
}