212 lines
6.1 KiB
C
212 lines
6.1 KiB
C
#include "ww.h"
|
|
#include <string.h>
|
|
#include <stdlib.h>
|
|
#include <stdio.h>
|
|
|
|
static char *
|
|
toklit(Arena *a, Tok t)
|
|
{
|
|
switch (t.kind) {
|
|
case TK_IDENT: return aprintf(a, "IDENT(%s)", t.text);
|
|
case TK_INT: return aprintf(a, "INT(%llu)", (unsigned long long)t.v.uval);
|
|
case TK_FLOAT: return aprintf(a, "FLOAT(%g)", t.v.fval);
|
|
case TK_RUNE: return aprintf(a, "RUNE(%llu)", (unsigned long long)t.v.uval);
|
|
case TK_STR: return aprintf(a, "STR(%s)", t.text);
|
|
case TK_ERR: return aprintf(a, "ERR(%s)", t.text);
|
|
default: return (char *)tokname(t.kind);
|
|
}
|
|
}
|
|
|
|
static int
|
|
runrow(const char *src, const char *expect)
|
|
{
|
|
Arena *a = newarena();
|
|
Lex l;
|
|
lexinit(&l, a, "<test>", src, strlen(src));
|
|
|
|
char *got = amalloc(a, 1);
|
|
got[0] = '\0';
|
|
u64 cap = 1, n = 0;
|
|
|
|
for (;;) {
|
|
Tok t = lexnext(&l);
|
|
if (t.kind == TK_EOF)
|
|
break;
|
|
const char *piece = toklit(a, t);
|
|
u64 plen = strlen(piece);
|
|
u64 need = n + plen + 2;
|
|
if (need >= cap) {
|
|
u64 nc = need * 2;
|
|
char *nb = amalloc(a, nc);
|
|
memcpy(nb, got, n);
|
|
got = nb;
|
|
cap = nc;
|
|
}
|
|
if (n) got[n++] = ' ';
|
|
memcpy(got + n, piece, plen);
|
|
n += plen;
|
|
got[n] = '\0';
|
|
}
|
|
|
|
int ok = strcmp(got, expect) == 0;
|
|
if (!ok) {
|
|
fprintf(stderr, "lex mismatch:\n src: %s\n"
|
|
" want: %s\n got: %s\n", src, expect, got);
|
|
}
|
|
freearena(a);
|
|
return ok;
|
|
}
|
|
|
|
struct row { const char *src, *expect; };
|
|
|
|
static const struct row rows[] = {
|
|
{ "", "" },
|
|
{ " \t\n ", "" },
|
|
{ "// comment\n", "" },
|
|
{ "/* a /b/ c */", "" },
|
|
|
|
{ "foo", "IDENT(foo)" },
|
|
{ "fn", "fn" },
|
|
{ "fn main", "fn IDENT(main)" },
|
|
{ "let x: i32 = 0;", "let IDENT(x) : IDENT(i32) = INT(0) ;" },
|
|
{ "export fn", "export fn" },
|
|
{ "if else for switch case return import type struct defer break continue proc chan nil true false package",
|
|
"if else for switch case return import type struct defer break continue proc chan nil true false package" },
|
|
|
|
{ "0", "INT(0)" },
|
|
{ "42", "INT(42)" },
|
|
{ "1_000_000", "INT(1000000)" },
|
|
{ "0xff", "INT(255)" },
|
|
{ "0xDE_AD_BE_EF", "INT(3735928559)" },
|
|
{ "0b1010", "INT(10)" },
|
|
{ "0o777", "INT(511)" },
|
|
{ "18446744073709551616", "ERR(18446744073709551616)" },
|
|
{ "3.14", "FLOAT(3.14)" },
|
|
{ "1.5e3", "FLOAT(1500)" },
|
|
|
|
{ "\"hello\"", "STR(hello)" },
|
|
{ "\"a\\nb\"", "STR(a\nb)" },
|
|
{ "'A'", "RUNE(65)" },
|
|
{ "'\\n'", "RUNE(10)" },
|
|
{ "'\\x7f'", "RUNE(127)" },
|
|
|
|
/* unicode escapes — \u (4 hex) / \U (8 hex) share the \x codepoint
|
|
* path (ref/hare/hare/lex/lex.ha:347 lex_unicode). Rune literals
|
|
* carry the raw codepoint; string literals UTF-8-encode it. */
|
|
{ "'\\u00e9'", "RUNE(233)" },
|
|
{ "'\\u20ac'", "RUNE(8364)" },
|
|
{ "'\\U0001F600'", "RUNE(128512)" },
|
|
{ "\"\\u00e9\"", "STR(\xc3\xa9)" },
|
|
{ "\"\\u20ac\"", "STR(\xe2\x82\xac)" },
|
|
{ "\"\\U0001F600\"", "STR(\xf0\x9f\x98\x80)" },
|
|
{ "\"caf\\u00e9\"", "STR(caf\xc3\xa9)" },
|
|
/* \x now also yields a codepoint that UTF-8-encodes in strings:
|
|
* \xe9 -> U+00E9 -> 0xC3 0xA9 (matches Hare's appendrune). */
|
|
{ "'\\xe9'", "RUNE(233)" },
|
|
{ "\"\\xe9\"", "STR(\xc3\xa9)" },
|
|
/* range boundaries: max valid codepoint, and the two edges that
|
|
* straddle the UTF-16 surrogate gap (0xD7FF ok / 0xE000 ok). */
|
|
{ "'\\U0010FFFF'", "RUNE(1114111)" },
|
|
{ "'\\uD7FF'", "RUNE(55295)" },
|
|
{ "'\\uE000'", "RUNE(57344)" },
|
|
|
|
{ "+ - * / % == != < > <= >= && || !",
|
|
"+ - * / % == != < > <= >= && || !" },
|
|
{ "= += -= *= /= %= &= |= ^= <<= >>=",
|
|
"= += -= *= /= %= &= |= ^= <<= >>=" },
|
|
{ "<< >> & | ^ ~ ?",
|
|
"<< >> & | ^ ~ ?" },
|
|
{ "( ) { } [ ] , ; : . ... @",
|
|
"( ) { } [ ] , ; : . ... @" },
|
|
{ "<- ->",
|
|
"<- ->" },
|
|
{ "@symbol(\"malloc\")",
|
|
"@ IDENT(symbol) ( STR(malloc) )" },
|
|
|
|
{ "fn add(a: i32, b: i32) i32 = { return a + b; };",
|
|
"fn IDENT(add) ( IDENT(a) : IDENT(i32) , IDENT(b) : IDENT(i32) ) IDENT(i32) = { return IDENT(a) + IDENT(b) ; } ;" },
|
|
};
|
|
|
|
/* A bad escape sets l.errs (the string/rune still lexes with the
|
|
* offending codepoint zeroed), so detection is via the error counter,
|
|
* not the token stream. The verbatim Hare messages live at lexunicode
|
|
* in cmd/wcc/lex.c. */
|
|
static int
|
|
runerr(const char *src)
|
|
{
|
|
Arena *a = newarena();
|
|
Lex l;
|
|
lexinit(&l, a, "<test>", src, strlen(src));
|
|
for (;;) {
|
|
Tok t = lexnext(&l);
|
|
if (t.kind == TK_EOF)
|
|
break;
|
|
}
|
|
int ok = l.errs > 0;
|
|
if (!ok)
|
|
fprintf(stderr, "expected escape error, none raised:\n src: %s\n",
|
|
src);
|
|
freearena(a);
|
|
return ok;
|
|
}
|
|
|
|
static int
|
|
runrewind(void)
|
|
{
|
|
const char *src = "1foo bar";
|
|
Arena *a = newarena();
|
|
Lex l;
|
|
lexinit(&l, a, "<test>", src, strlen(src));
|
|
Tok n = lexnext(&l);
|
|
Tok f = lexnext(&l);
|
|
Tok b = lexnext(&l);
|
|
int ok = n.kind == TK_INT && n.pos.col == 1 && n.v.uval == 1
|
|
&& f.kind == TK_IDENT && f.pos.col == 2
|
|
&& strcmp(f.text, "foo") == 0
|
|
&& b.kind == TK_IDENT && b.pos.col == 6
|
|
&& strcmp(b.text, "bar") == 0;
|
|
if (!ok)
|
|
fprintf(stderr, "suffix rewind columns: want 1,2,6 got %d,%d,%d\n",
|
|
n.pos.col, f.pos.col, b.pos.col);
|
|
freearena(a);
|
|
return ok;
|
|
}
|
|
|
|
static const char *const errrows[] = {
|
|
"'\\uZ'", /* unexpected rune scanning for escape */
|
|
"\"\\u00g0\"", /* non-hex digit inside a string escape */
|
|
"'\\u00", /* unexpected EOF scanning for escape */
|
|
"'\\UFFFFFFFF'", /* codepoint > 0x10FFFF (high bit set) */
|
|
"'\\U00110000'", /* exactly one past U+10FFFF */
|
|
"'\\uD800'", /* bottom of the UTF-16 surrogate range */
|
|
"'\\uDFFF'", /* top of the UTF-16 surrogate range */
|
|
};
|
|
|
|
int
|
|
main(void)
|
|
{
|
|
int fail = 0;
|
|
for (size_t i = 0; i < sizeof rows / sizeof rows[0]; i++) {
|
|
if (!runrow(rows[i].src, rows[i].expect)) {
|
|
fprintf(stderr, "row %zu failed\n", i);
|
|
fail++;
|
|
}
|
|
}
|
|
if (!runrewind())
|
|
fail++;
|
|
for (size_t i = 0; i < sizeof errrows / sizeof errrows[0]; i++) {
|
|
if (!runerr(errrows[i])) {
|
|
fprintf(stderr, "errrow %zu failed\n", i);
|
|
fail++;
|
|
}
|
|
}
|
|
if (fail) {
|
|
fprintf(stderr, "%d/%zu lex tests failed\n", fail,
|
|
sizeof rows / sizeof rows[0]);
|
|
return 1;
|
|
}
|
|
printf("lex: %zu/%zu ok\n", sizeof rows / sizeof rows[0],
|
|
sizeof rows / sizeof rows[0]);
|
|
return 0;
|
|
}
|