Hojun-Cho/ww
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

lex,wwi: \u/\U unicode escapes + wide-rune .wwi round-trip, both stages (#50)

Browse Source 
Hare-faithful \u (4 hex) / \U (8 hex) escapes; \x/\u/\U share one codepoint
path (ref/hare/hare/lex/lex.ha lex_unicode); string literals UTF-8-encode
multi-byte codepoints (cstage inline utf8enc, wwstage utf8.encoderune). The
.wwi producer rune serializer now emits \u/\U so exported wide-rune defs
round-trip (was a fatal >0xFF). Reject >0x10FFFF and surrogates with Hare-
verbatim error strings. Closes the int-cast spelling divergence (#48 RUNE_MAX).
Both stages byte-identical; 446 tests pass.
This commit is contained in:
Hojun-Cho
2026-06-18 22:47:53 +09:00
parent 64cf3c4094
commit 0197dfb9e6
7 changed files with 508 additions and 63 deletions
Download Patch File Download Diff File Expand all files Collapse all files

59
test/wcc/100_lex.c
View File

@@ -102,6 +102,26 @@ static const struct row rows[] = {
{ "'\\n'", "RUNE(10)" },
{ "'\\x7f'", "RUNE(127)" },
/* unicode escapes — \u (4 hex) / \U (8 hex) share the \x codepoint
* path (ref/hare/hare/lex/lex.ha:347 lex_unicode). Rune literals
* carry the raw codepoint; string literals UTF-8-encode it. */
{ "'\\u00e9'", "RUNE(233)" },
{ "'\\u20ac'", "RUNE(8364)" },
{ "'\\U0001F600'", "RUNE(128512)" },
{ "\"\\u00e9\"", "STR(\xc3\xa9)" },
{ "\"\\u20ac\"", "STR(\xe2\x82\xac)" },
{ "\"\\U0001F600\"", "STR(\xf0\x9f\x98\x80)" },
{ "\"caf\\u00e9\"", "STR(caf\xc3\xa9)" },
/* \x now also yields a codepoint that UTF-8-encodes in strings:
* \xe9 -> U+00E9 -> 0xC3 0xA9 (matches Hare's appendrune). */
{ "'\\xe9'", "RUNE(233)" },
{ "\"\\xe9\"", "STR(\xc3\xa9)" },
/* range boundaries: max valid codepoint, and the two edges that
* straddle the UTF-16 surrogate gap (0xD7FF ok / 0xE000 ok). */
{ "'\\U0010FFFF'", "RUNE(1114111)" },
{ "'\\uD7FF'", "RUNE(55295)" },
{ "'\\uE000'", "RUNE(57344)" },
/* operators & punct */
{ "+ - * / % == != < > <= >= && || !",
"+ - * / % == != < > <= >= && || !" },
@@ -121,6 +141,39 @@ static const struct row rows[] = {
"fn IDENT(add) ( IDENT(a) : IDENT(i32) , IDENT(b) : IDENT(i32) ) IDENT(i32) = { return IDENT(a) + IDENT(b) ; } ;" },
};
/* Escape error cases. A bad escape sets l.errs (the string/rune still
* lexes with the offending codepoint zeroed), so detection is via the
* error counter, not the token stream. The verbatim Hare messages live
* at lexunicode in cmd/wcc/lex.c. */
static int
runerr(const char *src)
{
Arena *a = newarena();
Lex l;
lexinit(&l, a, "<test>", src, strlen(src));
for (;;) {
Tok t = lexnext(&l);
if (t.kind == TK_EOF)
break;
}
int ok = l.errs > 0;
if (!ok)
fprintf(stderr, "expected escape error, none raised:\n src: %s\n",
src);
freearena(a);
return ok;
}
static const char *const errrows[] = {
"'\\uZ'", /* unexpected rune scanning for escape */
"\"\\u00g0\"", /* non-hex digit inside a string escape */
"'\\u00", /* unexpected EOF scanning for escape */
"'\\UFFFFFFFF'", /* codepoint > 0x10FFFF (high bit set) */
"'\\U00110000'", /* exactly one past U+10FFFF */
"'\\uD800'", /* bottom of the UTF-16 surrogate range */
"'\\uDFFF'", /* top of the UTF-16 surrogate range */
};
int
main(void)
{
@@ -131,6 +184,12 @@ main(void)
fail++;
}
}
for (size_t i = 0; i < sizeof errrows / sizeof errrows[0]; i++) {
if (!runerr(errrows[i])) {
fprintf(stderr, "errrow %zu failed\n", i);
fail++;
}
}
if (fail) {
fprintf(stderr, "%d/%zu lex tests failed\n", fail,
sizeof rows / sizeof rows[0]);