Hojun-Cho/ww
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

lex,wwi: \u/\U unicode escapes + wide-rune .wwi round-trip, both stages (#50)

Browse Source 
Hare-faithful \u (4 hex) / \U (8 hex) escapes; \x/\u/\U share one codepoint
path (ref/hare/hare/lex/lex.ha lex_unicode); string literals UTF-8-encode
multi-byte codepoints (cstage inline utf8enc, wwstage utf8.encoderune). The
.wwi producer rune serializer now emits \u/\U so exported wide-rune defs
round-trip (was a fatal >0xFF). Reject >0x10FFFF and surrogates with Hare-
verbatim error strings. Closes the int-cast spelling divergence (#48 RUNE_MAX).
Both stages byte-identical; 446 tests pass.
This commit is contained in:
Hojun-Cho
2026-06-18 22:47:53 +09:00
parent 64cf3c4094
commit 0197dfb9e6
7 changed files with 508 additions and 63 deletions
Download Patch File Download Diff File Expand all files Collapse all files

98
cmd/wcc/lex.c
View File

@@ -210,6 +210,68 @@ parseint(const char *s, u64 n, int base, int *ok)
return v;
}
/* shared escape decoder for \xHH (n=2), \uHHHH (n=4), \UHHHHHHHH (n=8).
* All three yield a codepoint, not a raw byte — mirrors
* ref/hare/hare/lex/lex.ha:347 fn lex_unicode. Error strings copied
* verbatim from that reference for diagnostic fidelity (#50). */
static int
lexunicode(Lex *l, int n, int *out)
{
u32 u = 0;
for (int i = 0; i < n; i++) {
int c = lget(l);
if (c < 0) {
Pos p = lpos(l);
errorf(p, "unexpected EOF scanning for escape");
l->errs++;
return -1;
}
if (!ishex(c)) {
Pos p = lpos(l);
errorf(p, "unexpected rune scanning for escape");
l->errs++;
return -1;
}
int d = (c <= '9' ? c - '0' : (c | 0x20) - 'a' + 10);
u = (u << 4) | (u32)d;
}
if (u > 0x10FFFF || (u >= 0xD800 && u < 0xE000)) {
Pos p = lpos(l);
errorf(p, "invalid unicode codepoint in escape");
l->errs++;
return -1;
}
*out = (int)u;
return 0;
}
/* utf8enc — encode codepoint cp (already validated <= 0x10FFFF and
* non-surrogate by lexunicode) into out (>= 4 bytes), return the byte
* count. The C bootstrap has no stdlib; this mirrors lib/encoding/utf8
* encoderune (the ww side calls that directly). */
static int
utf8enc(u32 cp, char *out)
{
if (cp < 0x80) {
out[0] = (char)cp;
return 1;
} else if (cp < 0x800) {
out[0] = (char)(0xC0 | (cp >> 6));
out[1] = (char)(0x80 | (cp & 0x3F));
return 2;
} else if (cp < 0x10000) {
out[0] = (char)(0xE0 | (cp >> 12));
out[1] = (char)(0x80 | ((cp >> 6) & 0x3F));
out[2] = (char)(0x80 | (cp & 0x3F));
return 3;
}
out[0] = (char)(0xF0 | (cp >> 18));
out[1] = (char)(0x80 | ((cp >> 12) & 0x3F));
out[2] = (char)(0x80 | ((cp >> 6) & 0x3F));
out[3] = (char)(0x80 | (cp & 0x3F));
return 4;
}
static int
escape(Lex *l, int *out)
{
@@ -227,19 +289,9 @@ escape(Lex *l, int *out)
case 'b': *out = '\b'; return 0;
case 'f': *out = '\f'; return 0;
case 'v': *out = '\v'; return 0;
case 'x': {
int hi = lget(l), lo = lget(l);
if (!ishex(hi) || !ishex(lo)) {
Pos p = lpos(l);
errorf(p, "bad \\x escape");
l->errs++;
return -1;
}
int h = (hi <= '9' ? hi - '0' : (hi | 0x20) - 'a' + 10);
int o = (lo <= '9' ? lo - '0' : (lo | 0x20) - 'a' + 10);
*out = (h << 4) | o;
return 0;
}
case 'x': return lexunicode(l, 2, out);
case 'u': return lexunicode(l, 4, out);
case 'U': return lexunicode(l, 8, out);
}
{ Pos p = lpos(l); errorf(p, "bad escape \\%c", c); l->errs++; }
return -1;
@@ -389,22 +441,34 @@ lexstr(Lex *l, Pos start)
return t;
}
if (c == '"') { lget(l); break; }
int ch;
/* Escape-decoded values are codepoints and UTF-8-encode into
* 1-4 bytes (mirrors Hare's memio::appendrune in lex_string,
* ref/hare/hare/lex/lex.ha:431). Raw source bytes are already
* UTF-8 and pass through unchanged — re-encoding them would
* double-encode the >0x7F continuation bytes. */
char enc[4];
int el;
if (c == '\\') {
int ch;
lget(l);
if (escape(l, &ch) < 0)
ch = 0;
el = utf8enc((u32)ch, enc);
} else {
ch = lget(l);
enc[0] = (char)lget(l);
el = 1;
}
if (n + 1 >= cap) {
if (n + el >= cap) {
u64 ncap = cap * 2;
while (n + el >= ncap)
ncap *= 2;
char *nb = amalloc(l->a, ncap);
memcpy(nb, buf, n);
buf = nb;
cap = ncap;
}
buf[n++] = (char)ch;
for (int i = 0; i < el; i++)
buf[n++] = enc[i];
}
buf[n] = '\0';
Tok t = (Tok){ TK_STR, start, buf, n, {0}, TK_NONE };