lex,wwi: \u/\U unicode escapes + wide-rune .wwi round-trip, both stages (#50)
Hare-faithful \u (4 hex) / \U (8 hex) escapes; \x/\u/\U share one codepoint path (ref/hare/hare/lex/lex.ha lex_unicode); string literals UTF-8-encode multi-byte codepoints (cstage inline utf8enc, wwstage utf8.encoderune). The .wwi producer rune serializer now emits \u/\U so exported wide-rune defs round-trip (was a fatal >0xFF). Reject >0x10FFFF and surrogates with Hare- verbatim error strings. Closes the int-cast spelling divergence (#48 RUNE_MAX). Both stages byte-identical; 446 tests pass.
This commit is contained in:
8
Makefile
8
Makefile
@@ -236,7 +236,7 @@ TESTS = $(BIN)/test_smoke $(BIN)/test_lex $(BIN)/test_parse $(BIN)/test_check \
|
|||||||
$(BIN)/test_driver_flagargs \
|
$(BIN)/test_driver_flagargs \
|
||||||
$(BIN)/test_let_global $(BIN)/test_def_neg_global \
|
$(BIN)/test_let_global $(BIN)/test_def_neg_global \
|
||||||
$(BIN)/test_def_const_fold \
|
$(BIN)/test_def_const_fold \
|
||||||
$(BIN)/test_int_cast_signed $(BIN)/test_dot_chain \
|
$(BIN)/test_int_cast_signed $(BIN)/test_uniesc_run $(BIN)/test_dot_chain \
|
||||||
$(BIN)/test_amp_dot $(BIN)/test_arr_elem_field \
|
$(BIN)/test_amp_dot $(BIN)/test_arr_elem_field \
|
||||||
$(BIN)/test_arr_elem_field_write \
|
$(BIN)/test_arr_elem_field_write \
|
||||||
$(BIN)/test_arr_enum_elem \
|
$(BIN)/test_arr_enum_elem \
|
||||||
@@ -1304,6 +1304,12 @@ $(BIN)/test_int_cast_signed: test/wcc/640_int_cast_signed.c $(BIN)/ww \
|
|||||||
$(LIB)/libwwrt.a | $(BIN)
|
$(LIB)/libwwrt.a | $(BIN)
|
||||||
$(CC) $(CFLAGS) -o $@ $<
|
$(CC) $(CFLAGS) -o $@ $<
|
||||||
|
|
||||||
|
$(BIN)/test_uniesc_run: test/wcc/110_uniesc_run.c $(BIN)/ww \
|
||||||
|
$(BIN)/w6c $(BIN)/w6a $(BIN)/w6l \
|
||||||
|
$(BIN)/ww_ww $(BIN)/w6c_ww $(BIN)/w6a_ww $(BIN)/w6l_ww \
|
||||||
|
$(LIB)/libwwrt.a | $(BIN)
|
||||||
|
$(CC) $(CFLAGS) -o $@ $<
|
||||||
|
|
||||||
$(BIN)/test_dot_chain: test/wcc/650_dot_chain.c $(BIN)/ww \
|
$(BIN)/test_dot_chain: test/wcc/650_dot_chain.c $(BIN)/ww \
|
||||||
$(BIN)/w6c $(BIN)/w6a $(BIN)/w6l \
|
$(BIN)/w6c $(BIN)/w6a $(BIN)/w6l \
|
||||||
$(BIN)/ww_ww $(BIN)/w6c_ww $(BIN)/w6a_ww $(BIN)/w6l_ww \
|
$(BIN)/ww_ww $(BIN)/w6c_ww $(BIN)/w6a_ww $(BIN)/w6l_ww \
|
||||||
|
|||||||
@@ -167,14 +167,17 @@ wwi_quote(FILE *of, const char *s, u64 n)
|
|||||||
static void
|
static void
|
||||||
wwi_rune(FILE *of, u64 cp)
|
wwi_rune(FILE *of, u64 cp)
|
||||||
{
|
{
|
||||||
/* The lexer's rune escapes stop at \xHH (no \u/\U — task #50), so a
|
/* #50: the lexer now reads \u/\U, so wide codepoints round-trip as
|
||||||
* codepoint above 0xff can't render as a re-parseable rune literal;
|
* those escapes (write-twin of lex.c lexunicode). Codepoints <=0xff
|
||||||
* fail loud rather than emit a malformed one. No exported def names
|
* keep the \xHH spelling they already round-tripped as. */
|
||||||
* such a rune today (RUNE_MAX is written as an int-cast for the same
|
if (cp > 0xffff) {
|
||||||
* reason). */
|
fprintf(of, "'\\U%08x'", (unsigned)cp);
|
||||||
if (cp > 0xff)
|
return;
|
||||||
fatal("wwi: rune codepoint U+%llx exceeds \\xHH (task #50)",
|
}
|
||||||
(unsigned long long)cp);
|
if (cp > 0xff) {
|
||||||
|
fprintf(of, "'\\u%04x'", (unsigned)cp);
|
||||||
|
return;
|
||||||
|
}
|
||||||
unsigned char c = (unsigned char)cp;
|
unsigned char c = (unsigned char)cp;
|
||||||
fputc('\'', of);
|
fputc('\'', of);
|
||||||
switch (c) {
|
switch (c) {
|
||||||
|
|||||||
@@ -210,6 +210,68 @@ parseint(const char *s, u64 n, int base, int *ok)
|
|||||||
return v;
|
return v;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/* shared escape decoder for \xHH (n=2), \uHHHH (n=4), \UHHHHHHHH (n=8).
|
||||||
|
* All three yield a codepoint, not a raw byte — mirrors
|
||||||
|
* ref/hare/hare/lex/lex.ha:347 fn lex_unicode. Error strings copied
|
||||||
|
* verbatim from that reference for diagnostic fidelity (#50). */
|
||||||
|
static int
|
||||||
|
lexunicode(Lex *l, int n, int *out)
|
||||||
|
{
|
||||||
|
u32 u = 0;
|
||||||
|
for (int i = 0; i < n; i++) {
|
||||||
|
int c = lget(l);
|
||||||
|
if (c < 0) {
|
||||||
|
Pos p = lpos(l);
|
||||||
|
errorf(p, "unexpected EOF scanning for escape");
|
||||||
|
l->errs++;
|
||||||
|
return -1;
|
||||||
|
}
|
||||||
|
if (!ishex(c)) {
|
||||||
|
Pos p = lpos(l);
|
||||||
|
errorf(p, "unexpected rune scanning for escape");
|
||||||
|
l->errs++;
|
||||||
|
return -1;
|
||||||
|
}
|
||||||
|
int d = (c <= '9' ? c - '0' : (c | 0x20) - 'a' + 10);
|
||||||
|
u = (u << 4) | (u32)d;
|
||||||
|
}
|
||||||
|
if (u > 0x10FFFF || (u >= 0xD800 && u < 0xE000)) {
|
||||||
|
Pos p = lpos(l);
|
||||||
|
errorf(p, "invalid unicode codepoint in escape");
|
||||||
|
l->errs++;
|
||||||
|
return -1;
|
||||||
|
}
|
||||||
|
*out = (int)u;
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* utf8enc — encode codepoint cp (already validated <= 0x10FFFF and
|
||||||
|
* non-surrogate by lexunicode) into out (>= 4 bytes), return the byte
|
||||||
|
* count. The C bootstrap has no stdlib; this mirrors lib/encoding/utf8
|
||||||
|
* encoderune (the ww side calls that directly). */
|
||||||
|
static int
|
||||||
|
utf8enc(u32 cp, char *out)
|
||||||
|
{
|
||||||
|
if (cp < 0x80) {
|
||||||
|
out[0] = (char)cp;
|
||||||
|
return 1;
|
||||||
|
} else if (cp < 0x800) {
|
||||||
|
out[0] = (char)(0xC0 | (cp >> 6));
|
||||||
|
out[1] = (char)(0x80 | (cp & 0x3F));
|
||||||
|
return 2;
|
||||||
|
} else if (cp < 0x10000) {
|
||||||
|
out[0] = (char)(0xE0 | (cp >> 12));
|
||||||
|
out[1] = (char)(0x80 | ((cp >> 6) & 0x3F));
|
||||||
|
out[2] = (char)(0x80 | (cp & 0x3F));
|
||||||
|
return 3;
|
||||||
|
}
|
||||||
|
out[0] = (char)(0xF0 | (cp >> 18));
|
||||||
|
out[1] = (char)(0x80 | ((cp >> 12) & 0x3F));
|
||||||
|
out[2] = (char)(0x80 | ((cp >> 6) & 0x3F));
|
||||||
|
out[3] = (char)(0x80 | (cp & 0x3F));
|
||||||
|
return 4;
|
||||||
|
}
|
||||||
|
|
||||||
static int
|
static int
|
||||||
escape(Lex *l, int *out)
|
escape(Lex *l, int *out)
|
||||||
{
|
{
|
||||||
@@ -227,19 +289,9 @@ escape(Lex *l, int *out)
|
|||||||
case 'b': *out = '\b'; return 0;
|
case 'b': *out = '\b'; return 0;
|
||||||
case 'f': *out = '\f'; return 0;
|
case 'f': *out = '\f'; return 0;
|
||||||
case 'v': *out = '\v'; return 0;
|
case 'v': *out = '\v'; return 0;
|
||||||
case 'x': {
|
case 'x': return lexunicode(l, 2, out);
|
||||||
int hi = lget(l), lo = lget(l);
|
case 'u': return lexunicode(l, 4, out);
|
||||||
if (!ishex(hi) || !ishex(lo)) {
|
case 'U': return lexunicode(l, 8, out);
|
||||||
Pos p = lpos(l);
|
|
||||||
errorf(p, "bad \\x escape");
|
|
||||||
l->errs++;
|
|
||||||
return -1;
|
|
||||||
}
|
|
||||||
int h = (hi <= '9' ? hi - '0' : (hi | 0x20) - 'a' + 10);
|
|
||||||
int o = (lo <= '9' ? lo - '0' : (lo | 0x20) - 'a' + 10);
|
|
||||||
*out = (h << 4) | o;
|
|
||||||
return 0;
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
{ Pos p = lpos(l); errorf(p, "bad escape \\%c", c); l->errs++; }
|
{ Pos p = lpos(l); errorf(p, "bad escape \\%c", c); l->errs++; }
|
||||||
return -1;
|
return -1;
|
||||||
@@ -389,22 +441,34 @@ lexstr(Lex *l, Pos start)
|
|||||||
return t;
|
return t;
|
||||||
}
|
}
|
||||||
if (c == '"') { lget(l); break; }
|
if (c == '"') { lget(l); break; }
|
||||||
int ch;
|
/* Escape-decoded values are codepoints and UTF-8-encode into
|
||||||
|
* 1-4 bytes (mirrors Hare's memio::appendrune in lex_string,
|
||||||
|
* ref/hare/hare/lex/lex.ha:431). Raw source bytes are already
|
||||||
|
* UTF-8 and pass through unchanged — re-encoding them would
|
||||||
|
* double-encode the >0x7F continuation bytes. */
|
||||||
|
char enc[4];
|
||||||
|
int el;
|
||||||
if (c == '\\') {
|
if (c == '\\') {
|
||||||
|
int ch;
|
||||||
lget(l);
|
lget(l);
|
||||||
if (escape(l, &ch) < 0)
|
if (escape(l, &ch) < 0)
|
||||||
ch = 0;
|
ch = 0;
|
||||||
|
el = utf8enc((u32)ch, enc);
|
||||||
} else {
|
} else {
|
||||||
ch = lget(l);
|
enc[0] = (char)lget(l);
|
||||||
|
el = 1;
|
||||||
}
|
}
|
||||||
if (n + 1 >= cap) {
|
if (n + el >= cap) {
|
||||||
u64 ncap = cap * 2;
|
u64 ncap = cap * 2;
|
||||||
|
while (n + el >= ncap)
|
||||||
|
ncap *= 2;
|
||||||
char *nb = amalloc(l->a, ncap);
|
char *nb = amalloc(l->a, ncap);
|
||||||
memcpy(nb, buf, n);
|
memcpy(nb, buf, n);
|
||||||
buf = nb;
|
buf = nb;
|
||||||
cap = ncap;
|
cap = ncap;
|
||||||
}
|
}
|
||||||
buf[n++] = (char)ch;
|
for (int i = 0; i < el; i++)
|
||||||
|
buf[n++] = enc[i];
|
||||||
}
|
}
|
||||||
buf[n] = '\0';
|
buf[n] = '\0';
|
||||||
Tok t = (Tok){ TK_STR, start, buf, n, {0}, TK_NONE };
|
Tok t = (Tok){ TK_STR, start, buf, n, {0}, TK_NONE };
|
||||||
|
|||||||
@@ -17,6 +17,7 @@ import os;
|
|||||||
import ascii;
|
import ascii;
|
||||||
import strings;
|
import strings;
|
||||||
import strconv;
|
import strconv;
|
||||||
|
import encoding.utf8;
|
||||||
|
|
||||||
// isidstart / isidpart — identifier classification. Lexer-local
|
// isidstart / isidpart — identifier classification. Lexer-local
|
||||||
// because the "alpha or '_' / alnum or '_'" set isn't part of Hare's
|
// because the "alpha or '_' / alnum or '_'" set isn't part of Hare's
|
||||||
@@ -306,6 +307,48 @@ fn parseint(p: *u8, n: u64, base: i32, ok: *bool) u64 = {
|
|||||||
return v;
|
return v;
|
||||||
};
|
};
|
||||||
|
|
||||||
|
// lexunicode — shared escape decoder for \xHH (n=2), \uHHHH (n=4),
|
||||||
|
// \UHHHHHHHH (n=8). All three yield a codepoint, not a raw byte —
|
||||||
|
// mirrors ref/hare/hare/lex/lex.ha:347 fn lex_unicode. Error strings
|
||||||
|
// copied verbatim from that reference for diagnostic fidelity (#50).
|
||||||
|
fn lexunicode(l: *lex, n: i32, out: *i32) bool = {
|
||||||
|
// u32 (not i32): an 8-digit \U with bit 31 set would go negative
|
||||||
|
// in i32 and slip past the `> 0x10FFFF` range check — cstage uses
|
||||||
|
// u32 here, so i32 would diverge (rule 10).
|
||||||
|
let u: u32 = 0u32;
|
||||||
|
let i: i32 = 0;
|
||||||
|
for (i < n) {
|
||||||
|
let c: i32 = lget(l);
|
||||||
|
if (c < 0) {
|
||||||
|
let cp: pos; curpos(l, &cp);
|
||||||
|
errat(l, &cp, "unexpected EOF scanning for escape");
|
||||||
|
return false;
|
||||||
|
};
|
||||||
|
if (!ascii.isxdigit(c: rune)) {
|
||||||
|
let cp: pos; curpos(l, &cp);
|
||||||
|
errat(l, &cp, "unexpected rune scanning for escape");
|
||||||
|
return false;
|
||||||
|
};
|
||||||
|
let d: i32 = hexval(c: rune)!;
|
||||||
|
u = (u << 4u32) | (d: u32);
|
||||||
|
i += 1;
|
||||||
|
};
|
||||||
|
if (u > 0x10FFFFu32) {
|
||||||
|
let cp: pos; curpos(l, &cp);
|
||||||
|
errat(l, &cp, "invalid unicode codepoint in escape");
|
||||||
|
return false;
|
||||||
|
};
|
||||||
|
if (u >= 0xD800u32) {
|
||||||
|
if (u < 0xE000u32) {
|
||||||
|
let cp: pos; curpos(l, &cp);
|
||||||
|
errat(l, &cp, "invalid unicode codepoint in escape");
|
||||||
|
return false;
|
||||||
|
};
|
||||||
|
};
|
||||||
|
*out = u: i32;
|
||||||
|
return true;
|
||||||
|
};
|
||||||
|
|
||||||
fn escape(l: *lex, out: *i32) bool = {
|
fn escape(l: *lex, out: *i32) bool = {
|
||||||
let c: i32 = lget(l);
|
let c: i32 = lget(l);
|
||||||
if (c < 0) { return false; };
|
if (c < 0) { return false; };
|
||||||
@@ -320,31 +363,9 @@ fn escape(l: *lex, out: *i32) bool = {
|
|||||||
if (c == 'b') { *out = '\b'; return true; };
|
if (c == 'b') { *out = '\b'; return true; };
|
||||||
if (c == 'f') { *out = '\f'; return true; };
|
if (c == 'f') { *out = '\f'; return true; };
|
||||||
if (c == 'v') { *out = '\v'; return true; };
|
if (c == 'v') { *out = '\v'; return true; };
|
||||||
if (c == 'x') {
|
if (c == 'x') { return lexunicode(l, 2, out); };
|
||||||
let hi: i32 = lget(l);
|
if (c == 'u') { return lexunicode(l, 4, out); };
|
||||||
let lo: i32 = lget(l);
|
if (c == 'U') { return lexunicode(l, 8, out); };
|
||||||
if (hi < 0) { return false; };
|
|
||||||
if (lo < 0) { return false; };
|
|
||||||
if (!ascii.isxdigit(hi: rune)) {
|
|
||||||
let cp: pos; curpos(l, &cp);
|
|
||||||
errat(l, &cp, "bad \\x escape");
|
|
||||||
return false;
|
|
||||||
};
|
|
||||||
if (!ascii.isxdigit(lo: rune)) {
|
|
||||||
let cp: pos; curpos(l, &cp);
|
|
||||||
errat(l, &cp, "bad \\x escape");
|
|
||||||
return false;
|
|
||||||
};
|
|
||||||
// Hex digits already validated by isxdigit above — `!`
|
|
||||||
// (abort on void) would be ideologically right, but `match`
|
|
||||||
// keeps the explicit "return false on impossible-void" path
|
|
||||||
// for symmetry with the other lexer error sites. Use `!`
|
|
||||||
// once we have a panic-with-position helper.
|
|
||||||
let h: i32 = hexval(hi: rune)!;
|
|
||||||
let lv: i32 = hexval(lo: rune)!;
|
|
||||||
*out = (h << 4) | lv;
|
|
||||||
return true;
|
|
||||||
};
|
|
||||||
let cp: pos; curpos(l, &cp);
|
let cp: pos; curpos(l, &cp);
|
||||||
errat(l, &cp, "bad escape");
|
errat(l, &cp, "bad escape");
|
||||||
return false;
|
return false;
|
||||||
@@ -636,15 +657,25 @@ fn lexstr(l: *lex, start: *pos, out: *tok) void = {
|
|||||||
return;
|
return;
|
||||||
};
|
};
|
||||||
if (c == '"') { lget(l); break; };
|
if (c == '"') { lget(l); break; };
|
||||||
let ch: i32 = 0;
|
// Escape-decoded values are codepoints and UTF-8-encode into
|
||||||
|
// 1-4 bytes (mirrors Hare's memio::appendrune in lex_string,
|
||||||
|
// ref/hare/hare/lex/lex.ha:431). Raw source bytes are already
|
||||||
|
// UTF-8 and pass through unchanged — re-encoding them would
|
||||||
|
// double-encode the >0x7F continuation bytes.
|
||||||
|
let enc: [4]u8;
|
||||||
|
let el: i32 = 1;
|
||||||
if (c == '\\') {
|
if (c == '\\') {
|
||||||
|
let ch: i32 = 0;
|
||||||
lget(l);
|
lget(l);
|
||||||
if (!escape(l, &ch)) { ch = 0; };
|
if (!escape(l, &ch)) { ch = 0; };
|
||||||
|
el = utf8.encoderune(enc, ch: rune);
|
||||||
} else {
|
} else {
|
||||||
ch = lget(l);
|
enc[0] = lget(l): u8;
|
||||||
|
el = 1;
|
||||||
};
|
};
|
||||||
if (nb + 1u64 >= cap) {
|
if (nb + (el: u64) >= cap) {
|
||||||
let ncap: u64 = cap * 2u64;
|
let ncap: u64 = cap * 2u64;
|
||||||
|
for (nb + (el: u64) >= ncap) { ncap = ncap * 2u64; };
|
||||||
let nb2: []u8 = alloc([], ncap)!;
|
let nb2: []u8 = alloc([], ncap)!;
|
||||||
let i: u64 = 0u64;
|
let i: u64 = 0u64;
|
||||||
for (i < nb) {
|
for (i < nb) {
|
||||||
@@ -655,9 +686,13 @@ fn lexstr(l: *lex, start: *pos, out: *tok) void = {
|
|||||||
buf = nb2;
|
buf = nb2;
|
||||||
cap = ncap;
|
cap = ncap;
|
||||||
};
|
};
|
||||||
let nbi: i32 = nb: i32;
|
let k: i32 = 0;
|
||||||
buf[nbi] = ch: u8;
|
for (k < el) {
|
||||||
nb += 1u64;
|
let nbi: i32 = nb: i32;
|
||||||
|
buf[nbi] = enc[k];
|
||||||
|
nb += 1u64;
|
||||||
|
k += 1;
|
||||||
|
};
|
||||||
};
|
};
|
||||||
out.kind = tkind.TK_STR;
|
out.kind = tkind.TK_STR;
|
||||||
out.file = start.file;
|
out.file = start.file;
|
||||||
|
|||||||
@@ -201,14 +201,36 @@ fn wwicheckdecl(c: *checker, d: *syntax.node) i32 = {
|
|||||||
|
|
||||||
// --- type-expr + const-expr unparser (rob §2.2/§2.4) ------------------
|
// --- type-expr + const-expr unparser (rob §2.2/§2.4) ------------------
|
||||||
|
|
||||||
// The lexer's rune escapes stop at \xHH (no \u/\U — task #50), so a
|
// wwihexdigits — emit the low `n` hex digits of `v`, most-significant
|
||||||
// codepoint above 0xff can't render as a re-parseable rune literal; fail
|
// first, lowercase. Byte-identical to cstage's fprintf("%0Nx").
|
||||||
// loud rather than emit a malformed one. No exported def names such a rune
|
fn wwihexdigits(fd: i32, v: u64, n: i32) void = {
|
||||||
// today (RUNE_MAX is written as an int-cast for the same reason).
|
let i: i32 = n - 1;
|
||||||
|
for (i >= 0) {
|
||||||
|
let d: u64 = (v >> ((i: u64) * 4u64)) & 15u64;
|
||||||
|
let c: u8 = 0u8;
|
||||||
|
if (d < 10u64) { c = (d: u8) + 48u8; } else { c = ((d: u8) - 10u8) + 97u8; };
|
||||||
|
wputb(fd, c);
|
||||||
|
i -= 1;
|
||||||
|
};
|
||||||
|
};
|
||||||
|
|
||||||
|
// #50: the lexer now reads \u/\U, so wide codepoints round-trip as those
|
||||||
|
// escapes (write-twin of lex.ww lexunicode). Codepoints <=0xff keep the
|
||||||
|
// \xHH spelling they already round-tripped as.
|
||||||
fn wwirune(fd: i32, cp: u64) void = {
|
fn wwirune(fd: i32, cp: u64) void = {
|
||||||
|
if (cp > 65535u64) {
|
||||||
|
wputb(fd, '\'');
|
||||||
|
wputs(fd, "\\U");
|
||||||
|
wwihexdigits(fd, cp, 8);
|
||||||
|
wputb(fd, '\'');
|
||||||
|
return;
|
||||||
|
};
|
||||||
if (cp > 255u64) {
|
if (cp > 255u64) {
|
||||||
wputs(2, "wwi: rune codepoint exceeds \\xHH (task #50)\n");
|
wputb(fd, '\'');
|
||||||
os.exit(1);
|
wputs(fd, "\\u");
|
||||||
|
wwihexdigits(fd, cp, 4);
|
||||||
|
wputb(fd, '\'');
|
||||||
|
return;
|
||||||
};
|
};
|
||||||
let c: u8 = cp: u8;
|
let c: u8 = cp: u8;
|
||||||
wputb(fd, '\'');
|
wputb(fd, '\'');
|
||||||
|
|||||||
@@ -102,6 +102,26 @@ static const struct row rows[] = {
|
|||||||
{ "'\\n'", "RUNE(10)" },
|
{ "'\\n'", "RUNE(10)" },
|
||||||
{ "'\\x7f'", "RUNE(127)" },
|
{ "'\\x7f'", "RUNE(127)" },
|
||||||
|
|
||||||
|
/* unicode escapes — \u (4 hex) / \U (8 hex) share the \x codepoint
|
||||||
|
* path (ref/hare/hare/lex/lex.ha:347 lex_unicode). Rune literals
|
||||||
|
* carry the raw codepoint; string literals UTF-8-encode it. */
|
||||||
|
{ "'\\u00e9'", "RUNE(233)" },
|
||||||
|
{ "'\\u20ac'", "RUNE(8364)" },
|
||||||
|
{ "'\\U0001F600'", "RUNE(128512)" },
|
||||||
|
{ "\"\\u00e9\"", "STR(\xc3\xa9)" },
|
||||||
|
{ "\"\\u20ac\"", "STR(\xe2\x82\xac)" },
|
||||||
|
{ "\"\\U0001F600\"", "STR(\xf0\x9f\x98\x80)" },
|
||||||
|
{ "\"caf\\u00e9\"", "STR(caf\xc3\xa9)" },
|
||||||
|
/* \x now also yields a codepoint that UTF-8-encodes in strings:
|
||||||
|
* \xe9 -> U+00E9 -> 0xC3 0xA9 (matches Hare's appendrune). */
|
||||||
|
{ "'\\xe9'", "RUNE(233)" },
|
||||||
|
{ "\"\\xe9\"", "STR(\xc3\xa9)" },
|
||||||
|
/* range boundaries: max valid codepoint, and the two edges that
|
||||||
|
* straddle the UTF-16 surrogate gap (0xD7FF ok / 0xE000 ok). */
|
||||||
|
{ "'\\U0010FFFF'", "RUNE(1114111)" },
|
||||||
|
{ "'\\uD7FF'", "RUNE(55295)" },
|
||||||
|
{ "'\\uE000'", "RUNE(57344)" },
|
||||||
|
|
||||||
/* operators & punct */
|
/* operators & punct */
|
||||||
{ "+ - * / % == != < > <= >= && || !",
|
{ "+ - * / % == != < > <= >= && || !",
|
||||||
"+ - * / % == != < > <= >= && || !" },
|
"+ - * / % == != < > <= >= && || !" },
|
||||||
@@ -121,6 +141,39 @@ static const struct row rows[] = {
|
|||||||
"fn IDENT(add) ( IDENT(a) : IDENT(i32) , IDENT(b) : IDENT(i32) ) IDENT(i32) = { return IDENT(a) + IDENT(b) ; } ;" },
|
"fn IDENT(add) ( IDENT(a) : IDENT(i32) , IDENT(b) : IDENT(i32) ) IDENT(i32) = { return IDENT(a) + IDENT(b) ; } ;" },
|
||||||
};
|
};
|
||||||
|
|
||||||
|
/* Escape error cases. A bad escape sets l.errs (the string/rune still
|
||||||
|
* lexes with the offending codepoint zeroed), so detection is via the
|
||||||
|
* error counter, not the token stream. The verbatim Hare messages live
|
||||||
|
* at lexunicode in cmd/wcc/lex.c. */
|
||||||
|
static int
|
||||||
|
runerr(const char *src)
|
||||||
|
{
|
||||||
|
Arena *a = newarena();
|
||||||
|
Lex l;
|
||||||
|
lexinit(&l, a, "<test>", src, strlen(src));
|
||||||
|
for (;;) {
|
||||||
|
Tok t = lexnext(&l);
|
||||||
|
if (t.kind == TK_EOF)
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
int ok = l.errs > 0;
|
||||||
|
if (!ok)
|
||||||
|
fprintf(stderr, "expected escape error, none raised:\n src: %s\n",
|
||||||
|
src);
|
||||||
|
freearena(a);
|
||||||
|
return ok;
|
||||||
|
}
|
||||||
|
|
||||||
|
static const char *const errrows[] = {
|
||||||
|
"'\\uZ'", /* unexpected rune scanning for escape */
|
||||||
|
"\"\\u00g0\"", /* non-hex digit inside a string escape */
|
||||||
|
"'\\u00", /* unexpected EOF scanning for escape */
|
||||||
|
"'\\UFFFFFFFF'", /* codepoint > 0x10FFFF (high bit set) */
|
||||||
|
"'\\U00110000'", /* exactly one past U+10FFFF */
|
||||||
|
"'\\uD800'", /* bottom of the UTF-16 surrogate range */
|
||||||
|
"'\\uDFFF'", /* top of the UTF-16 surrogate range */
|
||||||
|
};
|
||||||
|
|
||||||
int
|
int
|
||||||
main(void)
|
main(void)
|
||||||
{
|
{
|
||||||
@@ -131,6 +184,12 @@ main(void)
|
|||||||
fail++;
|
fail++;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
for (size_t i = 0; i < sizeof errrows / sizeof errrows[0]; i++) {
|
||||||
|
if (!runerr(errrows[i])) {
|
||||||
|
fprintf(stderr, "errrow %zu failed\n", i);
|
||||||
|
fail++;
|
||||||
|
}
|
||||||
|
}
|
||||||
if (fail) {
|
if (fail) {
|
||||||
fprintf(stderr, "%d/%zu lex tests failed\n", fail,
|
fprintf(stderr, "%d/%zu lex tests failed\n", fail,
|
||||||
sizeof rows / sizeof rows[0]);
|
sizeof rows / sizeof rows[0]);
|
||||||
|
|||||||
256
test/wcc/110_uniesc_run.c
Normal file
256
test/wcc/110_uniesc_run.c
Normal file
@@ -0,0 +1,256 @@
|
|||||||
|
/*
|
||||||
|
* 110_uniesc_run — \u / \U Unicode escapes, end-to-end through both
|
||||||
|
* the cstage `ww` and the wwstage `ww_ww` drivers (task #50).
|
||||||
|
*
|
||||||
|
* Rune literals carry the raw codepoint; string literals UTF-8-encode
|
||||||
|
* it into 1-4 bytes (ref/hare/hare/lex/lex.ha:347 lex_unicode + the
|
||||||
|
* appendrune in lex_string). \x shares the same codepoint path, so a
|
||||||
|
* >0x7F \x byte now widens to its 2-byte UTF-8 form in strings. Each
|
||||||
|
* fixture returns 42 on a correct decode/encode.
|
||||||
|
*
|
||||||
|
* The dual-driver loop mirrors 640_int_cast_signed: the wwstage arm is
|
||||||
|
* skipped when ww_ww is absent (e.g. a cstage-only build).
|
||||||
|
*
|
||||||
|
* The final per-driver case is a .wwi round-trip: an exported wide-rune
|
||||||
|
* `def` re-serialized by the producer must re-parse byte-exact (the
|
||||||
|
* write-twin of the lexer read-fix — cmd/w6c/wwi.c wwi_rune /
|
||||||
|
* selfhost/cmd/wcc/wwi.ww wwirune emit \u/\U above 0xFF).
|
||||||
|
*/
|
||||||
|
#include <stdio.h>
|
||||||
|
#include <stdlib.h>
|
||||||
|
#include <string.h>
|
||||||
|
#include <unistd.h>
|
||||||
|
#include <sys/stat.h>
|
||||||
|
#include <sys/wait.h>
|
||||||
|
|
||||||
|
static int
|
||||||
|
runwait(const char *cmd)
|
||||||
|
{
|
||||||
|
int rc = system(cmd);
|
||||||
|
if (rc == -1) return -1;
|
||||||
|
if (WIFEXITED(rc)) return WEXITSTATUS(rc);
|
||||||
|
return -1;
|
||||||
|
}
|
||||||
|
|
||||||
|
struct row { const char *label; const char *src; int want; };
|
||||||
|
|
||||||
|
static const struct row rows[] = {
|
||||||
|
/* \u (4 hex) rune → U+00E9 codepoint 233. */
|
||||||
|
{ "rune_u_2byte",
|
||||||
|
"fn main() i32 = {\n"
|
||||||
|
" let r: rune = '\\u00e9';\n"
|
||||||
|
" if ((r: u32) == 233u32) { return 42; };\n"
|
||||||
|
" return 0;\n"
|
||||||
|
"};\n",
|
||||||
|
42 },
|
||||||
|
/* \U (8 hex) rune → U+1F600 codepoint 128512 (> 0xFFFF, the case
|
||||||
|
* that forced the `0x...: rune` int-cast spelling before #50). */
|
||||||
|
{ "rune_U_emoji",
|
||||||
|
"fn main() i32 = {\n"
|
||||||
|
" let r: rune = '\\U0001F600';\n"
|
||||||
|
" if ((r: u32) == 128512u32) { return 42; };\n"
|
||||||
|
" return 0;\n"
|
||||||
|
"};\n",
|
||||||
|
42 },
|
||||||
|
/* \u é string → 2-byte UTF-8 0xC3 0xA9. */
|
||||||
|
{ "str_u_2byte",
|
||||||
|
"package main;\n"
|
||||||
|
"import strings;\n"
|
||||||
|
"fn main() i32 = {\n"
|
||||||
|
" let b: []u8 = strings.toutf8(\"\\u00e9\");\n"
|
||||||
|
" if (len(b) != 2) { return 1; };\n"
|
||||||
|
" if (b[0] != 0xc3u8) { return 2; };\n"
|
||||||
|
" if (b[1] != 0xa9u8) { return 3; };\n"
|
||||||
|
" return 42;\n"
|
||||||
|
"};\n",
|
||||||
|
42 },
|
||||||
|
/* \u € string → 3-byte UTF-8 0xE2 0x82 0xAC. */
|
||||||
|
{ "str_u_3byte",
|
||||||
|
"package main;\n"
|
||||||
|
"import strings;\n"
|
||||||
|
"fn main() i32 = {\n"
|
||||||
|
" let b: []u8 = strings.toutf8(\"\\u20ac\");\n"
|
||||||
|
" if (len(b) != 3) { return 1; };\n"
|
||||||
|
" if (b[0] != 0xe2u8) { return 2; };\n"
|
||||||
|
" if (b[1] != 0x82u8) { return 3; };\n"
|
||||||
|
" if (b[2] != 0xacu8) { return 4; };\n"
|
||||||
|
" return 42;\n"
|
||||||
|
"};\n",
|
||||||
|
42 },
|
||||||
|
/* \U 😀 string → 4-byte UTF-8 0xF0 0x9F 0x98 0x80. */
|
||||||
|
{ "str_U_4byte",
|
||||||
|
"package main;\n"
|
||||||
|
"import strings;\n"
|
||||||
|
"fn main() i32 = {\n"
|
||||||
|
" let b: []u8 = strings.toutf8(\"\\U0001F600\");\n"
|
||||||
|
" if (len(b) != 4) { return 1; };\n"
|
||||||
|
" if (b[0] != 0xf0u8) { return 2; };\n"
|
||||||
|
" if (b[1] != 0x9fu8) { return 3; };\n"
|
||||||
|
" if (b[2] != 0x98u8) { return 4; };\n"
|
||||||
|
" if (b[3] != 0x80u8) { return 5; };\n"
|
||||||
|
" return 42;\n"
|
||||||
|
"};\n",
|
||||||
|
42 },
|
||||||
|
/* \x ≥0x80 now yields a codepoint that UTF-8-encodes (\xe9 →
|
||||||
|
* U+00E9 → 0xC3 0xA9), matching Hare's shared escape path. */
|
||||||
|
{ "str_x_widens",
|
||||||
|
"package main;\n"
|
||||||
|
"import strings;\n"
|
||||||
|
"fn main() i32 = {\n"
|
||||||
|
" let b: []u8 = strings.toutf8(\"\\xe9\");\n"
|
||||||
|
" if (len(b) != 2) { return 1; };\n"
|
||||||
|
" if (b[0] != 0xc3u8) { return 2; };\n"
|
||||||
|
" if (b[1] != 0xa9u8) { return 3; };\n"
|
||||||
|
" return 42;\n"
|
||||||
|
"};\n",
|
||||||
|
42 },
|
||||||
|
};
|
||||||
|
|
||||||
|
static int
|
||||||
|
run_driver(const char *driver, const struct row *r, int i)
|
||||||
|
{
|
||||||
|
char src[64], tmpdir[64], cmd[1024];
|
||||||
|
snprintf(src, sizeof src, "/tmp/wwue_%d_%d.ww", getpid(), i);
|
||||||
|
snprintf(tmpdir, sizeof tmpdir, "/tmp/wwue_%d_d_%d", getpid(), i);
|
||||||
|
|
||||||
|
FILE *f = fopen(src, "wb");
|
||||||
|
if (!f) return -1;
|
||||||
|
fputs(r->src, f);
|
||||||
|
fclose(f);
|
||||||
|
|
||||||
|
mkdir(tmpdir, 0755);
|
||||||
|
snprintf(cmd, sizeof cmd, "cd %s && %s build %s",
|
||||||
|
tmpdir, driver, src);
|
||||||
|
if (runwait(cmd) != 0) {
|
||||||
|
fprintf(stderr, "row[%s]: build via %s failed\n",
|
||||||
|
r->label, driver);
|
||||||
|
unlink(src); rmdir(tmpdir);
|
||||||
|
return -1;
|
||||||
|
}
|
||||||
|
|
||||||
|
const char *base = strrchr(src, '/');
|
||||||
|
base = base ? base + 1 : src;
|
||||||
|
char outbin[128];
|
||||||
|
snprintf(outbin, sizeof outbin, "%s/%s", tmpdir, base);
|
||||||
|
char *dot = strrchr(outbin, '.');
|
||||||
|
if (dot && strcmp(dot, ".ww") == 0) *dot = '\0';
|
||||||
|
int got = runwait(outbin);
|
||||||
|
|
||||||
|
unlink(src); unlink(outbin); rmdir(tmpdir);
|
||||||
|
return got;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* run_roundtrip — sep-build a two-package program where the imported
|
||||||
|
* package exports wide-rune defs; the producer must serialize them as
|
||||||
|
* \u/\U so the importer re-parses the same codepoint. Returns the built
|
||||||
|
* program's exit code (42 on a correct round-trip). */
|
||||||
|
static int
|
||||||
|
run_roundtrip(const char *driver, int id)
|
||||||
|
{
|
||||||
|
char dir[80], wrdir[112], wrp[176], mainp[160], cmd[1024], outbin[160];
|
||||||
|
snprintf(dir, sizeof dir, "/tmp/wwue_rt_%d_%d", getpid(), id);
|
||||||
|
snprintf(wrdir, sizeof wrdir, "%s/wr", dir);
|
||||||
|
mkdir(dir, 0755);
|
||||||
|
mkdir(wrdir, 0755);
|
||||||
|
snprintf(wrp, sizeof wrp, "%s/wr.ww", wrdir);
|
||||||
|
snprintf(mainp, sizeof mainp, "%s/main.ww", dir);
|
||||||
|
|
||||||
|
FILE *f = fopen(wrp, "wb");
|
||||||
|
if (!f) return -1;
|
||||||
|
/* one def per serializer branch: \x (<=0xFF), \u (<=0xFFFF), \U. */
|
||||||
|
fputs("package wr;\n"
|
||||||
|
"export def EACUTE: rune = '\\u00e9';\n"
|
||||||
|
"export def EURO: rune = '\\u20ac';\n"
|
||||||
|
"export def SMILEY: rune = '\\U0001F600';\n", f);
|
||||||
|
fclose(f);
|
||||||
|
f = fopen(mainp, "wb");
|
||||||
|
if (!f) return -1;
|
||||||
|
fputs("package main;\n"
|
||||||
|
"import wr;\n"
|
||||||
|
"fn main() i32 = {\n"
|
||||||
|
" let a: rune = wr.EACUTE;\n"
|
||||||
|
" let b: rune = wr.EURO;\n"
|
||||||
|
" let c: rune = wr.SMILEY;\n"
|
||||||
|
" if ((a: u32) != 233u32) { return 1; };\n"
|
||||||
|
" if ((b: u32) != 8364u32) { return 2; };\n"
|
||||||
|
" if ((c: u32) != 128512u32) { return 3; };\n"
|
||||||
|
" return 42;\n"
|
||||||
|
"};\n", f);
|
||||||
|
fclose(f);
|
||||||
|
|
||||||
|
snprintf(cmd, sizeof cmd, "cd %s && %s build main.ww", dir, driver);
|
||||||
|
if (runwait(cmd) != 0) {
|
||||||
|
fprintf(stderr, "roundtrip: build via %s failed\n", driver);
|
||||||
|
snprintf(cmd, sizeof cmd, "rm -rf %s", dir);
|
||||||
|
(void)system(cmd);
|
||||||
|
return -1;
|
||||||
|
}
|
||||||
|
snprintf(outbin, sizeof outbin, "%s/main", dir);
|
||||||
|
int got = runwait(outbin);
|
||||||
|
snprintf(cmd, sizeof cmd, "rm -rf %s", dir);
|
||||||
|
(void)system(cmd);
|
||||||
|
return got;
|
||||||
|
}
|
||||||
|
|
||||||
|
int
|
||||||
|
main(void)
|
||||||
|
{
|
||||||
|
const char *bin = getenv("BIN");
|
||||||
|
if (!bin) bin = "out/bin";
|
||||||
|
char absbin[1024];
|
||||||
|
if (bin[0] != '/') {
|
||||||
|
char cwd[1024];
|
||||||
|
if (getcwd(cwd, sizeof cwd) == NULL) return 1;
|
||||||
|
snprintf(absbin, sizeof absbin, "%s/%s", cwd, bin);
|
||||||
|
bin = absbin;
|
||||||
|
}
|
||||||
|
|
||||||
|
char cdrv[1024];
|
||||||
|
snprintf(cdrv, sizeof cdrv, "%s/ww", bin);
|
||||||
|
char wdrv[1024];
|
||||||
|
snprintf(wdrv, sizeof wdrv, "%s/ww_ww", bin);
|
||||||
|
|
||||||
|
struct { const char *name; const char *path; int gated_on_existence; }
|
||||||
|
drivers[] = {
|
||||||
|
{ "cstage", cdrv, 0 },
|
||||||
|
{ "wwstage", wdrv, 1 },
|
||||||
|
{ NULL, NULL, 0 },
|
||||||
|
};
|
||||||
|
|
||||||
|
int n = (int)(sizeof rows / sizeof rows[0]);
|
||||||
|
int total = 0, fail = 0;
|
||||||
|
for (int d = 0; drivers[d].name; d++) {
|
||||||
|
if (drivers[d].gated_on_existence
|
||||||
|
&& access(drivers[d].path, X_OK) != 0) {
|
||||||
|
fprintf(stderr, "uniesc_run: skip %s (no %s)\n",
|
||||||
|
drivers[d].name, drivers[d].path);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
for (int i = 0; i < n; i++) {
|
||||||
|
int got = run_driver(drivers[d].path, &rows[i], i);
|
||||||
|
total++;
|
||||||
|
if (got != rows[i].want) {
|
||||||
|
fprintf(stderr,
|
||||||
|
"uniesc_run[%s][%s]: exit=%d want=%d\n",
|
||||||
|
drivers[d].name, rows[i].label,
|
||||||
|
got, rows[i].want);
|
||||||
|
fail++;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
int rt = run_roundtrip(drivers[d].path, d);
|
||||||
|
total++;
|
||||||
|
if (rt != 42) {
|
||||||
|
fprintf(stderr,
|
||||||
|
"uniesc_run[%s][wwi_roundtrip]: exit=%d want=42\n",
|
||||||
|
drivers[d].name, rt);
|
||||||
|
fail++;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (fail) {
|
||||||
|
fprintf(stderr, "uniesc_run: %d/%d fixtures failed\n",
|
||||||
|
fail, total);
|
||||||
|
return 1;
|
||||||
|
}
|
||||||
|
printf("uniesc_run: %d/%d ok\n", total, total);
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
Reference in New Issue
Block a user