lex,wwi: \u/\U unicode escapes + wide-rune .wwi round-trip, both stages (#50)
Hare-faithful \u (4 hex) / \U (8 hex) escapes; \x/\u/\U share one codepoint path (ref/hare/hare/lex/lex.ha lex_unicode); string literals UTF-8-encode multi-byte codepoints (cstage inline utf8enc, wwstage utf8.encoderune). The .wwi producer rune serializer now emits \u/\U so exported wide-rune defs round-trip (was a fatal >0xFF). Reject >0x10FFFF and surrogates with Hare- verbatim error strings. Closes the int-cast spelling divergence (#48 RUNE_MAX). Both stages byte-identical; 446 tests pass.
This commit is contained in:
@@ -102,6 +102,26 @@ static const struct row rows[] = {
|
||||
{ "'\\n'", "RUNE(10)" },
|
||||
{ "'\\x7f'", "RUNE(127)" },
|
||||
|
||||
/* unicode escapes — \u (4 hex) / \U (8 hex) share the \x codepoint
|
||||
* path (ref/hare/hare/lex/lex.ha:347 lex_unicode). Rune literals
|
||||
* carry the raw codepoint; string literals UTF-8-encode it. */
|
||||
{ "'\\u00e9'", "RUNE(233)" },
|
||||
{ "'\\u20ac'", "RUNE(8364)" },
|
||||
{ "'\\U0001F600'", "RUNE(128512)" },
|
||||
{ "\"\\u00e9\"", "STR(\xc3\xa9)" },
|
||||
{ "\"\\u20ac\"", "STR(\xe2\x82\xac)" },
|
||||
{ "\"\\U0001F600\"", "STR(\xf0\x9f\x98\x80)" },
|
||||
{ "\"caf\\u00e9\"", "STR(caf\xc3\xa9)" },
|
||||
/* \x now also yields a codepoint that UTF-8-encodes in strings:
|
||||
* \xe9 -> U+00E9 -> 0xC3 0xA9 (matches Hare's appendrune). */
|
||||
{ "'\\xe9'", "RUNE(233)" },
|
||||
{ "\"\\xe9\"", "STR(\xc3\xa9)" },
|
||||
/* range boundaries: max valid codepoint, and the two edges that
|
||||
* straddle the UTF-16 surrogate gap (0xD7FF ok / 0xE000 ok). */
|
||||
{ "'\\U0010FFFF'", "RUNE(1114111)" },
|
||||
{ "'\\uD7FF'", "RUNE(55295)" },
|
||||
{ "'\\uE000'", "RUNE(57344)" },
|
||||
|
||||
/* operators & punct */
|
||||
{ "+ - * / % == != < > <= >= && || !",
|
||||
"+ - * / % == != < > <= >= && || !" },
|
||||
@@ -121,6 +141,39 @@ static const struct row rows[] = {
|
||||
"fn IDENT(add) ( IDENT(a) : IDENT(i32) , IDENT(b) : IDENT(i32) ) IDENT(i32) = { return IDENT(a) + IDENT(b) ; } ;" },
|
||||
};
|
||||
|
||||
/* Escape error cases. A bad escape sets l.errs (the string/rune still
|
||||
* lexes with the offending codepoint zeroed), so detection is via the
|
||||
* error counter, not the token stream. The verbatim Hare messages live
|
||||
* at lexunicode in cmd/wcc/lex.c. */
|
||||
static int
|
||||
runerr(const char *src)
|
||||
{
|
||||
Arena *a = newarena();
|
||||
Lex l;
|
||||
lexinit(&l, a, "<test>", src, strlen(src));
|
||||
for (;;) {
|
||||
Tok t = lexnext(&l);
|
||||
if (t.kind == TK_EOF)
|
||||
break;
|
||||
}
|
||||
int ok = l.errs > 0;
|
||||
if (!ok)
|
||||
fprintf(stderr, "expected escape error, none raised:\n src: %s\n",
|
||||
src);
|
||||
freearena(a);
|
||||
return ok;
|
||||
}
|
||||
|
||||
static const char *const errrows[] = {
|
||||
"'\\uZ'", /* unexpected rune scanning for escape */
|
||||
"\"\\u00g0\"", /* non-hex digit inside a string escape */
|
||||
"'\\u00", /* unexpected EOF scanning for escape */
|
||||
"'\\UFFFFFFFF'", /* codepoint > 0x10FFFF (high bit set) */
|
||||
"'\\U00110000'", /* exactly one past U+10FFFF */
|
||||
"'\\uD800'", /* bottom of the UTF-16 surrogate range */
|
||||
"'\\uDFFF'", /* top of the UTF-16 surrogate range */
|
||||
};
|
||||
|
||||
int
|
||||
main(void)
|
||||
{
|
||||
@@ -131,6 +184,12 @@ main(void)
|
||||
fail++;
|
||||
}
|
||||
}
|
||||
for (size_t i = 0; i < sizeof errrows / sizeof errrows[0]; i++) {
|
||||
if (!runerr(errrows[i])) {
|
||||
fprintf(stderr, "errrow %zu failed\n", i);
|
||||
fail++;
|
||||
}
|
||||
}
|
||||
if (fail) {
|
||||
fprintf(stderr, "%d/%zu lex tests failed\n", fail,
|
||||
sizeof rows / sizeof rows[0]);
|
||||
|
||||
256
test/wcc/110_uniesc_run.c
Normal file
256
test/wcc/110_uniesc_run.c
Normal file
@@ -0,0 +1,256 @@
|
||||
/*
|
||||
* 110_uniesc_run — \u / \U Unicode escapes, end-to-end through both
|
||||
* the cstage `ww` and the wwstage `ww_ww` drivers (task #50).
|
||||
*
|
||||
* Rune literals carry the raw codepoint; string literals UTF-8-encode
|
||||
* it into 1-4 bytes (ref/hare/hare/lex/lex.ha:347 lex_unicode + the
|
||||
* appendrune in lex_string). \x shares the same codepoint path, so a
|
||||
* >0x7F \x byte now widens to its 2-byte UTF-8 form in strings. Each
|
||||
* fixture returns 42 on a correct decode/encode.
|
||||
*
|
||||
* The dual-driver loop mirrors 640_int_cast_signed: the wwstage arm is
|
||||
* skipped when ww_ww is absent (e.g. a cstage-only build).
|
||||
*
|
||||
* The final per-driver case is a .wwi round-trip: an exported wide-rune
|
||||
* `def` re-serialized by the producer must re-parse byte-exact (the
|
||||
* write-twin of the lexer read-fix — cmd/w6c/wwi.c wwi_rune /
|
||||
* selfhost/cmd/wcc/wwi.ww wwirune emit \u/\U above 0xFF).
|
||||
*/
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
#include <unistd.h>
|
||||
#include <sys/stat.h>
|
||||
#include <sys/wait.h>
|
||||
|
||||
static int
|
||||
runwait(const char *cmd)
|
||||
{
|
||||
int rc = system(cmd);
|
||||
if (rc == -1) return -1;
|
||||
if (WIFEXITED(rc)) return WEXITSTATUS(rc);
|
||||
return -1;
|
||||
}
|
||||
|
||||
struct row { const char *label; const char *src; int want; };
|
||||
|
||||
static const struct row rows[] = {
|
||||
/* \u (4 hex) rune → U+00E9 codepoint 233. */
|
||||
{ "rune_u_2byte",
|
||||
"fn main() i32 = {\n"
|
||||
" let r: rune = '\\u00e9';\n"
|
||||
" if ((r: u32) == 233u32) { return 42; };\n"
|
||||
" return 0;\n"
|
||||
"};\n",
|
||||
42 },
|
||||
/* \U (8 hex) rune → U+1F600 codepoint 128512 (> 0xFFFF, the case
|
||||
* that forced the `0x...: rune` int-cast spelling before #50). */
|
||||
{ "rune_U_emoji",
|
||||
"fn main() i32 = {\n"
|
||||
" let r: rune = '\\U0001F600';\n"
|
||||
" if ((r: u32) == 128512u32) { return 42; };\n"
|
||||
" return 0;\n"
|
||||
"};\n",
|
||||
42 },
|
||||
/* \u é string → 2-byte UTF-8 0xC3 0xA9. */
|
||||
{ "str_u_2byte",
|
||||
"package main;\n"
|
||||
"import strings;\n"
|
||||
"fn main() i32 = {\n"
|
||||
" let b: []u8 = strings.toutf8(\"\\u00e9\");\n"
|
||||
" if (len(b) != 2) { return 1; };\n"
|
||||
" if (b[0] != 0xc3u8) { return 2; };\n"
|
||||
" if (b[1] != 0xa9u8) { return 3; };\n"
|
||||
" return 42;\n"
|
||||
"};\n",
|
||||
42 },
|
||||
/* \u € string → 3-byte UTF-8 0xE2 0x82 0xAC. */
|
||||
{ "str_u_3byte",
|
||||
"package main;\n"
|
||||
"import strings;\n"
|
||||
"fn main() i32 = {\n"
|
||||
" let b: []u8 = strings.toutf8(\"\\u20ac\");\n"
|
||||
" if (len(b) != 3) { return 1; };\n"
|
||||
" if (b[0] != 0xe2u8) { return 2; };\n"
|
||||
" if (b[1] != 0x82u8) { return 3; };\n"
|
||||
" if (b[2] != 0xacu8) { return 4; };\n"
|
||||
" return 42;\n"
|
||||
"};\n",
|
||||
42 },
|
||||
/* \U 😀 string → 4-byte UTF-8 0xF0 0x9F 0x98 0x80. */
|
||||
{ "str_U_4byte",
|
||||
"package main;\n"
|
||||
"import strings;\n"
|
||||
"fn main() i32 = {\n"
|
||||
" let b: []u8 = strings.toutf8(\"\\U0001F600\");\n"
|
||||
" if (len(b) != 4) { return 1; };\n"
|
||||
" if (b[0] != 0xf0u8) { return 2; };\n"
|
||||
" if (b[1] != 0x9fu8) { return 3; };\n"
|
||||
" if (b[2] != 0x98u8) { return 4; };\n"
|
||||
" if (b[3] != 0x80u8) { return 5; };\n"
|
||||
" return 42;\n"
|
||||
"};\n",
|
||||
42 },
|
||||
/* \x ≥0x80 now yields a codepoint that UTF-8-encodes (\xe9 →
|
||||
* U+00E9 → 0xC3 0xA9), matching Hare's shared escape path. */
|
||||
{ "str_x_widens",
|
||||
"package main;\n"
|
||||
"import strings;\n"
|
||||
"fn main() i32 = {\n"
|
||||
" let b: []u8 = strings.toutf8(\"\\xe9\");\n"
|
||||
" if (len(b) != 2) { return 1; };\n"
|
||||
" if (b[0] != 0xc3u8) { return 2; };\n"
|
||||
" if (b[1] != 0xa9u8) { return 3; };\n"
|
||||
" return 42;\n"
|
||||
"};\n",
|
||||
42 },
|
||||
};
|
||||
|
||||
static int
|
||||
run_driver(const char *driver, const struct row *r, int i)
|
||||
{
|
||||
char src[64], tmpdir[64], cmd[1024];
|
||||
snprintf(src, sizeof src, "/tmp/wwue_%d_%d.ww", getpid(), i);
|
||||
snprintf(tmpdir, sizeof tmpdir, "/tmp/wwue_%d_d_%d", getpid(), i);
|
||||
|
||||
FILE *f = fopen(src, "wb");
|
||||
if (!f) return -1;
|
||||
fputs(r->src, f);
|
||||
fclose(f);
|
||||
|
||||
mkdir(tmpdir, 0755);
|
||||
snprintf(cmd, sizeof cmd, "cd %s && %s build %s",
|
||||
tmpdir, driver, src);
|
||||
if (runwait(cmd) != 0) {
|
||||
fprintf(stderr, "row[%s]: build via %s failed\n",
|
||||
r->label, driver);
|
||||
unlink(src); rmdir(tmpdir);
|
||||
return -1;
|
||||
}
|
||||
|
||||
const char *base = strrchr(src, '/');
|
||||
base = base ? base + 1 : src;
|
||||
char outbin[128];
|
||||
snprintf(outbin, sizeof outbin, "%s/%s", tmpdir, base);
|
||||
char *dot = strrchr(outbin, '.');
|
||||
if (dot && strcmp(dot, ".ww") == 0) *dot = '\0';
|
||||
int got = runwait(outbin);
|
||||
|
||||
unlink(src); unlink(outbin); rmdir(tmpdir);
|
||||
return got;
|
||||
}
|
||||
|
||||
/* run_roundtrip — sep-build a two-package program where the imported
|
||||
* package exports wide-rune defs; the producer must serialize them as
|
||||
* \u/\U so the importer re-parses the same codepoint. Returns the built
|
||||
* program's exit code (42 on a correct round-trip). */
|
||||
static int
|
||||
run_roundtrip(const char *driver, int id)
|
||||
{
|
||||
char dir[80], wrdir[112], wrp[176], mainp[160], cmd[1024], outbin[160];
|
||||
snprintf(dir, sizeof dir, "/tmp/wwue_rt_%d_%d", getpid(), id);
|
||||
snprintf(wrdir, sizeof wrdir, "%s/wr", dir);
|
||||
mkdir(dir, 0755);
|
||||
mkdir(wrdir, 0755);
|
||||
snprintf(wrp, sizeof wrp, "%s/wr.ww", wrdir);
|
||||
snprintf(mainp, sizeof mainp, "%s/main.ww", dir);
|
||||
|
||||
FILE *f = fopen(wrp, "wb");
|
||||
if (!f) return -1;
|
||||
/* one def per serializer branch: \x (<=0xFF), \u (<=0xFFFF), \U. */
|
||||
fputs("package wr;\n"
|
||||
"export def EACUTE: rune = '\\u00e9';\n"
|
||||
"export def EURO: rune = '\\u20ac';\n"
|
||||
"export def SMILEY: rune = '\\U0001F600';\n", f);
|
||||
fclose(f);
|
||||
f = fopen(mainp, "wb");
|
||||
if (!f) return -1;
|
||||
fputs("package main;\n"
|
||||
"import wr;\n"
|
||||
"fn main() i32 = {\n"
|
||||
" let a: rune = wr.EACUTE;\n"
|
||||
" let b: rune = wr.EURO;\n"
|
||||
" let c: rune = wr.SMILEY;\n"
|
||||
" if ((a: u32) != 233u32) { return 1; };\n"
|
||||
" if ((b: u32) != 8364u32) { return 2; };\n"
|
||||
" if ((c: u32) != 128512u32) { return 3; };\n"
|
||||
" return 42;\n"
|
||||
"};\n", f);
|
||||
fclose(f);
|
||||
|
||||
snprintf(cmd, sizeof cmd, "cd %s && %s build main.ww", dir, driver);
|
||||
if (runwait(cmd) != 0) {
|
||||
fprintf(stderr, "roundtrip: build via %s failed\n", driver);
|
||||
snprintf(cmd, sizeof cmd, "rm -rf %s", dir);
|
||||
(void)system(cmd);
|
||||
return -1;
|
||||
}
|
||||
snprintf(outbin, sizeof outbin, "%s/main", dir);
|
||||
int got = runwait(outbin);
|
||||
snprintf(cmd, sizeof cmd, "rm -rf %s", dir);
|
||||
(void)system(cmd);
|
||||
return got;
|
||||
}
|
||||
|
||||
int
|
||||
main(void)
|
||||
{
|
||||
const char *bin = getenv("BIN");
|
||||
if (!bin) bin = "out/bin";
|
||||
char absbin[1024];
|
||||
if (bin[0] != '/') {
|
||||
char cwd[1024];
|
||||
if (getcwd(cwd, sizeof cwd) == NULL) return 1;
|
||||
snprintf(absbin, sizeof absbin, "%s/%s", cwd, bin);
|
||||
bin = absbin;
|
||||
}
|
||||
|
||||
char cdrv[1024];
|
||||
snprintf(cdrv, sizeof cdrv, "%s/ww", bin);
|
||||
char wdrv[1024];
|
||||
snprintf(wdrv, sizeof wdrv, "%s/ww_ww", bin);
|
||||
|
||||
struct { const char *name; const char *path; int gated_on_existence; }
|
||||
drivers[] = {
|
||||
{ "cstage", cdrv, 0 },
|
||||
{ "wwstage", wdrv, 1 },
|
||||
{ NULL, NULL, 0 },
|
||||
};
|
||||
|
||||
int n = (int)(sizeof rows / sizeof rows[0]);
|
||||
int total = 0, fail = 0;
|
||||
for (int d = 0; drivers[d].name; d++) {
|
||||
if (drivers[d].gated_on_existence
|
||||
&& access(drivers[d].path, X_OK) != 0) {
|
||||
fprintf(stderr, "uniesc_run: skip %s (no %s)\n",
|
||||
drivers[d].name, drivers[d].path);
|
||||
continue;
|
||||
}
|
||||
for (int i = 0; i < n; i++) {
|
||||
int got = run_driver(drivers[d].path, &rows[i], i);
|
||||
total++;
|
||||
if (got != rows[i].want) {
|
||||
fprintf(stderr,
|
||||
"uniesc_run[%s][%s]: exit=%d want=%d\n",
|
||||
drivers[d].name, rows[i].label,
|
||||
got, rows[i].want);
|
||||
fail++;
|
||||
}
|
||||
}
|
||||
int rt = run_roundtrip(drivers[d].path, d);
|
||||
total++;
|
||||
if (rt != 42) {
|
||||
fprintf(stderr,
|
||||
"uniesc_run[%s][wwi_roundtrip]: exit=%d want=42\n",
|
||||
drivers[d].name, rt);
|
||||
fail++;
|
||||
}
|
||||
}
|
||||
if (fail) {
|
||||
fprintf(stderr, "uniesc_run: %d/%d fixtures failed\n",
|
||||
fail, total);
|
||||
return 1;
|
||||
}
|
||||
printf("uniesc_run: %d/%d ok\n", total, total);
|
||||
return 0;
|
||||
}
|
||||
Reference in New Issue
Block a user