Hojun-Cho/ww
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

lex,wwi: \u/\U unicode escapes + wide-rune .wwi round-trip, both stages (#50)

Browse Source 
Hare-faithful \u (4 hex) / \U (8 hex) escapes; \x/\u/\U share one codepoint
path (ref/hare/hare/lex/lex.ha lex_unicode); string literals UTF-8-encode
multi-byte codepoints (cstage inline utf8enc, wwstage utf8.encoderune). The
.wwi producer rune serializer now emits \u/\U so exported wide-rune defs
round-trip (was a fatal >0xFF). Reject >0x10FFFF and surrogates with Hare-
verbatim error strings. Closes the int-cast spelling divergence (#48 RUNE_MAX).
Both stages byte-identical; 446 tests pass.
This commit is contained in:
Hojun-Cho
2026-06-18 22:47:53 +09:00
parent 64cf3c4094
commit 0197dfb9e6
7 changed files with 508 additions and 63 deletions
Download Patch File Download Diff File Expand all files Collapse all files

97
lib/ww/syntax/lex.ww
View File

@@ -17,6 +17,7 @@ import os;
import ascii;
import strings;
import strconv;
import encoding.utf8;
// isidstart / isidpart — identifier classification. Lexer-local
// because the "alpha or '_' / alnum or '_'" set isn't part of Hare's
@@ -306,6 +307,48 @@ fn parseint(p: *u8, n: u64, base: i32, ok: *bool) u64 = {
return v;
};
// lexunicode — shared escape decoder for \xHH (n=2), \uHHHH (n=4),
// \UHHHHHHHH (n=8). All three yield a codepoint, not a raw byte —
// mirrors ref/hare/hare/lex/lex.ha:347 fn lex_unicode. Error strings
// copied verbatim from that reference for diagnostic fidelity (#50).
fn lexunicode(l: *lex, n: i32, out: *i32) bool = {
// u32 (not i32): an 8-digit \U with bit 31 set would go negative
// in i32 and slip past the `> 0x10FFFF` range check — cstage uses
// u32 here, so i32 would diverge (rule 10).
let u: u32 = 0u32;
let i: i32 = 0;
for (i < n) {
let c: i32 = lget(l);
if (c < 0) {
let cp: pos; curpos(l, &cp);
errat(l, &cp, "unexpected EOF scanning for escape");
return false;
};
if (!ascii.isxdigit(c: rune)) {
let cp: pos; curpos(l, &cp);
errat(l, &cp, "unexpected rune scanning for escape");
return false;
};
let d: i32 = hexval(c: rune)!;
u = (u << 4u32) | (d: u32);
i += 1;
};
if (u > 0x10FFFFu32) {
let cp: pos; curpos(l, &cp);
errat(l, &cp, "invalid unicode codepoint in escape");
return false;
};
if (u >= 0xD800u32) {
if (u < 0xE000u32) {
let cp: pos; curpos(l, &cp);
errat(l, &cp, "invalid unicode codepoint in escape");
return false;
};
};
*out = u: i32;
return true;
};
fn escape(l: *lex, out: *i32) bool = {
let c: i32 = lget(l);
if (c < 0) { return false; };
@@ -320,31 +363,9 @@ fn escape(l: *lex, out: *i32) bool = {
if (c == 'b') { *out = '\b'; return true; };
if (c == 'f') { *out = '\f'; return true; };
if (c == 'v') { *out = '\v'; return true; };
if (c == 'x') {
let hi: i32 = lget(l);
let lo: i32 = lget(l);
if (hi < 0) { return false; };
if (lo < 0) { return false; };
if (!ascii.isxdigit(hi: rune)) {
let cp: pos; curpos(l, &cp);
errat(l, &cp, "bad \\x escape");
return false;
};
if (!ascii.isxdigit(lo: rune)) {
let cp: pos; curpos(l, &cp);
errat(l, &cp, "bad \\x escape");
return false;
};
// Hex digits already validated by isxdigit above — `!`
// (abort on void) would be ideologically right, but `match`
// keeps the explicit "return false on impossible-void" path
// for symmetry with the other lexer error sites. Use `!`
// once we have a panic-with-position helper.
let h: i32 = hexval(hi: rune)!;
let lv: i32 = hexval(lo: rune)!;
*out = (h << 4) | lv;
return true;
};
if (c == 'x') { return lexunicode(l, 2, out); };
if (c == 'u') { return lexunicode(l, 4, out); };
if (c == 'U') { return lexunicode(l, 8, out); };
let cp: pos; curpos(l, &cp);
errat(l, &cp, "bad escape");
return false;
@@ -636,15 +657,25 @@ fn lexstr(l: *lex, start: *pos, out: *tok) void = {
return;
};
if (c == '"') { lget(l); break; };
let ch: i32 = 0;
// Escape-decoded values are codepoints and UTF-8-encode into
// 1-4 bytes (mirrors Hare's memio::appendrune in lex_string,
// ref/hare/hare/lex/lex.ha:431). Raw source bytes are already
// UTF-8 and pass through unchanged — re-encoding them would
// double-encode the >0x7F continuation bytes.
let enc: [4]u8;
let el: i32 = 1;
if (c == '\\') {
let ch: i32 = 0;
lget(l);
if (!escape(l, &ch)) { ch = 0; };
el = utf8.encoderune(enc, ch: rune);
} else {
ch = lget(l);
enc[0] = lget(l): u8;
el = 1;
};
if (nb + 1u64 >= cap) {
if (nb + (el: u64) >= cap) {
let ncap: u64 = cap * 2u64;
for (nb + (el: u64) >= ncap) { ncap = ncap * 2u64; };
let nb2: []u8 = alloc([], ncap)!;
let i: u64 = 0u64;
for (i < nb) {
@@ -655,9 +686,13 @@ fn lexstr(l: *lex, start: *pos, out: *tok) void = {
buf = nb2;
cap = ncap;
};
let nbi: i32 = nb: i32;
buf[nbi] = ch: u8;
nb += 1u64;
let k: i32 = 0;
for (k < el) {
let nbi: i32 = nb: i32;
buf[nbi] = enc[k];
nb += 1u64;
k += 1;
};
};
out.kind = tkind.TK_STR;
out.file = start.file;