ww source: align UTF-8 BOM placement

This commit is contained in:
2026-08-21 20:34:10 +09:00
parent ee4573bf55
commit 9381f8fb8e
11 changed files with 764 additions and 5 deletions

View File

@@ -91,6 +91,45 @@ fn rewritefile(path: str, content: str) void = {
assert(os.close(fd) == 0);
};
fn putbomfile(path: str, before: str, after: str, rewrite: bool) void = {
let flags: os.flag = os.flag.WRONLY;
if (rewrite) { flags |= os.flag.TRUNC; }
else { flags |= os.flag.CREATE | os.flag.EXCL; };
let fd: i32 = os.open(path, flags, 384i32);
assert(fd >= 0);
match (os.writeall(fd, before.ptr, before.len: u64)) {
case let n: i64 => assert(n == before.len: i64);
case let e: os.oserror => abort("write before BOM failed");
};
let bom: [3]u8;
bom[0] = 0xefu8; bom[1] = 0xbbu8; bom[2] = 0xbfu8;
match (os.writeall(fd, bom.ptr, 3u64)) {
case let n: i64 => assert(n == 3i64);
case let e: os.oserror => abort("write BOM failed");
};
match (os.writeall(fd, after.ptr, after.len: u64)) {
case let n: i64 => assert(n == after.len: i64);
case let e: os.oserror => abort("write after BOM failed");
};
assert(os.close(fd) == 0);
};
fn writebomfile(path: str, content: str) void = {
putbomfile(path, "", content, false);
};
fn rewritebomfile(path: str, content: str) void = {
putbomfile(path, "", content, true);
};
fn writemidbomfile(path: str, before: str, after: str) void = {
putbomfile(path, before, after, false);
};
fn rewritemidbomfile(path: str, before: str, after: str) void = {
putbomfile(path, before, after, true);
};
fn writeexecutable(path: str, content: str) void = {
let fd: i32 = os.open(path,
os.flag.WRONLY | os.flag.CREATE | os.flag.EXCL, 448i32);
@@ -246,6 +285,16 @@ fn same(a: str, b: str) bool = {
return true;
};
fn hasbom(s: str) bool = {
let i: i32 = 0;
for (i + 2 < s.len) {
if (s[i] == 0xefu8 && s[i + 1] == 0xbbu8
&& s[i + 2] == 0xbfu8) { return true; };
i += 1;
};
return false;
};
fn occurrences(haystack: str, needle: str) i32 = {
if (needle.len == 0 || haystack.len < needle.len) { return 0; };
let count: i32 = 0;
@@ -16531,3 +16580,391 @@ fn runtimepath(relative: str) str = {
assert(!os.exists("/dev/null.sepwork"));
clean(root);
};
// Go 1.26.5 ignores one UTF-8 BOM at the first codepoint of each physical
// source and rejects U+FEFF everywhere else. The lexer owns direct and
// imports-only parsing; the directory composer preserves that per-file
// boundary when it builds one synthetic unit.
@test fn utf8_bom_is_per_source_and_only_first() void = {
let root: str = fresh();
let source: str = strings.concat(root, "/source");
let dep: str = strings.concat(source, "/dep");
let app: str = strings.concat(source, "/app");
let tests: str = strings.concat(source, "/bomtests");
let onlytests: str = strings.concat(source, "/onlytests");
let bad: str = strings.concat(source, "/bad");
let badtest: str = strings.concat(source, "/badtest");
let warm: str = strings.concat(source, "/warm");
mkdirall(dep); mkdirall(app); mkdirall(tests); mkdirall(onlytests);
mkdirall(bad); mkdirall(badtest); mkdirall(warm);
writebomfile(strings.concat(dep, "/dep.ww"), strings.concat(
"package dep;\n",
"export fn value() i32 = { return 41; };\n"));
writebomfile(strings.concat(dep, "/more.ww"), strings.concat(
"package dep;\n",
"export fn more() i32 = { return 1; };\n"));
let appmain: str = strings.concat(app, "/main.ww");
let appvalid: str = strings.concat(
"package main;\nimport dep;\n",
"fn main() i32 = { return dep.value() + dep.more() + local() - 43; };\n");
writebomfile(appmain, appvalid);
writebomfile(strings.concat(app, "/local.ww"), strings.concat(
"package main;\n",
"fn local() i32 = { return 1; };\n"));
// Wrong-target sources are excluded before source decoding.
writemidbomfile(strings.concat(app, "/ignored_windows.ww"),
"not a package ", " and not valid WW\n");
// Direct frontend ownership: the initial marker is invisible at 1:1 and
// both stages emit the same assembly; a later marker is one codepoint error.
let direct: str = strings.concat(root, "/direct.ww");
let directbad: str = strings.concat(root, "/direct-bad.ww");
writebomfile(direct,
"package main;\nfn main() i32 = { return 0; };\n");
writemidbomfile(directbad,
"package main;\nfn main() i32 = { let s: str = \"",
"\"; return s.len; };\n");
let directouts: []str = [strings.concat(root, "/direct-c.s"),
strings.concat(root, "/direct-ww.s")];
let directbadouts: []str = [strings.concat(root, "/direct-bad-c.s"),
strings.concat(root, "/direct-bad-ww.s")];
let compilers: []str = ["w6c", "w6c_ww"];
let tags: []str = ["c", "ww"];
let middiags: []str = ["", ""];
let i: i32 = 0;
for (i < compilers.len) {
let av: []str = [driver(compilers[i]), "-c", "--command-package",
"-o", directouts[i], direct];
let out: commandout;
runcommand(root, strings.concat("bom-direct-valid-", tags[i]), av,
(30i64 * (time.second: i64)): time.duration, &out);
expectexit(&out, 0);
assert(out.stdout.len == 0 && out.stderr.len == 0);
let badav: []str = [driver(compilers[i]), "-c", "--command-package",
"-o", directbadouts[i], directbad];
runcommand(root, strings.concat("bom-direct-invalid-", tags[i]), badav,
(30i64 * (time.second: i64)): time.duration, &out);
expectexit(&out, 1);
assert(out.stdout.len == 0);
assert(has(out.stderr,
":2:33: error: invalid BOM in the middle of the file\n"));
assert(occurrences(out.stderr,
"invalid BOM in the middle of the file") == 1);
middiags[i] = strings.dup(out.stderr);
assert(!os.exists(directbadouts[i]));
i += 1;
};
assert(same(middiags[0], middiags[1]));
assert(same(readfile(directouts[0]), readfile(directouts[1])));
// Build consumes multiple BOM-prefixed sources in both the root and its
// imported dependency. The marker never enters unit bytes or identity.
let stages: []str = ["ww", "ww_ww"];
let works: []str = [strings.concat(root, "/work-c"),
strings.concat(root, "/work-ww")];
let bins: []str = [strings.concat(root, "/app-c"),
strings.concat(root, "/app-ww")];
i = 0;
for (i < stages.len) {
mkdirall(works[i]);
let av: []str = [driver(stages[i]), "build", "-w", works[i],
"-I", source, "-o", bins[i], "app"];
let out: commandout;
runcommand(root, strings.concat("bom-build-", tags[i]), av,
(120i64 * (time.second: i64)): time.duration, &out);
expectexit(&out, 0);
assert(out.stdout.len == 0 && out.stderr.len == 0);
assert(!directoryhasnew(works[i])
&& !directoryhasfragment(works[i], ".wwtxn."));
let runav: []str = [bins[i]];
runcommand(root, strings.concat("bom-run-", tags[i]), runav,
time.second, &out);
expectexit(&out, 0);
assert(out.stdout.len == 0 && out.stderr.len == 0);
i += 1;
};
assert(same(readfile(bins[0]), readfile(bins[1])));
let actions: []str = ["dep", "app"];
let suffixes: []str = [".unit.ww", ".wwi", ".s", ".o", ".a"];
let commandsuffixes: []str = [".unit.ww", ".wwi", ".s", ".o", ".a",
".init.unit.ww", ".init.s", ".init.o"];
let ai: i32 = 0;
for (ai < actions.len) {
let unit: str = readfile(strings.concat(works[0], "/", actions[ai],
".unit.ww"));
assert(!hasbom(unit) && has(unit, "\n package "));
let si: i32 = 0;
for (si < suffixes.len) {
assert(same(readfile(strings.concat(works[0], "/", actions[ai],
suffixes[si])), readfile(strings.concat(works[1], "/",
actions[ai], suffixes[si]))));
si += 1;
};
ai += 1;
};
let ci: i32 = suffixes.len;
for (ci < commandsuffixes.len) {
assert(same(readfile(strings.concat(works[0], "/app",
commandsuffixes[ci])), readfile(strings.concat(works[1], "/app",
commandsuffixes[ci]))));
ci += 1;
};
// Production, same-package, external-package, and test-only roles all get
// their own offset-zero allowance and execute through the ordinary harness.
writebomfile(strings.concat(tests, "/prod.ww"), strings.concat(
"package bomtests;\n",
"export fn value() i32 = { return 42; };\n"));
writebomfile(strings.concat(tests, "/same_test.ww"), strings.concat(
"package bomtests;\n",
"@test fn same_bom() void = { assert(value() == 42); };\n"));
writebomfile(strings.concat(tests, "/external_test.ww"), strings.concat(
"package bomtests_test;\nimport bomtests;\n",
"@test fn external_bom() void = { assert(bomtests.value() == 42); };\n"));
writebomfile(strings.concat(onlytests, "/only_test.ww"),
"package onlytests;\n@test fn only_bom() void = { assert(true); };\n");
let testrefs: []str = ["", ""];
let onlyrefs: []str = ["", ""];
let testworks: []str = [strings.concat(root, "/test-work-c"),
strings.concat(root, "/test-work-ww")];
i = 0;
for (i < stages.len) {
mkdirall(testworks[i]);
let av: []str = [driver(stages[i]), "test", "-w", testworks[i],
"-I", source, "bomtests"];
let out: commandout;
runcommand(root, strings.concat("bom-test-", tags[i]), av,
(120i64 * (time.second: i64)): time.duration, &out);
expectexit(&out, 0);
assert(out.stderr.len == 0 && has(out.stdout, "2 passed, 0 failed"));
testrefs[i] = strings.dup(out.stdout);
let onlyav: []str = [driver(stages[i]), "test", "-I", source,
"onlytests"];
runcommand(root, strings.concat("bom-test-only-", tags[i]), onlyav,
(120i64 * (time.second: i64)): time.duration, &out);
expectexit(&out, 0);
assert(out.stderr.len == 0 && has(out.stdout, "1 passed, 0 failed"));
onlyrefs[i] = strings.dup(out.stdout);
i += 1;
};
assert(same(testrefs[0], testrefs[1]) && same(onlyrefs[0], onlyrefs[1]));
let testactions: []str = ["bomtests-internal-test",
"bomtests_test-external-test", "bomtests-test-main"];
ai = 0;
for (ai < testactions.len) {
let testsuffixes: []str = suffixes;
if (ai == 2) { testsuffixes = commandsuffixes; };
let tsi: i32 = 0;
for (tsi < testsuffixes.len) {
assert(same(readfile(strings.concat(testworks[0], "/",
testactions[ai], testsuffixes[tsi])),
readfile(strings.concat(testworks[1], "/", testactions[ai],
testsuffixes[tsi]))));
tsi += 1;
};
if (ai < 2) {
let testunit: str = readfile(strings.concat(testworks[0], "/",
testactions[ai], ".unit.ww"));
assert(!hasbom(testunit) && has(testunit, "\n package "));
};
ai += 1;
};
assert(!directoryhasnew(testworks[0]) && !directoryhasnew(testworks[1]));
// A later BOM is rejected during request parsing. Missing-import graph
// resolution, compiler/assembler/linker producers, runtime, and publication
// are therefore never reached; cold workdirs and products stay empty.
writemidbomfile(strings.concat(bad, "/main.ww"),
"package main;\n// ",
"\nimport missing;\nfn main() i32 = { return 0; };\n");
writefile(strings.concat(badtest, "/prod.ww"),
"package badtest;\nfn value() i32 = { return 42; };\n");
writemidbomfile(strings.concat(badtest, "/prod_test.ww"),
"package badtest;\n// ",
"\n@test fn must_not_run() void = { abort(\"ran\"); };\n");
let baddiags: []str = ["", ""];
let badtestdiags: []str = ["", ""];
i = 0;
for (i < stages.len) {
let badwork: str = strings.concat(root, "/bad-work-", tags[i]);
let badout: str = strings.concat(root, "/bad-output-", tags[i]);
mkdirall(badwork);
let av: []str = [driver(stages[i]), "build", "-w", badwork,
"-I", source, "-o", badout, "bad"];
let out: commandout;
runcommand(root, strings.concat("bom-bad-build-", tags[i]), av,
(60i64 * (time.second: i64)): time.duration, &out);
expectexit(&out, 1);
assert(out.stdout.len == 0);
assert(has(out.stderr,
":2:4: error: invalid BOM in the middle of the file\n"));
assert(!has(out.stderr, "cannot find import")
&& !has(out.stderr, "w6c failed"));
baddiags[i] = strings.dup(out.stderr);
assert(!os.exists(badout) && directoryisempty(badwork));
let testav: []str = [driver(stages[i]), "test", "-I", source,
"badtest"];
runcommand(root, strings.concat("bom-bad-test-", tags[i]), testav,
(60i64 * (time.second: i64)): time.duration, &out);
expectexit(&out, 1);
assert(same(out.stdout, "FAIL\n"));
assert(has(out.stderr, "invalid BOM in the middle of the file")
&& !has(out.stderr, "must_not_run ..."));
badtestdiags[i] = strings.dup(out.stderr);
i += 1;
};
assert(same(baddiags[0], baddiags[1])
&& same(badtestdiags[0], badtestdiags[1]));
// A warm invalid edit cannot mutate committed action bytes or publication.
// Restoring the exact leading-BOM source composes the same unit and reuses
// the generation without invoking the compiler.
let warmpath: str = strings.concat(warm, "/main.ww");
let warmvalid: str =
"package main;\nfn main() i32 = { return 0; };\n";
writebomfile(warmpath, warmvalid);
let wrapper: str = strings.concat(root, "/compiler-wrapper.sh");
writeexecutable(wrapper, strings.concat(
"#!/bin/sh\n",
"printf 'compile\\n' >> \"$WW_BOM_TRACE\"\n",
"exec \"$WW_BOM_REAL\" \"$@\"\n"));
i = 0;
for (i < stages.len) {
let warmwork: str = strings.concat(root, "/warm-work-", tags[i]);
let warmout: str = strings.concat(root, "/warm-output-", tags[i]);
let trace: str = strings.concat(root, "/warm-trace-", tags[i]);
mkdirall(warmwork); writefile(trace, "");
let env: []str = os.getenvs();
append(env, strings.concat("WW_W6C=", wrapper));
append(env, strings.concat("WW_BOM_TRACE=", trace));
append(env, strings.concat("WW_BOM_REAL=", driver(compilers[i])));
let av: []str = [driver(stages[i]), "build", "-w", warmwork,
"-I", source, "-o", warmout, "warm"];
let out: commandout;
runcommandenv(root, strings.concat("bom-warm-cold-", tags[i]), av,
env, (120i64 * (time.second: i64)): time.duration, &out);
expectexit(&out, 0);
assert(out.stdout.len == 0 && out.stderr.len == 0
&& readfile(trace).len != 0);
let refs: []str = alloc([], commandsuffixes.len: u64)!;
let si: i32 = 0;
for (si < commandsuffixes.len) {
append(refs, strings.dup(readfile(strings.concat(warmwork,
"/warm", commandsuffixes[si]))));
si += 1;
};
let binref: str = strings.dup(readfile(warmout));
rewritefile(trace, "");
rewritemidbomfile(warmpath, "package main;\n// ",
"\nfn main() i32 = { return 0; };\n");
runcommandenv(root, strings.concat("bom-warm-invalid-", tags[i]), av,
env, (60i64 * (time.second: i64)): time.duration, &out);
expectexit(&out, 1);
assert(out.stdout.len == 0
&& has(out.stderr, "invalid BOM in the middle of the file"));
assert(readfile(trace).len == 0 && same(binref, readfile(warmout)));
si = 0;
for (si < commandsuffixes.len) {
assert(same(refs[si], readfile(strings.concat(warmwork,
"/warm", commandsuffixes[si]))));
si += 1;
};
assert(!directoryhasnew(warmwork)
&& !directoryhasfragment(warmwork, ".wwtxn."));
rewritebomfile(warmpath, warmvalid);
runcommandenv(root, strings.concat("bom-warm-restored-", tags[i]), av,
env, (120i64 * (time.second: i64)): time.duration, &out);
expectexit(&out, 0);
assert(out.stdout.len == 0 && out.stderr.len == 0);
assert(readfile(trace).len == 0 && same(binref, readfile(warmout))
&& !directoryhasnew(warmwork));
// The marker is semantically whitespace but its three source bytes are
// observable input: removing and re-adding it invalidates the unit just
// as Go's source-content action does, without changing output bytes.
rewritefile(trace, "");
rewritefile(warmpath, warmvalid);
runcommandenv(root, strings.concat("bom-warm-marker-removed-", tags[i]),
av, env, (120i64 * (time.second: i64)): time.duration, &out);
expectexit(&out, 0);
assert(out.stdout.len == 0 && out.stderr.len == 0);
assert(readfile(trace).len != 0 && same(binref, readfile(warmout)));
assert(!same(refs[0], readfile(strings.concat(warmwork,
"/warm.unit.ww"))));
rewritefile(trace, "");
rewritebomfile(warmpath, warmvalid);
runcommandenv(root, strings.concat("bom-warm-marker-restored-", tags[i]),
av, env, (120i64 * (time.second: i64)): time.duration, &out);
expectexit(&out, 0);
assert(out.stdout.len == 0 && out.stderr.len == 0);
assert(readfile(trace).len != 0 && same(binref, readfile(warmout)));
si = 0;
for (si < commandsuffixes.len) {
assert(same(refs[si], readfile(strings.concat(warmwork,
"/warm", commandsuffixes[si]))));
si += 1;
};
assert(!directoryhasnew(warmwork));
i += 1;
};
// Request-local source state: a valid Cstage build and an invalid WWstage
// build overlap without sharing diagnostics, products, or transaction data.
let pcwork: str = strings.concat(root, "/parallel-c-work");
let pwwork: str = strings.concat(root, "/parallel-ww-work");
let pcout: str = strings.concat(root, "/parallel-c-output");
let pwout: str = strings.concat(root, "/parallel-ww-output");
mkdirall(pcwork); mkdirall(pwwork);
let pcav: []str = [driver("ww"), "build", "-w", pcwork,
"-I", source, "-o", pcout, "app"];
let pwav: []str = [driver("ww_ww"), "build", "-w", pwwork,
"-I", source, "-o", pwout, "bad"];
let pc: exec.command;
pc.path = pcav[0]; pc.argv = pcav; pc.env = os.getenvs(); pc.dir = repo();
pc.stdoutpath = strings.concat(root, "/parallel-c.stdout");
pc.stderrpath = strings.concat(root, "/parallel-c.stderr");
pc.deadline = time.add(time.now(time.clock.monotonic),
(120i64 * (time.second: i64)): time.duration);
pc.grace = (100i64 * (time.millisecond: i64)): time.duration;
let pw: exec.command;
pw.path = pwav[0]; pw.argv = pwav; pw.env = os.getenvs(); pw.dir = repo();
pw.stdoutpath = strings.concat(root, "/parallel-ww.stdout");
pw.stderrpath = strings.concat(root, "/parallel-ww.stderr");
pw.deadline = time.add(time.now(time.clock.monotonic),
(120i64 * (time.second: i64)): time.duration);
pw.grace = (100i64 * (time.millisecond: i64)): time.duration;
let pcp: exec.process;
let pwp: exec.process;
exec.start(&pcp, &pc); exec.start(&pwp, &pw);
let pcdone: bool = false;
let pwdone: bool = false;
for (!pcdone || !pwdone) {
if (!pcdone) { pcdone = exec.poll(&pcp); };
if (!pwdone) { pwdone = exec.poll(&pwp); };
if (!pcdone || !pwdone) {
time.sleep(time.millisecond, time.clock.monotonic);
};
};
assert(pcp.result.errno == 0 && pcp.result.cleanuperrno == 0
&& pcp.result.termination == exec.termination.EXIT
&& pcp.result.code == 0);
assert(pwp.result.errno == 0 && pwp.result.cleanuperrno == 0
&& pwp.result.termination == exec.termination.EXIT
&& pwp.result.code == 1);
assert(readfile(pc.stdoutpath).len == 0 && readfile(pc.stderrpath).len == 0);
assert(readfile(pw.stdoutpath).len == 0
&& has(readfile(pw.stderrpath), "invalid BOM in the middle of the file"));
assert(same(readfile(pcout), readfile(bins[0])) && !os.exists(pwout));
assert(directoryisempty(pwwork) && !directoryhasnew(pcwork)
&& !directoryhasfragment(pcwork, ".wwtxn."));
let runav: []str = [pcout];
let out: commandout;
runcommand(root, "bom-parallel-run", runav, time.second, &out);
expectexit(&out, 0);
assert(out.stdout.len == 0 && out.stderr.len == 0);
assert(!directoryhasfragment(root, ".wwtxn.")
&& !directoryhasfragment(root, ".install")
&& !directoryhasnew(root));
clean(root);
};

View File

@@ -61,6 +61,7 @@ struct row { const char *src, *expect; };
static const struct row rows[] = {
{ "", "" },
{ "\xef\xbb\xbf" "package main;", "package IDENT(main) ;" },
{ " \t\n ", "" },
{ "// comment\n", "" },
{ "/* a /b/ c */", "" },
@@ -172,7 +173,29 @@ runrewind(void)
return ok;
}
static int
runbompos(void)
{
const char *src = "\xef\xbb\xbf" "package main;";
Arena *a = newarena();
Lex l;
lexinit(&l, a, "<test>", src, strlen(src));
Tok t = lexnext(&l);
int ok = t.kind == TK_MODULE && t.pos.line == 1 && t.pos.col == 4;
if (!ok)
fprintf(stderr, "leading BOM position: want 1:4 got %d:%d\n",
t.pos.line, t.pos.col);
freearena(a);
return ok;
}
static const char *const errrows[] = {
"\xef\xbb\xbf" "\xef\xbb\xbf" "package main;",
"package \xef\xbb\xbf" "main;",
"// \xef\xbb\xbf\n",
"/* \xef\xbb\xbf */",
"\"\xef\xbb\xbf\"",
"'\xef\xbb\xbf'",
"'\\uZ'", /* unexpected rune scanning for escape */
"\"\\u00g0\"", /* non-hex digit inside a string escape */
"'\\u00", /* unexpected EOF scanning for escape */
@@ -194,6 +217,8 @@ main(void)
}
if (!runrewind())
fail++;
if (!runbompos())
fail++;
for (size_t i = 0; i < sizeof errrows / sizeof errrows[0]; i++) {
if (!runerr(errrows[i])) {
fprintf(stderr, "errrow %zu failed\n", i);