ww source: reject malformed UTF-8
This commit is contained in:
@@ -161,6 +161,38 @@ fn rewritenulfile(path: str, before: str, after: str) void = {
|
||||
putnulfile(path, before, after, true);
|
||||
};
|
||||
|
||||
fn putinvalidutf8file(path: str, before: str, after: str,
|
||||
rewrite: bool) void = {
|
||||
let flags: os.flag = os.flag.WRONLY;
|
||||
if (rewrite) { flags |= os.flag.TRUNC; }
|
||||
else { flags |= os.flag.CREATE | os.flag.EXCL; };
|
||||
let fd: i32 = os.open(path, flags, 384i32);
|
||||
assert(fd >= 0);
|
||||
match (os.writeall(fd, before.ptr, before.len: u64)) {
|
||||
case let n: i64 => assert(n == before.len: i64);
|
||||
case let e: os.oserror => abort("write before invalid UTF-8 failed");
|
||||
};
|
||||
let invalid: [1]u8;
|
||||
invalid[0] = 0xffu8;
|
||||
match (os.writeall(fd, invalid.ptr, 1u64)) {
|
||||
case let n: i64 => assert(n == 1i64);
|
||||
case let e: os.oserror => abort("write invalid UTF-8 failed");
|
||||
};
|
||||
match (os.writeall(fd, after.ptr, after.len: u64)) {
|
||||
case let n: i64 => assert(n == after.len: i64);
|
||||
case let e: os.oserror => abort("write after invalid UTF-8 failed");
|
||||
};
|
||||
assert(os.close(fd) == 0);
|
||||
};
|
||||
|
||||
fn writeinvalidutf8file(path: str, before: str, after: str) void = {
|
||||
putinvalidutf8file(path, before, after, false);
|
||||
};
|
||||
|
||||
fn rewriteinvalidutf8file(path: str, before: str, after: str) void = {
|
||||
putinvalidutf8file(path, before, after, true);
|
||||
};
|
||||
|
||||
fn writedoublenulfile(path: str, before: str, after: str) void = {
|
||||
let fd: i32 = os.open(path,
|
||||
os.flag.WRONLY | os.flag.CREATE | os.flag.EXCL, 384i32);
|
||||
@@ -17925,3 +17957,382 @@ fn runtimepath(relative: str) str = {
|
||||
&& !directoryhasnew(root));
|
||||
clean(root);
|
||||
};
|
||||
|
||||
// Go 1.26.5 removes each malformed UTF-8 byte from the logical source after
|
||||
// reporting it. WW applies that decoder rule only after physical-file
|
||||
// eligibility and before package clauses or imports can affect the graph.
|
||||
@test fn malformed_utf8_is_rejected_in_every_selected_source() void = {
|
||||
let root: str = fresh();
|
||||
let source: str = strings.concat(root, "/source");
|
||||
mkdirall(source);
|
||||
|
||||
// Filtering the invalid byte reconstructs the keyword, so the decoder's
|
||||
// positioned error is the only failure and neither frontend emits assembly.
|
||||
let invaliddirect: str = strings.concat(root, "/invalid-direct.ww");
|
||||
let validdirect: str = strings.concat(root, "/valid-direct.ww");
|
||||
writeinvalidutf8file(invaliddirect, "pack",
|
||||
"age main;\nfn main() i32 = { return 0; };\n");
|
||||
writefile(validdirect, strings.concat(
|
||||
"package main;\n",
|
||||
"// 한국어, café, and <20> are valid UTF-8 source bytes.\n",
|
||||
"fn main() i32 = { return 0; };\n"));
|
||||
let compilers: []str = ["w6c", "w6c_ww"];
|
||||
let stages: []str = ["ww", "ww_ww"];
|
||||
let tags: []str = ["c", "ww"];
|
||||
let invaliddiags: []str = ["", ""];
|
||||
let validasms: []str = [strings.concat(root, "/valid-c.s"),
|
||||
strings.concat(root, "/valid-ww.s")];
|
||||
let si: i32 = 0;
|
||||
for (si < compilers.len) {
|
||||
let invalidasm: str = strings.concat(root, "/invalid-", tags[si], ".s");
|
||||
let invalidav: []str = [driver(compilers[si]), "-c",
|
||||
"--command-package", "-o", invalidasm, invaliddirect];
|
||||
let out: commandout;
|
||||
runcommand(root, strings.concat("utf8-direct-invalid-", tags[si]),
|
||||
invalidav, (30i64 * (time.second: i64)): time.duration, &out);
|
||||
expectexit(&out, 1);
|
||||
assert(out.stdout.len == 0
|
||||
&& has(out.stderr, ":1:5: error: invalid UTF-8 encoding\n")
|
||||
&& occurrences(out.stderr, "invalid UTF-8 encoding") == 1
|
||||
&& !has(out.stderr, "unexpected character")
|
||||
&& !has(out.stderr, "invalid or missing package clause")
|
||||
&& !os.exists(invalidasm));
|
||||
invaliddiags[si] = strings.dup(out.stderr);
|
||||
|
||||
let validav: []str = [driver(compilers[si]), "-c",
|
||||
"--command-package", "-o", validasms[si], validdirect];
|
||||
runcommand(root, strings.concat("utf8-direct-valid-", tags[si]),
|
||||
validav, (30i64 * (time.second: i64)): time.duration, &out);
|
||||
expectexit(&out, 0);
|
||||
assert(out.stdout.len == 0 && out.stderr.len == 0);
|
||||
si += 1;
|
||||
};
|
||||
assert(same(invaliddiags[0], invaliddiags[1]));
|
||||
assert(same(readfile(validasms[0]), readfile(validasms[1])));
|
||||
|
||||
let dep: str = strings.concat(source, "/dep");
|
||||
let app: str = strings.concat(source, "/app");
|
||||
let bad: str = strings.concat(source, "/bad");
|
||||
let good: str = strings.concat(source, "/good");
|
||||
let ordered: str = strings.concat(source, "/ordered");
|
||||
mkdirall(dep); mkdirall(app); mkdirall(bad); mkdirall(good);
|
||||
mkdirall(ordered);
|
||||
writeinvalidutf8file(strings.concat(dep, "/dep.ww"),
|
||||
"package dep;\n// bad ",
|
||||
" dependency\nexport fn value() i32 = { return 0; };\n");
|
||||
writefile(strings.concat(app, "/main.ww"), strings.concat(
|
||||
"package main;\nimport dep;\n",
|
||||
"fn main() i32 = { return dep.value(); };\n"));
|
||||
writeinvalidutf8file(strings.concat(bad, "/main.ww"),
|
||||
"package main;\n// bad ", strings.concat(
|
||||
" root\nimport missing;\n",
|
||||
"fn main() i32 = { return 0; };\n"));
|
||||
writefile(strings.concat(good, "/main.ww"),
|
||||
"package main;\nfn main() i32 = { return 0; };\n");
|
||||
writeinvalidutf8file(strings.concat(good, "/ignored_windows.ww"),
|
||||
"not package ", " and not valid WW\n");
|
||||
writeinvalidutf8file(strings.concat(good, "/ignored_test.ww"),
|
||||
"not package ", " and not valid WW\n");
|
||||
writeinvalidutf8file(strings.concat(ordered, "/z.ww"),
|
||||
"package main;\n// z ", "\nfn z() void = {};\n");
|
||||
writefile(strings.concat(ordered, "/b.ww"),
|
||||
"package main;\nfn main() i32 = { return 0; };\n");
|
||||
writeinvalidutf8file(strings.concat(ordered, "/a.ww"),
|
||||
"package main;\n// a ", "\nfn a() void = {};\n");
|
||||
|
||||
// Root and dependency decoding precede graph resolution and all producers.
|
||||
// Excluded files are not semantic inputs, and selection remains byte-sorted.
|
||||
let goodbins: []str = [strings.concat(root, "/good-c"),
|
||||
strings.concat(root, "/good-ww")];
|
||||
let depdiags: []str = ["", ""];
|
||||
let rootdiags: []str = ["", ""];
|
||||
let orderdiags: []str = ["", ""];
|
||||
si = 0;
|
||||
for (si < stages.len) {
|
||||
let out: commandout;
|
||||
let depwork: str = strings.concat(root, "/dep-work-", tags[si]);
|
||||
let depout: str = strings.concat(root, "/dep-output-", tags[si]);
|
||||
mkdirall(depwork);
|
||||
let depav: []str = [driver(stages[si]), "build", "-w", depwork,
|
||||
"-I", source, "-o", depout, "app"];
|
||||
runcommand(root, strings.concat("utf8-dependency-", tags[si]), depav,
|
||||
(60i64 * (time.second: i64)): time.duration, &out);
|
||||
expectexit(&out, 1);
|
||||
assert(out.stdout.len == 0
|
||||
&& has(out.stderr,
|
||||
"/dep/dep.ww:2:8: error: invalid UTF-8 encoding\n")
|
||||
&& occurrences(out.stderr, "invalid UTF-8 encoding") == 1
|
||||
&& !has(out.stderr, "w6c failed")
|
||||
&& !os.exists(depout) && directoryisempty(depwork));
|
||||
depdiags[si] = strings.dup(out.stderr);
|
||||
|
||||
let badwork: str = strings.concat(root, "/bad-work-", tags[si]);
|
||||
let badout: str = strings.concat(root, "/bad-output-", tags[si]);
|
||||
mkdirall(badwork);
|
||||
let badav: []str = [driver(stages[si]), "build", "-w", badwork,
|
||||
"-I", source, "-o", badout, "bad"];
|
||||
runcommand(root, strings.concat("utf8-root-", tags[si]), badav,
|
||||
(60i64 * (time.second: i64)): time.duration, &out);
|
||||
expectexit(&out, 1);
|
||||
assert(out.stdout.len == 0
|
||||
&& has(out.stderr,
|
||||
"/bad/main.ww:2:8: error: invalid UTF-8 encoding\n")
|
||||
&& occurrences(out.stderr, "invalid UTF-8 encoding") == 1
|
||||
&& !has(out.stderr, "cannot find import")
|
||||
&& !os.exists(badout) && directoryisempty(badwork));
|
||||
rootdiags[si] = strings.dup(out.stderr);
|
||||
|
||||
let orderwork: str = strings.concat(root, "/order-work-", tags[si]);
|
||||
let orderout: str = strings.concat(root, "/order-output-", tags[si]);
|
||||
mkdirall(orderwork);
|
||||
let orderav: []str = [driver(stages[si]), "build", "-w", orderwork,
|
||||
"-I", source, "-o", orderout, "ordered"];
|
||||
runcommand(root, strings.concat("utf8-order-", tags[si]), orderav,
|
||||
(60i64 * (time.second: i64)): time.duration, &out);
|
||||
expectexit(&out, 1);
|
||||
assert(out.stdout.len == 0
|
||||
&& has(out.stderr,
|
||||
"/ordered/a.ww:2:6: error: invalid UTF-8 encoding\n")
|
||||
&& !has(out.stderr, "/ordered/z.ww:")
|
||||
&& !os.exists(orderout) && directoryisempty(orderwork));
|
||||
orderdiags[si] = strings.dup(out.stderr);
|
||||
|
||||
let goodwork: str = strings.concat(root, "/good-work-", tags[si]);
|
||||
mkdirall(goodwork);
|
||||
let goodav: []str = [driver(stages[si]), "build", "-w", goodwork,
|
||||
"-I", source, "-o", goodbins[si], "good"];
|
||||
runcommand(root, strings.concat("utf8-excluded-", tags[si]), goodav,
|
||||
(120i64 * (time.second: i64)): time.duration, &out);
|
||||
expectexit(&out, 0);
|
||||
assert(out.stdout.len == 0 && out.stderr.len == 0
|
||||
&& !directoryhasnew(goodwork)
|
||||
&& !directoryhasfragment(goodwork, ".wwtxn."));
|
||||
let runav: []str = [goodbins[si]];
|
||||
runcommand(root, strings.concat("utf8-good-run-", tags[si]), runav,
|
||||
time.second, &out);
|
||||
expectexit(&out, 0);
|
||||
assert(out.stdout.len == 0 && out.stderr.len == 0);
|
||||
si += 1;
|
||||
};
|
||||
assert(same(depdiags[0], depdiags[1])
|
||||
&& same(rootdiags[0], rootdiags[1])
|
||||
&& same(orderdiags[0], orderdiags[1])
|
||||
&& same(readfile(goodbins[0]), readfile(goodbins[1])));
|
||||
|
||||
// Every selected test-source role rejects before generated-main creation,
|
||||
// execution, accounting, and retained-binary publication.
|
||||
let prodcase: str = strings.concat(source, "/prodcase");
|
||||
let samecase: str = strings.concat(source, "/samecase");
|
||||
let externalcase: str = strings.concat(source, "/externalcase");
|
||||
let onlycase: str = strings.concat(source, "/onlycase");
|
||||
mkdirall(prodcase); mkdirall(samecase); mkdirall(externalcase);
|
||||
mkdirall(onlycase);
|
||||
writeinvalidutf8file(strings.concat(prodcase, "/prod.ww"),
|
||||
"package prodcase;\n// bad ",
|
||||
" production\nfn value() i32 = { return 1; };\n");
|
||||
writefile(strings.concat(prodcase, "/prod_test.ww"),
|
||||
"package prodcase;\n@test fn must_not_run() void = { abort(); };\n");
|
||||
writefile(strings.concat(samecase, "/prod.ww"),
|
||||
"package samecase;\nfn value() i32 = { return 1; };\n");
|
||||
writeinvalidutf8file(strings.concat(samecase, "/same_test.ww"),
|
||||
"package samecase;\n// bad ",
|
||||
" same\n@test fn must_not_run() void = { abort(); };\n");
|
||||
writefile(strings.concat(externalcase, "/prod.ww"),
|
||||
"package externalcase;\nexport fn value() i32 = { return 1; };\n");
|
||||
writeinvalidutf8file(strings.concat(externalcase, "/external_test.ww"),
|
||||
"package externalcase_test;\n// bad ", strings.concat(
|
||||
" external\nimport externalcase;\n",
|
||||
"@test fn must_not_run() void = { abort(); };\n"));
|
||||
writeinvalidutf8file(strings.concat(onlycase, "/only_test.ww"), "",
|
||||
"package onlycase;\n@test fn must_not_run() void = { abort(); };\n");
|
||||
let testids: []str = ["prodcase", "samecase", "externalcase", "onlycase"];
|
||||
let testpositions: []str = [
|
||||
"/prodcase/prod.ww:2:8: error: invalid UTF-8 encoding\n",
|
||||
"/samecase/same_test.ww:2:8: error: invalid UTF-8 encoding\n",
|
||||
"/externalcase/external_test.ww:2:8: error: invalid UTF-8 encoding\n",
|
||||
"/onlycase/only_test.ww:1:1: error: invalid UTF-8 encoding\n"];
|
||||
let testdiags: []str = ["", "", "", ""];
|
||||
si = 0;
|
||||
for (si < stages.len) {
|
||||
let ti: i32 = 0;
|
||||
for (ti < testids.len) {
|
||||
let testwork: str = strings.concat(root, "/test-work-", tags[si],
|
||||
"-", boundarypkgname(ti));
|
||||
let testout: str = strings.concat(root, "/test-output-", tags[si],
|
||||
"-", boundarypkgname(ti));
|
||||
mkdirall(testwork);
|
||||
let av: []str = [driver(stages[si]), "test", "-w", testwork,
|
||||
"-I", source, "-o", testout, testids[ti]];
|
||||
let out: commandout;
|
||||
runcommand(root, strings.concat("utf8-test-", tags[si], "-",
|
||||
boundarypkgname(ti)), av,
|
||||
(60i64 * (time.second: i64)): time.duration, &out);
|
||||
expectexit(&out, 1);
|
||||
assert(same(out.stdout, "FAIL\n")
|
||||
&& has(out.stderr, testpositions[ti])
|
||||
&& occurrences(out.stderr, "invalid UTF-8 encoding") == 1
|
||||
&& !has(out.stdout, "must_not_run")
|
||||
&& !has(out.stdout, "discovered")
|
||||
&& !has(out.stdout, "passed") && !has(out.stdout, "failed")
|
||||
&& !has(out.stdout, "ok ")
|
||||
&& !os.exists(testout) && directoryisempty(testwork));
|
||||
if (si == 0) { testdiags[ti] = strings.dup(out.stderr); }
|
||||
else { assert(same(testdiags[ti], out.stderr)); };
|
||||
ti += 1;
|
||||
};
|
||||
si += 1;
|
||||
};
|
||||
|
||||
// Invalid warm input reaches no tool and preserves every committed action,
|
||||
// tool record, and public byte. Exact source restoration is a cache hit.
|
||||
let warm: str = strings.concat(source, "/warm");
|
||||
mkdirall(warm);
|
||||
let warmpath: str = strings.concat(warm, "/main.ww");
|
||||
let warmvalid: str =
|
||||
"package main;\nfn main() i32 = { return 0; };\n";
|
||||
writefile(warmpath, warmvalid);
|
||||
let wrapper: str = strings.concat(root, "/utf8-compiler-wrapper.sh");
|
||||
writeexecutable(wrapper, strings.concat(
|
||||
"#!/bin/sh\n",
|
||||
"printf 'compile\\n' >> \"$WW_UTF8_TRACE\"\n",
|
||||
"exec \"$WW_UTF8_REAL\" \"$@\"\n"));
|
||||
let commandsuffixes: []str = [".unit.ww", ".wwi", ".s", ".o", ".a",
|
||||
".init.unit.ww", ".init.s", ".init.o"];
|
||||
let toolpaths: []str = ["/.wwtool.ww", "/.wwtool.w6c",
|
||||
"/.wwtool.w6a", "/.wwtool.stamp"];
|
||||
si = 0;
|
||||
for (si < stages.len) {
|
||||
let warmwork: str = strings.concat(root, "/warm-work-", tags[si]);
|
||||
let warmout: str = strings.concat(root, "/warm-output-", tags[si]);
|
||||
let trace: str = strings.concat(root, "/warm-trace-", tags[si]);
|
||||
mkdirall(warmwork); writefile(trace, "");
|
||||
let env: []str = os.getenvs();
|
||||
append(env, strings.concat("WW_W6C=", wrapper));
|
||||
append(env, strings.concat("WW_UTF8_TRACE=", trace));
|
||||
append(env, strings.concat("WW_UTF8_REAL=", driver(compilers[si])));
|
||||
let av: []str = [driver(stages[si]), "build", "-w", warmwork,
|
||||
"-I", source, "-o", warmout, "warm"];
|
||||
let out: commandout;
|
||||
runcommandenv(root, strings.concat("utf8-warm-cold-", tags[si]), av,
|
||||
env, (120i64 * (time.second: i64)): time.duration, &out);
|
||||
expectexit(&out, 0);
|
||||
assert(out.stdout.len == 0 && out.stderr.len == 0
|
||||
&& readfile(trace).len != 0);
|
||||
let refs: []str = alloc([], commandsuffixes.len: u64)!;
|
||||
let fi: i32 = 0;
|
||||
for (fi < commandsuffixes.len) {
|
||||
append(refs, strings.dup(readfile(strings.concat(warmwork,
|
||||
"/warm", commandsuffixes[fi]))));
|
||||
fi += 1;
|
||||
};
|
||||
let toolrefs: []str = alloc([], toolpaths.len: u64)!;
|
||||
fi = 0;
|
||||
for (fi < toolpaths.len) {
|
||||
append(toolrefs, strings.dup(readfile(strings.concat(warmwork,
|
||||
toolpaths[fi]))));
|
||||
fi += 1;
|
||||
};
|
||||
let binref: str = strings.dup(readfile(warmout));
|
||||
rewritefile(trace, "");
|
||||
rewriteinvalidutf8file(warmpath, "package main;\n// bad ",
|
||||
" warm\nfn main() i32 = { return 0; };\n");
|
||||
runcommandenv(root, strings.concat("utf8-warm-invalid-", tags[si]), av,
|
||||
env, (60i64 * (time.second: i64)): time.duration, &out);
|
||||
expectexit(&out, 1);
|
||||
assert(out.stdout.len == 0
|
||||
&& has(out.stderr, ":2:8: error: invalid UTF-8 encoding\n")
|
||||
&& readfile(trace).len == 0 && same(binref, readfile(warmout)));
|
||||
fi = 0;
|
||||
for (fi < commandsuffixes.len) {
|
||||
assert(same(refs[fi], readfile(strings.concat(warmwork,
|
||||
"/warm", commandsuffixes[fi]))));
|
||||
fi += 1;
|
||||
};
|
||||
fi = 0;
|
||||
for (fi < toolpaths.len) {
|
||||
assert(same(toolrefs[fi], readfile(strings.concat(warmwork,
|
||||
toolpaths[fi]))));
|
||||
fi += 1;
|
||||
};
|
||||
assert(!directoryhasnew(warmwork)
|
||||
&& !directoryhasfragment(warmwork, ".wwtxn."));
|
||||
rewritefile(warmpath, warmvalid);
|
||||
runcommandenv(root, strings.concat("utf8-warm-restored-", tags[si]), av,
|
||||
env, (120i64 * (time.second: i64)): time.duration, &out);
|
||||
expectexit(&out, 0);
|
||||
assert(out.stdout.len == 0 && out.stderr.len == 0
|
||||
&& readfile(trace).len == 0 && same(binref, readfile(warmout)));
|
||||
fi = 0;
|
||||
for (fi < commandsuffixes.len) {
|
||||
assert(same(refs[fi], readfile(strings.concat(warmwork,
|
||||
"/warm", commandsuffixes[fi]))));
|
||||
fi += 1;
|
||||
};
|
||||
fi = 0;
|
||||
for (fi < toolpaths.len) {
|
||||
assert(same(toolrefs[fi], readfile(strings.concat(warmwork,
|
||||
toolpaths[fi]))));
|
||||
fi += 1;
|
||||
};
|
||||
si += 1;
|
||||
};
|
||||
|
||||
// Validation is request-local: an overlapping valid Cstage request is not
|
||||
// cancelled or contaminated by an invalid WWstage request.
|
||||
let pcwork: str = strings.concat(root, "/parallel-c-work");
|
||||
let pwwork: str = strings.concat(root, "/parallel-ww-work");
|
||||
let pcout: str = strings.concat(root, "/parallel-c-output");
|
||||
let pwout: str = strings.concat(root, "/parallel-ww-output");
|
||||
mkdirall(pcwork); mkdirall(pwwork);
|
||||
let pcav: []str = [driver("ww"), "build", "-w", pcwork,
|
||||
"-I", source, "-o", pcout, "good"];
|
||||
let pwav: []str = [driver("ww_ww"), "build", "-w", pwwork,
|
||||
"-I", source, "-o", pwout, "bad"];
|
||||
let pc: exec.command;
|
||||
pc.path = pcav[0]; pc.argv = pcav; pc.env = os.getenvs(); pc.dir = repo();
|
||||
pc.stdoutpath = strings.concat(root, "/parallel-c.stdout");
|
||||
pc.stderrpath = strings.concat(root, "/parallel-c.stderr");
|
||||
pc.deadline = time.add(time.now(time.clock.monotonic),
|
||||
(120i64 * (time.second: i64)): time.duration);
|
||||
pc.grace = (100i64 * (time.millisecond: i64)): time.duration;
|
||||
let pw: exec.command;
|
||||
pw.path = pwav[0]; pw.argv = pwav; pw.env = os.getenvs(); pw.dir = repo();
|
||||
pw.stdoutpath = strings.concat(root, "/parallel-ww.stdout");
|
||||
pw.stderrpath = strings.concat(root, "/parallel-ww.stderr");
|
||||
pw.deadline = time.add(time.now(time.clock.monotonic),
|
||||
(120i64 * (time.second: i64)): time.duration);
|
||||
pw.grace = (100i64 * (time.millisecond: i64)): time.duration;
|
||||
let pcp: exec.process;
|
||||
let pwp: exec.process;
|
||||
exec.start(&pcp, &pc); exec.start(&pwp, &pw);
|
||||
let pcdone: bool = false;
|
||||
let pwdone: bool = false;
|
||||
for (!pcdone || !pwdone) {
|
||||
if (!pcdone) { pcdone = exec.poll(&pcp); };
|
||||
if (!pwdone) { pwdone = exec.poll(&pwp); };
|
||||
if (!pcdone || !pwdone) {
|
||||
time.sleep(time.millisecond, time.clock.monotonic);
|
||||
};
|
||||
};
|
||||
assert(pcp.result.errno == 0 && pcp.result.cleanuperrno == 0
|
||||
&& pcp.result.termination == exec.termination.EXIT
|
||||
&& pcp.result.code == 0);
|
||||
assert(pwp.result.errno == 0 && pwp.result.cleanuperrno == 0
|
||||
&& pwp.result.termination == exec.termination.EXIT
|
||||
&& pwp.result.code == 1);
|
||||
assert(readfile(pc.stdoutpath).len == 0 && readfile(pc.stderrpath).len == 0);
|
||||
assert(readfile(pw.stdoutpath).len == 0
|
||||
&& has(readfile(pw.stderrpath), "invalid UTF-8 encoding"));
|
||||
assert(same(readfile(pcout), readfile(goodbins[0])) && !os.exists(pwout));
|
||||
assert(directoryisempty(pwwork) && !directoryhasnew(pcwork)
|
||||
&& !directoryhasfragment(pcwork, ".wwtxn."));
|
||||
let runav: []str = [pcout];
|
||||
let out: commandout;
|
||||
runcommand(root, "utf8-parallel-run", runav, time.second, &out);
|
||||
expectexit(&out, 0);
|
||||
assert(out.stdout.len == 0 && out.stderr.len == 0);
|
||||
assert(!directoryhasfragment(root, ".wwtxn.")
|
||||
&& !directoryhasfragment(root, ".install")
|
||||
&& !directoryhasnew(root));
|
||||
clean(root);
|
||||
};
|
||||
|
||||
@@ -240,6 +240,113 @@ static const struct {
|
||||
{ "f\0\0n", sizeof "f\0\0n" - 1, "fn", 2 },
|
||||
};
|
||||
|
||||
/* Go 1.26.5 source.nextch reports each DecodeRune width-one result and
|
||||
* removes that byte before scanner recovery. */
|
||||
static const struct {
|
||||
const char *src;
|
||||
size_t len;
|
||||
const char *expect;
|
||||
int errs;
|
||||
} utf8rows[] = {
|
||||
{ "f" "\xff" "n bar", sizeof "f" "\xff" "n bar" - 1,
|
||||
"fn IDENT(bar)", 1 },
|
||||
{ "f" "\x80" "n bar", sizeof "f" "\x80" "n bar" - 1,
|
||||
"fn IDENT(bar)", 1 },
|
||||
{ "f" "\xc0\x80" "n bar", sizeof "f" "\xc0\x80" "n bar" - 1,
|
||||
"fn IDENT(bar)", 2 },
|
||||
{ "f" "\xed\xa0\x80" "n bar",
|
||||
sizeof "f" "\xed\xa0\x80" "n bar" - 1,
|
||||
"fn IDENT(bar)", 3 },
|
||||
{ "f" "\xf4\x90\x80\x80" "n bar",
|
||||
sizeof "f" "\xf4\x90\x80\x80" "n bar" - 1,
|
||||
"fn IDENT(bar)", 4 },
|
||||
{ "f" "\xe2\x82" "n bar", sizeof "f" "\xe2\x82" "n bar" - 1,
|
||||
"fn IDENT(bar)", 2 },
|
||||
{ "1" "\xff" "_0", sizeof "1" "\xff" "_0" - 1,
|
||||
"INT(10)", 1 },
|
||||
{ "=" "\xff" "=", sizeof "=" "\xff" "=" - 1,
|
||||
"==", 1 },
|
||||
{ "/" "\xff" "/ comment\nfn",
|
||||
sizeof "/" "\xff" "/ comment\nfn" - 1, "fn", 1 },
|
||||
{ "/* end *" "\xff" "/ fn",
|
||||
sizeof "/* end *" "\xff" "/ fn" - 1, "fn", 1 },
|
||||
{ "\"a" "\xff" "b\"", sizeof "\"a" "\xff" "b\"" - 1,
|
||||
"STR(ab)", 1 },
|
||||
};
|
||||
|
||||
static int
|
||||
runutf8valid(void)
|
||||
{
|
||||
static const char strsrc[] =
|
||||
"\"" "\xc3\xa9" "\xea\xb0\x80" "\xef\xbf\xbd"
|
||||
"\xf0\x9f\x98\x80" "\" fn";
|
||||
static const char commentsrc[] =
|
||||
"// " "\xc3\xa9" "\xea\xb0\x80" "\xef\xbf\xbd"
|
||||
"\xf0\x9f\x98\x80" "\nfn";
|
||||
static const unsigned char want[] = {
|
||||
0xc3, 0xa9, 0xea, 0xb0, 0x80, 0xef, 0xbf, 0xbd,
|
||||
0xf0, 0x9f, 0x98, 0x80,
|
||||
};
|
||||
Arena *a = newarena();
|
||||
Lex l;
|
||||
lexinit(&l, a, "<test>", strsrc, sizeof strsrc - 1);
|
||||
Tok s = lexnext(&l);
|
||||
Tok f = lexnext(&l);
|
||||
int ok = s.kind == TK_STR && s.tlen == sizeof want
|
||||
&& memcmp(s.text, want, sizeof want) == 0
|
||||
&& f.kind == TK_FN && f.pos.col == 16 && l.errs == 0
|
||||
&& lexnext(&l).kind == TK_EOF;
|
||||
lexinit(&l, a, "<test>", commentsrc, sizeof commentsrc - 1);
|
||||
f = lexnext(&l);
|
||||
ok = ok && f.kind == TK_FN && f.pos.line == 2 && f.pos.col == 1
|
||||
&& l.errs == 0 && lexnext(&l).kind == TK_EOF;
|
||||
if (!ok)
|
||||
fprintf(stderr, "valid UTF-8 preservation failed\n");
|
||||
freearena(a);
|
||||
return ok;
|
||||
}
|
||||
|
||||
static int
|
||||
runutf8cols(void)
|
||||
{
|
||||
static const char src[] =
|
||||
"\xef\xbb\xbf" "f" "\xff" "\0" "n bar";
|
||||
Arena *a = newarena();
|
||||
Lex l;
|
||||
lexinit(&l, a, "<test>", src, sizeof src - 1);
|
||||
Tok f = lexnext(&l);
|
||||
Tok b = lexnext(&l);
|
||||
int ok = f.kind == TK_FN && f.pos.line == 1 && f.pos.col == 4
|
||||
&& b.kind == TK_IDENT && b.pos.line == 1 && b.pos.col == 9
|
||||
&& strcmp(b.text, "bar") == 0 && l.errs == 2
|
||||
&& l.nulcount == 1 && lexnext(&l).kind == TK_EOF;
|
||||
if (!ok)
|
||||
fprintf(stderr, "UTF-8 raw-byte columns or BOM/NUL recovery failed\n");
|
||||
freearena(a);
|
||||
return ok;
|
||||
}
|
||||
|
||||
static int
|
||||
runutf8escape(void)
|
||||
{
|
||||
static const char line[] = "\"\\" "\xff" "n\"";
|
||||
static const char hex[] = "\"\\x0" "\xff" "0\"";
|
||||
Arena *a = newarena();
|
||||
Lex l;
|
||||
lexinit(&l, a, "<test>", line, sizeof line - 1);
|
||||
Tok t = lexnext(&l);
|
||||
int ok = t.kind == TK_STR && t.tlen == 1 && t.text[0] == '\n'
|
||||
&& l.errs == 1 && lexnext(&l).kind == TK_EOF;
|
||||
lexinit(&l, a, "<test>", hex, sizeof hex - 1);
|
||||
t = lexnext(&l);
|
||||
ok = ok && t.kind == TK_STR && t.tlen == 1 && t.text[0] == '\0'
|
||||
&& l.errs == 1 && lexnext(&l).kind == TK_EOF;
|
||||
if (!ok)
|
||||
fprintf(stderr, "UTF-8 escape recovery failed\n");
|
||||
freearena(a);
|
||||
return ok;
|
||||
}
|
||||
|
||||
static int
|
||||
runescapenul(void)
|
||||
{
|
||||
@@ -333,10 +440,23 @@ main(void)
|
||||
fail++;
|
||||
}
|
||||
}
|
||||
for (size_t i = 0; i < sizeof utf8rows / sizeof utf8rows[0]; i++) {
|
||||
if (!runrown(utf8rows[i].src, utf8rows[i].len,
|
||||
utf8rows[i].expect, utf8rows[i].errs)) {
|
||||
fprintf(stderr, "UTF-8 row %zu failed\n", i);
|
||||
fail++;
|
||||
}
|
||||
}
|
||||
if (!runescapenul())
|
||||
fail++;
|
||||
if (!runsuffixnul())
|
||||
fail++;
|
||||
if (!runutf8valid())
|
||||
fail++;
|
||||
if (!runutf8cols())
|
||||
fail++;
|
||||
if (!runutf8escape())
|
||||
fail++;
|
||||
if (!runlongdirective())
|
||||
fail++;
|
||||
if (fail) {
|
||||
|
||||
Reference in New Issue
Block a user