From 9381f8fb8e29a29ef139f68ec06df4d204b16e8d Mon Sep 17 00:00:00 2001 From: Hojun-Cho Date: Fri, 21 Aug 2026 20:34:10 +0900 Subject: [PATCH] ww source: align UTF-8 BOM placement --- cmd/wcc/lex.c | 30 +++ cmd/ww/main.c | 13 +- docs/build-system.md | 118 +++++++++ docs/spec.md | 9 +- docs/test-system-v2.md | 14 ++ internal/wwpackage/package.ww | 7 +- lib/ww/syntax/lex.ww | 27 +++ lib/ww/syntax/tok_test.ww | 75 ++++++ selfhost/cmd/ww/main.ww | 14 +- test/package/package_test.ww | 437 ++++++++++++++++++++++++++++++++++ test/wcc/100_lex.c | 25 ++ 11 files changed, 764 insertions(+), 5 deletions(-) diff --git a/cmd/wcc/lex.c b/cmd/wcc/lex.c index 886906bf..1a89c5a0 100644 --- a/cmd/wcc/lex.c +++ b/cmd/wcc/lex.c @@ -7,6 +7,15 @@ #include #include +static int +bomat(const char *src, u64 len, u64 pos) +{ + return pos <= len && len - pos >= 3 + && (unsigned char)src[pos] == 0xef + && (unsigned char)src[pos + 1] == 0xbb + && (unsigned char)src[pos + 2] == 0xbf; +} + void lexinit(Lex *l, Arena *a, const char *file, const char *src, u64 len) { @@ -17,6 +26,12 @@ lexinit(Lex *l, Arena *a, const char *file, const char *src, u64 len) l->line = 1; l->col = 1; l->a = a; + /* Go 1.26.5 syntax.source.nextch ignores U+FEFF only at the first + * source position but counts its three bytes for the next column. */ + if (bomat(src, len, 0)) { + l->pos = 3; + l->col = 4; + } } static int @@ -25,6 +40,8 @@ lpeek(Lex *l, u64 ahead) u64 p = l->pos + ahead; if (p >= l->srclen) return -1; + if (bomat(l->src, l->srclen, p)) + return 0xfeff; return (unsigned char)l->src[p]; } @@ -33,6 +50,14 @@ lget(Lex *l) { if (l->pos >= l->srclen) return -1; + if (bomat(l->src, l->srclen, l->pos)) { + Pos p = { l->file, l->line, l->col }; + l->pos += 3; + l->col += 3; + errorf(p, "invalid BOM in the middle of the file"); + l->errs++; + return 0xfeff; + } int c = (unsigned char)l->src[l->pos++]; if (c == '\n') { l->line++; @@ -529,6 +554,11 @@ lexnext(Lex *l) return t; } int c = lpeek(l, 0); + if (c == 0xfeff) { + lget(l); + Tok t = (Tok){ TK_ERR, start, "", 0, {0}, TK_NONE }; + return t; + } if (isidstart(c)) return lexident(l, start); diff --git a/cmd/ww/main.c b/cmd/ww/main.c index ad29cfb5..c634fc5e 100644 --- a/cmd/ww/main.c +++ b/cmd/ww/main.c @@ -4740,7 +4740,18 @@ sep_emit_body(FILE *out, const char *path, const char *modpath) } else if (fputs("//ww:module-reset\n", out) == EOF) { bad = 1; } - if (fwrite(buf, 1, (size_t)len, out) != (size_t)len + /* Each physical source owns its own first-codepoint position. The + * compiler sees one synthetic aggregate, so replace the optional UTF-8 + * BOM with position-preserving whitespace rather than turning a later + * source's marker into a mid-unit BOM. */ + u64 off = len >= 3 + && (unsigned char)buf[0] == 0xef + && (unsigned char)buf[1] == 0xbb + && (unsigned char)buf[2] == 0xbf ? 3 : 0; + if (off != 0 && fwrite(" ", 1, 3, out) != 3) + bad = 1; + if (fwrite(buf + off, 1, (size_t)(len - off), out) + != (size_t)(len - off) || fputc('\n', out) == EOF) bad = 1; free(buf); diff --git a/docs/build-system.md b/docs/build-system.md index 53336b38..dbd43d02 100644 --- a/docs/build-system.md +++ b/docs/build-system.md @@ -8427,6 +8427,124 @@ Rejected source creates no persisted byte contract, while valid source bytes are unchanged. Build workdir format remains `18`, test workdir format remains `19`, and semantic storage format remains `3`. +### 11.42 Implemented per-source UTF-8 BOM placement + +Every eligible physical WW source may begin with one UTF-8-encoded U+FEFF byte +order mark (`EF BB BF`). That marker is ignored, and the following token keeps +its three-byte source position at line 1, column 4. U+FEFF at any later raw +source position is +rejected once as `invalid BOM in the middle of the file`, including inside a +line/block comment, string, or rune. Two leading markers therefore ignore the +first and reject the second. A truncated marker or another invalid UTF-8 byte +sequence retains the ordinary byte-error path; UTF-16 source is not introduced. + +#### Pinned Go evidence and applicability + +The sole authority is official Go 1.26.5 at commit +`c19862e5f8415b4f24b189d065ed739517c548ba`: + +- compiler reader `(*source).nextch` decodes UTF-8, skips U+FEFF at its first + source position, and reports `invalid BOM in the middle of the file` + elsewhere + ([`cmd/compile/internal/syntax/source.go`, lines 113–165](https://github.com/golang/go/blob/c19862e5f8415b4f24b189d065ed739517c548ba/src/cmd/compile/internal/syntax/source.go#L113-L165)); +- public scanner `(*Scanner).next` rejects later U+FEFF and `(*Scanner).Init` + consumes the first one + ([`go/scanner/scanner.go`, lines 58–100 and 128–164](https://github.com/golang/go/blob/c19862e5f8415b4f24b189d065ed739517c548ba/src/go/scanner/scanner.go#L58-L164)); +- scanner tests assert the ignored first marker and later markers between + tokens, in comments, runes, and strings + ([`go/scanner/scanner_test.go`, lines 371–374 and 812–815](https://github.com/golang/go/blob/c19862e5f8415b4f24b189d065ed739517c548ba/src/go/scanner/scanner_test.go#L371-L374), + [`go/scanner/scanner_test.go`, lines 812–815](https://github.com/golang/go/blob/c19862e5f8415b4f24b189d065ed739517c548ba/src/go/scanner/scanner_test.go#L812-L815)); +- compiler-scanner `TestScanErrors` asserts the positioned later-marker error + ([`cmd/compile/internal/syntax/scanner_test.go`, lines 587–599](https://github.com/golang/go/blob/c19862e5f8415b4f24b189d065ed739517c548ba/src/cmd/compile/internal/syntax/scanner_test.go#L587-L599)); and +- official command testdata places the marker before `package main` and loads + that source's imports and embedded file + ([`cmd/go/testdata/script/build_ignore_leading_bom.txt`, lines 1–25](https://github.com/golang/go/blob/c19862e5f8415b4f24b189d065ed739517c548ba/src/cmd/go/testdata/script/build_ignore_leading_bom.txt#L1-L25)). + +Those source branches and assertions are **behavior directly implemented or +asserted by pinned Go**. Applying the first position independently to every WW +physical source before its existing synthetic aggregate boundary is **behavior +derived from the pinned implementation**. The pre-fix Cstage/WWstage failures +on leading markers and successes for markers in comments/strings were +**directly measured WW behavior**. + +The behavior honestly applies inside WW's local, dotted-import, manifest-free +model: it is source representation before package and import interpretation. +It requires no module, manifest, registry, lock, cache, database, CAS, network +resolution, generalized import syntax, build expression, or normalized +identity. + +#### Ownership and four-axis result + +`lexinit`, `lpeek`, `lget`, and `lexnext` in `cmd/wcc/lex.c`, with their twins +in `lib/ww/syntax/lex.ww`, are the language owners. They skip the exact initial +marker while advancing the logical column by its encoded width, recognize a +later marker as one code point in every lexical context, and emit one +stage-identical diagnostic. +`sep_emit_body` in `cmd/ww/main.c` and `sepemitbody` in +`selfhost/cmd/ww/main.ww` replace the optional marker with three spaces in every +physical body placed after a synthetic `//ww:module-reset`, preserving columns; +direct and imports-only lexer input remains independently correct. The shared package +coordinator's `pkgclause` in `internal/wwpackage/package.ww` begins its +pre-discovery package-name scan after the same optional marker. It does not +replace the complete stage-driver scan. + +- **Go-like build:** a selected command, library, or imported dependency may + use the marker in each eligible source. Multiple physical marked sources + compose normally, command publication succeeds, and the executable runs + normally. A later marker rejects during source loading before graph/action + construction or compiler, assembler, archiver, linker, install, or runtime + work. A source error keeps precedence over a missing import. +- **Go-like test:** production, same-package, external-package, and test-only + files each receive the offset-zero allowance. Valid variants build and run + through the ordinary single directory product. A later marker yields the + existing attributable `FAIL\n` result without a variant, generated main, + test process, accounting, or retained binary. +- **Go-like package:** package-clause recognition now begins at the pinned + logical source start in direct compilers, directory drivers, and the shared + coordinator. Declared names, source roles, package conflicts, command/test + family selection, and canonical identity do not change. +- **Go-like import:** imports following a legal marker and imports in a marked + dependency retain their exact source spelling, file scope, qualifier, + contextual local/vendor resolution, case-sensitive canonical dotted + identity, visibility, cycle, and initialization behavior. The marker never + becomes an edge or identity component. + +Fixed-target filename selection remains earlier than parsing: an excluded +`_windows.ww` or `_windows_test.ww` contributes no marker diagnostic, package, +import, graph, action, artifact, or invalidation state. For valid inputs, graph +nodes, scheduling, producer arguments, initialization, runtime, result order, +and publication are unchanged. Legal marker bytes become three +position-preserving spaces only in the synthetic unit. Adding or removing the +marker therefore changes unit content and invalidates source-derived actions, +while semantic interface, assembly, object, archive, initializer, and +executable content remains the same; every form remains Cstage/WWstage +byte-identical. + +Cold rejection leaves no product, work generation, adjacent scratch, capture, +`.new`, `.install`, or `.wwtxn.*`. A warm later-marker edit preserves the +entire committed generation, tool vouchers, stamp, and public executable. +Restoring the exact legal leading form recreates the same unit and reuses committed +compiler/assembler/archive work before the normal link/publication boundary. +Concurrent valid and invalid requests keep independent lexer/coordinator state, +workdirs, captures, diagnostics, and products. Because later-marker rejection +occurs before a producer or test child, producer/runtime failure, signals, +timeouts, interruption, and process-group cleanup gain no new branch; their +existing owners remain authoritative. + +The C lexer unit, WW syntax unit, and WW-native +`utf8_bom_is_per_source_and_only_first` observer prove initial position, +later-marker lexical contexts, direct frontend diagnostics and assembly, +selected/imported multi-source build, wrong-target exclusion, every test source +role, test-only execution, syntax-before-resolution precedence, cold cleanup, +warm rollback and reuse, concurrent isolation, publication/runtime behavior, +and complete stage diagnostic/artifact parity. + +No persisted format changes. Previously valid marker-free bytes are +unchanged; previously leading-marked requests could not commit a generation; +and diagnostic text is not a persisted-byte contract. Build workdir format +remains `18`, test workdir format remains `19`, and semantic storage format +remains `3`. + ## 12. Candidate architectures and hard-gate decision Five candidates were developed as coherent systems, not as feature bins. diff --git a/docs/spec.md b/docs/spec.md index 91289bac..60eaf3c4 100644 --- a/docs/spec.md +++ b/docs/spec.md @@ -52,8 +52,13 @@ classes from §2 (`ident`, `int_lit`, …). ### 2.1 Source representation -Source is UTF-8. The lexer operates on bytes; non-ASCII bytes are legal -only inside string and rune literals and comments. +Source is UTF-8. One UTF-8-encoded byte order mark (U+FEFF, bytes +`EF BB BF`) is ignored when it is the first code point of a physical source +file. Its three encoded bytes still count in source positions, so a following +token on the first line begins at column 4. U+FEFF is invalid at +every other source position, including inside string and rune literals and +comments. Apart from that marker rule, the lexer operates on bytes and +non-ASCII bytes are legal only inside string and rune literals and comments. ### 2.2 Comments diff --git a/docs/test-system-v2.md b/docs/test-system-v2.md index 319176ea..08addab2 100644 --- a/docs/test-system-v2.md +++ b/docs/test-system-v2.md @@ -415,6 +415,20 @@ generation. The directory command retains its existing attributable final diagnostic. This ordering state is per source and never package, import, variant, action, artifact, publication, or persistence identity. +Every eligible source also owns an independent source-start position. An exact +UTF-8 BOM (`EF BB BF`) at byte offset zero is ignored before package-clause and +import parsing, so production, same-package, external-package, and test-only +sources all count the marker's three bytes and begin the following token at +logical line 1 column 4. The same code point anywhere else, +including a comment or literal, is a source error before variant actions, +generated main, producers, test runtime, accounting, or retained publication. +The shared coordinator applies the offset-zero rule while classifying package +clauses; the selected stage driver scans the complete source and preserves the +per-file rule while composing package units. Wrong-target files remain excluded +before either operation. The marker is source representation only and never +package/import/action/artifact/publication/persistence identity; no test result +is cached. + After that eligibility boundary and the coordinator's required package-clause classification and production `@test` validation parses, the delegated loader performs selected-basename Go 1.26.5 simple-fold preflight before its graph diff --git a/internal/wwpackage/package.ww b/internal/wwpackage/package.ww index f2e67caa..d24dd37f 100644 --- a/internal/wwpackage/package.ww +++ b/internal/wwpackage/package.ww @@ -724,7 +724,12 @@ fn pkgskipspace(src: str, start: i32) i32 = { }; fn pkgclause(src: str, out: *str) bool = { - let i: i32 = pkgskipspace(src, 0); + // Match the compiler readers: one UTF-8 BOM is invisible only at the + // first raw source position. The full stage driver owns later-BOM errors. + let start: i32 = 0; + if (src.len >= 3 && src[0] == 0xefu8 && src[1] == 0xbbu8 + && src[2] == 0xbfu8) { start = 3; }; + let i: i32 = pkgskipspace(src, start); let word: str = "package"; if (i + word.len >= src.len) { return false; }; let j: i32 = 0; diff --git a/lib/ww/syntax/lex.ww b/lib/ww/syntax/lex.ww index d9400657..1103e42f 100644 --- a/lib/ww/syntax/lex.ww +++ b/lib/ww/syntax/lex.ww @@ -65,6 +65,14 @@ export type lex = struct { modresetpath: str, }; +fn bomat(src: *u8, len: u64, off: u64) bool = { + if (off > len || len - off < 3u64) { return false; }; + let i: i32 = off: i32; + return src[i] == 0xefu8 + && src[i + 1] == 0xbbu8 + && src[i + 2] == 0xbfu8; +}; + export fn lexinit(l: *lex, file: str, src: *u8, len: u64) void = { l.file = file; l.src = src; @@ -76,6 +84,9 @@ export fn lexinit(l: *lex, file: str, src: *u8, len: u64) void = { l.modreset = 0; l.modpathset = 0; l.modresetpathset = 0; + // Go 1.26.5 syntax.source.nextch ignores U+FEFF only at the first + // source position but counts its three bytes for the next column. + if (bomat(src, len, 0u64)) { l.lpos = 3u64; l.col = 4; }; }; fn srcb(l: *lex, off: u64) i32 = { @@ -87,11 +98,22 @@ fn srcb(l: *lex, off: u64) i32 = { fn lpeek(l: *lex, ahead: u64) i32 = { let p: u64 = l.lpos + ahead; if (p >= l.srclen) { return -1; }; + if (bomat(l.src, l.srclen, p)) { return 0xFEFF; }; return srcb(l, p); }; fn lget(l: *lex) i32 = { if (l.lpos >= l.srclen) { return -1; }; + if (bomat(l.src, l.srclen, l.lpos)) { + let bp: pos; + bp.file = l.file; + bp.line = l.line; + bp.col = l.col; + l.lpos += 3u64; + l.col += 3; + errat(l, &bp, "invalid BOM in the middle of the file"); + return 0xFEFF; + }; let c: i32 = srcb(l, l.lpos); l.lpos += 1u64; if (c == '\n') { @@ -801,6 +823,11 @@ export fn lexnext(l: *lex, out: *tok) void = { return; }; let c: i32 = lpeek(l, 0u64); + if (c == 0xFEFF) { + lget(l); + emitsimple(&start, tkind.TK_ERR, out); + return; + }; if (c >= 0) { if (isidstart(c: rune)) { lexident(l, &start, out); return; }; diff --git a/lib/ww/syntax/tok_test.ww b/lib/ww/syntax/tok_test.ww index 585cf0ec..0719e851 100644 --- a/lib/ww/syntax/tok_test.ww +++ b/lib/ww/syntax/tok_test.ww @@ -338,3 +338,78 @@ fn checkfloat(src: str, want: u64) void = { assert(!(t.col != 6)); assert(!(t.text != "bar")); }; + +fn bomsource(dst: *u8, before: str, after: str) u64 = { + let n: u64 = 0u64; + let i: i32 = 0; + for (i < before.len) { + let ni: i32 = n: i32; + dst[ni] = before[i]; + n += 1u64; + i += 1; + }; + let bi: i32 = n: i32; + dst[bi] = 0xefu8; + dst[bi + 1] = 0xbbu8; + dst[bi + 2] = 0xbfu8; + n += 3u64; + i = 0; + for (i < after.len) { + let ni: i32 = n: i32; + dst[ni] = after[i]; + n += 1u64; + i += 1; + }; + return n; +}; + +fn bomerror(before: str, after: str) bool = { + let src: [128]u8; + let n: u64 = bomsource(src.ptr, before, after); + let l: syntax.lex; + syntax.lexinit(&l, "t", src.ptr, n); + let t: syntax.tok; + for (true) { + syntax.lexnext(&l, &t); + if (t.kind == syntax.tkind.TK_EOF) { break; }; + }; + return l.errs > 0; +}; + +fn doublebomerror() bool = { + let src: [6]u8; + let i: i32 = 0; + for (i < 6) { + if (i % 3 == 0) { src[i] = 0xefu8; }; + if (i % 3 == 1) { src[i] = 0xbbu8; }; + if (i % 3 == 2) { src[i] = 0xbfu8; }; + i += 1; + }; + let l: syntax.lex; + syntax.lexinit(&l, "t", src.ptr, 6u64); + let t: syntax.tok; + for (true) { + syntax.lexnext(&l, &t); + if (t.kind == syntax.tkind.TK_EOF) { break; }; + }; + return l.errs > 0; +}; + +@test fn utf8_bom_is_ignored_only_at_source_start() void = { + let src: [128]u8; + let n: u64 = bomsource(src.ptr, "", "package main;"); + let l: syntax.lex; + syntax.lexinit(&l, "t", src.ptr, n); + let t: syntax.tok; + syntax.lexnext(&l, &t); + assert(t.kind == syntax.tkind.TK_MODULE); + assert(t.line == 1 && t.col == 4); + assert(l.errs == 0); + + assert(doublebomerror()); + assert(bomerror("package ", "main;")); + assert(bomerror("// ", "\n")); + assert(bomerror("/* ", " */")); + assert(bomerror("\"", "\"")); + assert(bomerror("'", "'")); +}; diff --git a/selfhost/cmd/ww/main.ww b/selfhost/cmd/ww/main.ww index cf3a4f26..6f7709f1 100644 --- a/selfhost/cmd/ww/main.ww +++ b/selfhost/cmd/ww/main.ww @@ -5594,7 +5594,19 @@ fn sepemitbody(fd: i32, path: *u8, modpath: *u8) i32 = { return -1; }; }; - if (!sepwriteall(fd, bufp, blen) + // Each physical source owns its own first-codepoint position. The compiler + // sees one synthetic aggregate, so replace the optional UTF-8 BOM with + // position-preserving whitespace rather than making it a mid-unit BOM. + let off: u64 = 0u64; + if (blen >= 3u64 && bufp[0] == 0xefu8 + && bufp[1] == 0xbbu8 && bufp[2] == 0xbfu8) { + off = 3u64; + }; + if (off != 0u64 && !sepwriteall(fd, " ".ptr, 3u64)) { + cerr("ww: cannot write package unit\n"); + return -1; + }; + if (!sepwriteall(fd, bufp + off, blen - off) || !sepwriteall(fd, "\n".ptr, 1u64)) { cerr("ww: cannot write package unit\n"); return -1; diff --git a/test/package/package_test.ww b/test/package/package_test.ww index 20319e5e..a67979f3 100644 --- a/test/package/package_test.ww +++ b/test/package/package_test.ww @@ -91,6 +91,45 @@ fn rewritefile(path: str, content: str) void = { assert(os.close(fd) == 0); }; +fn putbomfile(path: str, before: str, after: str, rewrite: bool) void = { + let flags: os.flag = os.flag.WRONLY; + if (rewrite) { flags |= os.flag.TRUNC; } + else { flags |= os.flag.CREATE | os.flag.EXCL; }; + let fd: i32 = os.open(path, flags, 384i32); + assert(fd >= 0); + match (os.writeall(fd, before.ptr, before.len: u64)) { + case let n: i64 => assert(n == before.len: i64); + case let e: os.oserror => abort("write before BOM failed"); + }; + let bom: [3]u8; + bom[0] = 0xefu8; bom[1] = 0xbbu8; bom[2] = 0xbfu8; + match (os.writeall(fd, bom.ptr, 3u64)) { + case let n: i64 => assert(n == 3i64); + case let e: os.oserror => abort("write BOM failed"); + }; + match (os.writeall(fd, after.ptr, after.len: u64)) { + case let n: i64 => assert(n == after.len: i64); + case let e: os.oserror => abort("write after BOM failed"); + }; + assert(os.close(fd) == 0); +}; + +fn writebomfile(path: str, content: str) void = { + putbomfile(path, "", content, false); +}; + +fn rewritebomfile(path: str, content: str) void = { + putbomfile(path, "", content, true); +}; + +fn writemidbomfile(path: str, before: str, after: str) void = { + putbomfile(path, before, after, false); +}; + +fn rewritemidbomfile(path: str, before: str, after: str) void = { + putbomfile(path, before, after, true); +}; + fn writeexecutable(path: str, content: str) void = { let fd: i32 = os.open(path, os.flag.WRONLY | os.flag.CREATE | os.flag.EXCL, 448i32); @@ -246,6 +285,16 @@ fn same(a: str, b: str) bool = { return true; }; +fn hasbom(s: str) bool = { + let i: i32 = 0; + for (i + 2 < s.len) { + if (s[i] == 0xefu8 && s[i + 1] == 0xbbu8 + && s[i + 2] == 0xbfu8) { return true; }; + i += 1; + }; + return false; +}; + fn occurrences(haystack: str, needle: str) i32 = { if (needle.len == 0 || haystack.len < needle.len) { return 0; }; let count: i32 = 0; @@ -16531,3 +16580,391 @@ fn runtimepath(relative: str) str = { assert(!os.exists("/dev/null.sepwork")); clean(root); }; + +// Go 1.26.5 ignores one UTF-8 BOM at the first codepoint of each physical +// source and rejects U+FEFF everywhere else. The lexer owns direct and +// imports-only parsing; the directory composer preserves that per-file +// boundary when it builds one synthetic unit. +@test fn utf8_bom_is_per_source_and_only_first() void = { + let root: str = fresh(); + let source: str = strings.concat(root, "/source"); + let dep: str = strings.concat(source, "/dep"); + let app: str = strings.concat(source, "/app"); + let tests: str = strings.concat(source, "/bomtests"); + let onlytests: str = strings.concat(source, "/onlytests"); + let bad: str = strings.concat(source, "/bad"); + let badtest: str = strings.concat(source, "/badtest"); + let warm: str = strings.concat(source, "/warm"); + mkdirall(dep); mkdirall(app); mkdirall(tests); mkdirall(onlytests); + mkdirall(bad); mkdirall(badtest); mkdirall(warm); + + writebomfile(strings.concat(dep, "/dep.ww"), strings.concat( + "package dep;\n", + "export fn value() i32 = { return 41; };\n")); + writebomfile(strings.concat(dep, "/more.ww"), strings.concat( + "package dep;\n", + "export fn more() i32 = { return 1; };\n")); + let appmain: str = strings.concat(app, "/main.ww"); + let appvalid: str = strings.concat( + "package main;\nimport dep;\n", + "fn main() i32 = { return dep.value() + dep.more() + local() - 43; };\n"); + writebomfile(appmain, appvalid); + writebomfile(strings.concat(app, "/local.ww"), strings.concat( + "package main;\n", + "fn local() i32 = { return 1; };\n")); + // Wrong-target sources are excluded before source decoding. + writemidbomfile(strings.concat(app, "/ignored_windows.ww"), + "not a package ", " and not valid WW\n"); + + // Direct frontend ownership: the initial marker is invisible at 1:1 and + // both stages emit the same assembly; a later marker is one codepoint error. + let direct: str = strings.concat(root, "/direct.ww"); + let directbad: str = strings.concat(root, "/direct-bad.ww"); + writebomfile(direct, + "package main;\nfn main() i32 = { return 0; };\n"); + writemidbomfile(directbad, + "package main;\nfn main() i32 = { let s: str = \"", + "\"; return s.len; };\n"); + let directouts: []str = [strings.concat(root, "/direct-c.s"), + strings.concat(root, "/direct-ww.s")]; + let directbadouts: []str = [strings.concat(root, "/direct-bad-c.s"), + strings.concat(root, "/direct-bad-ww.s")]; + let compilers: []str = ["w6c", "w6c_ww"]; + let tags: []str = ["c", "ww"]; + let middiags: []str = ["", ""]; + let i: i32 = 0; + for (i < compilers.len) { + let av: []str = [driver(compilers[i]), "-c", "--command-package", + "-o", directouts[i], direct]; + let out: commandout; + runcommand(root, strings.concat("bom-direct-valid-", tags[i]), av, + (30i64 * (time.second: i64)): time.duration, &out); + expectexit(&out, 0); + assert(out.stdout.len == 0 && out.stderr.len == 0); + let badav: []str = [driver(compilers[i]), "-c", "--command-package", + "-o", directbadouts[i], directbad]; + runcommand(root, strings.concat("bom-direct-invalid-", tags[i]), badav, + (30i64 * (time.second: i64)): time.duration, &out); + expectexit(&out, 1); + assert(out.stdout.len == 0); + assert(has(out.stderr, + ":2:33: error: invalid BOM in the middle of the file\n")); + assert(occurrences(out.stderr, + "invalid BOM in the middle of the file") == 1); + middiags[i] = strings.dup(out.stderr); + assert(!os.exists(directbadouts[i])); + i += 1; + }; + assert(same(middiags[0], middiags[1])); + assert(same(readfile(directouts[0]), readfile(directouts[1]))); + + // Build consumes multiple BOM-prefixed sources in both the root and its + // imported dependency. The marker never enters unit bytes or identity. + let stages: []str = ["ww", "ww_ww"]; + let works: []str = [strings.concat(root, "/work-c"), + strings.concat(root, "/work-ww")]; + let bins: []str = [strings.concat(root, "/app-c"), + strings.concat(root, "/app-ww")]; + i = 0; + for (i < stages.len) { + mkdirall(works[i]); + let av: []str = [driver(stages[i]), "build", "-w", works[i], + "-I", source, "-o", bins[i], "app"]; + let out: commandout; + runcommand(root, strings.concat("bom-build-", tags[i]), av, + (120i64 * (time.second: i64)): time.duration, &out); + expectexit(&out, 0); + assert(out.stdout.len == 0 && out.stderr.len == 0); + assert(!directoryhasnew(works[i]) + && !directoryhasfragment(works[i], ".wwtxn.")); + let runav: []str = [bins[i]]; + runcommand(root, strings.concat("bom-run-", tags[i]), runav, + time.second, &out); + expectexit(&out, 0); + assert(out.stdout.len == 0 && out.stderr.len == 0); + i += 1; + }; + assert(same(readfile(bins[0]), readfile(bins[1]))); + let actions: []str = ["dep", "app"]; + let suffixes: []str = [".unit.ww", ".wwi", ".s", ".o", ".a"]; + let commandsuffixes: []str = [".unit.ww", ".wwi", ".s", ".o", ".a", + ".init.unit.ww", ".init.s", ".init.o"]; + let ai: i32 = 0; + for (ai < actions.len) { + let unit: str = readfile(strings.concat(works[0], "/", actions[ai], + ".unit.ww")); + assert(!hasbom(unit) && has(unit, "\n package ")); + let si: i32 = 0; + for (si < suffixes.len) { + assert(same(readfile(strings.concat(works[0], "/", actions[ai], + suffixes[si])), readfile(strings.concat(works[1], "/", + actions[ai], suffixes[si])))); + si += 1; + }; + ai += 1; + }; + let ci: i32 = suffixes.len; + for (ci < commandsuffixes.len) { + assert(same(readfile(strings.concat(works[0], "/app", + commandsuffixes[ci])), readfile(strings.concat(works[1], "/app", + commandsuffixes[ci])))); + ci += 1; + }; + + // Production, same-package, external-package, and test-only roles all get + // their own offset-zero allowance and execute through the ordinary harness. + writebomfile(strings.concat(tests, "/prod.ww"), strings.concat( + "package bomtests;\n", + "export fn value() i32 = { return 42; };\n")); + writebomfile(strings.concat(tests, "/same_test.ww"), strings.concat( + "package bomtests;\n", + "@test fn same_bom() void = { assert(value() == 42); };\n")); + writebomfile(strings.concat(tests, "/external_test.ww"), strings.concat( + "package bomtests_test;\nimport bomtests;\n", + "@test fn external_bom() void = { assert(bomtests.value() == 42); };\n")); + writebomfile(strings.concat(onlytests, "/only_test.ww"), + "package onlytests;\n@test fn only_bom() void = { assert(true); };\n"); + let testrefs: []str = ["", ""]; + let onlyrefs: []str = ["", ""]; + let testworks: []str = [strings.concat(root, "/test-work-c"), + strings.concat(root, "/test-work-ww")]; + i = 0; + for (i < stages.len) { + mkdirall(testworks[i]); + let av: []str = [driver(stages[i]), "test", "-w", testworks[i], + "-I", source, "bomtests"]; + let out: commandout; + runcommand(root, strings.concat("bom-test-", tags[i]), av, + (120i64 * (time.second: i64)): time.duration, &out); + expectexit(&out, 0); + assert(out.stderr.len == 0 && has(out.stdout, "2 passed, 0 failed")); + testrefs[i] = strings.dup(out.stdout); + let onlyav: []str = [driver(stages[i]), "test", "-I", source, + "onlytests"]; + runcommand(root, strings.concat("bom-test-only-", tags[i]), onlyav, + (120i64 * (time.second: i64)): time.duration, &out); + expectexit(&out, 0); + assert(out.stderr.len == 0 && has(out.stdout, "1 passed, 0 failed")); + onlyrefs[i] = strings.dup(out.stdout); + i += 1; + }; + assert(same(testrefs[0], testrefs[1]) && same(onlyrefs[0], onlyrefs[1])); + let testactions: []str = ["bomtests-internal-test", + "bomtests_test-external-test", "bomtests-test-main"]; + ai = 0; + for (ai < testactions.len) { + let testsuffixes: []str = suffixes; + if (ai == 2) { testsuffixes = commandsuffixes; }; + let tsi: i32 = 0; + for (tsi < testsuffixes.len) { + assert(same(readfile(strings.concat(testworks[0], "/", + testactions[ai], testsuffixes[tsi])), + readfile(strings.concat(testworks[1], "/", testactions[ai], + testsuffixes[tsi])))); + tsi += 1; + }; + if (ai < 2) { + let testunit: str = readfile(strings.concat(testworks[0], "/", + testactions[ai], ".unit.ww")); + assert(!hasbom(testunit) && has(testunit, "\n package ")); + }; + ai += 1; + }; + assert(!directoryhasnew(testworks[0]) && !directoryhasnew(testworks[1])); + + // A later BOM is rejected during request parsing. Missing-import graph + // resolution, compiler/assembler/linker producers, runtime, and publication + // are therefore never reached; cold workdirs and products stay empty. + writemidbomfile(strings.concat(bad, "/main.ww"), + "package main;\n// ", + "\nimport missing;\nfn main() i32 = { return 0; };\n"); + writefile(strings.concat(badtest, "/prod.ww"), + "package badtest;\nfn value() i32 = { return 42; };\n"); + writemidbomfile(strings.concat(badtest, "/prod_test.ww"), + "package badtest;\n// ", + "\n@test fn must_not_run() void = { abort(\"ran\"); };\n"); + let baddiags: []str = ["", ""]; + let badtestdiags: []str = ["", ""]; + i = 0; + for (i < stages.len) { + let badwork: str = strings.concat(root, "/bad-work-", tags[i]); + let badout: str = strings.concat(root, "/bad-output-", tags[i]); + mkdirall(badwork); + let av: []str = [driver(stages[i]), "build", "-w", badwork, + "-I", source, "-o", badout, "bad"]; + let out: commandout; + runcommand(root, strings.concat("bom-bad-build-", tags[i]), av, + (60i64 * (time.second: i64)): time.duration, &out); + expectexit(&out, 1); + assert(out.stdout.len == 0); + assert(has(out.stderr, + ":2:4: error: invalid BOM in the middle of the file\n")); + assert(!has(out.stderr, "cannot find import") + && !has(out.stderr, "w6c failed")); + baddiags[i] = strings.dup(out.stderr); + assert(!os.exists(badout) && directoryisempty(badwork)); + let testav: []str = [driver(stages[i]), "test", "-I", source, + "badtest"]; + runcommand(root, strings.concat("bom-bad-test-", tags[i]), testav, + (60i64 * (time.second: i64)): time.duration, &out); + expectexit(&out, 1); + assert(same(out.stdout, "FAIL\n")); + assert(has(out.stderr, "invalid BOM in the middle of the file") + && !has(out.stderr, "must_not_run ...")); + badtestdiags[i] = strings.dup(out.stderr); + i += 1; + }; + assert(same(baddiags[0], baddiags[1]) + && same(badtestdiags[0], badtestdiags[1])); + + // A warm invalid edit cannot mutate committed action bytes or publication. + // Restoring the exact leading-BOM source composes the same unit and reuses + // the generation without invoking the compiler. + let warmpath: str = strings.concat(warm, "/main.ww"); + let warmvalid: str = + "package main;\nfn main() i32 = { return 0; };\n"; + writebomfile(warmpath, warmvalid); + let wrapper: str = strings.concat(root, "/compiler-wrapper.sh"); + writeexecutable(wrapper, strings.concat( + "#!/bin/sh\n", + "printf 'compile\\n' >> \"$WW_BOM_TRACE\"\n", + "exec \"$WW_BOM_REAL\" \"$@\"\n")); + i = 0; + for (i < stages.len) { + let warmwork: str = strings.concat(root, "/warm-work-", tags[i]); + let warmout: str = strings.concat(root, "/warm-output-", tags[i]); + let trace: str = strings.concat(root, "/warm-trace-", tags[i]); + mkdirall(warmwork); writefile(trace, ""); + let env: []str = os.getenvs(); + append(env, strings.concat("WW_W6C=", wrapper)); + append(env, strings.concat("WW_BOM_TRACE=", trace)); + append(env, strings.concat("WW_BOM_REAL=", driver(compilers[i]))); + let av: []str = [driver(stages[i]), "build", "-w", warmwork, + "-I", source, "-o", warmout, "warm"]; + let out: commandout; + runcommandenv(root, strings.concat("bom-warm-cold-", tags[i]), av, + env, (120i64 * (time.second: i64)): time.duration, &out); + expectexit(&out, 0); + assert(out.stdout.len == 0 && out.stderr.len == 0 + && readfile(trace).len != 0); + let refs: []str = alloc([], commandsuffixes.len: u64)!; + let si: i32 = 0; + for (si < commandsuffixes.len) { + append(refs, strings.dup(readfile(strings.concat(warmwork, + "/warm", commandsuffixes[si])))); + si += 1; + }; + let binref: str = strings.dup(readfile(warmout)); + rewritefile(trace, ""); + rewritemidbomfile(warmpath, "package main;\n// ", + "\nfn main() i32 = { return 0; };\n"); + runcommandenv(root, strings.concat("bom-warm-invalid-", tags[i]), av, + env, (60i64 * (time.second: i64)): time.duration, &out); + expectexit(&out, 1); + assert(out.stdout.len == 0 + && has(out.stderr, "invalid BOM in the middle of the file")); + assert(readfile(trace).len == 0 && same(binref, readfile(warmout))); + si = 0; + for (si < commandsuffixes.len) { + assert(same(refs[si], readfile(strings.concat(warmwork, + "/warm", commandsuffixes[si])))); + si += 1; + }; + assert(!directoryhasnew(warmwork) + && !directoryhasfragment(warmwork, ".wwtxn.")); + rewritebomfile(warmpath, warmvalid); + runcommandenv(root, strings.concat("bom-warm-restored-", tags[i]), av, + env, (120i64 * (time.second: i64)): time.duration, &out); + expectexit(&out, 0); + assert(out.stdout.len == 0 && out.stderr.len == 0); + assert(readfile(trace).len == 0 && same(binref, readfile(warmout)) + && !directoryhasnew(warmwork)); + + // The marker is semantically whitespace but its three source bytes are + // observable input: removing and re-adding it invalidates the unit just + // as Go's source-content action does, without changing output bytes. + rewritefile(trace, ""); + rewritefile(warmpath, warmvalid); + runcommandenv(root, strings.concat("bom-warm-marker-removed-", tags[i]), + av, env, (120i64 * (time.second: i64)): time.duration, &out); + expectexit(&out, 0); + assert(out.stdout.len == 0 && out.stderr.len == 0); + assert(readfile(trace).len != 0 && same(binref, readfile(warmout))); + assert(!same(refs[0], readfile(strings.concat(warmwork, + "/warm.unit.ww")))); + rewritefile(trace, ""); + rewritebomfile(warmpath, warmvalid); + runcommandenv(root, strings.concat("bom-warm-marker-restored-", tags[i]), + av, env, (120i64 * (time.second: i64)): time.duration, &out); + expectexit(&out, 0); + assert(out.stdout.len == 0 && out.stderr.len == 0); + assert(readfile(trace).len != 0 && same(binref, readfile(warmout))); + si = 0; + for (si < commandsuffixes.len) { + assert(same(refs[si], readfile(strings.concat(warmwork, + "/warm", commandsuffixes[si])))); + si += 1; + }; + assert(!directoryhasnew(warmwork)); + i += 1; + }; + + // Request-local source state: a valid Cstage build and an invalid WWstage + // build overlap without sharing diagnostics, products, or transaction data. + let pcwork: str = strings.concat(root, "/parallel-c-work"); + let pwwork: str = strings.concat(root, "/parallel-ww-work"); + let pcout: str = strings.concat(root, "/parallel-c-output"); + let pwout: str = strings.concat(root, "/parallel-ww-output"); + mkdirall(pcwork); mkdirall(pwwork); + let pcav: []str = [driver("ww"), "build", "-w", pcwork, + "-I", source, "-o", pcout, "app"]; + let pwav: []str = [driver("ww_ww"), "build", "-w", pwwork, + "-I", source, "-o", pwout, "bad"]; + let pc: exec.command; + pc.path = pcav[0]; pc.argv = pcav; pc.env = os.getenvs(); pc.dir = repo(); + pc.stdoutpath = strings.concat(root, "/parallel-c.stdout"); + pc.stderrpath = strings.concat(root, "/parallel-c.stderr"); + pc.deadline = time.add(time.now(time.clock.monotonic), + (120i64 * (time.second: i64)): time.duration); + pc.grace = (100i64 * (time.millisecond: i64)): time.duration; + let pw: exec.command; + pw.path = pwav[0]; pw.argv = pwav; pw.env = os.getenvs(); pw.dir = repo(); + pw.stdoutpath = strings.concat(root, "/parallel-ww.stdout"); + pw.stderrpath = strings.concat(root, "/parallel-ww.stderr"); + pw.deadline = time.add(time.now(time.clock.monotonic), + (120i64 * (time.second: i64)): time.duration); + pw.grace = (100i64 * (time.millisecond: i64)): time.duration; + let pcp: exec.process; + let pwp: exec.process; + exec.start(&pcp, &pc); exec.start(&pwp, &pw); + let pcdone: bool = false; + let pwdone: bool = false; + for (!pcdone || !pwdone) { + if (!pcdone) { pcdone = exec.poll(&pcp); }; + if (!pwdone) { pwdone = exec.poll(&pwp); }; + if (!pcdone || !pwdone) { + time.sleep(time.millisecond, time.clock.monotonic); + }; + }; + assert(pcp.result.errno == 0 && pcp.result.cleanuperrno == 0 + && pcp.result.termination == exec.termination.EXIT + && pcp.result.code == 0); + assert(pwp.result.errno == 0 && pwp.result.cleanuperrno == 0 + && pwp.result.termination == exec.termination.EXIT + && pwp.result.code == 1); + assert(readfile(pc.stdoutpath).len == 0 && readfile(pc.stderrpath).len == 0); + assert(readfile(pw.stdoutpath).len == 0 + && has(readfile(pw.stderrpath), "invalid BOM in the middle of the file")); + assert(same(readfile(pcout), readfile(bins[0])) && !os.exists(pwout)); + assert(directoryisempty(pwwork) && !directoryhasnew(pcwork) + && !directoryhasfragment(pcwork, ".wwtxn.")); + let runav: []str = [pcout]; + let out: commandout; + runcommand(root, "bom-parallel-run", runav, time.second, &out); + expectexit(&out, 0); + assert(out.stdout.len == 0 && out.stderr.len == 0); + assert(!directoryhasfragment(root, ".wwtxn.") + && !directoryhasfragment(root, ".install") + && !directoryhasnew(root)); + clean(root); +}; diff --git a/test/wcc/100_lex.c b/test/wcc/100_lex.c index e549d666..969b6444 100644 --- a/test/wcc/100_lex.c +++ b/test/wcc/100_lex.c @@ -61,6 +61,7 @@ struct row { const char *src, *expect; }; static const struct row rows[] = { { "", "" }, + { "\xef\xbb\xbf" "package main;", "package IDENT(main) ;" }, { " \t\n ", "" }, { "// comment\n", "" }, { "/* a /b/ c */", "" }, @@ -172,7 +173,29 @@ runrewind(void) return ok; } +static int +runbompos(void) +{ + const char *src = "\xef\xbb\xbf" "package main;"; + Arena *a = newarena(); + Lex l; + lexinit(&l, a, "", src, strlen(src)); + Tok t = lexnext(&l); + int ok = t.kind == TK_MODULE && t.pos.line == 1 && t.pos.col == 4; + if (!ok) + fprintf(stderr, "leading BOM position: want 1:4 got %d:%d\n", + t.pos.line, t.pos.col); + freearena(a); + return ok; +} + static const char *const errrows[] = { + "\xef\xbb\xbf" "\xef\xbb\xbf" "package main;", + "package \xef\xbb\xbf" "main;", + "// \xef\xbb\xbf\n", + "/* \xef\xbb\xbf */", + "\"\xef\xbb\xbf\"", + "'\xef\xbb\xbf'", "'\\uZ'", /* unexpected rune scanning for escape */ "\"\\u00g0\"", /* non-hex digit inside a string escape */ "'\\u00", /* unexpected EOF scanning for escape */ @@ -194,6 +217,8 @@ main(void) } if (!runrewind()) fail++; + if (!runbompos()) + fail++; for (size_t i = 0; i < sizeof errrows / sizeof errrows[0]; i++) { if (!runerr(errrows[i])) { fprintf(stderr, "errrow %zu failed\n", i);