ww source: align UTF-8 BOM placement

cmd/wcc/lex.c

@@ -7,6 +7,15 @@
 #include <string.h>
 #include <errno.h>
 
+static int
+bomat(const char *src, u64 len, u64 pos)
+{
+	return pos <= len && len - pos >= 3
+	    && (unsigned char)src[pos] == 0xef
+	    && (unsigned char)src[pos + 1] == 0xbb
+	    && (unsigned char)src[pos + 2] == 0xbf;
+}
+
 void
 lexinit(Lex *l, Arena *a, const char *file, const char *src, u64 len)
 {
@@ -17,6 +26,12 @@ lexinit(Lex *l, Arena *a, const char *file, const char *src, u64 len)
 	l->line = 1;
 	l->col = 1;
 	l->a = a;
+	/* Go 1.26.5 syntax.source.nextch ignores U+FEFF only at the first
+	 * source position but counts its three bytes for the next column. */
+	if (bomat(src, len, 0)) {
+		l->pos = 3;
+		l->col = 4;
+	}
 }
 
 static int
@@ -25,6 +40,8 @@ lpeek(Lex *l, u64 ahead)
 	u64 p = l->pos + ahead;
 	if (p >= l->srclen)
 		return -1;
+	if (bomat(l->src, l->srclen, p))
+		return 0xfeff;
 	return (unsigned char)l->src[p];
 }
 
@@ -33,6 +50,14 @@ lget(Lex *l)
 {
 	if (l->pos >= l->srclen)
 		return -1;
+	if (bomat(l->src, l->srclen, l->pos)) {
+		Pos p = { l->file, l->line, l->col };
+		l->pos += 3;
+		l->col += 3;
+		errorf(p, "invalid BOM in the middle of the file");
+		l->errs++;
+		return 0xfeff;
+	}
 	int c = (unsigned char)l->src[l->pos++];
 	if (c == '\n') {
 		l->line++;
@@ -529,6 +554,11 @@ lexnext(Lex *l)
 		return t;
 	}
 	int c = lpeek(l, 0);
+	if (c == 0xfeff) {
+		lget(l);
+		Tok t = (Tok){ TK_ERR, start, "", 0, {0}, TK_NONE };
+		return t;
+	}
 
 	if (isidstart(c))
 		return lexident(l, start);

cmd/ww/main.c

@@ -4740,7 +4740,18 @@ sep_emit_body(FILE *out, const char *path, const char *modpath)
 	} else if (fputs("//ww:module-reset\n", out) == EOF) {
 		bad = 1;
 	}
-	if (fwrite(buf, 1, (size_t)len, out) != (size_t)len
+	/* Each physical source owns its own first-codepoint position. The
+	 * compiler sees one synthetic aggregate, so replace the optional UTF-8
+	 * BOM with position-preserving whitespace rather than turning a later
+	 * source's marker into a mid-unit BOM. */
+	u64 off = len >= 3
+	    && (unsigned char)buf[0] == 0xef
+	    && (unsigned char)buf[1] == 0xbb
+	    && (unsigned char)buf[2] == 0xbf ? 3 : 0;
+	if (off != 0 && fwrite("   ", 1, 3, out) != 3)
+		bad = 1;
+	if (fwrite(buf + off, 1, (size_t)(len - off), out)
+	    != (size_t)(len - off)
 	    || fputc('\n', out) == EOF)
 		bad = 1;
 	free(buf);

docs/build-system.md

@@ -8427,6 +8427,124 @@ Rejected source creates no persisted byte contract, while valid source bytes
 are unchanged.  Build workdir format remains `18`, test workdir format remains
 `19`, and semantic storage format remains `3`.
 
+### 11.42 Implemented per-source UTF-8 BOM placement
+
+Every eligible physical WW source may begin with one UTF-8-encoded U+FEFF byte
+order mark (`EF BB BF`).  That marker is ignored, and the following token keeps
+its three-byte source position at line 1, column 4.  U+FEFF at any later raw
+source position is
+rejected once as `invalid BOM in the middle of the file`, including inside a
+line/block comment, string, or rune.  Two leading markers therefore ignore the
+first and reject the second.  A truncated marker or another invalid UTF-8 byte
+sequence retains the ordinary byte-error path; UTF-16 source is not introduced.
+
+#### Pinned Go evidence and applicability
+
+The sole authority is official Go 1.26.5 at commit
+`c19862e5f8415b4f24b189d065ed739517c548ba`:
+
+- compiler reader `(*source).nextch` decodes UTF-8, skips U+FEFF at its first
+  source position, and reports `invalid BOM in the middle of the file`
+  elsewhere
+  ([`cmd/compile/internal/syntax/source.go`, lines 113–165](https://github.com/golang/go/blob/c19862e5f8415b4f24b189d065ed739517c548ba/src/cmd/compile/internal/syntax/source.go#L113-L165));
+- public scanner `(*Scanner).next` rejects later U+FEFF and `(*Scanner).Init`
+  consumes the first one
+  ([`go/scanner/scanner.go`, lines 58–100 and 128–164](https://github.com/golang/go/blob/c19862e5f8415b4f24b189d065ed739517c548ba/src/go/scanner/scanner.go#L58-L164));
+- scanner tests assert the ignored first marker and later markers between
+  tokens, in comments, runes, and strings
+  ([`go/scanner/scanner_test.go`, lines 371–374 and 812–815](https://github.com/golang/go/blob/c19862e5f8415b4f24b189d065ed739517c548ba/src/go/scanner/scanner_test.go#L371-L374),
+  [`go/scanner/scanner_test.go`, lines 812–815](https://github.com/golang/go/blob/c19862e5f8415b4f24b189d065ed739517c548ba/src/go/scanner/scanner_test.go#L812-L815));
+- compiler-scanner `TestScanErrors` asserts the positioned later-marker error
+  ([`cmd/compile/internal/syntax/scanner_test.go`, lines 587–599](https://github.com/golang/go/blob/c19862e5f8415b4f24b189d065ed739517c548ba/src/cmd/compile/internal/syntax/scanner_test.go#L587-L599)); and
+- official command testdata places the marker before `package main` and loads
+  that source's imports and embedded file
+  ([`cmd/go/testdata/script/build_ignore_leading_bom.txt`, lines 1–25](https://github.com/golang/go/blob/c19862e5f8415b4f24b189d065ed739517c548ba/src/cmd/go/testdata/script/build_ignore_leading_bom.txt#L1-L25)).
+
+Those source branches and assertions are **behavior directly implemented or
+asserted by pinned Go**.  Applying the first position independently to every WW
+physical source before its existing synthetic aggregate boundary is **behavior
+derived from the pinned implementation**.  The pre-fix Cstage/WWstage failures
+on leading markers and successes for markers in comments/strings were
+**directly measured WW behavior**.
+
+The behavior honestly applies inside WW's local, dotted-import, manifest-free
+model: it is source representation before package and import interpretation.
+It requires no module, manifest, registry, lock, cache, database, CAS, network
+resolution, generalized import syntax, build expression, or normalized
+identity.
+
+#### Ownership and four-axis result
+
+`lexinit`, `lpeek`, `lget`, and `lexnext` in `cmd/wcc/lex.c`, with their twins
+in `lib/ww/syntax/lex.ww`, are the language owners.  They skip the exact initial
+marker while advancing the logical column by its encoded width, recognize a
+later marker as one code point in every lexical context, and emit one
+stage-identical diagnostic.
+`sep_emit_body` in `cmd/ww/main.c` and `sepemitbody` in
+`selfhost/cmd/ww/main.ww` replace the optional marker with three spaces in every
+physical body placed after a synthetic `//ww:module-reset`, preserving columns;
+direct and imports-only lexer input remains independently correct.  The shared package
+coordinator's `pkgclause` in `internal/wwpackage/package.ww` begins its
+pre-discovery package-name scan after the same optional marker.  It does not
+replace the complete stage-driver scan.
+
+- **Go-like build:** a selected command, library, or imported dependency may
+  use the marker in each eligible source.  Multiple physical marked sources
+  compose normally, command publication succeeds, and the executable runs
+  normally.  A later marker rejects during source loading before graph/action
+  construction or compiler, assembler, archiver, linker, install, or runtime
+  work.  A source error keeps precedence over a missing import.
+- **Go-like test:** production, same-package, external-package, and test-only
+  files each receive the offset-zero allowance.  Valid variants build and run
+  through the ordinary single directory product.  A later marker yields the
+  existing attributable `FAIL\n` result without a variant, generated main,
+  test process, accounting, or retained binary.
+- **Go-like package:** package-clause recognition now begins at the pinned
+  logical source start in direct compilers, directory drivers, and the shared
+  coordinator.  Declared names, source roles, package conflicts, command/test
+  family selection, and canonical identity do not change.
+- **Go-like import:** imports following a legal marker and imports in a marked
+  dependency retain their exact source spelling, file scope, qualifier,
+  contextual local/vendor resolution, case-sensitive canonical dotted
+  identity, visibility, cycle, and initialization behavior.  The marker never
+  becomes an edge or identity component.
+
+Fixed-target filename selection remains earlier than parsing: an excluded
+`_windows.ww` or `_windows_test.ww` contributes no marker diagnostic, package,
+import, graph, action, artifact, or invalidation state.  For valid inputs, graph
+nodes, scheduling, producer arguments, initialization, runtime, result order,
+and publication are unchanged.  Legal marker bytes become three
+position-preserving spaces only in the synthetic unit.  Adding or removing the
+marker therefore changes unit content and invalidates source-derived actions,
+while semantic interface, assembly, object, archive, initializer, and
+executable content remains the same; every form remains Cstage/WWstage
+byte-identical.
+
+Cold rejection leaves no product, work generation, adjacent scratch, capture,
+`.new`, `.install`, or `.wwtxn.*`.  A warm later-marker edit preserves the
+entire committed generation, tool vouchers, stamp, and public executable.
+Restoring the exact legal leading form recreates the same unit and reuses committed
+compiler/assembler/archive work before the normal link/publication boundary.
+Concurrent valid and invalid requests keep independent lexer/coordinator state,
+workdirs, captures, diagnostics, and products.  Because later-marker rejection
+occurs before a producer or test child, producer/runtime failure, signals,
+timeouts, interruption, and process-group cleanup gain no new branch; their
+existing owners remain authoritative.
+
+The C lexer unit, WW syntax unit, and WW-native
+`utf8_bom_is_per_source_and_only_first` observer prove initial position,
+later-marker lexical contexts, direct frontend diagnostics and assembly,
+selected/imported multi-source build, wrong-target exclusion, every test source
+role, test-only execution, syntax-before-resolution precedence, cold cleanup,
+warm rollback and reuse, concurrent isolation, publication/runtime behavior,
+and complete stage diagnostic/artifact parity.
+
+No persisted format changes.  Previously valid marker-free bytes are
+unchanged; previously leading-marked requests could not commit a generation;
+and diagnostic text is not a persisted-byte contract.  Build workdir format
+remains `18`, test workdir format remains `19`, and semantic storage format
+remains `3`.
+
 ## 12. Candidate architectures and hard-gate decision
 
 Five candidates were developed as coherent systems, not as feature bins.

docs/spec.md

@@ -52,8 +52,13 @@ classes from §2 (`ident`, `int_lit`, …).
 
 ### 2.1 Source representation
 
-Source is UTF-8. The lexer operates on bytes; non-ASCII bytes are legal
+Source is UTF-8. One UTF-8-encoded byte order mark (U+FEFF, bytes
-only inside string and rune literals and comments.
+`EF BB BF`) is ignored when it is the first code point of a physical source
+file. Its three encoded bytes still count in source positions, so a following
+token on the first line begins at column 4. U+FEFF is invalid at
+every other source position, including inside string and rune literals and
+comments. Apart from that marker rule, the lexer operates on bytes and
+non-ASCII bytes are legal only inside string and rune literals and comments.
 
 ### 2.2 Comments
 

docs/test-system-v2.md

@@ -415,6 +415,20 @@ generation. The directory command retains its existing attributable final
 diagnostic. This ordering state is per source and never package, import,
 variant, action, artifact, publication, or persistence identity.
 
+Every eligible source also owns an independent source-start position. An exact
+UTF-8 BOM (`EF BB BF`) at byte offset zero is ignored before package-clause and
+import parsing, so production, same-package, external-package, and test-only
+sources all count the marker's three bytes and begin the following token at
+logical line 1 column 4. The same code point anywhere else,
+including a comment or literal, is a source error before variant actions,
+generated main, producers, test runtime, accounting, or retained publication.
+The shared coordinator applies the offset-zero rule while classifying package
+clauses; the selected stage driver scans the complete source and preserves the
+per-file rule while composing package units. Wrong-target files remain excluded
+before either operation. The marker is source representation only and never
+package/import/action/artifact/publication/persistence identity; no test result
+is cached.
+
 After that eligibility boundary and the coordinator's required package-clause
 classification and production `@test` validation parses, the delegated loader
 performs selected-basename Go 1.26.5 simple-fold preflight before its graph

internal/wwpackage/package.ww

@@ -724,7 +724,12 @@ fn pkgskipspace(src: str, start: i32) i32 = {
 };
 
 fn pkgclause(src: str, out: *str) bool = {
-	let i: i32 = pkgskipspace(src, 0);
+	// Match the compiler readers: one UTF-8 BOM is invisible only at the
+	// first raw source position. The full stage driver owns later-BOM errors.
+	let start: i32 = 0;
+	if (src.len >= 3 && src[0] == 0xefu8 && src[1] == 0xbbu8
+	    && src[2] == 0xbfu8) { start = 3; };
+	let i: i32 = pkgskipspace(src, start);
 	let word: str = "package";
 	if (i + word.len >= src.len) { return false; };
 	let j: i32 = 0;

lib/ww/syntax/lex.ww

@@ -65,6 +65,14 @@ export type lex = struct {
 	modresetpath: str,
 };
 
+fn bomat(src: *u8, len: u64, off: u64) bool = {
+	if (off > len || len - off < 3u64) { return false; };
+	let i: i32 = off: i32;
+	return src[i] == 0xefu8
+		&& src[i + 1] == 0xbbu8
+		&& src[i + 2] == 0xbfu8;
+};
+
 export fn lexinit(l: *lex, file: str, src: *u8, len: u64) void = {
 	l.file = file;
 	l.src = src;
@@ -76,6 +84,9 @@ export fn lexinit(l: *lex, file: str, src: *u8, len: u64) void = {
 	l.modreset = 0;
 	l.modpathset = 0;
 	l.modresetpathset = 0;
+	// Go 1.26.5 syntax.source.nextch ignores U+FEFF only at the first
+	// source position but counts its three bytes for the next column.
+	if (bomat(src, len, 0u64)) { l.lpos = 3u64; l.col = 4; };
 };
 
 fn srcb(l: *lex, off: u64) i32 = {
@@ -87,11 +98,22 @@ fn srcb(l: *lex, off: u64) i32 = {
 fn lpeek(l: *lex, ahead: u64) i32 = {
 	let p: u64 = l.lpos + ahead;
 	if (p >= l.srclen) { return -1; };
+	if (bomat(l.src, l.srclen, p)) { return 0xFEFF; };
 	return srcb(l, p);
 };
 
 fn lget(l: *lex) i32 = {
 	if (l.lpos >= l.srclen) { return -1; };
+	if (bomat(l.src, l.srclen, l.lpos)) {
+		let bp: pos;
+		bp.file = l.file;
+		bp.line = l.line;
+		bp.col = l.col;
+		l.lpos += 3u64;
+		l.col += 3;
+		errat(l, &bp, "invalid BOM in the middle of the file");
+		return 0xFEFF;
+	};
 	let c: i32 = srcb(l, l.lpos);
 	l.lpos += 1u64;
 	if (c == '\n') {
@@ -801,6 +823,11 @@ export fn lexnext(l: *lex, out: *tok) void = {
 		return;
 	};
 	let c: i32 = lpeek(l, 0u64);
+	if (c == 0xFEFF) {
+		lget(l);
+		emitsimple(&start, tkind.TK_ERR, out);
+		return;
+	};
 
 	if (c >= 0) {
 		if (isidstart(c: rune)) { lexident(l, &start, out); return; };

lib/ww/syntax/tok_test.ww

@@ -338,3 +338,78 @@ fn checkfloat(src: str, want: u64) void = {
 	assert(!(t.col != 6));
 	assert(!(t.text != "bar"));
 };
+
+fn bomsource(dst: *u8, before: str, after: str) u64 = {
+	let n: u64 = 0u64;
+	let i: i32 = 0;
+	for (i < before.len) {
+		let ni: i32 = n: i32;
+		dst[ni] = before[i];
+		n += 1u64;
+		i += 1;
+	};
+	let bi: i32 = n: i32;
+	dst[bi] = 0xefu8;
+	dst[bi + 1] = 0xbbu8;
+	dst[bi + 2] = 0xbfu8;
+	n += 3u64;
+	i = 0;
+	for (i < after.len) {
+		let ni: i32 = n: i32;
+		dst[ni] = after[i];
+		n += 1u64;
+		i += 1;
+	};
+	return n;
+};
+
+fn bomerror(before: str, after: str) bool = {
+	let src: [128]u8;
+	let n: u64 = bomsource(src.ptr, before, after);
+	let l: syntax.lex;
+	syntax.lexinit(&l, "t", src.ptr, n);
+	let t: syntax.tok;
+	for (true) {
+		syntax.lexnext(&l, &t);
+		if (t.kind == syntax.tkind.TK_EOF) { break; };
+	};
+	return l.errs > 0;
+};
+
+fn doublebomerror() bool = {
+	let src: [6]u8;
+	let i: i32 = 0;
+	for (i < 6) {
+		if (i % 3 == 0) { src[i] = 0xefu8; };
+		if (i % 3 == 1) { src[i] = 0xbbu8; };
+		if (i % 3 == 2) { src[i] = 0xbfu8; };
+		i += 1;
+	};
+	let l: syntax.lex;
+	syntax.lexinit(&l, "t", src.ptr, 6u64);
+	let t: syntax.tok;
+	for (true) {
+		syntax.lexnext(&l, &t);
+		if (t.kind == syntax.tkind.TK_EOF) { break; };
+	};
+	return l.errs > 0;
+};
+
+@test fn utf8_bom_is_ignored_only_at_source_start() void = {
+	let src: [128]u8;
+	let n: u64 = bomsource(src.ptr, "", "package main;");
+	let l: syntax.lex;
+	syntax.lexinit(&l, "t", src.ptr, n);
+	let t: syntax.tok;
+	syntax.lexnext(&l, &t);
+	assert(t.kind == syntax.tkind.TK_MODULE);
+	assert(t.line == 1 && t.col == 4);
+	assert(l.errs == 0);
+
+	assert(doublebomerror());
+	assert(bomerror("package ", "main;"));
+	assert(bomerror("// ", "\n"));
+	assert(bomerror("/* ", " */"));
+	assert(bomerror("\"", "\""));
+	assert(bomerror("'", "'"));
+};

selfhost/cmd/ww/main.ww

@@ -5594,7 +5594,19 @@ fn sepemitbody(fd: i32, path: *u8, modpath: *u8) i32 = {
 			return -1;
 		};
 	};
-	if (!sepwriteall(fd, bufp, blen)
+	// Each physical source owns its own first-codepoint position. The compiler
+	// sees one synthetic aggregate, so replace the optional UTF-8 BOM with
+	// position-preserving whitespace rather than making it a mid-unit BOM.
+	let off: u64 = 0u64;
+	if (blen >= 3u64 && bufp[0] == 0xefu8
+	    && bufp[1] == 0xbbu8 && bufp[2] == 0xbfu8) {
+		off = 3u64;
+	};
+	if (off != 0u64 && !sepwriteall(fd, "   ".ptr, 3u64)) {
+		cerr("ww: cannot write package unit\n");
+		return -1;
+	};
+	if (!sepwriteall(fd, bufp + off, blen - off)
 	    || !sepwriteall(fd, "\n".ptr, 1u64)) {
 		cerr("ww: cannot write package unit\n");
 		return -1;

test/package/package_test.ww

@@ -91,6 +91,45 @@ fn rewritefile(path: str, content: str) void = {
 	assert(os.close(fd) == 0);
 };
 
+fn putbomfile(path: str, before: str, after: str, rewrite: bool) void = {
+	let flags: os.flag = os.flag.WRONLY;
+	if (rewrite) { flags |= os.flag.TRUNC; }
+	else { flags |= os.flag.CREATE | os.flag.EXCL; };
+	let fd: i32 = os.open(path, flags, 384i32);
+	assert(fd >= 0);
+	match (os.writeall(fd, before.ptr, before.len: u64)) {
+	case let n: i64 => assert(n == before.len: i64);
+	case let e: os.oserror => abort("write before BOM failed");
+	};
+	let bom: [3]u8;
+	bom[0] = 0xefu8; bom[1] = 0xbbu8; bom[2] = 0xbfu8;
+	match (os.writeall(fd, bom.ptr, 3u64)) {
+	case let n: i64 => assert(n == 3i64);
+	case let e: os.oserror => abort("write BOM failed");
+	};
+	match (os.writeall(fd, after.ptr, after.len: u64)) {
+	case let n: i64 => assert(n == after.len: i64);
+	case let e: os.oserror => abort("write after BOM failed");
+	};
+	assert(os.close(fd) == 0);
+};
+
+fn writebomfile(path: str, content: str) void = {
+	putbomfile(path, "", content, false);
+};
+
+fn rewritebomfile(path: str, content: str) void = {
+	putbomfile(path, "", content, true);
+};
+
+fn writemidbomfile(path: str, before: str, after: str) void = {
+	putbomfile(path, before, after, false);
+};
+
+fn rewritemidbomfile(path: str, before: str, after: str) void = {
+	putbomfile(path, before, after, true);
+};
+
 fn writeexecutable(path: str, content: str) void = {
 	let fd: i32 = os.open(path,
 		os.flag.WRONLY | os.flag.CREATE | os.flag.EXCL, 448i32);
@@ -246,6 +285,16 @@ fn same(a: str, b: str) bool = {
 	return true;
 };
 
+fn hasbom(s: str) bool = {
+	let i: i32 = 0;
+	for (i + 2 < s.len) {
+		if (s[i] == 0xefu8 && s[i + 1] == 0xbbu8
+		    && s[i + 2] == 0xbfu8) { return true; };
+		i += 1;
+	};
+	return false;
+};
+
 fn occurrences(haystack: str, needle: str) i32 = {
 	if (needle.len == 0 || haystack.len < needle.len) { return 0; };
 	let count: i32 = 0;
@@ -16531,3 +16580,391 @@ fn runtimepath(relative: str) str = {
 	assert(!os.exists("/dev/null.sepwork"));
 	clean(root);
 };
+
+// Go 1.26.5 ignores one UTF-8 BOM at the first codepoint of each physical
+// source and rejects U+FEFF everywhere else. The lexer owns direct and
+// imports-only parsing; the directory composer preserves that per-file
+// boundary when it builds one synthetic unit.
+@test fn utf8_bom_is_per_source_and_only_first() void = {
+	let root: str = fresh();
+	let source: str = strings.concat(root, "/source");
+	let dep: str = strings.concat(source, "/dep");
+	let app: str = strings.concat(source, "/app");
+	let tests: str = strings.concat(source, "/bomtests");
+	let onlytests: str = strings.concat(source, "/onlytests");
+	let bad: str = strings.concat(source, "/bad");
+	let badtest: str = strings.concat(source, "/badtest");
+	let warm: str = strings.concat(source, "/warm");
+	mkdirall(dep); mkdirall(app); mkdirall(tests); mkdirall(onlytests);
+	mkdirall(bad); mkdirall(badtest); mkdirall(warm);
+
+	writebomfile(strings.concat(dep, "/dep.ww"), strings.concat(
+		"package dep;\n",
+		"export fn value() i32 = { return 41; };\n"));
+	writebomfile(strings.concat(dep, "/more.ww"), strings.concat(
+		"package dep;\n",
+		"export fn more() i32 = { return 1; };\n"));
+	let appmain: str = strings.concat(app, "/main.ww");
+	let appvalid: str = strings.concat(
+		"package main;\nimport dep;\n",
+		"fn main() i32 = { return dep.value() + dep.more() + local() - 43; };\n");
+	writebomfile(appmain, appvalid);
+	writebomfile(strings.concat(app, "/local.ww"), strings.concat(
+		"package main;\n",
+		"fn local() i32 = { return 1; };\n"));
+	// Wrong-target sources are excluded before source decoding.
+	writemidbomfile(strings.concat(app, "/ignored_windows.ww"),
+		"not a package ", " and not valid WW\n");
+
+	// Direct frontend ownership: the initial marker is invisible at 1:1 and
+	// both stages emit the same assembly; a later marker is one codepoint error.
+	let direct: str = strings.concat(root, "/direct.ww");
+	let directbad: str = strings.concat(root, "/direct-bad.ww");
+	writebomfile(direct,
+		"package main;\nfn main() i32 = { return 0; };\n");
+	writemidbomfile(directbad,
+		"package main;\nfn main() i32 = { let s: str = \"",
+		"\"; return s.len; };\n");
+	let directouts: []str = [strings.concat(root, "/direct-c.s"),
+		strings.concat(root, "/direct-ww.s")];
+	let directbadouts: []str = [strings.concat(root, "/direct-bad-c.s"),
+		strings.concat(root, "/direct-bad-ww.s")];
+	let compilers: []str = ["w6c", "w6c_ww"];
+	let tags: []str = ["c", "ww"];
+	let middiags: []str = ["", ""];
+	let i: i32 = 0;
+	for (i < compilers.len) {
+		let av: []str = [driver(compilers[i]), "-c", "--command-package",
+			"-o", directouts[i], direct];
+		let out: commandout;
+		runcommand(root, strings.concat("bom-direct-valid-", tags[i]), av,
+			(30i64 * (time.second: i64)): time.duration, &out);
+		expectexit(&out, 0);
+		assert(out.stdout.len == 0 && out.stderr.len == 0);
+		let badav: []str = [driver(compilers[i]), "-c", "--command-package",
+			"-o", directbadouts[i], directbad];
+		runcommand(root, strings.concat("bom-direct-invalid-", tags[i]), badav,
+			(30i64 * (time.second: i64)): time.duration, &out);
+		expectexit(&out, 1);
+		assert(out.stdout.len == 0);
+		assert(has(out.stderr,
+			":2:33: error: invalid BOM in the middle of the file\n"));
+		assert(occurrences(out.stderr,
+			"invalid BOM in the middle of the file") == 1);
+		middiags[i] = strings.dup(out.stderr);
+		assert(!os.exists(directbadouts[i]));
+		i += 1;
+	};
+	assert(same(middiags[0], middiags[1]));
+	assert(same(readfile(directouts[0]), readfile(directouts[1])));
+
+	// Build consumes multiple BOM-prefixed sources in both the root and its
+	// imported dependency. The marker never enters unit bytes or identity.
+	let stages: []str = ["ww", "ww_ww"];
+	let works: []str = [strings.concat(root, "/work-c"),
+		strings.concat(root, "/work-ww")];
+	let bins: []str = [strings.concat(root, "/app-c"),
+		strings.concat(root, "/app-ww")];
+	i = 0;
+	for (i < stages.len) {
+		mkdirall(works[i]);
+		let av: []str = [driver(stages[i]), "build", "-w", works[i],
+			"-I", source, "-o", bins[i], "app"];
+		let out: commandout;
+		runcommand(root, strings.concat("bom-build-", tags[i]), av,
+			(120i64 * (time.second: i64)): time.duration, &out);
+		expectexit(&out, 0);
+		assert(out.stdout.len == 0 && out.stderr.len == 0);
+		assert(!directoryhasnew(works[i])
+			&& !directoryhasfragment(works[i], ".wwtxn."));
+		let runav: []str = [bins[i]];
+		runcommand(root, strings.concat("bom-run-", tags[i]), runav,
+			time.second, &out);
+		expectexit(&out, 0);
+		assert(out.stdout.len == 0 && out.stderr.len == 0);
+		i += 1;
+	};
+	assert(same(readfile(bins[0]), readfile(bins[1])));
+	let actions: []str = ["dep", "app"];
+	let suffixes: []str = [".unit.ww", ".wwi", ".s", ".o", ".a"];
+	let commandsuffixes: []str = [".unit.ww", ".wwi", ".s", ".o", ".a",
+		".init.unit.ww", ".init.s", ".init.o"];
+	let ai: i32 = 0;
+	for (ai < actions.len) {
+		let unit: str = readfile(strings.concat(works[0], "/", actions[ai],
+			".unit.ww"));
+		assert(!hasbom(unit) && has(unit, "\n   package "));
+		let si: i32 = 0;
+		for (si < suffixes.len) {
+			assert(same(readfile(strings.concat(works[0], "/", actions[ai],
+				suffixes[si])), readfile(strings.concat(works[1], "/",
+				actions[ai], suffixes[si]))));
+			si += 1;
+		};
+		ai += 1;
+	};
+	let ci: i32 = suffixes.len;
+	for (ci < commandsuffixes.len) {
+		assert(same(readfile(strings.concat(works[0], "/app",
+			commandsuffixes[ci])), readfile(strings.concat(works[1], "/app",
+			commandsuffixes[ci]))));
+		ci += 1;
+	};
+
+	// Production, same-package, external-package, and test-only roles all get
+	// their own offset-zero allowance and execute through the ordinary harness.
+	writebomfile(strings.concat(tests, "/prod.ww"), strings.concat(
+		"package bomtests;\n",
+		"export fn value() i32 = { return 42; };\n"));
+	writebomfile(strings.concat(tests, "/same_test.ww"), strings.concat(
+		"package bomtests;\n",
+		"@test fn same_bom() void = { assert(value() == 42); };\n"));
+	writebomfile(strings.concat(tests, "/external_test.ww"), strings.concat(
+		"package bomtests_test;\nimport bomtests;\n",
+		"@test fn external_bom() void = { assert(bomtests.value() == 42); };\n"));
+	writebomfile(strings.concat(onlytests, "/only_test.ww"),
+		"package onlytests;\n@test fn only_bom() void = { assert(true); };\n");
+	let testrefs: []str = ["", ""];
+	let onlyrefs: []str = ["", ""];
+	let testworks: []str = [strings.concat(root, "/test-work-c"),
+		strings.concat(root, "/test-work-ww")];
+	i = 0;
+	for (i < stages.len) {
+		mkdirall(testworks[i]);
+		let av: []str = [driver(stages[i]), "test", "-w", testworks[i],
+			"-I", source, "bomtests"];
+		let out: commandout;
+		runcommand(root, strings.concat("bom-test-", tags[i]), av,
+			(120i64 * (time.second: i64)): time.duration, &out);
+		expectexit(&out, 0);
+		assert(out.stderr.len == 0 && has(out.stdout, "2 passed, 0 failed"));
+		testrefs[i] = strings.dup(out.stdout);
+		let onlyav: []str = [driver(stages[i]), "test", "-I", source,
+			"onlytests"];
+		runcommand(root, strings.concat("bom-test-only-", tags[i]), onlyav,
+			(120i64 * (time.second: i64)): time.duration, &out);
+		expectexit(&out, 0);
+		assert(out.stderr.len == 0 && has(out.stdout, "1 passed, 0 failed"));
+		onlyrefs[i] = strings.dup(out.stdout);
+		i += 1;
+	};
+	assert(same(testrefs[0], testrefs[1]) && same(onlyrefs[0], onlyrefs[1]));
+	let testactions: []str = ["bomtests-internal-test",
+		"bomtests_test-external-test", "bomtests-test-main"];
+	ai = 0;
+	for (ai < testactions.len) {
+		let testsuffixes: []str = suffixes;
+		if (ai == 2) { testsuffixes = commandsuffixes; };
+		let tsi: i32 = 0;
+		for (tsi < testsuffixes.len) {
+			assert(same(readfile(strings.concat(testworks[0], "/",
+				testactions[ai], testsuffixes[tsi])),
+				readfile(strings.concat(testworks[1], "/", testactions[ai],
+				testsuffixes[tsi]))));
+			tsi += 1;
+		};
+		if (ai < 2) {
+			let testunit: str = readfile(strings.concat(testworks[0], "/",
+				testactions[ai], ".unit.ww"));
+			assert(!hasbom(testunit) && has(testunit, "\n   package "));
+		};
+		ai += 1;
+	};
+	assert(!directoryhasnew(testworks[0]) && !directoryhasnew(testworks[1]));
+
+	// A later BOM is rejected during request parsing. Missing-import graph
+	// resolution, compiler/assembler/linker producers, runtime, and publication
+	// are therefore never reached; cold workdirs and products stay empty.
+	writemidbomfile(strings.concat(bad, "/main.ww"),
+		"package main;\n// ",
+		"\nimport missing;\nfn main() i32 = { return 0; };\n");
+	writefile(strings.concat(badtest, "/prod.ww"),
+		"package badtest;\nfn value() i32 = { return 42; };\n");
+	writemidbomfile(strings.concat(badtest, "/prod_test.ww"),
+		"package badtest;\n// ",
+		"\n@test fn must_not_run() void = { abort(\"ran\"); };\n");
+	let baddiags: []str = ["", ""];
+	let badtestdiags: []str = ["", ""];
+	i = 0;
+	for (i < stages.len) {
+		let badwork: str = strings.concat(root, "/bad-work-", tags[i]);
+		let badout: str = strings.concat(root, "/bad-output-", tags[i]);
+		mkdirall(badwork);
+		let av: []str = [driver(stages[i]), "build", "-w", badwork,
+			"-I", source, "-o", badout, "bad"];
+		let out: commandout;
+		runcommand(root, strings.concat("bom-bad-build-", tags[i]), av,
+			(60i64 * (time.second: i64)): time.duration, &out);
+		expectexit(&out, 1);
+		assert(out.stdout.len == 0);
+		assert(has(out.stderr,
+			":2:4: error: invalid BOM in the middle of the file\n"));
+		assert(!has(out.stderr, "cannot find import")
+			&& !has(out.stderr, "w6c failed"));
+		baddiags[i] = strings.dup(out.stderr);
+		assert(!os.exists(badout) && directoryisempty(badwork));
+		let testav: []str = [driver(stages[i]), "test", "-I", source,
+			"badtest"];
+		runcommand(root, strings.concat("bom-bad-test-", tags[i]), testav,
+			(60i64 * (time.second: i64)): time.duration, &out);
+		expectexit(&out, 1);
+		assert(same(out.stdout, "FAIL\n"));
+		assert(has(out.stderr, "invalid BOM in the middle of the file")
+			&& !has(out.stderr, "must_not_run ..."));
+		badtestdiags[i] = strings.dup(out.stderr);
+		i += 1;
+	};
+	assert(same(baddiags[0], baddiags[1])
+		&& same(badtestdiags[0], badtestdiags[1]));
+
+	// A warm invalid edit cannot mutate committed action bytes or publication.
+	// Restoring the exact leading-BOM source composes the same unit and reuses
+	// the generation without invoking the compiler.
+	let warmpath: str = strings.concat(warm, "/main.ww");
+	let warmvalid: str =
+		"package main;\nfn main() i32 = { return 0; };\n";
+	writebomfile(warmpath, warmvalid);
+	let wrapper: str = strings.concat(root, "/compiler-wrapper.sh");
+	writeexecutable(wrapper, strings.concat(
+		"#!/bin/sh\n",
+		"printf 'compile\\n' >> \"$WW_BOM_TRACE\"\n",
+		"exec \"$WW_BOM_REAL\" \"$@\"\n"));
+	i = 0;
+	for (i < stages.len) {
+		let warmwork: str = strings.concat(root, "/warm-work-", tags[i]);
+		let warmout: str = strings.concat(root, "/warm-output-", tags[i]);
+		let trace: str = strings.concat(root, "/warm-trace-", tags[i]);
+		mkdirall(warmwork); writefile(trace, "");
+		let env: []str = os.getenvs();
+		append(env, strings.concat("WW_W6C=", wrapper));
+		append(env, strings.concat("WW_BOM_TRACE=", trace));
+		append(env, strings.concat("WW_BOM_REAL=", driver(compilers[i])));
+		let av: []str = [driver(stages[i]), "build", "-w", warmwork,
+			"-I", source, "-o", warmout, "warm"];
+		let out: commandout;
+		runcommandenv(root, strings.concat("bom-warm-cold-", tags[i]), av,
+			env, (120i64 * (time.second: i64)): time.duration, &out);
+		expectexit(&out, 0);
+		assert(out.stdout.len == 0 && out.stderr.len == 0
+			&& readfile(trace).len != 0);
+		let refs: []str = alloc([], commandsuffixes.len: u64)!;
+		let si: i32 = 0;
+		for (si < commandsuffixes.len) {
+			append(refs, strings.dup(readfile(strings.concat(warmwork,
+				"/warm", commandsuffixes[si]))));
+			si += 1;
+		};
+		let binref: str = strings.dup(readfile(warmout));
+		rewritefile(trace, "");
+		rewritemidbomfile(warmpath, "package main;\n// ",
+			"\nfn main() i32 = { return 0; };\n");
+		runcommandenv(root, strings.concat("bom-warm-invalid-", tags[i]), av,
+			env, (60i64 * (time.second: i64)): time.duration, &out);
+		expectexit(&out, 1);
+		assert(out.stdout.len == 0
+			&& has(out.stderr, "invalid BOM in the middle of the file"));
+		assert(readfile(trace).len == 0 && same(binref, readfile(warmout)));
+		si = 0;
+		for (si < commandsuffixes.len) {
+			assert(same(refs[si], readfile(strings.concat(warmwork,
+				"/warm", commandsuffixes[si]))));
+			si += 1;
+		};
+		assert(!directoryhasnew(warmwork)
+			&& !directoryhasfragment(warmwork, ".wwtxn."));
+		rewritebomfile(warmpath, warmvalid);
+		runcommandenv(root, strings.concat("bom-warm-restored-", tags[i]), av,
+			env, (120i64 * (time.second: i64)): time.duration, &out);
+		expectexit(&out, 0);
+		assert(out.stdout.len == 0 && out.stderr.len == 0);
+		assert(readfile(trace).len == 0 && same(binref, readfile(warmout))
+			&& !directoryhasnew(warmwork));
+
+		// The marker is semantically whitespace but its three source bytes are
+		// observable input: removing and re-adding it invalidates the unit just
+		// as Go's source-content action does, without changing output bytes.
+		rewritefile(trace, "");
+		rewritefile(warmpath, warmvalid);
+		runcommandenv(root, strings.concat("bom-warm-marker-removed-", tags[i]),
+			av, env, (120i64 * (time.second: i64)): time.duration, &out);
+		expectexit(&out, 0);
+		assert(out.stdout.len == 0 && out.stderr.len == 0);
+		assert(readfile(trace).len != 0 && same(binref, readfile(warmout)));
+		assert(!same(refs[0], readfile(strings.concat(warmwork,
+			"/warm.unit.ww"))));
+		rewritefile(trace, "");
+		rewritebomfile(warmpath, warmvalid);
+		runcommandenv(root, strings.concat("bom-warm-marker-restored-", tags[i]),
+			av, env, (120i64 * (time.second: i64)): time.duration, &out);
+		expectexit(&out, 0);
+		assert(out.stdout.len == 0 && out.stderr.len == 0);
+		assert(readfile(trace).len != 0 && same(binref, readfile(warmout)));
+		si = 0;
+		for (si < commandsuffixes.len) {
+			assert(same(refs[si], readfile(strings.concat(warmwork,
+				"/warm", commandsuffixes[si]))));
+			si += 1;
+		};
+		assert(!directoryhasnew(warmwork));
+		i += 1;
+	};
+
+	// Request-local source state: a valid Cstage build and an invalid WWstage
+	// build overlap without sharing diagnostics, products, or transaction data.
+	let pcwork: str = strings.concat(root, "/parallel-c-work");
+	let pwwork: str = strings.concat(root, "/parallel-ww-work");
+	let pcout: str = strings.concat(root, "/parallel-c-output");
+	let pwout: str = strings.concat(root, "/parallel-ww-output");
+	mkdirall(pcwork); mkdirall(pwwork);
+	let pcav: []str = [driver("ww"), "build", "-w", pcwork,
+		"-I", source, "-o", pcout, "app"];
+	let pwav: []str = [driver("ww_ww"), "build", "-w", pwwork,
+		"-I", source, "-o", pwout, "bad"];
+	let pc: exec.command;
+	pc.path = pcav[0]; pc.argv = pcav; pc.env = os.getenvs(); pc.dir = repo();
+	pc.stdoutpath = strings.concat(root, "/parallel-c.stdout");
+	pc.stderrpath = strings.concat(root, "/parallel-c.stderr");
+	pc.deadline = time.add(time.now(time.clock.monotonic),
+		(120i64 * (time.second: i64)): time.duration);
+	pc.grace = (100i64 * (time.millisecond: i64)): time.duration;
+	let pw: exec.command;
+	pw.path = pwav[0]; pw.argv = pwav; pw.env = os.getenvs(); pw.dir = repo();
+	pw.stdoutpath = strings.concat(root, "/parallel-ww.stdout");
+	pw.stderrpath = strings.concat(root, "/parallel-ww.stderr");
+	pw.deadline = time.add(time.now(time.clock.monotonic),
+		(120i64 * (time.second: i64)): time.duration);
+	pw.grace = (100i64 * (time.millisecond: i64)): time.duration;
+	let pcp: exec.process;
+	let pwp: exec.process;
+	exec.start(&pcp, &pc); exec.start(&pwp, &pw);
+	let pcdone: bool = false;
+	let pwdone: bool = false;
+	for (!pcdone || !pwdone) {
+		if (!pcdone) { pcdone = exec.poll(&pcp); };
+		if (!pwdone) { pwdone = exec.poll(&pwp); };
+		if (!pcdone || !pwdone) {
+			time.sleep(time.millisecond, time.clock.monotonic);
+		};
+	};
+	assert(pcp.result.errno == 0 && pcp.result.cleanuperrno == 0
+		&& pcp.result.termination == exec.termination.EXIT
+		&& pcp.result.code == 0);
+	assert(pwp.result.errno == 0 && pwp.result.cleanuperrno == 0
+		&& pwp.result.termination == exec.termination.EXIT
+		&& pwp.result.code == 1);
+	assert(readfile(pc.stdoutpath).len == 0 && readfile(pc.stderrpath).len == 0);
+	assert(readfile(pw.stdoutpath).len == 0
+		&& has(readfile(pw.stderrpath), "invalid BOM in the middle of the file"));
+	assert(same(readfile(pcout), readfile(bins[0])) && !os.exists(pwout));
+	assert(directoryisempty(pwwork) && !directoryhasnew(pcwork)
+		&& !directoryhasfragment(pcwork, ".wwtxn."));
+	let runav: []str = [pcout];
+	let out: commandout;
+	runcommand(root, "bom-parallel-run", runav, time.second, &out);
+	expectexit(&out, 0);
+	assert(out.stdout.len == 0 && out.stderr.len == 0);
+	assert(!directoryhasfragment(root, ".wwtxn.")
+		&& !directoryhasfragment(root, ".install")
+		&& !directoryhasnew(root));
+	clean(root);
+};

test/wcc/100_lex.c

@@ -61,6 +61,7 @@ struct row { const char *src, *expect; };
 
 static const struct row rows[] = {
 	{ "",                "" },
+	{ "\xef\xbb\xbf" "package main;", "package IDENT(main) ;" },
 	{ "  \t\n  ",        "" },
 	{ "// comment\n",    "" },
 	{ "/* a /b/ c */",   "" },
@@ -172,7 +173,29 @@ runrewind(void)
 	return ok;
 }
 
+static int
+runbompos(void)
+{
+	const char *src = "\xef\xbb\xbf" "package main;";
+	Arena *a = newarena();
+	Lex l;
+	lexinit(&l, a, "<test>", src, strlen(src));
+	Tok t = lexnext(&l);
+	int ok = t.kind == TK_MODULE && t.pos.line == 1 && t.pos.col == 4;
+	if (!ok)
+		fprintf(stderr, "leading BOM position: want 1:4 got %d:%d\n",
+		    t.pos.line, t.pos.col);
+	freearena(a);
+	return ok;
+}
+
 static const char *const errrows[] = {
+	"\xef\xbb\xbf" "\xef\xbb\xbf" "package main;",
+	"package \xef\xbb\xbf" "main;",
+	"// \xef\xbb\xbf\n",
+	"/* \xef\xbb\xbf */",
+	"\"\xef\xbb\xbf\"",
+	"'\xef\xbb\xbf'",
 	"'\\uZ'",		/* unexpected rune scanning for escape */
 	"\"\\u00g0\"",		/* non-hex digit inside a string escape */
 	"'\\u00",		/* unexpected EOF scanning for escape */
@@ -194,6 +217,8 @@ main(void)
 	}
 	if (!runrewind())
 		fail++;
+	if (!runbompos())
+		fail++;
 	for (size_t i = 0; i < sizeof errrows / sizeof errrows[0]; i++) {
 		if (!runerr(errrows[i])) {
 			fprintf(stderr, "errrow %zu failed\n", i);