ww source: align UTF-8 BOM placement
This commit is contained in:
@@ -7,6 +7,15 @@
|
||||
#include <string.h>
|
||||
#include <errno.h>
|
||||
|
||||
static int
|
||||
bomat(const char *src, u64 len, u64 pos)
|
||||
{
|
||||
return pos <= len && len - pos >= 3
|
||||
&& (unsigned char)src[pos] == 0xef
|
||||
&& (unsigned char)src[pos + 1] == 0xbb
|
||||
&& (unsigned char)src[pos + 2] == 0xbf;
|
||||
}
|
||||
|
||||
void
|
||||
lexinit(Lex *l, Arena *a, const char *file, const char *src, u64 len)
|
||||
{
|
||||
@@ -17,6 +26,12 @@ lexinit(Lex *l, Arena *a, const char *file, const char *src, u64 len)
|
||||
l->line = 1;
|
||||
l->col = 1;
|
||||
l->a = a;
|
||||
/* Go 1.26.5 syntax.source.nextch ignores U+FEFF only at the first
|
||||
* source position but counts its three bytes for the next column. */
|
||||
if (bomat(src, len, 0)) {
|
||||
l->pos = 3;
|
||||
l->col = 4;
|
||||
}
|
||||
}
|
||||
|
||||
static int
|
||||
@@ -25,6 +40,8 @@ lpeek(Lex *l, u64 ahead)
|
||||
u64 p = l->pos + ahead;
|
||||
if (p >= l->srclen)
|
||||
return -1;
|
||||
if (bomat(l->src, l->srclen, p))
|
||||
return 0xfeff;
|
||||
return (unsigned char)l->src[p];
|
||||
}
|
||||
|
||||
@@ -33,6 +50,14 @@ lget(Lex *l)
|
||||
{
|
||||
if (l->pos >= l->srclen)
|
||||
return -1;
|
||||
if (bomat(l->src, l->srclen, l->pos)) {
|
||||
Pos p = { l->file, l->line, l->col };
|
||||
l->pos += 3;
|
||||
l->col += 3;
|
||||
errorf(p, "invalid BOM in the middle of the file");
|
||||
l->errs++;
|
||||
return 0xfeff;
|
||||
}
|
||||
int c = (unsigned char)l->src[l->pos++];
|
||||
if (c == '\n') {
|
||||
l->line++;
|
||||
@@ -529,6 +554,11 @@ lexnext(Lex *l)
|
||||
return t;
|
||||
}
|
||||
int c = lpeek(l, 0);
|
||||
if (c == 0xfeff) {
|
||||
lget(l);
|
||||
Tok t = (Tok){ TK_ERR, start, "", 0, {0}, TK_NONE };
|
||||
return t;
|
||||
}
|
||||
|
||||
if (isidstart(c))
|
||||
return lexident(l, start);
|
||||
|
||||
@@ -4740,7 +4740,18 @@ sep_emit_body(FILE *out, const char *path, const char *modpath)
|
||||
} else if (fputs("//ww:module-reset\n", out) == EOF) {
|
||||
bad = 1;
|
||||
}
|
||||
if (fwrite(buf, 1, (size_t)len, out) != (size_t)len
|
||||
/* Each physical source owns its own first-codepoint position. The
|
||||
* compiler sees one synthetic aggregate, so replace the optional UTF-8
|
||||
* BOM with position-preserving whitespace rather than turning a later
|
||||
* source's marker into a mid-unit BOM. */
|
||||
u64 off = len >= 3
|
||||
&& (unsigned char)buf[0] == 0xef
|
||||
&& (unsigned char)buf[1] == 0xbb
|
||||
&& (unsigned char)buf[2] == 0xbf ? 3 : 0;
|
||||
if (off != 0 && fwrite(" ", 1, 3, out) != 3)
|
||||
bad = 1;
|
||||
if (fwrite(buf + off, 1, (size_t)(len - off), out)
|
||||
!= (size_t)(len - off)
|
||||
|| fputc('\n', out) == EOF)
|
||||
bad = 1;
|
||||
free(buf);
|
||||
|
||||
@@ -8427,6 +8427,124 @@ Rejected source creates no persisted byte contract, while valid source bytes
|
||||
are unchanged. Build workdir format remains `18`, test workdir format remains
|
||||
`19`, and semantic storage format remains `3`.
|
||||
|
||||
### 11.42 Implemented per-source UTF-8 BOM placement
|
||||
|
||||
Every eligible physical WW source may begin with one UTF-8-encoded U+FEFF byte
|
||||
order mark (`EF BB BF`). That marker is ignored, and the following token keeps
|
||||
its three-byte source position at line 1, column 4. U+FEFF at any later raw
|
||||
source position is
|
||||
rejected once as `invalid BOM in the middle of the file`, including inside a
|
||||
line/block comment, string, or rune. Two leading markers therefore ignore the
|
||||
first and reject the second. A truncated marker or another invalid UTF-8 byte
|
||||
sequence retains the ordinary byte-error path; UTF-16 source is not introduced.
|
||||
|
||||
#### Pinned Go evidence and applicability
|
||||
|
||||
The sole authority is official Go 1.26.5 at commit
|
||||
`c19862e5f8415b4f24b189d065ed739517c548ba`:
|
||||
|
||||
- compiler reader `(*source).nextch` decodes UTF-8, skips U+FEFF at its first
|
||||
source position, and reports `invalid BOM in the middle of the file`
|
||||
elsewhere
|
||||
([`cmd/compile/internal/syntax/source.go`, lines 113–165](https://github.com/golang/go/blob/c19862e5f8415b4f24b189d065ed739517c548ba/src/cmd/compile/internal/syntax/source.go#L113-L165));
|
||||
- public scanner `(*Scanner).next` rejects later U+FEFF and `(*Scanner).Init`
|
||||
consumes the first one
|
||||
([`go/scanner/scanner.go`, lines 58–100 and 128–164](https://github.com/golang/go/blob/c19862e5f8415b4f24b189d065ed739517c548ba/src/go/scanner/scanner.go#L58-L164));
|
||||
- scanner tests assert the ignored first marker and later markers between
|
||||
tokens, in comments, runes, and strings
|
||||
([`go/scanner/scanner_test.go`, lines 371–374 and 812–815](https://github.com/golang/go/blob/c19862e5f8415b4f24b189d065ed739517c548ba/src/go/scanner/scanner_test.go#L371-L374),
|
||||
[`go/scanner/scanner_test.go`, lines 812–815](https://github.com/golang/go/blob/c19862e5f8415b4f24b189d065ed739517c548ba/src/go/scanner/scanner_test.go#L812-L815));
|
||||
- compiler-scanner `TestScanErrors` asserts the positioned later-marker error
|
||||
([`cmd/compile/internal/syntax/scanner_test.go`, lines 587–599](https://github.com/golang/go/blob/c19862e5f8415b4f24b189d065ed739517c548ba/src/cmd/compile/internal/syntax/scanner_test.go#L587-L599)); and
|
||||
- official command testdata places the marker before `package main` and loads
|
||||
that source's imports and embedded file
|
||||
([`cmd/go/testdata/script/build_ignore_leading_bom.txt`, lines 1–25](https://github.com/golang/go/blob/c19862e5f8415b4f24b189d065ed739517c548ba/src/cmd/go/testdata/script/build_ignore_leading_bom.txt#L1-L25)).
|
||||
|
||||
Those source branches and assertions are **behavior directly implemented or
|
||||
asserted by pinned Go**. Applying the first position independently to every WW
|
||||
physical source before its existing synthetic aggregate boundary is **behavior
|
||||
derived from the pinned implementation**. The pre-fix Cstage/WWstage failures
|
||||
on leading markers and successes for markers in comments/strings were
|
||||
**directly measured WW behavior**.
|
||||
|
||||
The behavior honestly applies inside WW's local, dotted-import, manifest-free
|
||||
model: it is source representation before package and import interpretation.
|
||||
It requires no module, manifest, registry, lock, cache, database, CAS, network
|
||||
resolution, generalized import syntax, build expression, or normalized
|
||||
identity.
|
||||
|
||||
#### Ownership and four-axis result
|
||||
|
||||
`lexinit`, `lpeek`, `lget`, and `lexnext` in `cmd/wcc/lex.c`, with their twins
|
||||
in `lib/ww/syntax/lex.ww`, are the language owners. They skip the exact initial
|
||||
marker while advancing the logical column by its encoded width, recognize a
|
||||
later marker as one code point in every lexical context, and emit one
|
||||
stage-identical diagnostic.
|
||||
`sep_emit_body` in `cmd/ww/main.c` and `sepemitbody` in
|
||||
`selfhost/cmd/ww/main.ww` replace the optional marker with three spaces in every
|
||||
physical body placed after a synthetic `//ww:module-reset`, preserving columns;
|
||||
direct and imports-only lexer input remains independently correct. The shared package
|
||||
coordinator's `pkgclause` in `internal/wwpackage/package.ww` begins its
|
||||
pre-discovery package-name scan after the same optional marker. It does not
|
||||
replace the complete stage-driver scan.
|
||||
|
||||
- **Go-like build:** a selected command, library, or imported dependency may
|
||||
use the marker in each eligible source. Multiple physical marked sources
|
||||
compose normally, command publication succeeds, and the executable runs
|
||||
normally. A later marker rejects during source loading before graph/action
|
||||
construction or compiler, assembler, archiver, linker, install, or runtime
|
||||
work. A source error keeps precedence over a missing import.
|
||||
- **Go-like test:** production, same-package, external-package, and test-only
|
||||
files each receive the offset-zero allowance. Valid variants build and run
|
||||
through the ordinary single directory product. A later marker yields the
|
||||
existing attributable `FAIL\n` result without a variant, generated main,
|
||||
test process, accounting, or retained binary.
|
||||
- **Go-like package:** package-clause recognition now begins at the pinned
|
||||
logical source start in direct compilers, directory drivers, and the shared
|
||||
coordinator. Declared names, source roles, package conflicts, command/test
|
||||
family selection, and canonical identity do not change.
|
||||
- **Go-like import:** imports following a legal marker and imports in a marked
|
||||
dependency retain their exact source spelling, file scope, qualifier,
|
||||
contextual local/vendor resolution, case-sensitive canonical dotted
|
||||
identity, visibility, cycle, and initialization behavior. The marker never
|
||||
becomes an edge or identity component.
|
||||
|
||||
Fixed-target filename selection remains earlier than parsing: an excluded
|
||||
`_windows.ww` or `_windows_test.ww` contributes no marker diagnostic, package,
|
||||
import, graph, action, artifact, or invalidation state. For valid inputs, graph
|
||||
nodes, scheduling, producer arguments, initialization, runtime, result order,
|
||||
and publication are unchanged. Legal marker bytes become three
|
||||
position-preserving spaces only in the synthetic unit. Adding or removing the
|
||||
marker therefore changes unit content and invalidates source-derived actions,
|
||||
while semantic interface, assembly, object, archive, initializer, and
|
||||
executable content remains the same; every form remains Cstage/WWstage
|
||||
byte-identical.
|
||||
|
||||
Cold rejection leaves no product, work generation, adjacent scratch, capture,
|
||||
`.new`, `.install`, or `.wwtxn.*`. A warm later-marker edit preserves the
|
||||
entire committed generation, tool vouchers, stamp, and public executable.
|
||||
Restoring the exact legal leading form recreates the same unit and reuses committed
|
||||
compiler/assembler/archive work before the normal link/publication boundary.
|
||||
Concurrent valid and invalid requests keep independent lexer/coordinator state,
|
||||
workdirs, captures, diagnostics, and products. Because later-marker rejection
|
||||
occurs before a producer or test child, producer/runtime failure, signals,
|
||||
timeouts, interruption, and process-group cleanup gain no new branch; their
|
||||
existing owners remain authoritative.
|
||||
|
||||
The C lexer unit, WW syntax unit, and WW-native
|
||||
`utf8_bom_is_per_source_and_only_first` observer prove initial position,
|
||||
later-marker lexical contexts, direct frontend diagnostics and assembly,
|
||||
selected/imported multi-source build, wrong-target exclusion, every test source
|
||||
role, test-only execution, syntax-before-resolution precedence, cold cleanup,
|
||||
warm rollback and reuse, concurrent isolation, publication/runtime behavior,
|
||||
and complete stage diagnostic/artifact parity.
|
||||
|
||||
No persisted format changes. Previously valid marker-free bytes are
|
||||
unchanged; previously leading-marked requests could not commit a generation;
|
||||
and diagnostic text is not a persisted-byte contract. Build workdir format
|
||||
remains `18`, test workdir format remains `19`, and semantic storage format
|
||||
remains `3`.
|
||||
|
||||
## 12. Candidate architectures and hard-gate decision
|
||||
|
||||
Five candidates were developed as coherent systems, not as feature bins.
|
||||
|
||||
@@ -52,8 +52,13 @@ classes from §2 (`ident`, `int_lit`, …).
|
||||
|
||||
### 2.1 Source representation
|
||||
|
||||
Source is UTF-8. The lexer operates on bytes; non-ASCII bytes are legal
|
||||
only inside string and rune literals and comments.
|
||||
Source is UTF-8. One UTF-8-encoded byte order mark (U+FEFF, bytes
|
||||
`EF BB BF`) is ignored when it is the first code point of a physical source
|
||||
file. Its three encoded bytes still count in source positions, so a following
|
||||
token on the first line begins at column 4. U+FEFF is invalid at
|
||||
every other source position, including inside string and rune literals and
|
||||
comments. Apart from that marker rule, the lexer operates on bytes and
|
||||
non-ASCII bytes are legal only inside string and rune literals and comments.
|
||||
|
||||
### 2.2 Comments
|
||||
|
||||
|
||||
@@ -415,6 +415,20 @@ generation. The directory command retains its existing attributable final
|
||||
diagnostic. This ordering state is per source and never package, import,
|
||||
variant, action, artifact, publication, or persistence identity.
|
||||
|
||||
Every eligible source also owns an independent source-start position. An exact
|
||||
UTF-8 BOM (`EF BB BF`) at byte offset zero is ignored before package-clause and
|
||||
import parsing, so production, same-package, external-package, and test-only
|
||||
sources all count the marker's three bytes and begin the following token at
|
||||
logical line 1 column 4. The same code point anywhere else,
|
||||
including a comment or literal, is a source error before variant actions,
|
||||
generated main, producers, test runtime, accounting, or retained publication.
|
||||
The shared coordinator applies the offset-zero rule while classifying package
|
||||
clauses; the selected stage driver scans the complete source and preserves the
|
||||
per-file rule while composing package units. Wrong-target files remain excluded
|
||||
before either operation. The marker is source representation only and never
|
||||
package/import/action/artifact/publication/persistence identity; no test result
|
||||
is cached.
|
||||
|
||||
After that eligibility boundary and the coordinator's required package-clause
|
||||
classification and production `@test` validation parses, the delegated loader
|
||||
performs selected-basename Go 1.26.5 simple-fold preflight before its graph
|
||||
|
||||
@@ -724,7 +724,12 @@ fn pkgskipspace(src: str, start: i32) i32 = {
|
||||
};
|
||||
|
||||
fn pkgclause(src: str, out: *str) bool = {
|
||||
let i: i32 = pkgskipspace(src, 0);
|
||||
// Match the compiler readers: one UTF-8 BOM is invisible only at the
|
||||
// first raw source position. The full stage driver owns later-BOM errors.
|
||||
let start: i32 = 0;
|
||||
if (src.len >= 3 && src[0] == 0xefu8 && src[1] == 0xbbu8
|
||||
&& src[2] == 0xbfu8) { start = 3; };
|
||||
let i: i32 = pkgskipspace(src, start);
|
||||
let word: str = "package";
|
||||
if (i + word.len >= src.len) { return false; };
|
||||
let j: i32 = 0;
|
||||
|
||||
@@ -65,6 +65,14 @@ export type lex = struct {
|
||||
modresetpath: str,
|
||||
};
|
||||
|
||||
fn bomat(src: *u8, len: u64, off: u64) bool = {
|
||||
if (off > len || len - off < 3u64) { return false; };
|
||||
let i: i32 = off: i32;
|
||||
return src[i] == 0xefu8
|
||||
&& src[i + 1] == 0xbbu8
|
||||
&& src[i + 2] == 0xbfu8;
|
||||
};
|
||||
|
||||
export fn lexinit(l: *lex, file: str, src: *u8, len: u64) void = {
|
||||
l.file = file;
|
||||
l.src = src;
|
||||
@@ -76,6 +84,9 @@ export fn lexinit(l: *lex, file: str, src: *u8, len: u64) void = {
|
||||
l.modreset = 0;
|
||||
l.modpathset = 0;
|
||||
l.modresetpathset = 0;
|
||||
// Go 1.26.5 syntax.source.nextch ignores U+FEFF only at the first
|
||||
// source position but counts its three bytes for the next column.
|
||||
if (bomat(src, len, 0u64)) { l.lpos = 3u64; l.col = 4; };
|
||||
};
|
||||
|
||||
fn srcb(l: *lex, off: u64) i32 = {
|
||||
@@ -87,11 +98,22 @@ fn srcb(l: *lex, off: u64) i32 = {
|
||||
fn lpeek(l: *lex, ahead: u64) i32 = {
|
||||
let p: u64 = l.lpos + ahead;
|
||||
if (p >= l.srclen) { return -1; };
|
||||
if (bomat(l.src, l.srclen, p)) { return 0xFEFF; };
|
||||
return srcb(l, p);
|
||||
};
|
||||
|
||||
fn lget(l: *lex) i32 = {
|
||||
if (l.lpos >= l.srclen) { return -1; };
|
||||
if (bomat(l.src, l.srclen, l.lpos)) {
|
||||
let bp: pos;
|
||||
bp.file = l.file;
|
||||
bp.line = l.line;
|
||||
bp.col = l.col;
|
||||
l.lpos += 3u64;
|
||||
l.col += 3;
|
||||
errat(l, &bp, "invalid BOM in the middle of the file");
|
||||
return 0xFEFF;
|
||||
};
|
||||
let c: i32 = srcb(l, l.lpos);
|
||||
l.lpos += 1u64;
|
||||
if (c == '\n') {
|
||||
@@ -801,6 +823,11 @@ export fn lexnext(l: *lex, out: *tok) void = {
|
||||
return;
|
||||
};
|
||||
let c: i32 = lpeek(l, 0u64);
|
||||
if (c == 0xFEFF) {
|
||||
lget(l);
|
||||
emitsimple(&start, tkind.TK_ERR, out);
|
||||
return;
|
||||
};
|
||||
|
||||
if (c >= 0) {
|
||||
if (isidstart(c: rune)) { lexident(l, &start, out); return; };
|
||||
|
||||
@@ -338,3 +338,78 @@ fn checkfloat(src: str, want: u64) void = {
|
||||
assert(!(t.col != 6));
|
||||
assert(!(t.text != "bar"));
|
||||
};
|
||||
|
||||
fn bomsource(dst: *u8, before: str, after: str) u64 = {
|
||||
let n: u64 = 0u64;
|
||||
let i: i32 = 0;
|
||||
for (i < before.len) {
|
||||
let ni: i32 = n: i32;
|
||||
dst[ni] = before[i];
|
||||
n += 1u64;
|
||||
i += 1;
|
||||
};
|
||||
let bi: i32 = n: i32;
|
||||
dst[bi] = 0xefu8;
|
||||
dst[bi + 1] = 0xbbu8;
|
||||
dst[bi + 2] = 0xbfu8;
|
||||
n += 3u64;
|
||||
i = 0;
|
||||
for (i < after.len) {
|
||||
let ni: i32 = n: i32;
|
||||
dst[ni] = after[i];
|
||||
n += 1u64;
|
||||
i += 1;
|
||||
};
|
||||
return n;
|
||||
};
|
||||
|
||||
fn bomerror(before: str, after: str) bool = {
|
||||
let src: [128]u8;
|
||||
let n: u64 = bomsource(src.ptr, before, after);
|
||||
let l: syntax.lex;
|
||||
syntax.lexinit(&l, "t", src.ptr, n);
|
||||
let t: syntax.tok;
|
||||
for (true) {
|
||||
syntax.lexnext(&l, &t);
|
||||
if (t.kind == syntax.tkind.TK_EOF) { break; };
|
||||
};
|
||||
return l.errs > 0;
|
||||
};
|
||||
|
||||
fn doublebomerror() bool = {
|
||||
let src: [6]u8;
|
||||
let i: i32 = 0;
|
||||
for (i < 6) {
|
||||
if (i % 3 == 0) { src[i] = 0xefu8; };
|
||||
if (i % 3 == 1) { src[i] = 0xbbu8; };
|
||||
if (i % 3 == 2) { src[i] = 0xbfu8; };
|
||||
i += 1;
|
||||
};
|
||||
let l: syntax.lex;
|
||||
syntax.lexinit(&l, "t", src.ptr, 6u64);
|
||||
let t: syntax.tok;
|
||||
for (true) {
|
||||
syntax.lexnext(&l, &t);
|
||||
if (t.kind == syntax.tkind.TK_EOF) { break; };
|
||||
};
|
||||
return l.errs > 0;
|
||||
};
|
||||
|
||||
@test fn utf8_bom_is_ignored_only_at_source_start() void = {
|
||||
let src: [128]u8;
|
||||
let n: u64 = bomsource(src.ptr, "", "package main;");
|
||||
let l: syntax.lex;
|
||||
syntax.lexinit(&l, "t", src.ptr, n);
|
||||
let t: syntax.tok;
|
||||
syntax.lexnext(&l, &t);
|
||||
assert(t.kind == syntax.tkind.TK_MODULE);
|
||||
assert(t.line == 1 && t.col == 4);
|
||||
assert(l.errs == 0);
|
||||
|
||||
assert(doublebomerror());
|
||||
assert(bomerror("package ", "main;"));
|
||||
assert(bomerror("// ", "\n"));
|
||||
assert(bomerror("/* ", " */"));
|
||||
assert(bomerror("\"", "\""));
|
||||
assert(bomerror("'", "'"));
|
||||
};
|
||||
|
||||
@@ -5594,7 +5594,19 @@ fn sepemitbody(fd: i32, path: *u8, modpath: *u8) i32 = {
|
||||
return -1;
|
||||
};
|
||||
};
|
||||
if (!sepwriteall(fd, bufp, blen)
|
||||
// Each physical source owns its own first-codepoint position. The compiler
|
||||
// sees one synthetic aggregate, so replace the optional UTF-8 BOM with
|
||||
// position-preserving whitespace rather than making it a mid-unit BOM.
|
||||
let off: u64 = 0u64;
|
||||
if (blen >= 3u64 && bufp[0] == 0xefu8
|
||||
&& bufp[1] == 0xbbu8 && bufp[2] == 0xbfu8) {
|
||||
off = 3u64;
|
||||
};
|
||||
if (off != 0u64 && !sepwriteall(fd, " ".ptr, 3u64)) {
|
||||
cerr("ww: cannot write package unit\n");
|
||||
return -1;
|
||||
};
|
||||
if (!sepwriteall(fd, bufp + off, blen - off)
|
||||
|| !sepwriteall(fd, "\n".ptr, 1u64)) {
|
||||
cerr("ww: cannot write package unit\n");
|
||||
return -1;
|
||||
|
||||
@@ -91,6 +91,45 @@ fn rewritefile(path: str, content: str) void = {
|
||||
assert(os.close(fd) == 0);
|
||||
};
|
||||
|
||||
fn putbomfile(path: str, before: str, after: str, rewrite: bool) void = {
|
||||
let flags: os.flag = os.flag.WRONLY;
|
||||
if (rewrite) { flags |= os.flag.TRUNC; }
|
||||
else { flags |= os.flag.CREATE | os.flag.EXCL; };
|
||||
let fd: i32 = os.open(path, flags, 384i32);
|
||||
assert(fd >= 0);
|
||||
match (os.writeall(fd, before.ptr, before.len: u64)) {
|
||||
case let n: i64 => assert(n == before.len: i64);
|
||||
case let e: os.oserror => abort("write before BOM failed");
|
||||
};
|
||||
let bom: [3]u8;
|
||||
bom[0] = 0xefu8; bom[1] = 0xbbu8; bom[2] = 0xbfu8;
|
||||
match (os.writeall(fd, bom.ptr, 3u64)) {
|
||||
case let n: i64 => assert(n == 3i64);
|
||||
case let e: os.oserror => abort("write BOM failed");
|
||||
};
|
||||
match (os.writeall(fd, after.ptr, after.len: u64)) {
|
||||
case let n: i64 => assert(n == after.len: i64);
|
||||
case let e: os.oserror => abort("write after BOM failed");
|
||||
};
|
||||
assert(os.close(fd) == 0);
|
||||
};
|
||||
|
||||
fn writebomfile(path: str, content: str) void = {
|
||||
putbomfile(path, "", content, false);
|
||||
};
|
||||
|
||||
fn rewritebomfile(path: str, content: str) void = {
|
||||
putbomfile(path, "", content, true);
|
||||
};
|
||||
|
||||
fn writemidbomfile(path: str, before: str, after: str) void = {
|
||||
putbomfile(path, before, after, false);
|
||||
};
|
||||
|
||||
fn rewritemidbomfile(path: str, before: str, after: str) void = {
|
||||
putbomfile(path, before, after, true);
|
||||
};
|
||||
|
||||
fn writeexecutable(path: str, content: str) void = {
|
||||
let fd: i32 = os.open(path,
|
||||
os.flag.WRONLY | os.flag.CREATE | os.flag.EXCL, 448i32);
|
||||
@@ -246,6 +285,16 @@ fn same(a: str, b: str) bool = {
|
||||
return true;
|
||||
};
|
||||
|
||||
fn hasbom(s: str) bool = {
|
||||
let i: i32 = 0;
|
||||
for (i + 2 < s.len) {
|
||||
if (s[i] == 0xefu8 && s[i + 1] == 0xbbu8
|
||||
&& s[i + 2] == 0xbfu8) { return true; };
|
||||
i += 1;
|
||||
};
|
||||
return false;
|
||||
};
|
||||
|
||||
fn occurrences(haystack: str, needle: str) i32 = {
|
||||
if (needle.len == 0 || haystack.len < needle.len) { return 0; };
|
||||
let count: i32 = 0;
|
||||
@@ -16531,3 +16580,391 @@ fn runtimepath(relative: str) str = {
|
||||
assert(!os.exists("/dev/null.sepwork"));
|
||||
clean(root);
|
||||
};
|
||||
|
||||
// Go 1.26.5 ignores one UTF-8 BOM at the first codepoint of each physical
|
||||
// source and rejects U+FEFF everywhere else. The lexer owns direct and
|
||||
// imports-only parsing; the directory composer preserves that per-file
|
||||
// boundary when it builds one synthetic unit.
|
||||
@test fn utf8_bom_is_per_source_and_only_first() void = {
|
||||
let root: str = fresh();
|
||||
let source: str = strings.concat(root, "/source");
|
||||
let dep: str = strings.concat(source, "/dep");
|
||||
let app: str = strings.concat(source, "/app");
|
||||
let tests: str = strings.concat(source, "/bomtests");
|
||||
let onlytests: str = strings.concat(source, "/onlytests");
|
||||
let bad: str = strings.concat(source, "/bad");
|
||||
let badtest: str = strings.concat(source, "/badtest");
|
||||
let warm: str = strings.concat(source, "/warm");
|
||||
mkdirall(dep); mkdirall(app); mkdirall(tests); mkdirall(onlytests);
|
||||
mkdirall(bad); mkdirall(badtest); mkdirall(warm);
|
||||
|
||||
writebomfile(strings.concat(dep, "/dep.ww"), strings.concat(
|
||||
"package dep;\n",
|
||||
"export fn value() i32 = { return 41; };\n"));
|
||||
writebomfile(strings.concat(dep, "/more.ww"), strings.concat(
|
||||
"package dep;\n",
|
||||
"export fn more() i32 = { return 1; };\n"));
|
||||
let appmain: str = strings.concat(app, "/main.ww");
|
||||
let appvalid: str = strings.concat(
|
||||
"package main;\nimport dep;\n",
|
||||
"fn main() i32 = { return dep.value() + dep.more() + local() - 43; };\n");
|
||||
writebomfile(appmain, appvalid);
|
||||
writebomfile(strings.concat(app, "/local.ww"), strings.concat(
|
||||
"package main;\n",
|
||||
"fn local() i32 = { return 1; };\n"));
|
||||
// Wrong-target sources are excluded before source decoding.
|
||||
writemidbomfile(strings.concat(app, "/ignored_windows.ww"),
|
||||
"not a package ", " and not valid WW\n");
|
||||
|
||||
// Direct frontend ownership: the initial marker is invisible at 1:1 and
|
||||
// both stages emit the same assembly; a later marker is one codepoint error.
|
||||
let direct: str = strings.concat(root, "/direct.ww");
|
||||
let directbad: str = strings.concat(root, "/direct-bad.ww");
|
||||
writebomfile(direct,
|
||||
"package main;\nfn main() i32 = { return 0; };\n");
|
||||
writemidbomfile(directbad,
|
||||
"package main;\nfn main() i32 = { let s: str = \"",
|
||||
"\"; return s.len; };\n");
|
||||
let directouts: []str = [strings.concat(root, "/direct-c.s"),
|
||||
strings.concat(root, "/direct-ww.s")];
|
||||
let directbadouts: []str = [strings.concat(root, "/direct-bad-c.s"),
|
||||
strings.concat(root, "/direct-bad-ww.s")];
|
||||
let compilers: []str = ["w6c", "w6c_ww"];
|
||||
let tags: []str = ["c", "ww"];
|
||||
let middiags: []str = ["", ""];
|
||||
let i: i32 = 0;
|
||||
for (i < compilers.len) {
|
||||
let av: []str = [driver(compilers[i]), "-c", "--command-package",
|
||||
"-o", directouts[i], direct];
|
||||
let out: commandout;
|
||||
runcommand(root, strings.concat("bom-direct-valid-", tags[i]), av,
|
||||
(30i64 * (time.second: i64)): time.duration, &out);
|
||||
expectexit(&out, 0);
|
||||
assert(out.stdout.len == 0 && out.stderr.len == 0);
|
||||
let badav: []str = [driver(compilers[i]), "-c", "--command-package",
|
||||
"-o", directbadouts[i], directbad];
|
||||
runcommand(root, strings.concat("bom-direct-invalid-", tags[i]), badav,
|
||||
(30i64 * (time.second: i64)): time.duration, &out);
|
||||
expectexit(&out, 1);
|
||||
assert(out.stdout.len == 0);
|
||||
assert(has(out.stderr,
|
||||
":2:33: error: invalid BOM in the middle of the file\n"));
|
||||
assert(occurrences(out.stderr,
|
||||
"invalid BOM in the middle of the file") == 1);
|
||||
middiags[i] = strings.dup(out.stderr);
|
||||
assert(!os.exists(directbadouts[i]));
|
||||
i += 1;
|
||||
};
|
||||
assert(same(middiags[0], middiags[1]));
|
||||
assert(same(readfile(directouts[0]), readfile(directouts[1])));
|
||||
|
||||
// Build consumes multiple BOM-prefixed sources in both the root and its
|
||||
// imported dependency. The marker never enters unit bytes or identity.
|
||||
let stages: []str = ["ww", "ww_ww"];
|
||||
let works: []str = [strings.concat(root, "/work-c"),
|
||||
strings.concat(root, "/work-ww")];
|
||||
let bins: []str = [strings.concat(root, "/app-c"),
|
||||
strings.concat(root, "/app-ww")];
|
||||
i = 0;
|
||||
for (i < stages.len) {
|
||||
mkdirall(works[i]);
|
||||
let av: []str = [driver(stages[i]), "build", "-w", works[i],
|
||||
"-I", source, "-o", bins[i], "app"];
|
||||
let out: commandout;
|
||||
runcommand(root, strings.concat("bom-build-", tags[i]), av,
|
||||
(120i64 * (time.second: i64)): time.duration, &out);
|
||||
expectexit(&out, 0);
|
||||
assert(out.stdout.len == 0 && out.stderr.len == 0);
|
||||
assert(!directoryhasnew(works[i])
|
||||
&& !directoryhasfragment(works[i], ".wwtxn."));
|
||||
let runav: []str = [bins[i]];
|
||||
runcommand(root, strings.concat("bom-run-", tags[i]), runav,
|
||||
time.second, &out);
|
||||
expectexit(&out, 0);
|
||||
assert(out.stdout.len == 0 && out.stderr.len == 0);
|
||||
i += 1;
|
||||
};
|
||||
assert(same(readfile(bins[0]), readfile(bins[1])));
|
||||
let actions: []str = ["dep", "app"];
|
||||
let suffixes: []str = [".unit.ww", ".wwi", ".s", ".o", ".a"];
|
||||
let commandsuffixes: []str = [".unit.ww", ".wwi", ".s", ".o", ".a",
|
||||
".init.unit.ww", ".init.s", ".init.o"];
|
||||
let ai: i32 = 0;
|
||||
for (ai < actions.len) {
|
||||
let unit: str = readfile(strings.concat(works[0], "/", actions[ai],
|
||||
".unit.ww"));
|
||||
assert(!hasbom(unit) && has(unit, "\n package "));
|
||||
let si: i32 = 0;
|
||||
for (si < suffixes.len) {
|
||||
assert(same(readfile(strings.concat(works[0], "/", actions[ai],
|
||||
suffixes[si])), readfile(strings.concat(works[1], "/",
|
||||
actions[ai], suffixes[si]))));
|
||||
si += 1;
|
||||
};
|
||||
ai += 1;
|
||||
};
|
||||
let ci: i32 = suffixes.len;
|
||||
for (ci < commandsuffixes.len) {
|
||||
assert(same(readfile(strings.concat(works[0], "/app",
|
||||
commandsuffixes[ci])), readfile(strings.concat(works[1], "/app",
|
||||
commandsuffixes[ci]))));
|
||||
ci += 1;
|
||||
};
|
||||
|
||||
// Production, same-package, external-package, and test-only roles all get
|
||||
// their own offset-zero allowance and execute through the ordinary harness.
|
||||
writebomfile(strings.concat(tests, "/prod.ww"), strings.concat(
|
||||
"package bomtests;\n",
|
||||
"export fn value() i32 = { return 42; };\n"));
|
||||
writebomfile(strings.concat(tests, "/same_test.ww"), strings.concat(
|
||||
"package bomtests;\n",
|
||||
"@test fn same_bom() void = { assert(value() == 42); };\n"));
|
||||
writebomfile(strings.concat(tests, "/external_test.ww"), strings.concat(
|
||||
"package bomtests_test;\nimport bomtests;\n",
|
||||
"@test fn external_bom() void = { assert(bomtests.value() == 42); };\n"));
|
||||
writebomfile(strings.concat(onlytests, "/only_test.ww"),
|
||||
"package onlytests;\n@test fn only_bom() void = { assert(true); };\n");
|
||||
let testrefs: []str = ["", ""];
|
||||
let onlyrefs: []str = ["", ""];
|
||||
let testworks: []str = [strings.concat(root, "/test-work-c"),
|
||||
strings.concat(root, "/test-work-ww")];
|
||||
i = 0;
|
||||
for (i < stages.len) {
|
||||
mkdirall(testworks[i]);
|
||||
let av: []str = [driver(stages[i]), "test", "-w", testworks[i],
|
||||
"-I", source, "bomtests"];
|
||||
let out: commandout;
|
||||
runcommand(root, strings.concat("bom-test-", tags[i]), av,
|
||||
(120i64 * (time.second: i64)): time.duration, &out);
|
||||
expectexit(&out, 0);
|
||||
assert(out.stderr.len == 0 && has(out.stdout, "2 passed, 0 failed"));
|
||||
testrefs[i] = strings.dup(out.stdout);
|
||||
let onlyav: []str = [driver(stages[i]), "test", "-I", source,
|
||||
"onlytests"];
|
||||
runcommand(root, strings.concat("bom-test-only-", tags[i]), onlyav,
|
||||
(120i64 * (time.second: i64)): time.duration, &out);
|
||||
expectexit(&out, 0);
|
||||
assert(out.stderr.len == 0 && has(out.stdout, "1 passed, 0 failed"));
|
||||
onlyrefs[i] = strings.dup(out.stdout);
|
||||
i += 1;
|
||||
};
|
||||
assert(same(testrefs[0], testrefs[1]) && same(onlyrefs[0], onlyrefs[1]));
|
||||
let testactions: []str = ["bomtests-internal-test",
|
||||
"bomtests_test-external-test", "bomtests-test-main"];
|
||||
ai = 0;
|
||||
for (ai < testactions.len) {
|
||||
let testsuffixes: []str = suffixes;
|
||||
if (ai == 2) { testsuffixes = commandsuffixes; };
|
||||
let tsi: i32 = 0;
|
||||
for (tsi < testsuffixes.len) {
|
||||
assert(same(readfile(strings.concat(testworks[0], "/",
|
||||
testactions[ai], testsuffixes[tsi])),
|
||||
readfile(strings.concat(testworks[1], "/", testactions[ai],
|
||||
testsuffixes[tsi]))));
|
||||
tsi += 1;
|
||||
};
|
||||
if (ai < 2) {
|
||||
let testunit: str = readfile(strings.concat(testworks[0], "/",
|
||||
testactions[ai], ".unit.ww"));
|
||||
assert(!hasbom(testunit) && has(testunit, "\n package "));
|
||||
};
|
||||
ai += 1;
|
||||
};
|
||||
assert(!directoryhasnew(testworks[0]) && !directoryhasnew(testworks[1]));
|
||||
|
||||
// A later BOM is rejected during request parsing. Missing-import graph
|
||||
// resolution, compiler/assembler/linker producers, runtime, and publication
|
||||
// are therefore never reached; cold workdirs and products stay empty.
|
||||
writemidbomfile(strings.concat(bad, "/main.ww"),
|
||||
"package main;\n// ",
|
||||
"\nimport missing;\nfn main() i32 = { return 0; };\n");
|
||||
writefile(strings.concat(badtest, "/prod.ww"),
|
||||
"package badtest;\nfn value() i32 = { return 42; };\n");
|
||||
writemidbomfile(strings.concat(badtest, "/prod_test.ww"),
|
||||
"package badtest;\n// ",
|
||||
"\n@test fn must_not_run() void = { abort(\"ran\"); };\n");
|
||||
let baddiags: []str = ["", ""];
|
||||
let badtestdiags: []str = ["", ""];
|
||||
i = 0;
|
||||
for (i < stages.len) {
|
||||
let badwork: str = strings.concat(root, "/bad-work-", tags[i]);
|
||||
let badout: str = strings.concat(root, "/bad-output-", tags[i]);
|
||||
mkdirall(badwork);
|
||||
let av: []str = [driver(stages[i]), "build", "-w", badwork,
|
||||
"-I", source, "-o", badout, "bad"];
|
||||
let out: commandout;
|
||||
runcommand(root, strings.concat("bom-bad-build-", tags[i]), av,
|
||||
(60i64 * (time.second: i64)): time.duration, &out);
|
||||
expectexit(&out, 1);
|
||||
assert(out.stdout.len == 0);
|
||||
assert(has(out.stderr,
|
||||
":2:4: error: invalid BOM in the middle of the file\n"));
|
||||
assert(!has(out.stderr, "cannot find import")
|
||||
&& !has(out.stderr, "w6c failed"));
|
||||
baddiags[i] = strings.dup(out.stderr);
|
||||
assert(!os.exists(badout) && directoryisempty(badwork));
|
||||
let testav: []str = [driver(stages[i]), "test", "-I", source,
|
||||
"badtest"];
|
||||
runcommand(root, strings.concat("bom-bad-test-", tags[i]), testav,
|
||||
(60i64 * (time.second: i64)): time.duration, &out);
|
||||
expectexit(&out, 1);
|
||||
assert(same(out.stdout, "FAIL\n"));
|
||||
assert(has(out.stderr, "invalid BOM in the middle of the file")
|
||||
&& !has(out.stderr, "must_not_run ..."));
|
||||
badtestdiags[i] = strings.dup(out.stderr);
|
||||
i += 1;
|
||||
};
|
||||
assert(same(baddiags[0], baddiags[1])
|
||||
&& same(badtestdiags[0], badtestdiags[1]));
|
||||
|
||||
// A warm invalid edit cannot mutate committed action bytes or publication.
|
||||
// Restoring the exact leading-BOM source composes the same unit and reuses
|
||||
// the generation without invoking the compiler.
|
||||
let warmpath: str = strings.concat(warm, "/main.ww");
|
||||
let warmvalid: str =
|
||||
"package main;\nfn main() i32 = { return 0; };\n";
|
||||
writebomfile(warmpath, warmvalid);
|
||||
let wrapper: str = strings.concat(root, "/compiler-wrapper.sh");
|
||||
writeexecutable(wrapper, strings.concat(
|
||||
"#!/bin/sh\n",
|
||||
"printf 'compile\\n' >> \"$WW_BOM_TRACE\"\n",
|
||||
"exec \"$WW_BOM_REAL\" \"$@\"\n"));
|
||||
i = 0;
|
||||
for (i < stages.len) {
|
||||
let warmwork: str = strings.concat(root, "/warm-work-", tags[i]);
|
||||
let warmout: str = strings.concat(root, "/warm-output-", tags[i]);
|
||||
let trace: str = strings.concat(root, "/warm-trace-", tags[i]);
|
||||
mkdirall(warmwork); writefile(trace, "");
|
||||
let env: []str = os.getenvs();
|
||||
append(env, strings.concat("WW_W6C=", wrapper));
|
||||
append(env, strings.concat("WW_BOM_TRACE=", trace));
|
||||
append(env, strings.concat("WW_BOM_REAL=", driver(compilers[i])));
|
||||
let av: []str = [driver(stages[i]), "build", "-w", warmwork,
|
||||
"-I", source, "-o", warmout, "warm"];
|
||||
let out: commandout;
|
||||
runcommandenv(root, strings.concat("bom-warm-cold-", tags[i]), av,
|
||||
env, (120i64 * (time.second: i64)): time.duration, &out);
|
||||
expectexit(&out, 0);
|
||||
assert(out.stdout.len == 0 && out.stderr.len == 0
|
||||
&& readfile(trace).len != 0);
|
||||
let refs: []str = alloc([], commandsuffixes.len: u64)!;
|
||||
let si: i32 = 0;
|
||||
for (si < commandsuffixes.len) {
|
||||
append(refs, strings.dup(readfile(strings.concat(warmwork,
|
||||
"/warm", commandsuffixes[si]))));
|
||||
si += 1;
|
||||
};
|
||||
let binref: str = strings.dup(readfile(warmout));
|
||||
rewritefile(trace, "");
|
||||
rewritemidbomfile(warmpath, "package main;\n// ",
|
||||
"\nfn main() i32 = { return 0; };\n");
|
||||
runcommandenv(root, strings.concat("bom-warm-invalid-", tags[i]), av,
|
||||
env, (60i64 * (time.second: i64)): time.duration, &out);
|
||||
expectexit(&out, 1);
|
||||
assert(out.stdout.len == 0
|
||||
&& has(out.stderr, "invalid BOM in the middle of the file"));
|
||||
assert(readfile(trace).len == 0 && same(binref, readfile(warmout)));
|
||||
si = 0;
|
||||
for (si < commandsuffixes.len) {
|
||||
assert(same(refs[si], readfile(strings.concat(warmwork,
|
||||
"/warm", commandsuffixes[si]))));
|
||||
si += 1;
|
||||
};
|
||||
assert(!directoryhasnew(warmwork)
|
||||
&& !directoryhasfragment(warmwork, ".wwtxn."));
|
||||
rewritebomfile(warmpath, warmvalid);
|
||||
runcommandenv(root, strings.concat("bom-warm-restored-", tags[i]), av,
|
||||
env, (120i64 * (time.second: i64)): time.duration, &out);
|
||||
expectexit(&out, 0);
|
||||
assert(out.stdout.len == 0 && out.stderr.len == 0);
|
||||
assert(readfile(trace).len == 0 && same(binref, readfile(warmout))
|
||||
&& !directoryhasnew(warmwork));
|
||||
|
||||
// The marker is semantically whitespace but its three source bytes are
|
||||
// observable input: removing and re-adding it invalidates the unit just
|
||||
// as Go's source-content action does, without changing output bytes.
|
||||
rewritefile(trace, "");
|
||||
rewritefile(warmpath, warmvalid);
|
||||
runcommandenv(root, strings.concat("bom-warm-marker-removed-", tags[i]),
|
||||
av, env, (120i64 * (time.second: i64)): time.duration, &out);
|
||||
expectexit(&out, 0);
|
||||
assert(out.stdout.len == 0 && out.stderr.len == 0);
|
||||
assert(readfile(trace).len != 0 && same(binref, readfile(warmout)));
|
||||
assert(!same(refs[0], readfile(strings.concat(warmwork,
|
||||
"/warm.unit.ww"))));
|
||||
rewritefile(trace, "");
|
||||
rewritebomfile(warmpath, warmvalid);
|
||||
runcommandenv(root, strings.concat("bom-warm-marker-restored-", tags[i]),
|
||||
av, env, (120i64 * (time.second: i64)): time.duration, &out);
|
||||
expectexit(&out, 0);
|
||||
assert(out.stdout.len == 0 && out.stderr.len == 0);
|
||||
assert(readfile(trace).len != 0 && same(binref, readfile(warmout)));
|
||||
si = 0;
|
||||
for (si < commandsuffixes.len) {
|
||||
assert(same(refs[si], readfile(strings.concat(warmwork,
|
||||
"/warm", commandsuffixes[si]))));
|
||||
si += 1;
|
||||
};
|
||||
assert(!directoryhasnew(warmwork));
|
||||
i += 1;
|
||||
};
|
||||
|
||||
// Request-local source state: a valid Cstage build and an invalid WWstage
|
||||
// build overlap without sharing diagnostics, products, or transaction data.
|
||||
let pcwork: str = strings.concat(root, "/parallel-c-work");
|
||||
let pwwork: str = strings.concat(root, "/parallel-ww-work");
|
||||
let pcout: str = strings.concat(root, "/parallel-c-output");
|
||||
let pwout: str = strings.concat(root, "/parallel-ww-output");
|
||||
mkdirall(pcwork); mkdirall(pwwork);
|
||||
let pcav: []str = [driver("ww"), "build", "-w", pcwork,
|
||||
"-I", source, "-o", pcout, "app"];
|
||||
let pwav: []str = [driver("ww_ww"), "build", "-w", pwwork,
|
||||
"-I", source, "-o", pwout, "bad"];
|
||||
let pc: exec.command;
|
||||
pc.path = pcav[0]; pc.argv = pcav; pc.env = os.getenvs(); pc.dir = repo();
|
||||
pc.stdoutpath = strings.concat(root, "/parallel-c.stdout");
|
||||
pc.stderrpath = strings.concat(root, "/parallel-c.stderr");
|
||||
pc.deadline = time.add(time.now(time.clock.monotonic),
|
||||
(120i64 * (time.second: i64)): time.duration);
|
||||
pc.grace = (100i64 * (time.millisecond: i64)): time.duration;
|
||||
let pw: exec.command;
|
||||
pw.path = pwav[0]; pw.argv = pwav; pw.env = os.getenvs(); pw.dir = repo();
|
||||
pw.stdoutpath = strings.concat(root, "/parallel-ww.stdout");
|
||||
pw.stderrpath = strings.concat(root, "/parallel-ww.stderr");
|
||||
pw.deadline = time.add(time.now(time.clock.monotonic),
|
||||
(120i64 * (time.second: i64)): time.duration);
|
||||
pw.grace = (100i64 * (time.millisecond: i64)): time.duration;
|
||||
let pcp: exec.process;
|
||||
let pwp: exec.process;
|
||||
exec.start(&pcp, &pc); exec.start(&pwp, &pw);
|
||||
let pcdone: bool = false;
|
||||
let pwdone: bool = false;
|
||||
for (!pcdone || !pwdone) {
|
||||
if (!pcdone) { pcdone = exec.poll(&pcp); };
|
||||
if (!pwdone) { pwdone = exec.poll(&pwp); };
|
||||
if (!pcdone || !pwdone) {
|
||||
time.sleep(time.millisecond, time.clock.monotonic);
|
||||
};
|
||||
};
|
||||
assert(pcp.result.errno == 0 && pcp.result.cleanuperrno == 0
|
||||
&& pcp.result.termination == exec.termination.EXIT
|
||||
&& pcp.result.code == 0);
|
||||
assert(pwp.result.errno == 0 && pwp.result.cleanuperrno == 0
|
||||
&& pwp.result.termination == exec.termination.EXIT
|
||||
&& pwp.result.code == 1);
|
||||
assert(readfile(pc.stdoutpath).len == 0 && readfile(pc.stderrpath).len == 0);
|
||||
assert(readfile(pw.stdoutpath).len == 0
|
||||
&& has(readfile(pw.stderrpath), "invalid BOM in the middle of the file"));
|
||||
assert(same(readfile(pcout), readfile(bins[0])) && !os.exists(pwout));
|
||||
assert(directoryisempty(pwwork) && !directoryhasnew(pcwork)
|
||||
&& !directoryhasfragment(pcwork, ".wwtxn."));
|
||||
let runav: []str = [pcout];
|
||||
let out: commandout;
|
||||
runcommand(root, "bom-parallel-run", runav, time.second, &out);
|
||||
expectexit(&out, 0);
|
||||
assert(out.stdout.len == 0 && out.stderr.len == 0);
|
||||
assert(!directoryhasfragment(root, ".wwtxn.")
|
||||
&& !directoryhasfragment(root, ".install")
|
||||
&& !directoryhasnew(root));
|
||||
clean(root);
|
||||
};
|
||||
|
||||
@@ -61,6 +61,7 @@ struct row { const char *src, *expect; };
|
||||
|
||||
static const struct row rows[] = {
|
||||
{ "", "" },
|
||||
{ "\xef\xbb\xbf" "package main;", "package IDENT(main) ;" },
|
||||
{ " \t\n ", "" },
|
||||
{ "// comment\n", "" },
|
||||
{ "/* a /b/ c */", "" },
|
||||
@@ -172,7 +173,29 @@ runrewind(void)
|
||||
return ok;
|
||||
}
|
||||
|
||||
static int
|
||||
runbompos(void)
|
||||
{
|
||||
const char *src = "\xef\xbb\xbf" "package main;";
|
||||
Arena *a = newarena();
|
||||
Lex l;
|
||||
lexinit(&l, a, "<test>", src, strlen(src));
|
||||
Tok t = lexnext(&l);
|
||||
int ok = t.kind == TK_MODULE && t.pos.line == 1 && t.pos.col == 4;
|
||||
if (!ok)
|
||||
fprintf(stderr, "leading BOM position: want 1:4 got %d:%d\n",
|
||||
t.pos.line, t.pos.col);
|
||||
freearena(a);
|
||||
return ok;
|
||||
}
|
||||
|
||||
static const char *const errrows[] = {
|
||||
"\xef\xbb\xbf" "\xef\xbb\xbf" "package main;",
|
||||
"package \xef\xbb\xbf" "main;",
|
||||
"// \xef\xbb\xbf\n",
|
||||
"/* \xef\xbb\xbf */",
|
||||
"\"\xef\xbb\xbf\"",
|
||||
"'\xef\xbb\xbf'",
|
||||
"'\\uZ'", /* unexpected rune scanning for escape */
|
||||
"\"\\u00g0\"", /* non-hex digit inside a string escape */
|
||||
"'\\u00", /* unexpected EOF scanning for escape */
|
||||
@@ -194,6 +217,8 @@ main(void)
|
||||
}
|
||||
if (!runrewind())
|
||||
fail++;
|
||||
if (!runbompos())
|
||||
fail++;
|
||||
for (size_t i = 0; i < sizeof errrows / sizeof errrows[0]; i++) {
|
||||
if (!runerr(errrows[i])) {
|
||||
fprintf(stderr, "errrow %zu failed\n", i);
|
||||
|
||||
Reference in New Issue
Block a user