Hojun-Cho/ww
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

cmd: share package builds across test directories

Browse Source
This commit is contained in:
Hojun-Cho
2026-08-12 13:54:15 +09:00
parent 02574c88f1
commit 29e97ff8a1
3 changed files with 1143 additions and 322 deletions
Download Patch File Download Diff File Expand all files Collapse all files

769
cmd/ww/main.c
View File

File diff suppressed because it is too large Load Diff

38
internal/wwpackage/package.ww
View File

@@ -106,8 +106,8 @@ fn pkgusage() void = {
let s: str = strings.concat(
"usage: wwtest package [-c] [-list] [-j N] [-I DIR] [-w DIR] [-run|-filter GLOB] [-timeout-ms=N] [DIR | DIR/...] [-- GLOB ...]\n",
" *_test.ww is the sole test-source form; @test elsewhere is rejected\n",
" -c retains the compiled package binaries; -j N runs up to N directory builds or test binaries at once\n",
" -w DIR keys a persistent shared build workdir per package directory\n");
" -c retains the compiled package binaries; -j N runs up to N build or test processes at once\n",
" -w DIR keys one persistent shared build workdir for the test request\n");
pkgput(os.STDERR_FILENO, s);
};
@@ -530,8 +530,8 @@ fn pkgsetplanpaths(p: *pkgplan, groups: []pkggroup, root: str, index: i32,
if (!pkgmakedir(p.root)) { return false; };
p.workdir = "";
if (workroot.len != 0) {
// The first byte-sorted product name is stable metadata in the
// injective directory key; every product in this plan shares the dir.
// The request root and first byte-sorted product name make one stable
// escaped key for the command-scoped driver workdir.
p.workdir = strings.concat(workroot, "/",
pkgworkkey(p.dir, groups[p.start].pkg));
match (os.mkdirs(p.workdir, 448)) {
@@ -600,7 +600,7 @@ fn pkgreportcommand(kind: str, g: *pkggroup, r: *exec.result) void = {
fn pkgstartbuild(p: *pkgplan, groups: []pkggroup, builder: str, includes: []str,
h: *exec.process) void = {
let ba: []str = alloc([],
(8 + (p.end - p.start) * 5 + includes.len * 2): u64)!;
(8 + (p.end - p.start) * 6 + includes.len * 2): u64)!;
append(ba, builder);
append(ba, "test");
append(ba, "-c");
@@ -611,6 +611,7 @@ fn pkgstartbuild(p: *pkgplan, groups: []pkggroup, builder: str, includes: []str,
if (g.external) { append(ba, "external"); }
else { append(ba, "same"); };
append(ba, g.pkg);
append(ba, g.dir);
append(ba, g.bin);
append(ba, g.buildok);
i += 1;
@@ -976,20 +977,13 @@ export fn packagecommand(args: []str) int = {
"wwtest package: cannot use -o with multiple packages");
return 2;
};
let plans: []pkgplan = alloc([], groups.len: u64)!;
i = 0;
for (i < groups.len) {
let p: pkgplan;
p.dir = groups[i].dir;
p.start = i;
for (i < groups.len
&& strings.compare(groups[i].dir, p.dir) == 0) {
i += 1;
};
p.end = i;
p.state = PKGQUEUED;
append(plans, p);
};
let plans: []pkgplan = alloc([], 1u64)!;
let plan: pkgplan;
plan.dir = discoverroot;
plan.start = 0;
plan.end = groups.len;
plan.state = PKGQUEUED;
append(plans, plan);
let borrowed: str = temp.dir();
let tmproot: str = strings.dup(borrowed);
@@ -1009,9 +1003,9 @@ export fn packagecommand(args: []str) int = {
i += 1;
};
// Build one shared package plan per directory. As soon as a plan completes,
// its successful products may run under the same global -j bound while
// other directory builds remain active. Emission stays ordered below.
// Build the complete request in one command-owned package universe. Once
// the union build completes, successful products run under the same global
// -j bound. Emission stays in byte-sorted group order below.
let handles: []exec.process = alloc([], plans.len: u64)!;
i = 0;
for (i < plans.len) {

610
selfhost/cmd/ww/main.ww
View File

@@ -417,8 +417,13 @@ fn dirfileattest(dirpath: *u8, name: *u8) i32 = {
def SEP_VARIANT_PRODUCTION: i32 = 0;
def SEP_VARIANT_SAME_TEST: i32 = 1;
def SEP_VARIANT_EXTERNAL: i32 = 2;
def SEP_ROLE_NORMAL: i32 = 0;
def SEP_ROLE_EXTERNAL_PRODUCTION: i32 = 1;
def SEP_ROLE_TEST_SUPPORT: i32 = 2;
def SEP_TEST_SUPPORT_MODULE: str = "__wwtest";
def SEP_MAXPRODUCT: i32 = 2;
def SEP_MAXPRODUCT: i32 = 256;
def SEP_MAXCONTEXT: i32 = 257;
def SEP_ARTIFACT_MAX: i32 = 1024;
// Classify a selected directory entry: 1 production, 2 test, 0 skipped,
// -1 @test outside *_test.ww, -2 non-regular source.
@@ -719,38 +724,62 @@ type lflags = struct {
def SEP_MAXPKG: i32 = 256;
type sepbind = struct {
kind: u8,
name: str,
target: *u8,
};
type seppkg = struct {
path: *u8, // dotted import path, NUL-term; root path[0]==0
entry: *u8, // resolved package dir (or file, file root), NUL-term
artifact: *u8, // non-importable product-root artifact key
name: *u8, // validated declared name; directory packages only
testpackage: *u8,
sources: **u8, // owned, byte-sorted selected paths; dirs only
nsources: i32,
isdir: i32,
variant: i32,
role: i32,
root: bool,
failed: bool,
testsupport: bool,
loaded: bool,
emitcontext: i32,
contextstate: []u8,
bindings: []sepbind,
deps: []i32, // direct-dep indices into sepgraph.pkg
ndeps: i32,
color: i32, // tri-color DFS: 0 white, 1 gray, 2 black
};
type sepcontext = struct {
root: *u8,
searchpath: *u8,
};
type sepgraph = struct {
pkg: []seppkg, // alloc'd SEP_MAXPKG
n: i32,
context: []sepcontext,
ncontext: i32,
supportcontext: i32,
};
type sepproduct = struct {
dir: *u8,
out: *u8,
testpackage: *u8,
status: *u8,
artifact: *u8,
variant: i32,
context: i32,
root: i32,
};
fn sepfindoraddvariant(g: *sepgraph, path: *u8, entry: *u8,
isdir: i32, variant: i32, testpackage: *u8, root: bool) i32 = {
isdir: i32, variant: i32, testpackage: *u8, role: i32,
artifact: *u8, root: bool) i32 = {
if (cstrlen(path) >= 256u64) {
cerr("ww: package path is too long (limit 255 bytes)\n");
return -1;
@@ -759,23 +788,69 @@ fn sepfindoraddvariant(g: *sepgraph, path: *u8, entry: *u8,
for (i < g.n) {
let samelocation: bool = os.samefile(pathstr(g.pkg[i].entry),
pathstr(entry));
let testrootpair: bool = samelocation && isdir != 0
&& g.pkg[i].isdir != 0
&& ((root && g.pkg[i].root
&& variant != g.pkg[i].variant)
|| (root && !g.pkg[i].root
&& variant != SEP_VARIANT_PRODUCTION
&& g.pkg[i].variant == SEP_VARIANT_PRODUCTION)
|| (!root && g.pkg[i].root
if (root || g.pkg[i].root) {
if (root && g.pkg[i].root) {
if (!samelocation) { i += 1; continue; };
if (variant != g.pkg[i].variant) { i += 1; continue; };
cerr("ww: duplicate package-test root ");
cerr(pathstr(entry)); cerr("\n");
return -1;
};
// A cycle back to an ordinary production root reuses that root after
// loading assigns its declared identity. Test roots stay distinct.
if (samelocation
&& variant == SEP_VARIANT_PRODUCTION
&& g.pkg[i].variant != SEP_VARIANT_PRODUCTION));
if (cstreq(g.pkg[i].path, path)) {
if (testrootpair) {
&& g.pkg[i].variant == SEP_VARIANT_PRODUCTION
&& cstreq(g.pkg[i].path, path)) {
return i;
};
if (!samelocation) { i += 1; continue; };
let rootvariant: i32 = variant;
let productionvariant: i32 = g.pkg[i].variant;
if (!root) {
rootvariant = g.pkg[i].variant;
productionvariant = variant;
};
if (isdir != 0 && g.pkg[i].isdir != 0
&& rootvariant != SEP_VARIANT_PRODUCTION
&& productionvariant == SEP_VARIANT_PRODUCTION) {
i += 1;
continue;
};
if (!os.samefile(pathstr(g.pkg[i].entry), pathstr(entry))
|| g.pkg[i].variant != variant) {
cerr("ww: package directory "); cerr(pathstr(entry));
cerr(" has incompatible root and production variants\n");
return -1;
};
let samepath: bool = cstreq(g.pkg[i].path, path);
let sameartifact: bool = true;
if (role == SEP_ROLE_EXTERNAL_PRODUCTION) {
sameartifact = g.pkg[i].artifact != nil && artifact != nil
&& cstreq(g.pkg[i].artifact, artifact);
};
let sameaction: bool = samepath && samelocation
&& g.pkg[i].variant == variant && g.pkg[i].role == role
&& sameartifact;
if (sameaction) { return i; };
// An external test consumes the one canonical production action for
// its directory. The role only separates physically different
// packages that happen to use the same source qualifier.
if (samepath && samelocation
&& g.pkg[i].variant == SEP_VARIANT_PRODUCTION
&& variant == SEP_VARIANT_PRODUCTION
&& ((role == SEP_ROLE_EXTERNAL_PRODUCTION
&& g.pkg[i].role == SEP_ROLE_NORMAL)
|| (role == SEP_ROLE_NORMAL
&& g.pkg[i].role == SEP_ROLE_EXTERNAL_PRODUCTION))) {
return i;
};
if (samepath && (role == SEP_ROLE_EXTERNAL_PRODUCTION
|| g.pkg[i].role == SEP_ROLE_EXTERNAL_PRODUCTION)) {
i += 1;
continue;
};
if (samepath) {
if (!samelocation || g.pkg[i].variant != variant
|| g.pkg[i].role != role) {
cerr("ww: package "); cerr(pathstr(path));
cerr(" resolves to more than one location\n");
return -1;
@@ -783,7 +858,8 @@ fn sepfindoraddvariant(g: *sepgraph, path: *u8, entry: *u8,
return i;
};
if (samelocation) {
if (testrootpair) {
if (role == SEP_ROLE_TEST_SUPPORT
|| g.pkg[i].role == SEP_ROLE_TEST_SUPPORT) {
i += 1;
continue;
};
@@ -807,6 +883,7 @@ fn sepfindoraddvariant(g: *sepgraph, path: *u8, entry: *u8,
let elen: u64 = cstrlen(entry);
g.pkg[g.n].path = arenadupcstr(path, plen);
g.pkg[g.n].entry = arenadupcstr(entry, elen);
g.pkg[g.n].artifact = artifact;
g.pkg[g.n].name = nil;
g.pkg[g.n].testpackage = nil;
if (testpackage != nil) {
@@ -817,9 +894,17 @@ fn sepfindoraddvariant(g: *sepgraph, path: *u8, entry: *u8,
g.pkg[g.n].nsources = 0;
g.pkg[g.n].isdir = isdir;
g.pkg[g.n].variant = variant;
g.pkg[g.n].role = role;
g.pkg[g.n].root = root;
g.pkg[g.n].failed = false;
g.pkg[g.n].testsupport = false;
g.pkg[g.n].loaded = false;
g.pkg[g.n].emitcontext = -1;
let cslot: []u8 = alloc([], SEP_MAXCONTEXT: u64)!;
cslot.len = SEP_MAXCONTEXT;
g.pkg[g.n].contextstate = cslot;
let emptybindings: []sepbind;
g.pkg[g.n].bindings = emptybindings;
let dslot: []i32 = alloc([], SEP_MAXPKG: u64)!;
dslot.len = SEP_MAXPKG;
g.pkg[g.n].deps = dslot;
@@ -832,7 +917,13 @@ fn sepfindoraddvariant(g: *sepgraph, path: *u8, entry: *u8,
fn sepfindoradd(g: *sepgraph, path: *u8, entry: *u8, isdir: i32) i32 = {
return sepfindoraddvariant(g, path, entry, isdir,
SEP_VARIANT_PRODUCTION, nil, false);
SEP_VARIANT_PRODUCTION, nil, SEP_ROLE_NORMAL, nil, false);
};
fn sepfindoraddrole(g: *sepgraph, path: *u8, entry: *u8, isdir: i32,
role: i32, artifact: *u8) i32 = {
return sepfindoraddvariant(g, path, entry, isdir,
SEP_VARIANT_PRODUCTION, nil, role, artifact, false);
};
// Release the package-owned directory-membership lists through one graph
@@ -859,29 +950,83 @@ fn sepgraphfree(g: *sepgraph) void = {
};
};
fn sepcontextfor(g: *sepgraph, root: *u8, incs: *u8,
toolsrcdir: *u8) i32 = {
let cap: u64 = (os.PATH_MAX: u64) * 2u64;
let need: u64 = cstrlen(root) + 1u64 + cstrlen(toolsrcdir) + 1u64;
if (incs != nil && incs[0u64] != 0u8) {
need += cstrlen(incs) + 1u64;
};
if (need > cap) {
cerr("ww: package import search path is too long\n");
return -1;
};
let search: []u8 = alloc([], cap)!;
search.len = cap: i32;
let off: u64 = cstrinto(search.ptr, 0u64, root);
off = byteinto(search.ptr, off, 58u8);
if (incs != nil && incs[0u64] != 0u8) {
off = cstrinto(search.ptr, off, incs);
off = byteinto(search.ptr, off, 58u8);
};
off = cstrinto(search.ptr, off, toolsrcdir);
cstrseal(search.ptr, off);
let i: i32 = 0;
for (i < g.ncontext) {
if (cstreq(g.context[i].searchpath, search.ptr)) { return i; };
i += 1;
};
if (g.ncontext >= SEP_MAXCONTEXT) {
cerr("ww: too many package import contexts\n");
return -1;
};
g.context[g.ncontext].root = root;
g.context[g.ncontext].searchpath = search.ptr;
let result: i32 = g.ncontext;
g.ncontext += 1;
return result;
};
// Build one artifact path. Test roots use distinct names even though both
// compiler units reset to the bare executable namespace.
fn sepfname(g: *sepgraph, pi: i32, scratch: *u8, suffix: str) *u8 = {
let buf: []u8 = alloc([], (os.PATH_MAX: u64))!;
buf.len = os.PATH_MAX;
let buf: []u8 = alloc([], SEP_ARTIFACT_MAX: u64)!;
buf.len = SEP_ARTIFACT_MAX;
let off: u64 = cstrinto(buf.ptr, 0u64, scratch);
off = byteinto(buf.ptr, off, 47u8); // '/'
if (g.pkg[pi].root
&& g.pkg[pi].variant == SEP_VARIANT_SAME_TEST) {
off = strinto(buf.ptr, off, "__ww-test-same");
} else { if (g.pkg[pi].root
&& g.pkg[pi].variant == SEP_VARIANT_EXTERNAL) {
off = strinto(buf.ptr, off, "__ww-test-external");
if (g.pkg[pi].artifact != nil) {
off = cstrinto(buf.ptr, off, g.pkg[pi].artifact);
} else { if (g.pkg[pi].path[0u64] != 0u8) {
off = cstrinto(buf.ptr, off, g.pkg[pi].path);
} else {
off = strinto(buf.ptr, off, "__root");
}; }; };
}; };
off = strinto(buf.ptr, off, suffix);
cstrseal(buf.ptr, off);
return buf.ptr;
};
// sepfname uses SEP_ARTIFACT_MAX storage. Validate the longest suffix once
// before opening files so two action identities can never alias by truncation.
fn sepvalidateartifactpaths(g: *sepgraph, scratch: *u8) i32 = {
let i: i32 = 0;
for (i < g.n) {
let base: *u8 = g.pkg[i].artifact;
if (base == nil) {
base = g.pkg[i].path;
if (base[0u64] == 0u8) { base = "__root\0".ptr; };
};
let need: u64 = cstrlen(scratch) + 1u64 + cstrlen(base)
+ ".unit.new".len: u64 + 1u64;
if (need > SEP_ARTIFACT_MAX: u64) {
cerr("ww: package artifact path is too long\n");
return -1;
};
i += 1;
};
return 0;
};
fn sepexternalname(pkg: *seppkg, path: *u8, n: u64,
leafonly: bool) bool = {
if (pkg.variant != SEP_VARIANT_EXTERNAL || pkg.testpackage == nil) {
@@ -912,13 +1057,59 @@ fn sepexternalproduction(pkg: *seppkg, path: *u8, n: u64) bool = {
return sepexternalname(pkg, path, n, false);
};
fn sepbindadd(bindings: *[]sepbind, kind: u8, name: str,
target: *u8) void = {
let i: i32 = 0;
for (i < len(*bindings)) {
let b: sepbind = (*bindings)[i];
if (b.kind == kind && syntax.streq(b.name, name)) {
if (target == nil && b.target == nil) { return; };
if (target != nil && b.target != nil
&& os.samefile(pathstr(target), pathstr(b.target))) {
return;
};
};
i += 1;
};
append(*bindings, sepbind {
kind = kind,
name = strings.dup(name),
target = target,
});
};
fn sepbindsame(a: []sepbind, b: []sepbind) bool = {
if (len(a) != len(b)) { return false; };
let i: i32 = 0;
for (i < len(a)) {
let found: bool = false;
let j: i32 = 0;
for (j < len(b)) {
if (a[i].kind == b[j].kind
&& syntax.streq(a[i].name, b[j].name)) {
if (a[i].target == nil && b[j].target == nil) {
found = true;
} else { if (a[i].target != nil && b[j].target != nil
&& os.samefile(pathstr(a[i].target),
pathstr(b[j].target))) {
found = true;
}; };
};
j += 1;
};
if (!found) { return false; };
i += 1;
};
return true;
};
// Scan one already-selected source file for its leading package clause
// (when it is an owned directory source) and top-level imports. A DIRECTORY
// import is a package boundary: add as a direct dep of pi. A FILE import is an
// intra-package split: fold its imports into pi. Mirrors cstage
// sep_scan_file (collects PATHS, not bytes).
fn sepscanfile(g: *sepgraph, pi: i32, file: *u8, searchpath: *u8,
fv: *expctx, ownedsource: i32) i32 = {
fv: *expctx, bindings: *[]sepbind, ownedsource: i32) i32 = {
let fview: str;
fview.ptr = file;
fview.len = cstrlen(file): i32;
@@ -1039,6 +1230,7 @@ fn sepscanfile(g: *sepgraph, pi: i32, file: *u8, searchpath: *u8,
};
if (ipath != nil) {
if (isdir != 0) {
sepbindadd(bindings, 'D': u8, u.usepath, ipath);
let self: bool = os.samefile(pathstr(ipath),
pathstr(g.pkg[pi].entry));
if (self && sepexternalname(&g.pkg[pi], idp, idn, true)) {
@@ -1054,11 +1246,33 @@ fn sepscanfile(g: *sepgraph, pi: i32, file: *u8, searchpath: *u8,
cerr("' cannot import itself\n");
return -1;
};
let runtimeproduction: bool = false;
if (externalproduction) {
let gi: i32 = 0;
for (gi < g.n) {
if (g.pkg[gi].testsupport
&& syntax.streq(pathstr(g.pkg[gi].path),
u.usepath)
&& os.samefile(pathstr(g.pkg[gi].entry),
pathstr(ipath))) {
runtimeproduction = true;
};
gi += 1;
};
};
let nm: []u8 = alloc([], idn + 1u64)!;
let k: u64 = 0u64;
for (k < idn) { nm[k] = idp[k]; k += 1u64; };
nm[idn] = 0u8;
let di: i32 = sepfindoradd(g, nm.ptr, ipath, 1);
let di: i32 = -1;
if (externalproduction && !runtimeproduction) {
let art: *u8 = appendlit(g.pkg[pi].artifact,
"-production");
di = sepfindoraddrole(g, nm.ptr, ipath, 1,
SEP_ROLE_EXTERNAL_PRODUCTION, art);
} else {
di = sepfindoradd(g, nm.ptr, ipath, 1);
};
if (di < 0) { return -1; };
let seen: bool = false;
let m: i32 = 0;
@@ -1072,7 +1286,9 @@ fn sepscanfile(g: *sepgraph, pi: i32, file: *u8, searchpath: *u8,
g.pkg[pi].ndeps += 1;
};
} else {
if (sepscanfile(g, pi, ipath, searchpath, fv, 0) < 0) {
sepbindadd(bindings, 'F': u8, u.usepath, ipath);
if (sepscanfile(g, pi, ipath, searchpath, fv,
bindings, 0) < 0) {
return -1;
};
};
@@ -1098,6 +1314,8 @@ fn sepscanfile(g: *sepgraph, pi: i32, file: *u8, searchpath: *u8,
os.write(2, idp, idn);
cerr("\n");
return -1;
} else {
sepbindadd(bindings, 'I': u8, u.usepath, nil);
};
};
};
@@ -1106,21 +1324,41 @@ fn sepscanfile(g: *sepgraph, pi: i32, file: *u8, searchpath: *u8,
return 0;
};
fn sepdepcmp(g: *sepgraph, a: i32, b: i32) i32 = {
let r: i32 = strings.compare(pathstr(g.pkg[a].path),
pathstr(g.pkg[b].path)): i32;
if (r != 0) { return r; };
if (g.pkg[a].role < g.pkg[b].role) { return -1; };
if (g.pkg[a].role > g.pkg[b].role) { return 1; };
if (g.pkg[a].artifact == nil && g.pkg[b].artifact == nil) { return 0; };
if (g.pkg[a].artifact == nil) { return -1; };
if (g.pkg[b].artifact == nil) { return 1; };
return strings.compare(pathstr(g.pkg[a].artifact),
pathstr(g.pkg[b].artifact)): i32;
};
// Load pi once: a directory node takes ownership of its sorted production
// paths, then the same stored list supplies package-name validation and
// dependency scanning. Recurse over the resulting edges. `color` doubles as
// a loaded marker (2); reset to white before topo.
fn seploadpkg(g: *sepgraph, pi: i32, searchpath: *u8) i32 = {
if (g.pkg[pi].color == 2) {
// paths, then every selected-root context verifies the same canonical import
// bindings before the package is compiled once.
fn seploadpkg(g: *sepgraph, pi: i32, context: i32) i32 = {
if (g.pkg[pi].testsupport
&& g.supportcontext >= 0) { context = g.supportcontext; };
if (context < 0 || context >= g.ncontext) { return -1; };
if (g.pkg[pi].contextstate[context] == 2u8) {
if (g.pkg[pi].failed) { return -1; };
return 0;
};
g.pkg[pi].color = 2;
if (g.pkg[pi].contextstate[context] == 1u8) { return 0; };
g.pkg[pi].contextstate[context] = 1u8;
let searchpath: *u8 = g.context[context].searchpath;
let fv: expctx;
fv.out = -1;
fv.dirs = searchpath;
fv.visit = nil;
let bindings: []sepbind;
let rc: i32 = 0;
if (!g.pkg[pi].loaded) {
g.pkg[pi].loaded = true;
if (g.pkg[pi].isdir != 0) {
let sources: **u8;
let nsources: i32;
@@ -1142,11 +1380,14 @@ fn seploadpkg(g: *sepgraph, pi: i32, searchpath: *u8) i32 = {
cerr(": directory contains no WW package sources\n");
rc = -1;
}; }; };
};
};
if (g.pkg[pi].isdir != 0) {
let i: i32 = 0;
for (i < g.pkg[pi].nsources) {
if (rc == 0) {
rc = sepscanfile(g, pi, g.pkg[pi].sources[i],
searchpath, &fv, 1);
searchpath, &fv, &bindings, 1);
};
i += 1;
};
@@ -1168,10 +1409,28 @@ fn seploadpkg(g: *sepgraph, pi: i32, searchpath: *u8) i32 = {
rc = -1;
};
};
} else {
rc = sepscanfile(g, pi, g.pkg[pi].entry, searchpath, &fv, 0);
} else { if (rc == 0) {
rc = sepscanfile(g, pi, g.pkg[pi].entry, searchpath,
&fv, &bindings, 0);
}; };
if (rc == 0 && g.pkg[pi].emitcontext < 0) {
g.pkg[pi].bindings = bindings;
g.pkg[pi].emitcontext = context;
} else { if (rc == 0 && !sepbindsame(g.pkg[pi].bindings, bindings)) {
cerr("ww: package ");
if (g.pkg[pi].path[0u64] != 0u8) {
cerr(pathstr(g.pkg[pi].path));
} else { cerr(pathstr(g.pkg[pi].name)); };
cerr(" resolves imports differently in ");
cerr(pathstr(g.context[g.pkg[pi].emitcontext].root));
cerr(" and "); cerr(pathstr(g.context[context].root)); cerr("\n");
rc = -1;
}; };
if (rc < 0) {
g.pkg[pi].contextstate[context] = 2u8;
g.pkg[pi].failed = true;
return rc;
};
if (rc < 0) { g.pkg[pi].failed = true; return rc; };
if (g.pkg[pi].root && g.pkg[pi].path[0u64] == 0u8
&& g.pkg[pi].name != nil) {
g.pkg[pi].path = arenadupcstr(g.pkg[pi].name,
@@ -1181,18 +1440,18 @@ fn seploadpkg(g: *sepgraph, pi: i32, searchpath: *u8) i32 = {
for (si < g.pkg[pi].ndeps) {
let v: i32 = g.pkg[pi].deps[si];
let sj: i32 = si;
for (sj > 0 && strings.compare(
pathstr(g.pkg[g.pkg[pi].deps[sj - 1]].path),
pathstr(g.pkg[v].path)) > 0) {
for (sj > 0 && sepdepcmp(g,
g.pkg[pi].deps[sj - 1], v) > 0) {
g.pkg[pi].deps[sj] = g.pkg[pi].deps[sj - 1];
sj -= 1;
};
g.pkg[pi].deps[sj] = v;
si += 1;
};
g.pkg[pi].contextstate[context] = 2u8;
let k: i32 = 0;
for (k < g.pkg[pi].ndeps) {
if (seploadpkg(g, g.pkg[pi].deps[k], searchpath) < 0) {
if (seploadpkg(g, g.pkg[pi].deps[k], context) < 0) {
g.pkg[pi].failed = true;
return -1;
};
@@ -1242,6 +1501,30 @@ fn septopovisit(g: *sepgraph, pi: i32, order: []i32, no: *i32,
return 0;
};
fn sepvalidatemoduleclosure(g: *sepgraph, order: []i32, n: i32,
includeroot: bool) i32 = {
let i: i32 = 0;
for (i < n) {
let a: i32 = order[i];
if ((includeroot || !g.pkg[a].root)
&& g.pkg[a].path[0u64] != 0u8) {
let j: i32 = i + 1;
for (j < n) {
let b: i32 = order[j];
if ((includeroot || !g.pkg[b].root)
&& cstreq(g.pkg[a].path, g.pkg[b].path)) {
cerr("ww: product closure contains multiple packages named ");
cerr(pathstr(g.pkg[a].path)); cerr("\n");
return -1;
};
j += 1;
};
};
i += 1;
};
return 0;
};
// Emit one of pi's own source files into the sep-unit under the
// //ww:module-reset primary boundary (so -c emits its decls, imported
// ==0). DIRECTORY imports are skipped (provided as `.wwi` ahead);
@@ -1357,7 +1640,10 @@ fn sepemitbody(fd: i32, path: *u8, visit: *expctx, searchpath: *u8,
// //ww:module-reset. Compiler exports are self-contained for public type
// facts; the linker separately retains the reachable archive closure.
fn sepcomposeunit(g: *sepgraph, pi: i32, scratch: *u8,
searchpath: *u8, unitf: *u8) i32 = {
unitf: *u8) i32 = {
if (g.pkg[pi].emitcontext < 0
|| g.pkg[pi].emitcontext >= g.ncontext) { return -1; };
let searchpath: *u8 = g.context[g.pkg[pi].emitcontext].searchpath;
let u: i32 = os.open(pathstr(unitf), os.flag.WRONLY | os.flag.CREATE | os.flag.TRUNC, 420i32); // 0o644
if (u < 0) {
cerr("ww: cannot open unit\n");
@@ -1607,9 +1893,9 @@ fn copyfileatomic(src: *u8, dst: *u8) i32 = {
fn workdirstamptext(istest: i32, emitasm: i32) str = {
if (istest != 0) {
if (emitasm != 0) {
return "ww workdir fmt 4 mode test asm 1\n";
return "ww workdir fmt 5 mode test asm 1\n";
};
return "ww workdir fmt 4 mode test asm 0\n";
return "ww workdir fmt 5 mode test asm 0\n";
};
if (emitasm != 0) {
return "ww workdir fmt 4 mode build asm 1\n";
@@ -1669,6 +1955,45 @@ fn recordproductstatus(path: *u8) i32 = {
return os.rename(pathstr(tmpp), pathstr(path));
};
// Remove every committed unit voucher before a stale-tool pass. Remaining
// artifacts cannot be reused without their matching unit, so a partial pass
// may safely record its new tool identity: successful actions have current
// units and failed/no-longer-requested actions have none.
fn invalidateworkdirunits(scratch: *u8) i32 = {
let fd: i32 = os.open(pathstr(scratch), os.flag.RDONLY, 0i32);
if (fd < 0) { return -1; };
let buf: []u8 = alloc([], 8192u64)!;
buf.len = 8192;
let rc: i32 = 0;
let r: i64 = os.getdents64(fd, buf.ptr, 8192u64);
for (r > 0i64 && rc == 0) {
let off: u64 = 0u64;
for (off < r: u64) {
let reclen: u64 = (buf[off + 16u64]): u64
+ ((buf[off + 17u64]): u64) * 256u64;
if (reclen == 0u64) { rc = -1; break; };
let name: *u8 = buf.ptr + off + 19u64;
let ns: str = pathstr(name);
if (strings.hassuffix(ns, ".unit.ww")) {
if (cstrlen(scratch) + 1u64 + cstrlen(name) + 1u64
> os.PATH_MAX: u64) {
rc = -1;
break;
};
let path: *u8 = joinpath(scratch, name);
let rr: i32 = os.remove(pathstr(path));
if (rr != 0 && rr != -2) { rc = -1; break; };
};
off += reclen;
};
if (rc == 0) { r = os.getdents64(fd, buf.ptr, 8192u64); };
};
if (r < 0i64) { rc = -1; };
if (os.close(fd) != 0) { rc = -1; };
if (rc != 0) { cerr("ww: cannot invalidate stale package units\n"); };
return rc;
};
// cerrpath — the "ww: <head><path>\n" diagnostic shape shared by the
// workdir error sites; byte-identical wording to the cstage twin's
// fprintf(..., "%s", path) forms.
@@ -1735,20 +2060,6 @@ fn buildonesepimpl(selfdir: *u8, src: *u8, entryisdir: i32,
};
};
// searchpath = srcd + ':' + incs + ':' + dotdotlib.
let searchpath: []u8 = alloc([], (os.PATH_MAX: u64) * 3u64)!;
searchpath.len = ((os.PATH_MAX: u64) * 3u64): i32;
{
let off: u64 = cstrinto(searchpath.ptr, 0u64, srcd.ptr);
off = byteinto(searchpath.ptr, off, 58u8); // ':'
if (incs[0u64] != 0u8) {
off = cstrinto(searchpath.ptr, off, incs);
off = byteinto(searchpath.ptr, off, 58u8);
};
off = cstrinto(searchpath.ptr, off, dotdotlib.ptr);
cstrseal(searchpath.ptr, off);
};
let stem: []u8 = alloc([], (os.PATH_MAX: u64))!;
stem.len = os.PATH_MAX;
if (entryisdir != 0) {
@@ -1820,6 +2131,7 @@ fn buildonesepimpl(selfdir: *u8, src: *u8, entryisdir: i32,
if (!fileequal(toola, a6)) { staleall = true; };
};
};
if (staleall && invalidateworkdirunits(scratch) != 0) { return 1; };
};
let libwwrt: []u8 = alloc([], (os.PATH_MAX: u64))!;
@@ -1832,15 +2144,33 @@ fn buildonesepimpl(selfdir: *u8, src: *u8, entryisdir: i32,
let pkgslot: []seppkg = alloc([], SEP_MAXPKG: u64)!;
pkgslot.len = SEP_MAXPKG;
let g: *sepgraph = alloc(sepgraph{pkg = pkgslot, n = 0})!;
let contextslot: []sepcontext = alloc([], SEP_MAXCONTEXT: u64)!;
contextslot.len = SEP_MAXCONTEXT;
let g: *sepgraph = alloc(sepgraph{
pkg = pkgslot,
n = 0,
context = contextslot,
ncontext = 0,
supportcontext = -1,
})!;
if (graphout != nil) { *graphout = g; };
let rootpath: *u8 = "\0".ptr;
if (packageonly != 0 && rootidentity != nil) { rootpath = rootidentity; };
let producti: i32 = 0;
for (producti < nproducts) {
products[producti].root = sepfindoraddvariant(g, rootpath, src,
let entry: *u8 = src;
if (products[producti].dir != nil) {
entry = products[producti].dir;
};
let contextroot: *u8 = entry;
if (entryisdir == 0) { contextroot = srcd.ptr; };
products[producti].context = sepcontextfor(g, contextroot,
incs, dotdotlib.ptr);
if (products[producti].context < 0) { return 1; };
products[producti].root = sepfindoraddvariant(g, rootpath, entry,
entryisdir, products[producti].variant,
products[producti].testpackage, true);
products[producti].testpackage,
SEP_ROLE_NORMAL, products[producti].artifact, true);
if (products[producti].root < 0) { return 1; };
producti += 1;
};
@@ -1855,40 +2185,60 @@ fn buildonesepimpl(selfdir: *u8, src: *u8, entryisdir: i32,
let tp: *u8 = locateimport(dotdotlib.ptr, "test".ptr,
"test".len: u64, &td);
if (tp != nil) {
let rootissupport: bool = entryisdir != 0
&& os.samefile(pathstr(tp), pathstr(src));
g.supportcontext = sepcontextfor(g, dotdotlib.ptr, nil,
dotdotlib.ptr);
if (g.supportcontext < 0) { return 1; };
let collision: bool = false;
producti = 0;
for (!rootissupport && producti < nproducts) {
for (producti < nproducts) {
let root: i32 = products[producti].root;
let rootissupport: bool = entryisdir != 0
&& os.samefile(pathstr(tp),
pathstr(g.pkg[root].entry));
let name: *u8 = products[producti].testpackage;
if (name != nil && (cstreqlit(name, "test")
if (!rootissupport && name != nil
&& (cstreqlit(name, "test")
|| cstreqlit(name, "test_test"))) {
collision = true;
};
producti += 1;
};
if (!rootissupport && !collision) {
producti = 0;
for (producti < nproducts && !collision) {
let ud: i32 = 0;
let up: *u8 = locateimport(searchpath.ptr, "test".ptr,
let up: *u8 = locateimport(
g.context[products[producti].context].searchpath,
"test".ptr,
"test".len: u64, &ud);
if (up != nil && !os.samefile(pathstr(tp), pathstr(up))) {
collision = true;
};
producti += 1;
};
if (collision) { testsupportmodule = SEP_TEST_SUPPORT_MODULE; };
producti = 0;
for (producti < nproducts) {
let root: i32 = products[producti].root;
let rootissupport: bool = entryisdir != 0
&& os.samefile(pathstr(tp),
pathstr(g.pkg[root].entry));
// A same-test build of the runtime package already owns run
// and its source imports. An external test still needs the
// colocated production node, also its support dependency.
if (rootissupport
&& syntax.streq(testsupportmodule, "test")
&& products[producti].variant != SEP_VARIANT_EXTERNAL) {
producti += 1;
continue;
};
let ti: i32 = sepfindoradd(g,
testsupportmodule.ptr, tp, td);
let ti: i32 = -1;
if (syntax.streq(testsupportmodule,
SEP_TEST_SUPPORT_MODULE)) {
ti = sepfindoraddrole(g, testsupportmodule.ptr, tp,
td, SEP_ROLE_TEST_SUPPORT, nil);
} else {
ti = sepfindoradd(g, testsupportmodule.ptr, tp, td);
};
if (ti < 0) { return 1; };
g.pkg[ti].testsupport = true;
let seen: bool = false;
@@ -1910,7 +2260,7 @@ fn buildonesepimpl(selfdir: *u8, src: *u8, entryisdir: i32,
producti = 0;
for (producti < nproducts) {
let root: i32 = products[producti].root;
if (seploadpkg(g, root, searchpath.ptr) < 0) {
if (seploadpkg(g, root, products[producti].context) < 0) {
g.pkg[root].failed = true;
producti += 1;
continue;
@@ -1924,6 +2274,7 @@ fn buildonesepimpl(selfdir: *u8, src: *u8, entryisdir: i32,
};
producti += 1;
};
if (sepvalidateartifactpaths(g, scratch) < 0) { return 1; };
let rootpackage: bool = packageonly != 0;
if (rootpackage && !g.pkg[products[0].root].failed
&& cstreqlit(g.pkg[products[0].root].name, "main")) {
@@ -1947,7 +2298,9 @@ fn buildonesepimpl(selfdir: *u8, src: *u8, entryisdir: i32,
for (ci < g.n) { g.pkg[ci].color = 0; ci += 1; };
let ignored: i32 = 0;
if (septopovisit(g, root, order,
&ignored, stack, 0) < 0) {
&ignored, stack, 0) < 0
|| sepvalidatemoduleclosure(g, order, ignored,
rootpackage) < 0) {
g.pkg[root].failed = true;
};
};
@@ -2016,7 +2369,7 @@ fn buildonesepimpl(selfdir: *u8, src: *u8, entryisdir: i32,
};
let needsexport: bool = !g.pkg[pi].root || rootpackage;
let needsarchive: bool = !g.pkg[pi].root || rootpackage;
if (sepcomposeunit(g, pi, scratch, searchpath.ptr, cu) < 0) {
if (sepcomposeunit(g, pi, scratch, cu) < 0) {
g.pkg[pi].failed = true;
anyfailed = true;
oi += 1;
@@ -2187,10 +2540,10 @@ fn buildonesepimpl(selfdir: *u8, src: *u8, entryisdir: i32,
};
oi += 1;
};
// Tool identity commits only after every package artifact it vouches
// for is itself committed; a killed pass leaves the old identity and
// forces a full recompile, never a false reuse.
if (warm && !anyfailed) {
// A stale pass removed every old unit voucher before compiling. Current
// successful units remain safe to vouch for when a sibling root fails;
// a killed pass retains the old identity and invalidates again next time.
if (warm) {
if (!fileequal(toolc, c6)) {
if (copyfileatomic(c6, toolc) != 0) {
cerrpath("ww: cannot record ", toolc, "\n");
@@ -2256,8 +2609,8 @@ fn buildonesepimpl(selfdir: *u8, src: *u8, entryisdir: i32,
let nlink: i32 = 0;
if (septopovisit(g, root, linkorder, &nlink,
linkstack, 0) < 0) { return 1; };
// argv: 3 fixed + closure + libwwrt + flags + nil.
let total: i32 = 3 + nlink + 1 + 2 * nldirs + 2 * nllibs + 1;
// argv: 3 fixed + closure + libwwrt + joined flags + nil.
let total: i32 = 3 + nlink + 1 + nldirs + nllibs + 1;
let largv: []*u8 = alloc([], total: u64)!;
largv.len = total;
largv[0] = "w6l\0".ptr;
@@ -2270,6 +2623,7 @@ fn buildonesepimpl(selfdir: *u8, src: *u8, entryisdir: i32,
if (g.pkg[root].variant == SEP_VARIANT_SAME_TEST
&& pi != root
&& g.pkg[pi].variant == SEP_VARIANT_PRODUCTION
&& g.pkg[pi].role != SEP_ROLE_TEST_SUPPORT
&& os.samefile(pathstr(g.pkg[pi].entry),
pathstr(g.pkg[root].entry))) {
li -= 1;
@@ -2284,16 +2638,26 @@ fn buildonesepimpl(selfdir: *u8, src: *u8, entryisdir: i32,
largv[pos] = libwwrt.ptr; pos += 1;
let k: i32 = 0;
for (k < nldirs) {
largv[pos] = "-L\0".ptr;
largv[pos + 1] = ldirs[k];
pos += 2;
let n: u64 = cstrlen(ldirs[k]);
let flag: []u8 = alloc([], n + 3u64)!;
flag.len = (n + 3u64): i32;
flag[0] = 45u8; flag[1] = 76u8;
bytecpy(flag.ptr + 2u64, ldirs[k], n);
flag[n + 2u64] = 0u8;
largv[pos] = flag.ptr;
pos += 1;
k += 1;
};
k = 0;
for (k < nllibs) {
largv[pos] = "-l\0".ptr;
largv[pos + 1] = llibs[k];
pos += 2;
let n: u64 = cstrlen(llibs[k]);
let flag: []u8 = alloc([], n + 3u64)!;
flag.len = (n + 3u64): i32;
flag[0] = 45u8; flag[1] = 108u8;
bytecpy(flag.ptr + 2u64, llibs[k], n);
flag[n + 2u64] = 0u8;
largv[pos] = flag.ptr;
pos += 1;
k += 1;
};
largv[pos] = nil;
@@ -2342,9 +2706,11 @@ fn buildonesep(selfdir: *u8, src: *u8, entryisdir: i32,
let scratch: *u8 = nil;
let g: *sepgraph = nil;
let product: sepproduct;
product.dir = src;
product.out = out;
product.testpackage = testpackage;
product.status = nil;
product.artifact = nil;
product.variant = rootvariant;
product.root = -1;
let r: i32 = buildonesepimpl(selfdir, src, entryisdir, rootidentity,
@@ -2376,8 +2742,8 @@ fn buildonesep(selfdir: *u8, src: *u8, entryisdir: i32,
return r;
};
// Build every selected test root for one directory inside one command-owned
// package universe. The first output owns the shared cold sepwork tree.
// Build every selected directory/variant root inside one command-owned package
// universe. The first output owns the shared cold sepwork tree.
fn buildpackagetests(selfdir: *u8, src: *u8, incs: *u8, workdir: *u8,
products: *sepproduct, nproducts: i32) i32 = {
let scratch: *u8 = nil;
@@ -2396,6 +2762,23 @@ fn cstrendswithlit(p: *u8, lit: str) bool = {
return strings.hassuffix(pathstr(p), lit);
};
fn productartifact(index: i32, variant: i32) *u8 = {
let buf: []u8 = alloc([], 64u64)!;
buf.len = 64;
let off: u64 = strinto(buf.ptr, 0u64, "__ww-test-");
buf[off] = (((index / 100) % 10) + 48): u8; off += 1u64;
buf[off] = (((index / 10) % 10) + 48): u8; off += 1u64;
buf[off] = ((index % 10) + 48): u8; off += 1u64;
off = byteinto(buf.ptr, off, '-': u8);
if (variant == SEP_VARIANT_SAME_TEST) {
off = strinto(buf.ptr, off, "same");
} else {
off = strinto(buf.ptr, off, "external");
};
cstrseal(buf.ptr, off);
return buf.ptr;
};
fn basenameoff(p: *u8, plen: u64) u64 = {
let start: u64 = 0u64;
let i: u64 = 0u64;
@@ -3030,14 +3413,15 @@ fn dotest(selfdir: *u8, argv: **u8, argc: i32, start: i32) i32 = {
packageopts = true; afterdash = true; i += 1; continue;
};
if (cstreqlit(p, "--ww-package-test")) {
if (i + 4 >= argc || products.len >= SEP_MAXPRODUCT) {
cerr("ww test: --ww-package-test needs kind, package, output, and status\n");
if (i + 5 >= argc || products.len >= SEP_MAXPRODUCT) {
cerr("ww test: --ww-package-test needs kind, package, directory, output, and status\n");
return 2;
};
let kind: *u8 = argv[i + 1];
let name: *u8 = argv[i + 2];
let output: *u8 = argv[i + 3];
let status: *u8 = argv[i + 4];
let dir: *u8 = argv[i + 3];
let output: *u8 = argv[i + 4];
let status: *u8 = argv[i + 5];
let pn: u64 = cstrlen(name);
let variant: i32 = SEP_VARIANT_EXTERNAL;
if (cstreqlit(kind, "same")) {
@@ -3046,7 +3430,8 @@ fn dotest(selfdir: *u8, argv: **u8, argc: i32, start: i32) i32 = {
if ((!cstreqlit(kind, "same")
&& !cstreqlit(kind, "external"))
|| pn == 0u64 || pn >= 256u64
|| output[0u64] == 0u8 || status[0u64] == 0u8
|| dir[0u64] == 0u8 || output[0u64] == 0u8
|| status[0u64] == 0u8
|| (cstreqlit(kind, "external")
&& (pn <= 5u64
|| !cstrendswithlit(name,
@@ -3054,26 +3439,16 @@ fn dotest(selfdir: *u8, argv: **u8, argc: i32, start: i32) i32 = {
cerr("ww test: invalid --ww-package-test variant\n");
return 2;
};
let duplicate: bool = false;
let producti: i32 = 0;
for (producti < products.len) {
if (products[producti].variant == variant) {
duplicate = true;
};
producti += 1;
};
if (duplicate) {
cerr("ww test: duplicate --ww-package-test variant\n");
return 2;
};
let product: sepproduct;
product.dir = dir;
product.out = output;
product.testpackage = name;
product.status = status;
product.artifact = nil;
product.variant = variant;
product.root = -1;
append(products, product);
i += 5;
i += 6;
continue;
};
if (p[1u64] == 73u8) { // '-I'
@@ -3159,13 +3534,30 @@ fn dotest(selfdir: *u8, argv: **u8, argc: i32, start: i32) i32 = {
for (producti < products.len) {
let product: sepproduct = products[producti];
let j: i32 = producti;
for (j > 0 && products[j - 1].variant > product.variant) {
for (j > 0 && (strings.compare(pathstr(products[j - 1].dir),
pathstr(product.dir)) > 0
|| (strings.compare(pathstr(products[j - 1].dir),
pathstr(product.dir)) == 0
&& products[j - 1].variant > product.variant))) {
products[j] = products[j - 1];
j -= 1;
};
products[j] = product;
producti += 1;
};
producti = 0;
for (producti < products.len) {
if (producti > 0
&& cstreq(products[producti - 1].dir, products[producti].dir)
&& products[producti - 1].variant
== products[producti].variant) {
cerr("ww test: duplicate --ww-package-test variant for directory\n");
return 2;
};
products[producti].artifact = productartifact(producti,
products[producti].variant);
producti += 1;
};
if (products.len != 0 && packageopts) {
cerr("ww test: package-test variant rejects package options\n");
return 2;
@@ -3205,8 +3597,8 @@ fn dotest(selfdir: *u8, argv: **u8, argc: i32, start: i32) i32 = {
cerr("ww test: pattern needs a single test file\n");
return 2;
};
// -w forwards: the coordinator keys one persistent driver
// workdir per directory plan under the given root.
// -w forwards: the coordinator keys one persistent driver workdir
// for the complete selected test request.
return execpackagetests(selfdir, argv, argc, start,
targetindex, nil, false);
};