cmd: share package builds across test products
This commit is contained in:
@@ -25,29 +25,34 @@ type pkggroup = struct {
|
||||
dir: str,
|
||||
pkg: str,
|
||||
external: bool,
|
||||
plan: i32,
|
||||
root: str,
|
||||
workdir: str,
|
||||
bin: str,
|
||||
buildout: str,
|
||||
builderr: str,
|
||||
buildok: str,
|
||||
runout: str,
|
||||
runerr: str,
|
||||
state: i32,
|
||||
fail: i32,
|
||||
buildres: exec.result,
|
||||
runres: exec.result,
|
||||
};
|
||||
|
||||
// pkggroup.state values for the bounded scheduler.
|
||||
type pkgplan = struct {
|
||||
dir: str,
|
||||
root: str,
|
||||
workdir: str,
|
||||
buildout: str,
|
||||
builderr: str,
|
||||
start: i32,
|
||||
end: i32,
|
||||
state: i32,
|
||||
buildres: exec.result,
|
||||
};
|
||||
|
||||
// Product and directory-plan process states for the bounded coordinator.
|
||||
def PKGQUEUED: i32 = 0;
|
||||
def PKGBUILDING: i32 = 1;
|
||||
def PKGRUNNING: i32 = 2;
|
||||
def PKGDONE: i32 = 3;
|
||||
|
||||
// pkggroup.fail values recorded at launch, reported at ordered emission.
|
||||
def PKGFAILNONE: i32 = 0;
|
||||
def PKGFAILSETUP: i32 = 1;
|
||||
|
||||
def pkgpoll: time.duration = 1000000i64: time.duration;
|
||||
|
||||
type pkgdiscover = struct {
|
||||
@@ -56,7 +61,7 @@ type pkgdiscover = struct {
|
||||
};
|
||||
|
||||
// Preserve the caller's toolchain environment while pinning the locale and
|
||||
// temporary directory used by the current package group.
|
||||
// temporary directory used by the current build plan or test product.
|
||||
fn toolenv(tmpdir: str) []str = {
|
||||
let inherited: []str = os.getenvs();
|
||||
let env: []str = alloc([], (inherited.len + 2): u64)!;
|
||||
@@ -101,8 +106,8 @@ fn pkgusage() void = {
|
||||
let s: str = strings.concat(
|
||||
"usage: wwtest package [-c] [-list] [-j N] [-I DIR] [-w DIR] [-run|-filter GLOB] [-timeout-ms=N] [DIR | DIR/...] [-- GLOB ...]\n",
|
||||
" *_test.ww is the sole test-source form; @test elsewhere is rejected\n",
|
||||
" -c retains the compiled package binaries; -j N runs up to N package groups at once\n",
|
||||
" -w DIR keys a persistent per-package-group build workdir under DIR\n");
|
||||
" -c retains the compiled package binaries; -j N runs up to N directory builds or test binaries at once\n",
|
||||
" -w DIR keys a persistent shared build workdir per package directory\n");
|
||||
pkgput(os.STDERR_FILENO, s);
|
||||
};
|
||||
|
||||
@@ -246,6 +251,14 @@ fn pkgisdir(path: str) bool = {
|
||||
};
|
||||
};
|
||||
|
||||
fn pkgisreg(path: str) bool = {
|
||||
let fi: os.filestat;
|
||||
match (os.lstat(&fi, path)) {
|
||||
case void => return pkgmodeis(fi.mode, os.mode.REG);
|
||||
case let e: os.oserror => return false;
|
||||
};
|
||||
};
|
||||
|
||||
fn pkgkeepfile(name: str) bool = {
|
||||
if (!strings.hassuffix(name, ".ww")) { return false; };
|
||||
return true;
|
||||
@@ -510,36 +523,45 @@ fn pkgworkkey(dir: str, pkg: str) str = {
|
||||
pkgworkescape(pkg));
|
||||
};
|
||||
|
||||
fn pkgsetpaths(g: *pkggroup, root: str, index: i32,
|
||||
fn pkgsetplanpaths(p: *pkgplan, groups: []pkggroup, root: str, index: i32,
|
||||
compileonly: bool, outname: str, workroot: str) bool = {
|
||||
let num: str = strconv.i32tos(index, strconv.base.DEC);
|
||||
g.root = strings.concat(root, "/group-", num);
|
||||
if (!pkgmakedir(g.root)) { return false; };
|
||||
// A caller-owned persistent workdir root keys one driver -w dir
|
||||
// per (dir, pkg) group; the driver's content-identity contract
|
||||
// owns every reuse decision, so this stays a pure path policy.
|
||||
g.workdir = "";
|
||||
p.root = strings.concat(root, "/plan-", num);
|
||||
if (!pkgmakedir(p.root)) { return false; };
|
||||
p.workdir = "";
|
||||
if (workroot.len != 0) {
|
||||
g.workdir = strings.concat(workroot, "/",
|
||||
pkgworkkey(g.dir, g.pkg));
|
||||
match (os.mkdirs(g.workdir, 448)) {
|
||||
// The first byte-sorted product name is stable metadata in the
|
||||
// injective directory key; every product in this plan shares the dir.
|
||||
p.workdir = strings.concat(workroot, "/",
|
||||
pkgworkkey(p.dir, groups[p.start].pkg));
|
||||
match (os.mkdirs(p.workdir, 448)) {
|
||||
case void => void;
|
||||
case let e: os.oserror => return false;
|
||||
};
|
||||
};
|
||||
if (compileonly) {
|
||||
if (outname.len != 0) {
|
||||
g.bin = outname;
|
||||
p.buildout = strings.concat(p.root, "/build.stdout");
|
||||
p.builderr = strings.concat(p.root, "/build.stderr");
|
||||
let i: i32 = p.start;
|
||||
for (i < p.end) {
|
||||
let g: *pkggroup = &groups[i];
|
||||
g.plan = index;
|
||||
g.root = strings.concat(p.root, "/product-",
|
||||
strconv.i32tos(i - p.start, strconv.base.DEC));
|
||||
if (!pkgmakedir(g.root)) { return false; };
|
||||
if (compileonly) {
|
||||
if (outname.len != 0) {
|
||||
g.bin = outname;
|
||||
} else {
|
||||
g.bin = strings.concat(g.dir, "/", g.pkg, ".test");
|
||||
};
|
||||
} else {
|
||||
g.bin = strings.concat(g.dir, "/", g.pkg, ".test");
|
||||
g.bin = strings.concat(g.root, "/package.test");
|
||||
};
|
||||
} else {
|
||||
g.bin = strings.concat(g.root, "/package.test");
|
||||
g.runout = strings.concat(g.root, "/test.stdout");
|
||||
g.runerr = strings.concat(g.root, "/test.stderr");
|
||||
g.buildok = strings.concat(g.root, "/build.ok");
|
||||
i += 1;
|
||||
};
|
||||
g.buildout = strings.concat(g.root, "/build.stdout");
|
||||
g.builderr = strings.concat(g.root, "/build.stderr");
|
||||
g.runout = strings.concat(g.root, "/test.stdout");
|
||||
g.runerr = strings.concat(g.root, "/test.stderr");
|
||||
return true;
|
||||
};
|
||||
|
||||
@@ -575,36 +597,42 @@ fn pkgreportcommand(kind: str, g: *pkggroup, r: *exec.result) void = {
|
||||
pkgputln(os.STDERR_FILENO, ")");
|
||||
};
|
||||
|
||||
fn pkgstartbuild(g: *pkggroup, builder: str, includes: []str,
|
||||
fn pkgstartbuild(p: *pkgplan, groups: []pkggroup, builder: str, includes: []str,
|
||||
h: *exec.process) void = {
|
||||
let ba: []str = alloc([], (13 + includes.len * 2): u64)!;
|
||||
let ba: []str = alloc([],
|
||||
(8 + (p.end - p.start) * 5 + includes.len * 2): u64)!;
|
||||
append(ba, builder);
|
||||
append(ba, "test");
|
||||
append(ba, "-c");
|
||||
append(ba, "-o");
|
||||
append(ba, g.bin);
|
||||
append(ba, "--ww-package-test");
|
||||
if (g.external) { append(ba, "external"); }
|
||||
else { append(ba, "same"); };
|
||||
append(ba, g.pkg);
|
||||
let i: i32 = p.start;
|
||||
for (i < p.end) {
|
||||
let g: *pkggroup = &groups[i];
|
||||
append(ba, "--ww-package-test");
|
||||
if (g.external) { append(ba, "external"); }
|
||||
else { append(ba, "same"); };
|
||||
append(ba, g.pkg);
|
||||
append(ba, g.bin);
|
||||
append(ba, g.buildok);
|
||||
i += 1;
|
||||
};
|
||||
let ii: i32 = 0;
|
||||
for (ii < includes.len) {
|
||||
append(ba, "-I");
|
||||
append(ba, includes[ii]);
|
||||
ii += 1;
|
||||
};
|
||||
if (g.workdir.len != 0) {
|
||||
if (p.workdir.len != 0) {
|
||||
append(ba, "-w");
|
||||
append(ba, g.workdir);
|
||||
append(ba, p.workdir);
|
||||
};
|
||||
append(ba, g.dir);
|
||||
append(ba, p.dir);
|
||||
let bcmd: exec.command;
|
||||
bcmd.path = builder;
|
||||
bcmd.argv = ba;
|
||||
bcmd.env = toolenv(g.root);
|
||||
bcmd.env = toolenv(p.root);
|
||||
bcmd.dir = "";
|
||||
bcmd.stdoutpath = g.buildout;
|
||||
bcmd.stderrpath = g.builderr;
|
||||
bcmd.stdoutpath = p.buildout;
|
||||
bcmd.stderrpath = p.builderr;
|
||||
bcmd.deadline.sec = 0i64;
|
||||
bcmd.deadline.nsec = 0i64;
|
||||
bcmd.grace = 0i64: time.duration;
|
||||
@@ -633,10 +661,8 @@ fn pkgstartrun(g: *pkggroup, filters: []str, timeoutarg: str,
|
||||
exec.start(h, &rcmd);
|
||||
};
|
||||
|
||||
fn pkgbuildok(g: *pkggroup) bool = {
|
||||
return g.buildres.errno == 0 && g.buildres.cleanuperrno == 0
|
||||
&& g.buildres.termination == exec.termination.EXIT
|
||||
&& g.buildres.code == 0;
|
||||
fn pkgproductbuilt(g: *pkggroup) bool = {
|
||||
return pkgisreg(g.buildok) && pkgisreg(g.bin);
|
||||
};
|
||||
|
||||
fn pkgrunok(g: *pkggroup) bool = {
|
||||
@@ -645,24 +671,7 @@ fn pkgrunok(g: *pkggroup) bool = {
|
||||
&& g.runres.code == 0;
|
||||
};
|
||||
|
||||
// Ordered emission: a group's captures, diagnostics, and verdict line are
|
||||
// written only here, strictly in group order, so concurrent scheduling
|
||||
// produces the byte stream sequential scheduling produced.
|
||||
fn pkgemitgroup(g: *pkggroup, tmproot: str, compileonly: bool) bool = {
|
||||
if (g.fail == PKGFAILSETUP) {
|
||||
pkgfailpath(tmproot, "cannot create group temporary directory");
|
||||
return false;
|
||||
};
|
||||
let buildcaptures: bool = pkgemitfile(g.buildout, os.STDOUT_FILENO);
|
||||
buildcaptures = pkgemitfile(g.builderr, os.STDERR_FILENO) && buildcaptures;
|
||||
if (!buildcaptures) {
|
||||
pkgfailpath(g.root, "cannot read build capture");
|
||||
return false;
|
||||
};
|
||||
if (!pkgbuildok(g)) {
|
||||
pkgreportcommand("build", g, &g.buildres);
|
||||
return false;
|
||||
};
|
||||
fn pkgemitgroup(g: *pkggroup, compileonly: bool) bool = {
|
||||
if (compileonly) {
|
||||
pkgput(os.STDOUT_FILENO, "built ");
|
||||
pkglabel(g);
|
||||
@@ -697,6 +706,30 @@ fn pkgemitgroup(g: *pkggroup, tmproot: str, compileonly: bool) bool = {
|
||||
return true;
|
||||
};
|
||||
|
||||
// A directory build capture is emitted once, followed by its independently
|
||||
// executed products in their existing byte-sorted group order.
|
||||
fn pkgemitplan(p: *pkgplan, groups: []pkggroup,
|
||||
compileonly: bool) i32 = {
|
||||
let buildcaptures: bool = pkgemitfile(p.buildout, os.STDOUT_FILENO);
|
||||
buildcaptures = pkgemitfile(p.builderr, os.STDERR_FILENO) && buildcaptures;
|
||||
if (!buildcaptures) {
|
||||
pkgfailpath(p.root, "cannot read build capture");
|
||||
return 1;
|
||||
};
|
||||
let failed: i32 = 0;
|
||||
let i: i32 = p.start;
|
||||
for (i < p.end) {
|
||||
if (!pkgproductbuilt(&groups[i])) {
|
||||
pkgreportcommand("build", &groups[i], &p.buildres);
|
||||
failed += 1;
|
||||
} else if (!pkgemitgroup(&groups[i], compileonly)) {
|
||||
failed += 1;
|
||||
};
|
||||
i += 1;
|
||||
};
|
||||
return failed;
|
||||
};
|
||||
|
||||
export fn packagecommand(args: []str) int = {
|
||||
let compileonly: bool = false;
|
||||
let list: bool = false;
|
||||
@@ -943,74 +976,130 @@ export fn packagecommand(args: []str) int = {
|
||||
"wwtest package: cannot use -o with multiple packages");
|
||||
return 2;
|
||||
};
|
||||
let plans: []pkgplan = alloc([], groups.len: u64)!;
|
||||
i = 0;
|
||||
for (i < groups.len) {
|
||||
let p: pkgplan;
|
||||
p.dir = groups[i].dir;
|
||||
p.start = i;
|
||||
for (i < groups.len
|
||||
&& strings.compare(groups[i].dir, p.dir) == 0) {
|
||||
i += 1;
|
||||
};
|
||||
p.end = i;
|
||||
p.state = PKGQUEUED;
|
||||
append(plans, p);
|
||||
};
|
||||
|
||||
let borrowed: str = temp.dir();
|
||||
let tmproot: str = strings.dup(borrowed);
|
||||
let failed: i32 = 0;
|
||||
|
||||
// Bounded scheduler: up to `jobs` groups in flight, each a
|
||||
// build-then-run process chain supervised with exec.start/poll.
|
||||
// Launch order, run-slot accounting, and ordered emission keep
|
||||
// -j 1 byte-identical to the former sequential loop; a setup
|
||||
// failure stops new launches exactly where that loop broke.
|
||||
let handles: []exec.process = alloc([], groups.len: u64)!;
|
||||
i = 0;
|
||||
for (i < groups.len) {
|
||||
for (i < plans.len) {
|
||||
if (!pkgsetplanpaths(&plans[i], groups, tmproot, i,
|
||||
compileonly, outname, workroot)) {
|
||||
pkgfailpath(tmproot, "cannot create directory-plan temporary path");
|
||||
if (!pkgremoveall(tmproot)) {
|
||||
pkgput(os.STDERR_FILENO,
|
||||
"wwtest package: cleanup failed; retained ");
|
||||
pkgputln(os.STDERR_FILENO, tmproot);
|
||||
};
|
||||
return 1;
|
||||
};
|
||||
i += 1;
|
||||
};
|
||||
|
||||
// Build one shared package plan per directory. As soon as a plan completes,
|
||||
// its successful products may run under the same global -j bound while
|
||||
// other directory builds remain active. Emission stays ordered below.
|
||||
let handles: []exec.process = alloc([], plans.len: u64)!;
|
||||
i = 0;
|
||||
for (i < plans.len) {
|
||||
let h: exec.process;
|
||||
append(handles, h);
|
||||
i += 1;
|
||||
};
|
||||
let launched: i32 = 0;
|
||||
let emitted: i32 = 0;
|
||||
let runhandles: []exec.process = alloc([], groups.len: u64)!;
|
||||
i = 0;
|
||||
for (i < groups.len) {
|
||||
let h: exec.process;
|
||||
append(runhandles, h);
|
||||
groups[i].state = PKGQUEUED;
|
||||
i += 1;
|
||||
};
|
||||
let planlaunched: i32 = 0;
|
||||
let plancompleted: i32 = 0;
|
||||
let productcompleted: i32 = 0;
|
||||
if (compileonly) { productcompleted = groups.len; };
|
||||
let active: i32 = 0;
|
||||
let stopped: bool = false;
|
||||
for (emitted < groups.len) {
|
||||
for (!stopped && launched < groups.len && active < jobs) {
|
||||
let g: *pkggroup = &groups[launched];
|
||||
if (!pkgsetpaths(g, tmproot, launched, compileonly,
|
||||
outname, workroot)) {
|
||||
g.fail = PKGFAILSETUP;
|
||||
g.state = PKGDONE;
|
||||
stopped = true;
|
||||
} else {
|
||||
pkgstartbuild(g, builder, includes,
|
||||
&handles[launched]);
|
||||
g.state = PKGBUILDING;
|
||||
active += 1;
|
||||
};
|
||||
launched += 1;
|
||||
};
|
||||
let k: i32 = emitted;
|
||||
for (k < launched) {
|
||||
let g: *pkggroup = &groups[k];
|
||||
if (g.state == PKGBUILDING && exec.poll(&handles[k])) {
|
||||
g.buildres = handles[k].result;
|
||||
if (pkgbuildok(g) && !compileonly) {
|
||||
pkgstartrun(g, filters, timeoutarg,
|
||||
list, &handles[k]);
|
||||
g.state = PKGRUNNING;
|
||||
} else {
|
||||
g.state = PKGDONE;
|
||||
active -= 1;
|
||||
for (plancompleted < plans.len || productcompleted < groups.len) {
|
||||
// Fill free slots with already-built products first, then the next
|
||||
// byte-sorted directory build. Missing product markers are completed
|
||||
// build failures and consume no process slot.
|
||||
let filling: bool = true;
|
||||
for (filling && active < jobs) {
|
||||
filling = false;
|
||||
if (!compileonly) {
|
||||
let gi: i32 = 0;
|
||||
for (gi < groups.len) {
|
||||
let g: *pkggroup = &groups[gi];
|
||||
if (g.state == PKGQUEUED
|
||||
&& plans[g.plan].state == PKGDONE) {
|
||||
if (!pkgproductbuilt(g)) {
|
||||
g.state = PKGDONE;
|
||||
productcompleted += 1;
|
||||
} else {
|
||||
pkgstartrun(g, filters, timeoutarg, list,
|
||||
&runhandles[gi]);
|
||||
g.state = PKGRUNNING;
|
||||
active += 1;
|
||||
};
|
||||
filling = true;
|
||||
break;
|
||||
};
|
||||
gi += 1;
|
||||
};
|
||||
} else if (g.state == PKGRUNNING && exec.poll(&handles[k])) {
|
||||
g.runres = handles[k].result;
|
||||
g.state = PKGDONE;
|
||||
};
|
||||
if (!filling && planlaunched < plans.len) {
|
||||
pkgstartbuild(&plans[planlaunched], groups, builder, includes,
|
||||
&handles[planlaunched]);
|
||||
plans[planlaunched].state = PKGBUILDING;
|
||||
active += 1;
|
||||
planlaunched += 1;
|
||||
filling = true;
|
||||
};
|
||||
};
|
||||
let k: i32 = 0;
|
||||
for (k < planlaunched) {
|
||||
let p: *pkgplan = &plans[k];
|
||||
if (p.state == PKGBUILDING && exec.poll(&handles[k])) {
|
||||
p.buildres = handles[k].result;
|
||||
p.state = PKGDONE;
|
||||
active -= 1;
|
||||
plancompleted += 1;
|
||||
};
|
||||
k += 1;
|
||||
};
|
||||
for (emitted < launched && groups[emitted].state == PKGDONE) {
|
||||
if (!pkgemitgroup(&groups[emitted], tmproot,
|
||||
compileonly)) {
|
||||
failed += 1;
|
||||
if (!compileonly) {
|
||||
k = 0;
|
||||
for (k < groups.len) {
|
||||
let g: *pkggroup = &groups[k];
|
||||
if (g.state == PKGRUNNING && exec.poll(&runhandles[k])) {
|
||||
g.runres = runhandles[k].result;
|
||||
g.state = PKGDONE;
|
||||
active -= 1;
|
||||
productcompleted += 1;
|
||||
};
|
||||
k += 1;
|
||||
};
|
||||
emitted += 1;
|
||||
};
|
||||
if (stopped && active == 0 && emitted == launched) { break; };
|
||||
if (active > 0) {
|
||||
time.sleep(pkgpoll, time.clock.monotonic);
|
||||
};
|
||||
if (active > 0) { time.sleep(pkgpoll, time.clock.monotonic); };
|
||||
};
|
||||
|
||||
i = 0;
|
||||
for (i < plans.len) {
|
||||
failed += pkgemitplan(&plans[i], groups, compileonly);
|
||||
i += 1;
|
||||
};
|
||||
if (!pkgremoveall(tmproot)) {
|
||||
pkgput(os.STDERR_FILENO, "wwtest package: cleanup failed; retained ");
|
||||
|
||||
Reference in New Issue
Block a user