Hojun-Cho/ww
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

test/wcc/995_self_rebuild: fork-parallel the 5-tool rebuild loop

Browse Source 
Was sequential `for (i = 0; i < 5; i++) rebuild_one(...)` at ~3:15
wall. Split into spawn_build (fork+system ww_ww build) phase and
waitpid+slurp_eq collect phase. The five tool builds now run
concurrently; wall drops from sum(builds) to max(builds).

Per-tool workdir already pid-keyed (/tmp/wwsr_<pid>_<tool>); add
build.err per workdir so concurrent diagnostics don't merge —
collect phase replays the err file on failure. 180s timeout still
inside the cd && ... && ww_ww build string.

All five tools' failure status collected before reporting, so a
multi-divergence run names every tool (not just the first).

make test wall: 4m31s → 3m17s on 8 cores. 132/132 green.
This commit is contained in:
Hojun-Cho
2026-05-21 16:41:03 +09:00
parent 65a6cac6a0
commit 31594e4cae
1 changed files with 104 additions and 48 deletions
Download Patch File Download Diff File Expand all files Collapse all files

152
test/wcc/995_self_rebuild.c
View File

@@ -10,6 +10,10 @@
* This is stricter than `make bootstrap`: that loop only proves the * This is stricter than `make bootstrap`: that loop only proves the
* wwdump cgen self-stabilises; this proves every wwstage tool round- * wwdump cgen self-stabilises; this proves every wwstage tool round-
* trips through the wwstage pipeline. * trips through the wwstage pipeline.
*
* The five tool builds run concurrently: fork() per tool, parent
* collects each pid with waitpid() and then byte-compares. Wall drops
* from sum(builds) to max(builds).
*/ */
#include <stdio.h> #include <stdio.h>
#include <stdlib.h> #include <stdlib.h>
@@ -67,47 +71,93 @@ slurp_eq(const char *a, const char *b)
* Some tools have a local module dir (w6a, w6l with sibling .ww files). * Some tools have a local module dir (w6a, w6l with sibling .ww files).
* inc_local is "" for tools without one (w6c, ww, wwdump). * inc_local is "" for tools without one (w6c, ww, wwdump).
*/ */
static int struct buildjob {
rebuild_one(const char *bin, const char *cwd, const char *tool, const char *tool;
const char *src_rel, const char *inc_local) const char *src_rel;
{ const char *inc_local;
char workdir[64]; char workdir[64];
snprintf(workdir, sizeof workdir, "/tmp/wwsr_%d_%s", getpid(), tool); pid_t pid;
char cmd[4096]; };
snprintf(cmd, sizeof cmd, "rm -rf %s && mkdir -p %s", workdir, workdir);
if (runwait(cmd) != 0) return -1;
if (inc_local && inc_local[0]) { /* Fork a child that runs the `ww_ww build` for this tool. The child
* inherits no concurrent siblings — system() spawns a fresh /bin/sh -c.
* stderr lands in <workdir>/build.err so concurrent builds don't merge
* their diagnostics. */
static int
spawn_build(const char *bin, const char *cwd, struct buildjob *j)
{
snprintf(j->workdir, sizeof j->workdir, "/tmp/wwsr_%d_%s",
getpid(), j->tool);
char setup[256];
snprintf(setup, sizeof setup, "rm -rf %s && mkdir -p %s",
j->workdir, j->workdir);
if (runwait(setup) != 0) return -1;
char cmd[4096];
if (j->inc_local && j->inc_local[0]) {
snprintf(cmd, sizeof cmd, snprintf(cmd, sizeof cmd,
"cd %s && timeout 180 %s/ww_ww build -I %s/%s -I %s/lib/ww -I %s/lib/ww/lex -I %s/lib/ww/parse -I %s/selfhost/cmd/wcc " "cd %s && timeout 180 %s/ww_ww build -I %s/%s "
"%s/%s >/dev/null 2>&1", "-I %s/lib/ww -I %s/lib/ww/lex -I %s/lib/ww/parse "
workdir, bin, cwd, inc_local, cwd, cwd, cwd, cwd, cwd, src_rel); "-I %s/selfhost/cmd/wcc %s/%s >/dev/null 2>%s/build.err",
j->workdir, bin, cwd, j->inc_local,
cwd, cwd, cwd, cwd, cwd, j->src_rel, j->workdir);
} else { } else {
snprintf(cmd, sizeof cmd, snprintf(cmd, sizeof cmd,
"cd %s && timeout 180 %s/ww_ww build -I %s/lib/ww -I %s/lib/ww/lex -I %s/lib/ww/parse -I %s/selfhost/cmd/wcc " "cd %s && timeout 180 %s/ww_ww build "
"%s/%s >/dev/null 2>&1", "-I %s/lib/ww -I %s/lib/ww/lex -I %s/lib/ww/parse "
workdir, bin, cwd, cwd, cwd, cwd, cwd, src_rel); "-I %s/selfhost/cmd/wcc %s/%s >/dev/null 2>%s/build.err",
} j->workdir, bin, cwd, cwd, cwd, cwd, cwd, j->src_rel, j->workdir);
if (runwait(cmd) != 0) {
fprintf(stderr, "self-rebuild FAIL: ww_ww build errored on %s\n", tool);
return -1;
} }
char rebuilt[256], canonical[256]; pid_t p = fork();
snprintf(rebuilt, sizeof rebuilt, "%s/main", workdir); if (p < 0) return -1;
snprintf(canonical, sizeof canonical, "%s/%s_ww", bin, tool); if (p == 0) {
int rc = system(cmd);
if (rc == -1) _exit(1);
_exit(WIFEXITED(rc) ? WEXITSTATUS(rc) : 1);
}
j->pid = p;
return 0;
}
int rc = slurp_eq(rebuilt, canonical); /* Reap one job, byte-diff rebuilt vs canonical, replay captured stderr
if (rc != 0) { * on build failure, rm workdir. Returns 0 on byte-match, -1 otherwise. */
fprintf(stderr, "self-rebuild FAIL: %s rebuilt != cstage %s\n", static int
tool, canonical); collect_build(const char *bin, struct buildjob *j)
{
int rc = 0;
int status;
if (waitpid(j->pid, &status, 0) < 0) {
fprintf(stderr, "self-rebuild FAIL: waitpid %s\n", j->tool);
rc = -1;
} else if (!WIFEXITED(status) || WEXITSTATUS(status) != 0) {
fprintf(stderr, "self-rebuild FAIL: ww_ww build errored on %s\n",
j->tool);
/* Replay the captured stderr so the failure is diagnosable. */
char errpath[256];
snprintf(errpath, sizeof errpath, "%s/build.err", j->workdir);
FILE *f = fopen(errpath, "r");
if (f) {
char buf[1024];
while (fgets(buf, sizeof buf, f))
fputs(buf, stderr);
fclose(f);
}
rc = -1;
} else {
char rebuilt[256], canonical[256];
snprintf(rebuilt, sizeof rebuilt, "%s/main", j->workdir);
snprintf(canonical, sizeof canonical, "%s/%s_ww", bin, j->tool);
rc = slurp_eq(rebuilt, canonical);
if (rc != 0) {
fprintf(stderr, "self-rebuild FAIL: %s rebuilt != cstage %s\n",
j->tool, canonical);
}
} }
/* Leave the driver's intermediates (.s/.o/.combined.ww) next to the char cleanup[128];
* source — 991/992/994 read those fixtures, and `make wwstage` had snprintf(cleanup, sizeof cleanup, "rm -rf %s", j->workdir);
* already produced byte-identical copies anyway. */ runwait(cleanup);
snprintf(cmd, sizeof cmd, "rm -rf %s", workdir);
runwait(cmd);
return rc; return rc;
} }
@@ -119,26 +169,32 @@ main(void)
char cwd[1024]; char cwd[1024];
if (getcwd(cwd, sizeof cwd) == NULL) return 1; if (getcwd(cwd, sizeof cwd) == NULL) return 1;
struct { struct buildjob jobs[] = {
const char *tool; { "w6c", "selfhost/cmd/w6c/main.ww", "", {0}, 0 },
const char *src; { "w6a", "selfhost/cmd/w6a/main.ww", "selfhost/cmd/w6a", {0}, 0 },
const char *inc_local; { "w6l", "selfhost/cmd/w6l/main.ww", "selfhost/cmd/w6l", {0}, 0 },
} tools[] = { { "ww", "selfhost/cmd/ww/main.ww", "", {0}, 0 },
{ "w6c", "selfhost/cmd/w6c/main.ww", "" }, { "wwdump", "selfhost/cmd/wwdump/main.ww", "", {0}, 0 },
{ "w6a", "selfhost/cmd/w6a/main.ww", "selfhost/cmd/w6a" },
{ "w6l", "selfhost/cmd/w6l/main.ww", "selfhost/cmd/w6l" },
{ "ww", "selfhost/cmd/ww/main.ww", "" },
{ "wwdump", "selfhost/cmd/wwdump/main.ww", "" },
{ NULL, NULL, NULL },
}; };
const int n = (int)(sizeof jobs / sizeof jobs[0]);
int fail = 0, n = 0; int spawned = 0;
for (int i = 0; tools[i].tool; i++) { for (int i = 0; i < n; i++) {
if (rebuild_one(bin, cwd, tools[i].tool, tools[i].src, if (spawn_build(bin, cwd, &jobs[i]) != 0) {
tools[i].inc_local) != 0) fprintf(stderr, "self-rebuild FAIL: spawn %s\n", jobs[i].tool);
fail++; jobs[i].pid = 0;
n++; } else {
spawned++;
}
} }
int fail = n - spawned;
for (int i = 0; i < n; i++) {
if (jobs[i].pid == 0) continue;
if (collect_build(bin, &jobs[i]) != 0)
fail++;
}
if (fail) { if (fail) {
fprintf(stderr, "self-rebuild: %d/%d tool(s) diverged\n", fail, n); fprintf(stderr, "self-rebuild: %d/%d tool(s) diverged\n", fail, n);
return 1; return 1;