Hojun-Cho/ww
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

cmd: share package builds across test products

Browse Source
This commit is contained in:
Hojun-Cho
2026-08-12 11:42:24 +09:00
parent 7d25ad9f9f
commit dd975d195c
3 changed files with 898 additions and 384 deletions
Download Patch File Download Diff File Expand all files Collapse all files

394
cmd/ww/main.c
View File

@@ -300,6 +300,7 @@ source_has_test_decl(const char *path)
#define SEP_VARIANT_SAME_TEST 1 #define SEP_VARIANT_SAME_TEST 1
#define SEP_VARIANT_EXTERNAL 2 #define SEP_VARIANT_EXTERNAL 2
#define SEP_TEST_SUPPORT_MODULE "__wwtest" #define SEP_TEST_SUPPORT_MODULE "__wwtest"
#define SEP_MAXPRODUCT 2
/* Test-file package classification uses the compiler's imports-only parser. /* Test-file package classification uses the compiler's imports-only parser.
* The coordinator chooses variants, but the command owns which real source * The coordinator chooses variants, but the command owns which real source
@@ -495,6 +496,8 @@ struct seppkg {
int nsources; int nsources;
int is_dir; int is_dir;
int variant; /* SEP_VARIANT_*; dependencies are production */ int variant; /* SEP_VARIANT_*; dependencies are production */
int root; /* independently compiled/linkable product root */
int failed; /* discovery/compile failure reaches this action */
int test_support; /* compiler-generated -T support package */ int test_support; /* compiler-generated -T support package */
int deps[SEP_MAXPKG]; /* direct-dep indices into sepgraph.pkg */ int deps[SEP_MAXPKG]; /* direct-dep indices into sepgraph.pkg */
int ndeps; int ndeps;
@@ -506,9 +509,18 @@ struct sepgraph {
int n; int n;
}; };
struct sepproduct {
const char *out;
const char *test_package;
const char *status;
int variant;
int root;
};
static int static int
sep_find_or_add_variant(struct sepgraph *g, const char *path, sep_find_or_add_variant(struct sepgraph *g, const char *path,
const char *entry, int is_dir, int variant, const char *test_package) const char *entry, int is_dir, int variant, const char *test_package,
int root)
{ {
if (strlen(path) >= sizeof g->pkg[0].path) { if (strlen(path) >= sizeof g->pkg[0].path) {
fprintf(stderr, "ww: package path is too long (limit %zu bytes)\n", fprintf(stderr, "ww: package path is too long (limit %zu bytes)\n",
@@ -526,13 +538,18 @@ sep_find_or_add_variant(struct sepgraph *g, const char *path,
return -1; return -1;
} }
for (int i = 0; i < g->n; i++) { for (int i = 0; i < g->n; i++) {
int test_production_pair = i == 0 int same_location = strcmp(g->pkg[i].canon, canon) == 0;
&& g->pkg[i].variant != SEP_VARIANT_PRODUCTION int test_root_pair = same_location && is_dir && g->pkg[i].is_dir
&& variant == SEP_VARIANT_PRODUCTION && is_dir && ((root && g->pkg[i].root
&& g->pkg[i].is_dir && variant != g->pkg[i].variant)
&& strcmp(g->pkg[i].canon, canon) == 0; || (root && !g->pkg[i].root
&& variant != SEP_VARIANT_PRODUCTION
&& g->pkg[i].variant == SEP_VARIANT_PRODUCTION)
|| (!root && g->pkg[i].root
&& variant == SEP_VARIANT_PRODUCTION
&& g->pkg[i].variant != SEP_VARIANT_PRODUCTION));
if (strcmp(g->pkg[i].path, path) == 0) { if (strcmp(g->pkg[i].path, path) == 0) {
if (test_production_pair) if (test_root_pair)
continue; continue;
if (strcmp(g->pkg[i].canon, canon) != 0 if (strcmp(g->pkg[i].canon, canon) != 0
|| g->pkg[i].variant != variant) { || g->pkg[i].variant != variant) {
@@ -546,11 +563,12 @@ sep_find_or_add_variant(struct sepgraph *g, const char *path,
free(canon); free(canon);
return i; return i;
} }
if (strcmp(g->pkg[i].canon, canon) == 0) { if (same_location) {
/* A test root and a production variant reached by its imports or /* A test root and a production variant reached by its imports or
* runtime closure intentionally may share one directory. No other * runtime closure, and the selected test roots themselves,
* physical-directory alias is permitted. */ * intentionally may share one directory. No other physical-directory
if (test_production_pair) * alias is permitted. */
if (test_root_pair)
continue; continue;
fprintf(stderr, fprintf(stderr,
"ww: package directory %s has identities %s and %s\n", "ww: package directory %s has identities %s and %s\n",
@@ -573,6 +591,8 @@ sep_find_or_add_variant(struct sepgraph *g, const char *path,
free(canon); free(canon);
p->is_dir = is_dir; p->is_dir = is_dir;
p->variant = variant; p->variant = variant;
p->root = root;
p->failed = 0;
p->test_support = 0; p->test_support = 0;
p->name[0] = '\0'; p->name[0] = '\0';
p->test_package[0] = '\0'; p->test_package[0] = '\0';
@@ -591,7 +611,7 @@ sep_find_or_add(struct sepgraph *g, const char *path, const char *entry,
int is_dir) int is_dir)
{ {
return sep_find_or_add_variant(g, path, entry, is_dir, return sep_find_or_add_variant(g, path, entry, is_dir,
SEP_VARIANT_PRODUCTION, NULL); SEP_VARIANT_PRODUCTION, NULL, 0);
} }
/* Release the one package-owned directory-membership list. Every graph exit /* Release the one package-owned directory-membership list. Every graph exit
@@ -608,12 +628,23 @@ sep_graph_free(struct sepgraph *g)
free(g); free(g);
} }
/* Dots stay (legal in filenames); the root's empty path becomes "__root". */ /* Dots stay (legal in filenames). Product roots have distinct artifact names
* even though each compiler unit resets to the bare executable namespace. */
static void static void
sep_fname(const struct sepgraph *g, int pi, const char *scratch, sep_fname(const struct sepgraph *g, int pi, const char *scratch,
const char *suffix, char *out, size_t outsz) const char *suffix, char *out, size_t outsz)
{ {
const char *base = g->pkg[pi].path[0] ? g->pkg[pi].path : "__root"; const char *base = g->pkg[pi].path;
if (g->pkg[pi].root) {
if (g->pkg[pi].variant == SEP_VARIANT_SAME_TEST)
base = "__ww-test-same";
else if (g->pkg[pi].variant == SEP_VARIANT_EXTERNAL)
base = "__ww-test-external";
else if (base[0] == '\0')
base = "__root";
} else if (base[0] == '\0') {
base = "__root";
}
snprintf(out, outsz, "%s/%s%s", scratch, base, suffix); snprintf(out, outsz, "%s/%s%s", scratch, base, suffix);
} }
@@ -861,7 +892,7 @@ sep_scan_file(struct sepgraph *g, int pi, const char *file,
static int static int
sep_load_pkg(struct sepgraph *g, int pi, const char *searchpath) sep_load_pkg(struct sepgraph *g, int pi, const char *searchpath)
{ {
if (g->pkg[pi].color == 2) return 0; if (g->pkg[pi].color == 2) return g->pkg[pi].failed ? -1 : 0;
g->pkg[pi].color = 2; g->pkg[pi].color = 2;
struct ImportSet fv = {0}; struct ImportSet fv = {0};
int rc = 0; int rc = 0;
@@ -901,12 +932,15 @@ sep_load_pkg(struct sepgraph *g, int pi, const char *searchpath)
} }
for (int i = 0; i < fv.n; i++) free(fv.paths[i]); for (int i = 0; i < fv.n; i++) free(fv.paths[i]);
free(fv.paths); free(fv.paths);
if (rc < 0) return rc; if (rc < 0) {
g->pkg[pi].failed = 1;
return rc;
}
/* Give a non-main root its declared identity before recursively loading /* Give a non-main root its declared identity before recursively loading
* dependencies. A back-edge can then reuse node 0 and reach the normal * dependencies. A back-edge can then reuse node 0 and reach the normal
* cycle detector instead of looking like a location alias. Executable * cycle detector instead of looking like a location alias. Executable
* roots are reset to the bare-root identity after loading. */ * roots are reset to the bare-root identity after loading. */
if (pi == 0 && g->pkg[pi].path[0] == '\0' if (g->pkg[pi].root && g->pkg[pi].path[0] == '\0'
&& g->pkg[pi].name[0] != '\0') { && g->pkg[pi].name[0] != '\0') {
size_t n = strlen(g->pkg[pi].name); size_t n = strlen(g->pkg[pi].name);
memcpy(g->pkg[pi].path, g->pkg[pi].name, n + 1); memcpy(g->pkg[pi].path, g->pkg[pi].name, n + 1);
@@ -924,8 +958,10 @@ sep_load_pkg(struct sepgraph *g, int pi, const char *searchpath)
/* recurse into freshly-added deps (sep_find_or_add may have grown /* recurse into freshly-added deps (sep_find_or_add may have grown
* g->n during the scan; iterate by index). */ * g->n during the scan; iterate by index). */
for (int k = 0; k < g->pkg[pi].ndeps; k++) for (int k = 0; k < g->pkg[pi].ndeps; k++)
if (sep_load_pkg(g, g->pkg[pi].deps[k], searchpath) < 0) if (sep_load_pkg(g, g->pkg[pi].deps[k], searchpath) < 0) {
g->pkg[pi].failed = 1;
return -1; return -1;
}
return 0; return 0;
} }
@@ -1239,20 +1275,38 @@ copy_file_atomic(const char *src, const char *dst)
return rename(tmp, dst); return rename(tmp, dst);
} }
/* A coordinator-private completion marker distinguishes a newly linked
* product from a caller-owned binary left behind by an earlier invocation. */
static int
record_product_status(const char *path)
{
if (path == NULL) return 0;
char tmp[1100];
snprintf(tmp, sizeof tmp, "%s.new", path);
FILE *f = fopen(tmp, "wb");
if (f == NULL) return -1;
int bad = fputs("ok\n", f) == EOF;
if (fclose(f) != 0) bad = 1;
if (bad) return -1;
return rename(tmp, path);
}
/* The stamp pins the non-content build inputs a unit compare cannot see: /* The stamp pins the non-content build inputs a unit compare cannot see:
* the -T/-S shape of the producer pass and the artifact protocol * the -T/-S shape of the producer pass and the artifact protocol
* revision (bump "fmt" when the unit/archive/commit format changes). */ * revision (bump "fmt" when the unit/archive/commit format changes). */
static void static void
workdir_stamp_text(char *buf, size_t bufsz, int is_test, int emit_asm) workdir_stamp_text(char *buf, size_t bufsz, int is_test, int emit_asm)
{ {
snprintf(buf, bufsz, "ww workdir fmt 3 mode %s asm %d\n", snprintf(buf, bufsz, "ww workdir fmt 4 mode %s asm %d\n",
is_test ? "test" : "build", emit_asm); is_test ? "test" : "build", emit_asm);
} }
/* build_one_sep — discover_deps, reverse_topo, /* build_sep_plan — discover dependencies for every requested product in one
* the transitive producer loop (one `w6c -c -I` per package, dep-first, * package universe, compile the dependency-first union once, then link each
* root from its own complete reachable archive closure. The transitive
* producer loop (one `w6c -c -I` per package, dep-first,
* each DEP `.o` wrapped in its own deterministic `.a`), then a * each DEP `.o` wrapped in its own deterministic `.a`), then a
* reverse-topo `w6l` of the root `.o` + dep `.a` set + libwwrt.a. Side * reverse-topo `w6l` of each root `.o` + dep `.a` set + libwwrt.a. Side
* files land in a cold `<stem>.sepwork` dir, or under the persistent * files land in a cold `<stem>.sepwork` dir, or under the persistent
* `-w` workdir with content-identity package reuse. */ * `-w` workdir with content-identity package reuse. */
static int static int
@@ -1260,10 +1314,15 @@ build_one_sep_impl(const char *src, int entry_is_dir,
const char *root_identity, const char *out, const char *root_identity, const char *out,
const char *objstem, const char *extra_includes, const char *extra_libs, const char *objstem, const char *extra_includes, const char *extra_libs,
const char *extra_libdirs, int package_only, int is_test, const char *extra_libdirs, int package_only, int is_test,
int root_variant, const char *test_package, int emit_asm, struct sepproduct *products, int nproducts, int emit_asm,
const char *workdir, char *scratchout, size_t scratchoutsz, const char *workdir, char *scratchout, size_t scratchoutsz,
struct sepgraph **graphout) struct sepgraph **graphout)
{ {
if (nproducts < 1 || nproducts > SEP_MAXPRODUCT) return 1;
for (int i = 0; i < nproducts; i++)
if (products[i].status != NULL
&& unlink(products[i].status) != 0 && errno != ENOENT)
return 1;
const char *c6 = toolpath("WW_W6C", "w6c"); const char *c6 = toolpath("WW_W6C", "w6c");
const char *a6 = toolpath("WW_W6A", "w6a"); const char *a6 = toolpath("WW_W6A", "w6a");
const char *l6 = toolpath("WW_W6L", "w6l"); const char *l6 = toolpath("WW_W6L", "w6l");
@@ -1365,9 +1424,11 @@ build_one_sep_impl(const char *src, int entry_is_dir,
if (graphout) *graphout = g; if (graphout) *graphout = g;
const char *rootpath = package_only && root_identity const char *rootpath = package_only && root_identity
? root_identity : ""; ? root_identity : "";
int root = sep_find_or_add_variant(g, rootpath, src, entry_is_dir, for (int i = 0; i < nproducts; i++) {
root_variant, test_package); products[i].root = sep_find_or_add_variant(g, rootpath, src,
if (root < 0) return 1; entry_is_dir, products[i].variant, products[i].test_package, 1);
if (products[i].root < 0) return 1;
}
const char *test_support_module = "test"; const char *test_support_module = "test";
/* -T generates a dispatcher whose support qualifier is selected by the /* -T generates a dispatcher whose support qualifier is selected by the
* command. Represent that compiler-generated requirement as a direct root * command. Represent that compiler-generated requirement as a direct root
@@ -1384,9 +1445,13 @@ build_one_sep_impl(const char *src, int entry_is_dir,
&& strcmp(tc, rc) == 0; && strcmp(tc, rc) == 0;
free(tc); free(tc);
free(rc); free(rc);
int collision = !root_is_support && test_package != NULL int collision = 0;
&& (strcmp(test_package, "test") == 0 for (int i = 0; !root_is_support && i < nproducts; i++) {
|| strcmp(test_package, "test_test") == 0); const char *name = products[i].test_package;
if (name != NULL && (strcmp(name, "test") == 0
|| strcmp(name, "test_test") == 0))
collision = 1;
}
char userpath[1024]; char userpath[1024];
int userdir = 0; int userdir = 0;
if (!root_is_support && !collision if (!root_is_support && !collision
@@ -1401,10 +1466,14 @@ build_one_sep_impl(const char *src, int entry_is_dir,
free(uc); free(uc);
} }
if (collision) test_support_module = SEP_TEST_SUPPORT_MODULE; if (collision) test_support_module = SEP_TEST_SUPPORT_MODULE;
for (int i = 0; i < nproducts; i++) {
int root = products[i].root;
/* A same-test build of the runtime package already owns run /* A same-test build of the runtime package already owns run
* and its source imports. An external test still needs the * and its source imports. An external test still needs the
* colocated production node, which is also its one support dep. */ * colocated production node, which is also its support dep. */
if (!root_is_support || root_variant == SEP_VARIANT_EXTERNAL) { if (root_is_support
&& products[i].variant != SEP_VARIANT_EXTERNAL)
continue;
int ti = sep_find_or_add(g, test_support_module, tpath, int ti = sep_find_or_add(g, test_support_module, tpath,
tdir); tdir);
if (ti < 0) return 1; if (ti < 0) return 1;
@@ -1419,32 +1488,69 @@ build_one_sep_impl(const char *src, int entry_is_dir,
} }
} }
} }
if (sep_load_pkg(g, root, srcdir) < 0) return 1; for (int i = 0; i < nproducts; i++) {
if (root_variant != SEP_VARIANT_PRODUCTION int root = products[i].root;
&& (test_package == NULL if (sep_load_pkg(g, root, srcdir) < 0) {
|| strcmp(g->pkg[root].name, test_package) != 0)) { g->pkg[root].failed = 1;
continue;
}
if (products[i].variant != SEP_VARIANT_PRODUCTION
&& (products[i].test_package == NULL
|| strcmp(g->pkg[root].name,
products[i].test_package) != 0)) {
fprintf(stderr, fprintf(stderr,
"ww: package-test selector does not match loaded package\n"); "ww: package-test selector does not match loaded package\n");
return 1; g->pkg[root].failed = 1;
}
} }
int root_package = package_only; int root_package = package_only;
if (root_package && strcmp(g->pkg[root].name, "main") == 0) { if (root_package && !g->pkg[products[0].root].failed
&& strcmp(g->pkg[products[0].root].name, "main") == 0) {
fprintf(stderr, "ww: -p requires a non-main package\n"); fprintf(stderr, "ww: -p requires a non-main package\n");
return 1; return 1;
} }
for (int i = 0; i < g->n; i++) g->pkg[i].color = 0;
int *order = calloc((size_t)g->n, sizeof *order); int *order = calloc((size_t)g->n, sizeof *order);
int *stack = calloc((size_t)g->n, sizeof *stack); int *stack = calloc((size_t)g->n, sizeof *stack);
int norder = 0; int norder = 0;
if (order == NULL || stack == NULL || if (order == NULL || stack == NULL) {
sep_topo_visit(g, root, order, &norder, stack, 0) < 0) {
free(stack); free(order); return 1; free(stack); free(order); return 1;
} }
/* Diagnose cycles per product before constructing the shared union. A
* variant-local cycle must not suppress an independent sibling root. */
for (int i = 0; i < nproducts; i++) {
int root = products[i].root;
if (g->pkg[root].failed) continue;
for (int pi = 0; pi < g->n; pi++) g->pkg[pi].color = 0;
int ignored = 0;
if (sep_topo_visit(g, root, order, &ignored, stack, 0) < 0)
g->pkg[root].failed = 1;
}
for (int pi = 0; pi < g->n; pi++) g->pkg[pi].color = 0;
for (int i = 0; i < nproducts; i++) {
int root = products[i].root;
if (!g->pkg[root].failed
&& sep_topo_visit(g, root, order, &norder, stack, 0) < 0) {
free(stack); free(order); return 1;
}
}
free(stack); free(stack);
if (!root_package) g->pkg[root].path[0] = '\0'; if (!root_package)
for (int i = 0; i < nproducts; i++)
if (g->pkg[products[i].root].path[0] != '\0')
g->pkg[products[i].root].path[0] = '\0';
int any_failed = 0;
for (int i = 0; i < nproducts; i++)
if (g->pkg[products[i].root].failed) any_failed = 1;
for (int oi = 0; oi < norder; oi++) { for (int oi = 0; oi < norder; oi++) {
int pi = order[oi]; int pi = order[oi];
for (int k = 0; k < g->pkg[pi].ndeps; k++)
if (g->pkg[g->pkg[pi].deps[k]].failed)
g->pkg[pi].failed = 1;
if (g->pkg[pi].failed) {
any_failed = 1;
continue;
}
char unitf[1024], wwi[1024], asmf[1024], obj[1024], apath[1024]; char unitf[1024], wwi[1024], asmf[1024], obj[1024], apath[1024];
char unitnew[1024], wwinew[1024], asmnew[1024], objnew[1024]; char unitnew[1024], wwinew[1024], asmnew[1024], objnew[1024];
char anew[1024], cmd[8192]; char anew[1024], cmd[8192];
@@ -1465,10 +1571,13 @@ build_one_sep_impl(const char *src, int entry_is_dir,
const char *cs = warm ? asmnew : asmf; const char *cs = warm ? asmnew : asmf;
const char *co = warm ? objnew : obj; const char *co = warm ? objnew : obj;
const char *ca = warm ? anew : apath; const char *ca = warm ? anew : apath;
int needs_export = pi != root || root_package; int needs_export = !g->pkg[pi].root || root_package;
int needs_archive = pi != root || root_package; int needs_archive = !g->pkg[pi].root || root_package;
if (sep_compose_unit(g, pi, scratch, srcdir, if (sep_compose_unit(g, pi, scratch, srcdir, cu) < 0) {
cu) < 0) { free(order); return 1; } g->pkg[pi].failed = 1;
any_failed = 1;
continue;
}
if (warm && !stale_all && file_equal(unitnew, unitf) if (warm && !stale_all && file_equal(unitnew, unitf)
&& file_is_reg(asmf) && file_is_reg(asmf)
&& (!needs_export || file_is_reg(wwi)) && (!needs_export || file_is_reg(wwi))
@@ -1477,7 +1586,8 @@ build_one_sep_impl(const char *src, int entry_is_dir,
if (unlink(unitnew) != 0) { if (unlink(unitnew) != 0) {
fprintf(stderr, "ww: cannot remove %s\n", fprintf(stderr, "ww: cannot remove %s\n",
unitnew); unitnew);
free(order); return 1; g->pkg[pi].failed = 1;
any_failed = 1;
} }
continue; continue;
} }
@@ -1492,7 +1602,7 @@ build_one_sep_impl(const char *src, int entry_is_dir,
/* #79: the root carries -T under `ww test` /* #79: the root carries -T under `ww test`
* so w6c synthesizes the test main. Deps never * so w6c synthesizes the test main. Deps never
* get -T. */ * get -T. */
if (is_test) if (is_test && g->pkg[pi].root)
snprintf(cmd, sizeof cmd, snprintf(cmd, sizeof cmd,
"%s -T --test-support-module %s -c -o %s %s", "%s -T --test-support-module %s -c -o %s %s",
c6, test_support_module, cs, cu); c6, test_support_module, cs, cu);
@@ -1509,14 +1619,18 @@ build_one_sep_impl(const char *src, int entry_is_dir,
if (run(cmd) != 0) { if (run(cmd) != 0) {
fprintf(stderr, "ww: w6c failed for %s\n", fprintf(stderr, "ww: w6c failed for %s\n",
g->pkg[pi].path[0] ? g->pkg[pi].path : "(root)"); g->pkg[pi].path[0] ? g->pkg[pi].path : "(root)");
free(order); return 1; g->pkg[pi].failed = 1;
any_failed = 1;
continue;
} }
if (!emit_asm) { if (!emit_asm) {
snprintf(cmd, sizeof cmd, "%s -o %s %s", a6, co, cs); snprintf(cmd, sizeof cmd, "%s -o %s %s", a6, co, cs);
if (run(cmd) != 0) { if (run(cmd) != 0) {
fprintf(stderr, "ww: w6a failed for %s\n", fprintf(stderr, "ww: w6a failed for %s\n",
g->pkg[pi].path[0] ? g->pkg[pi].path : "(root)"); g->pkg[pi].path[0] ? g->pkg[pi].path : "(root)");
free(order); return 1; g->pkg[pi].failed = 1;
any_failed = 1;
continue;
} }
} }
/* wrap each DEP package's `.o` in its own deterministic `.a` /* wrap each DEP package's `.o` in its own deterministic `.a`
@@ -1528,7 +1642,9 @@ build_one_sep_impl(const char *src, int entry_is_dir,
if (archive_o(co, ca) != 0) { if (archive_o(co, ca) != 0) {
fprintf(stderr, "ww: archive failed for %s\n", fprintf(stderr, "ww: archive failed for %s\n",
g->pkg[pi].path[0] ? g->pkg[pi].path : "(root)"); g->pkg[pi].path[0] ? g->pkg[pi].path : "(root)");
free(order); return 1; g->pkg[pi].failed = 1;
any_failed = 1;
continue;
} }
} }
/* Commit order: artifacts before the unit that vouches for /* Commit order: artifacts before the unit that vouches for
@@ -1542,14 +1658,16 @@ build_one_sep_impl(const char *src, int entry_is_dir,
|| rename(unitnew, unitf) != 0) { || rename(unitnew, unitf) != 0) {
fprintf(stderr, "ww: cannot commit %s\n", fprintf(stderr, "ww: cannot commit %s\n",
g->pkg[pi].path[0] ? g->pkg[pi].path : "(root)"); g->pkg[pi].path[0] ? g->pkg[pi].path : "(root)");
free(order); return 1; g->pkg[pi].failed = 1;
any_failed = 1;
continue;
} }
} }
} }
/* Tool identity commits only after every package artifact it vouches /* Tool identity commits only after every package artifact it vouches
* for is itself committed; a killed pass leaves the old identity and * for is itself committed; a killed pass leaves the old identity and
* forces a full recompile, never a false reuse. */ * forces a full recompile, never a false reuse. */
if (warm) { if (warm && !any_failed) {
if (!file_equal(toolc, c6) if (!file_equal(toolc, c6)
&& copy_file_atomic(c6, toolc) != 0) { && copy_file_atomic(c6, toolc) != 0) {
fprintf(stderr, "ww: cannot record %s\n", toolc); fprintf(stderr, "ww: cannot record %s\n", toolc);
@@ -1573,8 +1691,10 @@ build_one_sep_impl(const char *src, int entry_is_dir,
} }
} }
} }
if (emit_asm) { free(order); return 0; } if (emit_asm) { free(order); return any_failed ? 1 : 0; }
if (root_package) { if (root_package) {
int root = products[0].root;
if (g->pkg[root].failed) { free(order); return 1; }
char archive[1024], iface[1024], outiface[1100]; char archive[1024], iface[1024], outiface[1100];
sep_fname(g, root, scratch, ".a", archive, sizeof archive); sep_fname(g, root, scratch, ".a", archive, sizeof archive);
sep_fname(g, root, scratch, ".wwi", iface, sizeof iface); sep_fname(g, root, scratch, ".wwi", iface, sizeof iface);
@@ -1589,12 +1709,11 @@ build_one_sep_impl(const char *src, int entry_is_dir,
return 0; return 0;
} }
/* reverse-topo link: root `.o` first (order[norder-1], force-loaded), free(order);
* then transitive dep `.a` in reverse-topo order, then libwwrt.a — /* Each product gets its own reverse-topological closure: root `.o` first,
* each archive selectively pulls only members satisfying a live undef. * then every transitively reachable dependency `.a`, then libwwrt.a. A
* A same-test root already contains its production sources; if that * same-test root already contains its production sources, so its colocated
* production variant is also reached through the runtime closure, keep * production archive is omitted without dropping that node's dependencies. */
* traversing its dependencies but do not link its duplicate archive. */
char rtargs[2048] = {0}; char rtargs[2048] = {0};
char rtpath[1024]; char rtpath[1024];
snprintf(rtpath, sizeof rtpath, "%s/libwwrt.a", libdir); snprintf(rtpath, sizeof rtpath, "%s/libwwrt.a", libdir);
@@ -1606,9 +1725,24 @@ build_one_sep_impl(const char *src, int entry_is_dir,
snprintf(a2, sizeof a2, "%s/../obj/rt/syscall.o", self_dir); snprintf(a2, sizeof a2, "%s/../obj/rt/syscall.o", self_dir);
snprintf(rtargs, sizeof rtargs, "%s %s", a1, a2); snprintf(rtargs, sizeof rtargs, "%s %s", a1, a2);
} }
const char *libargs = (extra_libs && extra_libs[0]) ? extra_libs : "";
const char *libdirset = (extra_libdirs && extra_libdirs[0]) ? extra_libdirs : "";
for (int i = 0; i < nproducts; i++) {
int root = products[i].root;
if (g->pkg[root].failed) { any_failed = 1; continue; }
for (int pi = 0; pi < g->n; pi++) g->pkg[pi].color = 0;
int *linkorder = calloc((size_t)g->n, sizeof *linkorder);
int *linkstack = calloc((size_t)g->n, sizeof *linkstack);
int nlink = 0;
if (linkorder == NULL || linkstack == NULL
|| sep_topo_visit(g, root, linkorder, &nlink,
linkstack, 0) < 0) {
free(linkstack); free(linkorder); return 1;
}
free(linkstack);
char objs[8192] = {0}; char objs[8192] = {0};
for (int oi = norder - 1; oi >= 0; oi--) { for (int oi = nlink - 1; oi >= 0; oi--) {
int pi = order[oi]; int pi = linkorder[oi];
if (g->pkg[root].variant == SEP_VARIANT_SAME_TEST if (g->pkg[root].variant == SEP_VARIANT_SAME_TEST
&& pi != root && pi != root
&& g->pkg[pi].variant == SEP_VARIANT_PRODUCTION && g->pkg[pi].variant == SEP_VARIANT_PRODUCTION
@@ -1618,19 +1752,28 @@ build_one_sep_impl(const char *src, int entry_is_dir,
sep_fname(g, pi, scratch, sep_fname(g, pi, scratch,
pi == root ? ".o" : ".a", path, sizeof path); pi == root ? ".o" : ".a", path, sizeof path);
size_t n = strlen(objs); size_t n = strlen(objs);
snprintf(objs + n, sizeof objs - n, "%s%s", n ? " " : "", path); snprintf(objs + n, sizeof objs - n, "%s%s",
n ? " " : "", path);
} }
const char *libargs = (extra_libs && extra_libs[0]) ? extra_libs : ""; free(linkorder);
const char *libdirset = (extra_libdirs && extra_libdirs[0]) ? extra_libdirs : "";
char cmd[16384]; char cmd[16384];
snprintf(cmd, sizeof cmd, "%s -o %s %s %s%s%s%s%s", snprintf(cmd, sizeof cmd, "%s -o %s %s %s%s%s%s%s",
l6, out, objs, rtargs, l6, products[i].out, objs, rtargs,
libdirset[0] ? " " : "", libdirset, libdirset[0] ? " " : "", libdirset,
libargs[0] ? " " : "", libargs); libargs[0] ? " " : "", libargs);
int rc = run(cmd); if (run(cmd) != 0) {
free(order); fprintf(stderr, "ww: w6l failed\n");
if (rc != 0) { fprintf(stderr, "ww: w6l failed\n"); return 1; } g->pkg[root].failed = 1;
return 0; any_failed = 1;
continue;
}
if (record_product_status(products[i].status) != 0) {
fprintf(stderr, "ww: cannot record package-test product\n");
g->pkg[root].failed = 1;
any_failed = 1;
}
}
return any_failed ? 1 : 0;
} }
/* build_one_sep — thin wrapper over build_one_sep_impl. `ww build` and an /* build_one_sep — thin wrapper over build_one_sep_impl. `ww build` and an
@@ -1649,9 +1792,16 @@ build_one_sep(const char *src, int entry_is_dir, const char *root_identity,
{ {
char scratch[1100] = {0}; char scratch[1100] = {0};
struct sepgraph *g = NULL; struct sepgraph *g = NULL;
struct sepproduct product = {
.out = out,
.test_package = test_package,
.status = NULL,
.variant = root_variant,
.root = -1,
};
int r = build_one_sep_impl(src, entry_is_dir, root_identity, out, objstem, int r = build_one_sep_impl(src, entry_is_dir, root_identity, out, objstem,
extra_includes, extra_libs, extra_libdirs, package_only, is_test, extra_includes, extra_libs, extra_libdirs, package_only, is_test,
root_variant, test_package, emit_asm, workdir, scratch, &product, 1, emit_asm, workdir, scratch,
sizeof scratch, &g); sizeof scratch, &g);
sep_graph_free(g); sep_graph_free(g);
if (!keepscratch && scratch[0]) { if (!keepscratch && scratch[0]) {
@@ -1679,6 +1829,22 @@ build_one_sep(const char *src, int entry_is_dir, const char *root_identity,
return r; return r;
} }
/* The package coordinator submits all selected roots for one directory in one
* request. Its first output owns the shared cold sepwork tree; every product
* remains an independent root compile and link inside that tree. */
static int
build_package_tests(const char *src, const char *extra_includes,
const char *workdir, struct sepproduct *products, int nproducts)
{
char scratch[1100] = {0};
struct sepgraph *g = NULL;
int r = build_one_sep_impl(src, 1, NULL, products[0].out,
products[0].out, extra_includes, "", "", 0, 1,
products, nproducts, 0, workdir, scratch, sizeof scratch, &g);
sep_graph_free(g);
return r;
}
static int static int
do_version(void) do_version(void)
{ {
@@ -1985,8 +2151,8 @@ static int
do_test(int argc, char **argv) do_test(int argc, char **argv)
{ {
const char *src = NULL; const char *src = NULL;
const char *package_test_kind = NULL; struct sepproduct products[SEP_MAXPRODUCT];
const char *package_test_name = NULL; int nproducts = 0;
char incs[2048] = {0}; char incs[2048] = {0};
/* -c (Go's `go test -c`) builds the test binary without running it. /* -c (Go's `go test -c`) builds the test binary without running it.
* -S + -o <stem> stops after the lib/test-inclusive package `.s` * -S + -o <stem> stops after the lib/test-inclusive package `.s`
@@ -2032,25 +2198,42 @@ do_test(int argc, char **argv)
} else if (strcmp(argv[i], "-c") == 0) { } else if (strcmp(argv[i], "-c") == 0) {
compileonly = 1; compileonly = 1;
} else if (strcmp(argv[i], "--ww-package-test") == 0) { } else if (strcmp(argv[i], "--ww-package-test") == 0) {
if (i + 2 >= argc || package_test_kind != NULL) { if (i + 4 >= argc || nproducts >= SEP_MAXPRODUCT) {
fprintf(stderr, fprintf(stderr,
"ww test: --ww-package-test needs kind and package\n"); "ww test: --ww-package-test needs kind, package, output, and status\n");
return 2; return 2;
} }
package_test_kind = argv[++i]; const char *kind = argv[++i];
package_test_name = argv[++i]; const char *name = argv[++i];
size_t pn = strlen(package_test_name); const char *output = argv[++i];
if ((strcmp(package_test_kind, "same") != 0 const char *status = argv[++i];
&& strcmp(package_test_kind, "external") != 0) size_t pn = strlen(name);
int variant = strcmp(kind, "same") == 0
? SEP_VARIANT_SAME_TEST : SEP_VARIANT_EXTERNAL;
if ((strcmp(kind, "same") != 0
&& strcmp(kind, "external") != 0)
|| pn == 0 || pn >= sizeof ((struct seppkg *)0)->name || pn == 0 || pn >= sizeof ((struct seppkg *)0)->name
|| (strcmp(package_test_kind, "external") == 0 || output[0] == '\0' || status[0] == '\0'
|| (strcmp(kind, "external") == 0
&& (pn <= 5 && (pn <= 5
|| strcmp(package_test_name + pn - 5, || strcmp(name + pn - 5,
"_test") != 0))) { "_test") != 0))) {
fprintf(stderr, fprintf(stderr,
"ww test: invalid --ww-package-test variant\n"); "ww test: invalid --ww-package-test variant\n");
return 2; return 2;
} }
for (int p = 0; p < nproducts; p++)
if (products[p].variant == variant) {
fprintf(stderr,
"ww test: duplicate --ww-package-test variant\n");
return 2;
}
products[nproducts].out = output;
products[nproducts].test_package = name;
products[nproducts].status = status;
products[nproducts].variant = variant;
products[nproducts].root = -1;
nproducts++;
} else if (strcmp(argv[i], "-S") == 0) { } else if (strcmp(argv[i], "-S") == 0) {
emit_asm = 1; emit_asm = 1;
} else if (strcmp(argv[i], "-list") == 0) { } else if (strcmp(argv[i], "-list") == 0) {
@@ -2101,18 +2284,32 @@ do_test(int argc, char **argv)
fprintf(stderr, "ww test: -S needs -o\n"); fprintf(stderr, "ww test: -S needs -o\n");
return 2; return 2;
} }
if (package_test_kind != NULL && packageopts) { for (int i = 1; i < nproducts; i++) {
struct sepproduct p = products[i];
int j = i;
while (j > 0 && products[j - 1].variant > p.variant) {
products[j] = products[j - 1];
j--;
}
products[j] = p;
}
if (nproducts != 0 && packageopts) {
fprintf(stderr, fprintf(stderr,
"ww test: package-test variant rejects package options\n"); "ww test: package-test variant rejects package options\n");
return 2; return 2;
} }
if (nproducts != 0 && outstem[0]) {
fprintf(stderr,
"ww test: package-test products reject -o\n");
return 2;
}
/* Go's ./... form: a trailing "..." element is a package-tree /* Go's ./... form: a trailing "..." element is a package-tree
* request for the coordinator, never a literal path — recognized * request for the coordinator, never a literal path — recognized
* before stat, with the directory-mode rejects. */ * before stat, with the directory-mode rejects. */
size_t tlen = strlen(target); size_t tlen = strlen(target);
if (strcmp(target, "...") == 0 || if (strcmp(target, "...") == 0 ||
(tlen >= 4 && strcmp(target + tlen - 4, "/...") == 0)) { (tlen >= 4 && strcmp(target + tlen - 4, "/...") == 0)) {
if (package_test_kind != NULL) { if (nproducts != 0) {
fprintf(stderr, fprintf(stderr,
"ww test: package-test variant needs one directory\n"); "ww test: package-test variant needs one directory\n");
return 2; return 2;
@@ -2135,7 +2332,7 @@ do_test(int argc, char **argv)
return 2; return 2;
} }
/* -w forwards: the coordinator keys one persistent driver /* -w forwards: the coordinator keys one persistent driver
* workdir per package group under the given root. */ * workdir per directory plan under the given root. */
return exec_package_tests(argc, argv, src, NULL, 0); return exec_package_tests(argc, argv, src, NULL, 0);
} }
struct stat st; struct stat st;
@@ -2165,17 +2362,14 @@ do_test(int argc, char **argv)
"ww test: pattern needs a single test file\n"); "ww test: pattern needs a single test file\n");
return 2; return 2;
} }
if (package_test_kind != NULL) { if (nproducts != 0) {
if (!compileonly || !outstem[0]) { if (!compileonly) {
fprintf(stderr, fprintf(stderr,
"ww test: package-test variant needs -c -o\n"); "ww test: package-test products need -c\n");
return 2; return 2;
} }
int variant = strcmp(package_test_kind, "same") == 0 return build_package_tests(resolved, incs, workdir,
? SEP_VARIANT_SAME_TEST : SEP_VARIANT_EXTERNAL; products, nproducts);
return build_one_sep(resolved, 1, NULL, outstem, outstem,
incs, "", "", 0, 1, variant, package_test_name, 0,
1, workdir);
} }
return exec_package_tests(argc, argv, src, resolved, 0); return exec_package_tests(argc, argv, src, resolved, 0);
} }
@@ -2184,7 +2378,7 @@ do_test(int argc, char **argv)
"ww test: package options need a directory\n"); "ww test: package options need a directory\n");
return 2; return 2;
} }
if (package_test_kind != NULL) { if (nproducts != 0) {
fprintf(stderr, fprintf(stderr,
"ww test: package-test variant needs one directory\n"); "ww test: package-test variant needs one directory\n");
return 2; return 2;
@@ -2245,7 +2439,7 @@ do_test(int argc, char **argv)
return rc; return rc;
} }
if (S_ISREG(st.st_mode)) { if (S_ISREG(st.st_mode)) {
if (package_test_kind != NULL) { if (nproducts != 0) {
fprintf(stderr, fprintf(stderr,
"ww test: package-test variant needs one directory\n"); "ww test: package-test variant needs one directory\n");
return 2; return 2;
@@ -2324,16 +2518,14 @@ do_test(int argc, char **argv)
fprintf(stderr, "ww test: pattern needs a single test file\n"); fprintf(stderr, "ww test: pattern needs a single test file\n");
return 2; return 2;
} }
if (package_test_kind != NULL) { if (nproducts != 0) {
if (!compileonly || !outstem[0]) { if (!compileonly) {
fprintf(stderr, fprintf(stderr,
"ww test: package-test variant needs -c -o\n"); "ww test: package-test products need -c\n");
return 2; return 2;
} }
int variant = strcmp(package_test_kind, "same") == 0 return build_package_tests(target, incs, workdir,
? SEP_VARIANT_SAME_TEST : SEP_VARIANT_EXTERNAL; products, nproducts);
return build_one_sep(target, 1, NULL, outstem, outstem, incs,
"", "", 0, 1, variant, package_test_name, 0, 1, workdir);
} }
return exec_package_tests(argc, argv, src, NULL, src == NULL); return exec_package_tests(argc, argv, src, NULL, src == NULL);
} }

293
internal/wwpackage/package.ww
View File

@@ -25,29 +25,34 @@ type pkggroup = struct {
dir: str, dir: str,
pkg: str, pkg: str,
external: bool, external: bool,
plan: i32,
root: str, root: str,
workdir: str,
bin: str, bin: str,
buildout: str, buildok: str,
builderr: str,
runout: str, runout: str,
runerr: str, runerr: str,
state: i32, state: i32,
fail: i32,
buildres: exec.result,
runres: exec.result, runres: exec.result,
}; };
// pkggroup.state values for the bounded scheduler. type pkgplan = struct {
dir: str,
root: str,
workdir: str,
buildout: str,
builderr: str,
start: i32,
end: i32,
state: i32,
buildres: exec.result,
};
// Product and directory-plan process states for the bounded coordinator.
def PKGQUEUED: i32 = 0; def PKGQUEUED: i32 = 0;
def PKGBUILDING: i32 = 1; def PKGBUILDING: i32 = 1;
def PKGRUNNING: i32 = 2; def PKGRUNNING: i32 = 2;
def PKGDONE: i32 = 3; def PKGDONE: i32 = 3;
// pkggroup.fail values recorded at launch, reported at ordered emission.
def PKGFAILNONE: i32 = 0;
def PKGFAILSETUP: i32 = 1;
def pkgpoll: time.duration = 1000000i64: time.duration; def pkgpoll: time.duration = 1000000i64: time.duration;
type pkgdiscover = struct { type pkgdiscover = struct {
@@ -56,7 +61,7 @@ type pkgdiscover = struct {
}; };
// Preserve the caller's toolchain environment while pinning the locale and // Preserve the caller's toolchain environment while pinning the locale and
// temporary directory used by the current package group. // temporary directory used by the current build plan or test product.
fn toolenv(tmpdir: str) []str = { fn toolenv(tmpdir: str) []str = {
let inherited: []str = os.getenvs(); let inherited: []str = os.getenvs();
let env: []str = alloc([], (inherited.len + 2): u64)!; let env: []str = alloc([], (inherited.len + 2): u64)!;
@@ -101,8 +106,8 @@ fn pkgusage() void = {
let s: str = strings.concat( let s: str = strings.concat(
"usage: wwtest package [-c] [-list] [-j N] [-I DIR] [-w DIR] [-run|-filter GLOB] [-timeout-ms=N] [DIR | DIR/...] [-- GLOB ...]\n", "usage: wwtest package [-c] [-list] [-j N] [-I DIR] [-w DIR] [-run|-filter GLOB] [-timeout-ms=N] [DIR | DIR/...] [-- GLOB ...]\n",
" *_test.ww is the sole test-source form; @test elsewhere is rejected\n", " *_test.ww is the sole test-source form; @test elsewhere is rejected\n",
" -c retains the compiled package binaries; -j N runs up to N package groups at once\n", " -c retains the compiled package binaries; -j N runs up to N directory builds or test binaries at once\n",
" -w DIR keys a persistent per-package-group build workdir under DIR\n"); " -w DIR keys a persistent shared build workdir per package directory\n");
pkgput(os.STDERR_FILENO, s); pkgput(os.STDERR_FILENO, s);
}; };
@@ -246,6 +251,14 @@ fn pkgisdir(path: str) bool = {
}; };
}; };
fn pkgisreg(path: str) bool = {
let fi: os.filestat;
match (os.lstat(&fi, path)) {
case void => return pkgmodeis(fi.mode, os.mode.REG);
case let e: os.oserror => return false;
};
};
fn pkgkeepfile(name: str) bool = { fn pkgkeepfile(name: str) bool = {
if (!strings.hassuffix(name, ".ww")) { return false; }; if (!strings.hassuffix(name, ".ww")) { return false; };
return true; return true;
@@ -510,23 +523,31 @@ fn pkgworkkey(dir: str, pkg: str) str = {
pkgworkescape(pkg)); pkgworkescape(pkg));
}; };
fn pkgsetpaths(g: *pkggroup, root: str, index: i32, fn pkgsetplanpaths(p: *pkgplan, groups: []pkggroup, root: str, index: i32,
compileonly: bool, outname: str, workroot: str) bool = { compileonly: bool, outname: str, workroot: str) bool = {
let num: str = strconv.i32tos(index, strconv.base.DEC); let num: str = strconv.i32tos(index, strconv.base.DEC);
g.root = strings.concat(root, "/group-", num); p.root = strings.concat(root, "/plan-", num);
if (!pkgmakedir(g.root)) { return false; }; if (!pkgmakedir(p.root)) { return false; };
// A caller-owned persistent workdir root keys one driver -w dir p.workdir = "";
// per (dir, pkg) group; the driver's content-identity contract
// owns every reuse decision, so this stays a pure path policy.
g.workdir = "";
if (workroot.len != 0) { if (workroot.len != 0) {
g.workdir = strings.concat(workroot, "/", // The first byte-sorted product name is stable metadata in the
pkgworkkey(g.dir, g.pkg)); // injective directory key; every product in this plan shares the dir.
match (os.mkdirs(g.workdir, 448)) { p.workdir = strings.concat(workroot, "/",
pkgworkkey(p.dir, groups[p.start].pkg));
match (os.mkdirs(p.workdir, 448)) {
case void => void; case void => void;
case let e: os.oserror => return false; case let e: os.oserror => return false;
}; };
}; };
p.buildout = strings.concat(p.root, "/build.stdout");
p.builderr = strings.concat(p.root, "/build.stderr");
let i: i32 = p.start;
for (i < p.end) {
let g: *pkggroup = &groups[i];
g.plan = index;
g.root = strings.concat(p.root, "/product-",
strconv.i32tos(i - p.start, strconv.base.DEC));
if (!pkgmakedir(g.root)) { return false; };
if (compileonly) { if (compileonly) {
if (outname.len != 0) { if (outname.len != 0) {
g.bin = outname; g.bin = outname;
@@ -536,10 +557,11 @@ fn pkgsetpaths(g: *pkggroup, root: str, index: i32,
} else { } else {
g.bin = strings.concat(g.root, "/package.test"); g.bin = strings.concat(g.root, "/package.test");
}; };
g.buildout = strings.concat(g.root, "/build.stdout");
g.builderr = strings.concat(g.root, "/build.stderr");
g.runout = strings.concat(g.root, "/test.stdout"); g.runout = strings.concat(g.root, "/test.stdout");
g.runerr = strings.concat(g.root, "/test.stderr"); g.runerr = strings.concat(g.root, "/test.stderr");
g.buildok = strings.concat(g.root, "/build.ok");
i += 1;
};
return true; return true;
}; };
@@ -575,36 +597,42 @@ fn pkgreportcommand(kind: str, g: *pkggroup, r: *exec.result) void = {
pkgputln(os.STDERR_FILENO, ")"); pkgputln(os.STDERR_FILENO, ")");
}; };
fn pkgstartbuild(g: *pkggroup, builder: str, includes: []str, fn pkgstartbuild(p: *pkgplan, groups: []pkggroup, builder: str, includes: []str,
h: *exec.process) void = { h: *exec.process) void = {
let ba: []str = alloc([], (13 + includes.len * 2): u64)!; let ba: []str = alloc([],
(8 + (p.end - p.start) * 5 + includes.len * 2): u64)!;
append(ba, builder); append(ba, builder);
append(ba, "test"); append(ba, "test");
append(ba, "-c"); append(ba, "-c");
append(ba, "-o"); let i: i32 = p.start;
append(ba, g.bin); for (i < p.end) {
let g: *pkggroup = &groups[i];
append(ba, "--ww-package-test"); append(ba, "--ww-package-test");
if (g.external) { append(ba, "external"); } if (g.external) { append(ba, "external"); }
else { append(ba, "same"); }; else { append(ba, "same"); };
append(ba, g.pkg); append(ba, g.pkg);
append(ba, g.bin);
append(ba, g.buildok);
i += 1;
};
let ii: i32 = 0; let ii: i32 = 0;
for (ii < includes.len) { for (ii < includes.len) {
append(ba, "-I"); append(ba, "-I");
append(ba, includes[ii]); append(ba, includes[ii]);
ii += 1; ii += 1;
}; };
if (g.workdir.len != 0) { if (p.workdir.len != 0) {
append(ba, "-w"); append(ba, "-w");
append(ba, g.workdir); append(ba, p.workdir);
}; };
append(ba, g.dir); append(ba, p.dir);
let bcmd: exec.command; let bcmd: exec.command;
bcmd.path = builder; bcmd.path = builder;
bcmd.argv = ba; bcmd.argv = ba;
bcmd.env = toolenv(g.root); bcmd.env = toolenv(p.root);
bcmd.dir = ""; bcmd.dir = "";
bcmd.stdoutpath = g.buildout; bcmd.stdoutpath = p.buildout;
bcmd.stderrpath = g.builderr; bcmd.stderrpath = p.builderr;
bcmd.deadline.sec = 0i64; bcmd.deadline.sec = 0i64;
bcmd.deadline.nsec = 0i64; bcmd.deadline.nsec = 0i64;
bcmd.grace = 0i64: time.duration; bcmd.grace = 0i64: time.duration;
@@ -633,10 +661,8 @@ fn pkgstartrun(g: *pkggroup, filters: []str, timeoutarg: str,
exec.start(h, &rcmd); exec.start(h, &rcmd);
}; };
fn pkgbuildok(g: *pkggroup) bool = { fn pkgproductbuilt(g: *pkggroup) bool = {
return g.buildres.errno == 0 && g.buildres.cleanuperrno == 0 return pkgisreg(g.buildok) && pkgisreg(g.bin);
&& g.buildres.termination == exec.termination.EXIT
&& g.buildres.code == 0;
}; };
fn pkgrunok(g: *pkggroup) bool = { fn pkgrunok(g: *pkggroup) bool = {
@@ -645,24 +671,7 @@ fn pkgrunok(g: *pkggroup) bool = {
&& g.runres.code == 0; && g.runres.code == 0;
}; };
// Ordered emission: a group's captures, diagnostics, and verdict line are fn pkgemitgroup(g: *pkggroup, compileonly: bool) bool = {
// written only here, strictly in group order, so concurrent scheduling
// produces the byte stream sequential scheduling produced.
fn pkgemitgroup(g: *pkggroup, tmproot: str, compileonly: bool) bool = {
if (g.fail == PKGFAILSETUP) {
pkgfailpath(tmproot, "cannot create group temporary directory");
return false;
};
let buildcaptures: bool = pkgemitfile(g.buildout, os.STDOUT_FILENO);
buildcaptures = pkgemitfile(g.builderr, os.STDERR_FILENO) && buildcaptures;
if (!buildcaptures) {
pkgfailpath(g.root, "cannot read build capture");
return false;
};
if (!pkgbuildok(g)) {
pkgreportcommand("build", g, &g.buildres);
return false;
};
if (compileonly) { if (compileonly) {
pkgput(os.STDOUT_FILENO, "built "); pkgput(os.STDOUT_FILENO, "built ");
pkglabel(g); pkglabel(g);
@@ -697,6 +706,30 @@ fn pkgemitgroup(g: *pkggroup, tmproot: str, compileonly: bool) bool = {
return true; return true;
}; };
// A directory build capture is emitted once, followed by its independently
// executed products in their existing byte-sorted group order.
fn pkgemitplan(p: *pkgplan, groups: []pkggroup,
compileonly: bool) i32 = {
let buildcaptures: bool = pkgemitfile(p.buildout, os.STDOUT_FILENO);
buildcaptures = pkgemitfile(p.builderr, os.STDERR_FILENO) && buildcaptures;
if (!buildcaptures) {
pkgfailpath(p.root, "cannot read build capture");
return 1;
};
let failed: i32 = 0;
let i: i32 = p.start;
for (i < p.end) {
if (!pkgproductbuilt(&groups[i])) {
pkgreportcommand("build", &groups[i], &p.buildres);
failed += 1;
} else if (!pkgemitgroup(&groups[i], compileonly)) {
failed += 1;
};
i += 1;
};
return failed;
};
export fn packagecommand(args: []str) int = { export fn packagecommand(args: []str) int = {
let compileonly: bool = false; let compileonly: bool = false;
let list: bool = false; let list: bool = false;
@@ -943,74 +976,130 @@ export fn packagecommand(args: []str) int = {
"wwtest package: cannot use -o with multiple packages"); "wwtest package: cannot use -o with multiple packages");
return 2; return 2;
}; };
let plans: []pkgplan = alloc([], groups.len: u64)!;
i = 0;
for (i < groups.len) {
let p: pkgplan;
p.dir = groups[i].dir;
p.start = i;
for (i < groups.len
&& strings.compare(groups[i].dir, p.dir) == 0) {
i += 1;
};
p.end = i;
p.state = PKGQUEUED;
append(plans, p);
};
let borrowed: str = temp.dir(); let borrowed: str = temp.dir();
let tmproot: str = strings.dup(borrowed); let tmproot: str = strings.dup(borrowed);
let failed: i32 = 0; let failed: i32 = 0;
// Bounded scheduler: up to `jobs` groups in flight, each a
// build-then-run process chain supervised with exec.start/poll.
// Launch order, run-slot accounting, and ordered emission keep
// -j 1 byte-identical to the former sequential loop; a setup
// failure stops new launches exactly where that loop broke.
let handles: []exec.process = alloc([], groups.len: u64)!;
i = 0; i = 0;
for (i < groups.len) { for (i < plans.len) {
if (!pkgsetplanpaths(&plans[i], groups, tmproot, i,
compileonly, outname, workroot)) {
pkgfailpath(tmproot, "cannot create directory-plan temporary path");
if (!pkgremoveall(tmproot)) {
pkgput(os.STDERR_FILENO,
"wwtest package: cleanup failed; retained ");
pkgputln(os.STDERR_FILENO, tmproot);
};
return 1;
};
i += 1;
};
// Build one shared package plan per directory. As soon as a plan completes,
// its successful products may run under the same global -j bound while
// other directory builds remain active. Emission stays ordered below.
let handles: []exec.process = alloc([], plans.len: u64)!;
i = 0;
for (i < plans.len) {
let h: exec.process; let h: exec.process;
append(handles, h); append(handles, h);
i += 1; i += 1;
}; };
let launched: i32 = 0; let runhandles: []exec.process = alloc([], groups.len: u64)!;
let emitted: i32 = 0; i = 0;
for (i < groups.len) {
let h: exec.process;
append(runhandles, h);
groups[i].state = PKGQUEUED;
i += 1;
};
let planlaunched: i32 = 0;
let plancompleted: i32 = 0;
let productcompleted: i32 = 0;
if (compileonly) { productcompleted = groups.len; };
let active: i32 = 0; let active: i32 = 0;
let stopped: bool = false; for (plancompleted < plans.len || productcompleted < groups.len) {
for (emitted < groups.len) { // Fill free slots with already-built products first, then the next
for (!stopped && launched < groups.len && active < jobs) { // byte-sorted directory build. Missing product markers are completed
let g: *pkggroup = &groups[launched]; // build failures and consume no process slot.
if (!pkgsetpaths(g, tmproot, launched, compileonly, let filling: bool = true;
outname, workroot)) { for (filling && active < jobs) {
g.fail = PKGFAILSETUP; filling = false;
if (!compileonly) {
let gi: i32 = 0;
for (gi < groups.len) {
let g: *pkggroup = &groups[gi];
if (g.state == PKGQUEUED
&& plans[g.plan].state == PKGDONE) {
if (!pkgproductbuilt(g)) {
g.state = PKGDONE; g.state = PKGDONE;
stopped = true; productcompleted += 1;
} else { } else {
pkgstartbuild(g, builder, includes, pkgstartrun(g, filters, timeoutarg, list,
&handles[launched]); &runhandles[gi]);
g.state = PKGBUILDING; g.state = PKGRUNNING;
active += 1; active += 1;
}; };
launched += 1; filling = true;
break;
}; };
let k: i32 = emitted; gi += 1;
for (k < launched) {
let g: *pkggroup = &groups[k];
if (g.state == PKGBUILDING && exec.poll(&handles[k])) {
g.buildres = handles[k].result;
if (pkgbuildok(g) && !compileonly) {
pkgstartrun(g, filters, timeoutarg,
list, &handles[k]);
g.state = PKGRUNNING;
} else {
g.state = PKGDONE;
active -= 1;
}; };
} else if (g.state == PKGRUNNING && exec.poll(&handles[k])) { };
g.runres = handles[k].result; if (!filling && planlaunched < plans.len) {
g.state = PKGDONE; pkgstartbuild(&plans[planlaunched], groups, builder, includes,
&handles[planlaunched]);
plans[planlaunched].state = PKGBUILDING;
active += 1;
planlaunched += 1;
filling = true;
};
};
let k: i32 = 0;
for (k < planlaunched) {
let p: *pkgplan = &plans[k];
if (p.state == PKGBUILDING && exec.poll(&handles[k])) {
p.buildres = handles[k].result;
p.state = PKGDONE;
active -= 1; active -= 1;
plancompleted += 1;
}; };
k += 1; k += 1;
}; };
for (emitted < launched && groups[emitted].state == PKGDONE) { if (!compileonly) {
if (!pkgemitgroup(&groups[emitted], tmproot, k = 0;
compileonly)) { for (k < groups.len) {
failed += 1; let g: *pkggroup = &groups[k];
if (g.state == PKGRUNNING && exec.poll(&runhandles[k])) {
g.runres = runhandles[k].result;
g.state = PKGDONE;
active -= 1;
productcompleted += 1;
}; };
emitted += 1; k += 1;
}; };
if (stopped && active == 0 && emitted == launched) { break; };
if (active > 0) {
time.sleep(pkgpoll, time.clock.monotonic);
}; };
if (active > 0) { time.sleep(pkgpoll, time.clock.monotonic); };
};
i = 0;
for (i < plans.len) {
failed += pkgemitplan(&plans[i], groups, compileonly);
i += 1;
}; };
if (!pkgremoveall(tmproot)) { if (!pkgremoveall(tmproot)) {
pkgput(os.STDERR_FILENO, "wwtest package: cleanup failed; retained "); pkgput(os.STDERR_FILENO, "wwtest package: cleanup failed; retained ");

419
selfhost/cmd/ww/main.ww
View File

@@ -418,6 +418,7 @@ def SEP_VARIANT_PRODUCTION: i32 = 0;
def SEP_VARIANT_SAME_TEST: i32 = 1; def SEP_VARIANT_SAME_TEST: i32 = 1;
def SEP_VARIANT_EXTERNAL: i32 = 2; def SEP_VARIANT_EXTERNAL: i32 = 2;
def SEP_TEST_SUPPORT_MODULE: str = "__wwtest"; def SEP_TEST_SUPPORT_MODULE: str = "__wwtest";
def SEP_MAXPRODUCT: i32 = 2;
// Classify a selected directory entry: 1 production, 2 test, 0 skipped, // Classify a selected directory entry: 1 production, 2 test, 0 skipped,
// -1 @test outside *_test.ww, -2 non-regular source. // -1 @test outside *_test.ww, -2 non-regular source.
@@ -727,6 +728,8 @@ type seppkg = struct {
nsources: i32, nsources: i32,
isdir: i32, isdir: i32,
variant: i32, variant: i32,
root: bool,
failed: bool,
testsupport: bool, testsupport: bool,
deps: []i32, // direct-dep indices into sepgraph.pkg deps: []i32, // direct-dep indices into sepgraph.pkg
ndeps: i32, ndeps: i32,
@@ -738,21 +741,36 @@ type sepgraph = struct {
n: i32, n: i32,
}; };
type sepproduct = struct {
out: *u8,
testpackage: *u8,
status: *u8,
variant: i32,
root: i32,
};
fn sepfindoraddvariant(g: *sepgraph, path: *u8, entry: *u8, fn sepfindoraddvariant(g: *sepgraph, path: *u8, entry: *u8,
isdir: i32, variant: i32, testpackage: *u8) i32 = { isdir: i32, variant: i32, testpackage: *u8, root: bool) i32 = {
if (cstrlen(path) >= 256u64) { if (cstrlen(path) >= 256u64) {
cerr("ww: package path is too long (limit 255 bytes)\n"); cerr("ww: package path is too long (limit 255 bytes)\n");
return -1; return -1;
}; };
let i: i32 = 0; let i: i32 = 0;
for (i < g.n) { for (i < g.n) {
let testproductionpair: bool = i == 0 let samelocation: bool = os.samefile(pathstr(g.pkg[i].entry),
&& g.pkg[i].variant != SEP_VARIANT_PRODUCTION pathstr(entry));
&& variant == SEP_VARIANT_PRODUCTION && isdir != 0 let testrootpair: bool = samelocation && isdir != 0
&& g.pkg[i].isdir != 0 && g.pkg[i].isdir != 0
&& os.samefile(pathstr(g.pkg[i].entry), pathstr(entry)); && ((root && g.pkg[i].root
&& variant != g.pkg[i].variant)
|| (root && !g.pkg[i].root
&& variant != SEP_VARIANT_PRODUCTION
&& g.pkg[i].variant == SEP_VARIANT_PRODUCTION)
|| (!root && g.pkg[i].root
&& variant == SEP_VARIANT_PRODUCTION
&& g.pkg[i].variant != SEP_VARIANT_PRODUCTION));
if (cstreq(g.pkg[i].path, path)) { if (cstreq(g.pkg[i].path, path)) {
if (testproductionpair) { if (testrootpair) {
i += 1; i += 1;
continue; continue;
}; };
@@ -764,8 +782,8 @@ fn sepfindoraddvariant(g: *sepgraph, path: *u8, entry: *u8,
}; };
return i; return i;
}; };
if (os.samefile(pathstr(g.pkg[i].entry), pathstr(entry))) { if (samelocation) {
if (testproductionpair) { if (testrootpair) {
i += 1; i += 1;
continue; continue;
}; };
@@ -799,6 +817,8 @@ fn sepfindoraddvariant(g: *sepgraph, path: *u8, entry: *u8,
g.pkg[g.n].nsources = 0; g.pkg[g.n].nsources = 0;
g.pkg[g.n].isdir = isdir; g.pkg[g.n].isdir = isdir;
g.pkg[g.n].variant = variant; g.pkg[g.n].variant = variant;
g.pkg[g.n].root = root;
g.pkg[g.n].failed = false;
g.pkg[g.n].testsupport = false; g.pkg[g.n].testsupport = false;
let dslot: []i32 = alloc([], SEP_MAXPKG: u64)!; let dslot: []i32 = alloc([], SEP_MAXPKG: u64)!;
dslot.len = SEP_MAXPKG; dslot.len = SEP_MAXPKG;
@@ -812,7 +832,7 @@ fn sepfindoraddvariant(g: *sepgraph, path: *u8, entry: *u8,
fn sepfindoradd(g: *sepgraph, path: *u8, entry: *u8, isdir: i32) i32 = { fn sepfindoradd(g: *sepgraph, path: *u8, entry: *u8, isdir: i32) i32 = {
return sepfindoraddvariant(g, path, entry, isdir, return sepfindoraddvariant(g, path, entry, isdir,
SEP_VARIANT_PRODUCTION, nil); SEP_VARIANT_PRODUCTION, nil, false);
}; };
// Release the package-owned directory-membership lists through one graph // Release the package-owned directory-membership lists through one graph
@@ -839,18 +859,24 @@ fn sepgraphfree(g: *sepgraph) void = {
}; };
}; };
// Build "<scratch>/<base><suffix>" NUL-term; base = path, or "__root" // Build one artifact path. Test roots use distinct names even though both
// for the empty root path. // compiler units reset to the bare executable namespace.
fn sepfname(g: *sepgraph, pi: i32, scratch: *u8, suffix: str) *u8 = { fn sepfname(g: *sepgraph, pi: i32, scratch: *u8, suffix: str) *u8 = {
let buf: []u8 = alloc([], (os.PATH_MAX: u64))!; let buf: []u8 = alloc([], (os.PATH_MAX: u64))!;
buf.len = os.PATH_MAX; buf.len = os.PATH_MAX;
let off: u64 = cstrinto(buf.ptr, 0u64, scratch); let off: u64 = cstrinto(buf.ptr, 0u64, scratch);
off = byteinto(buf.ptr, off, 47u8); // '/' off = byteinto(buf.ptr, off, 47u8); // '/'
if (g.pkg[pi].path[0u64] != 0u8) { if (g.pkg[pi].root
&& g.pkg[pi].variant == SEP_VARIANT_SAME_TEST) {
off = strinto(buf.ptr, off, "__ww-test-same");
} else { if (g.pkg[pi].root
&& g.pkg[pi].variant == SEP_VARIANT_EXTERNAL) {
off = strinto(buf.ptr, off, "__ww-test-external");
} else { if (g.pkg[pi].path[0u64] != 0u8) {
off = cstrinto(buf.ptr, off, g.pkg[pi].path); off = cstrinto(buf.ptr, off, g.pkg[pi].path);
} else { } else {
off = strinto(buf.ptr, off, "__root"); off = strinto(buf.ptr, off, "__root");
}; }; }; };
off = strinto(buf.ptr, off, suffix); off = strinto(buf.ptr, off, suffix);
cstrseal(buf.ptr, off); cstrseal(buf.ptr, off);
return buf.ptr; return buf.ptr;
@@ -1085,7 +1111,10 @@ fn sepscanfile(g: *sepgraph, pi: i32, file: *u8, searchpath: *u8,
// dependency scanning. Recurse over the resulting edges. `color` doubles as // dependency scanning. Recurse over the resulting edges. `color` doubles as
// a loaded marker (2); reset to white before topo. // a loaded marker (2); reset to white before topo.
fn seploadpkg(g: *sepgraph, pi: i32, searchpath: *u8) i32 = { fn seploadpkg(g: *sepgraph, pi: i32, searchpath: *u8) i32 = {
if (g.pkg[pi].color == 2) { return 0; }; if (g.pkg[pi].color == 2) {
if (g.pkg[pi].failed) { return -1; };
return 0;
};
g.pkg[pi].color = 2; g.pkg[pi].color = 2;
let fv: expctx; let fv: expctx;
fv.out = -1; fv.out = -1;
@@ -1142,8 +1171,8 @@ fn seploadpkg(g: *sepgraph, pi: i32, searchpath: *u8) i32 = {
} else { } else {
rc = sepscanfile(g, pi, g.pkg[pi].entry, searchpath, &fv, 0); rc = sepscanfile(g, pi, g.pkg[pi].entry, searchpath, &fv, 0);
}; };
if (rc < 0) { return rc; }; if (rc < 0) { g.pkg[pi].failed = true; return rc; };
if (pi == 0 && g.pkg[pi].path[0u64] == 0u8 if (g.pkg[pi].root && g.pkg[pi].path[0u64] == 0u8
&& g.pkg[pi].name != nil) { && g.pkg[pi].name != nil) {
g.pkg[pi].path = arenadupcstr(g.pkg[pi].name, g.pkg[pi].path = arenadupcstr(g.pkg[pi].name,
cstrlen(g.pkg[pi].name)); cstrlen(g.pkg[pi].name));
@@ -1163,7 +1192,10 @@ fn seploadpkg(g: *sepgraph, pi: i32, searchpath: *u8) i32 = {
}; };
let k: i32 = 0; let k: i32 = 0;
for (k < g.pkg[pi].ndeps) { for (k < g.pkg[pi].ndeps) {
if (seploadpkg(g, g.pkg[pi].deps[k], searchpath) < 0) { return -1; }; if (seploadpkg(g, g.pkg[pi].deps[k], searchpath) < 0) {
g.pkg[pi].failed = true;
return -1;
};
k += 1; k += 1;
}; };
return 0; return 0;
@@ -1575,14 +1607,14 @@ fn copyfileatomic(src: *u8, dst: *u8) i32 = {
fn workdirstamptext(istest: i32, emitasm: i32) str = { fn workdirstamptext(istest: i32, emitasm: i32) str = {
if (istest != 0) { if (istest != 0) {
if (emitasm != 0) { if (emitasm != 0) {
return "ww workdir fmt 3 mode test asm 1\n"; return "ww workdir fmt 4 mode test asm 1\n";
}; };
return "ww workdir fmt 3 mode test asm 0\n"; return "ww workdir fmt 4 mode test asm 0\n";
}; };
if (emitasm != 0) { if (emitasm != 0) {
return "ww workdir fmt 3 mode build asm 1\n"; return "ww workdir fmt 4 mode build asm 1\n";
}; };
return "ww workdir fmt 3 mode build asm 0\n"; return "ww workdir fmt 4 mode build asm 0\n";
}; };
fn stampmatches(path: *u8, want: str) bool = { fn stampmatches(path: *u8, want: str) bool = {
@@ -1618,6 +1650,25 @@ fn writestampatomic(path: *u8, want: str) i32 = {
return os.rename(pathstr(tmpp), pathstr(path)); return os.rename(pathstr(tmpp), pathstr(path));
}; };
// A coordinator-private completion marker distinguishes a newly linked
// product from a caller-owned binary left by an earlier invocation.
fn recordproductstatus(path: *u8) i32 = {
if (path == nil) { return 0; };
let tmpp: *u8 = appendlit(path, ".new");
let fd: i32 = os.open(pathstr(tmpp),
os.flag.WRONLY | os.flag.CREATE | os.flag.TRUNC, 420i32);
if (fd < 0) { return -1; };
let body: str = "ok\n";
let bad: bool = false;
match (os.writeall(fd, body.ptr, body.len: u64)) {
case let n: i64 => { if (n != body.len: i64) { bad = true; }; };
case let e: os.oserror => { bad = true; };
};
if (os.close(fd) != 0) { bad = true; };
if (bad) { return -1; };
return os.rename(pathstr(tmpp), pathstr(path));
};
// cerrpath — the "ww: <head><path>\n" diagnostic shape shared by the // cerrpath — the "ww: <head><path>\n" diagnostic shape shared by the
// workdir error sites; byte-identical wording to the cstage twin's // workdir error sites; byte-identical wording to the cstage twin's
// fprintf(..., "%s", path) forms. // fprintf(..., "%s", path) forms.
@@ -1630,9 +1681,18 @@ fn cerrpath(head: str, path: *u8, tail: str) void = {
fn buildonesepimpl(selfdir: *u8, src: *u8, entryisdir: i32, fn buildonesepimpl(selfdir: *u8, src: *u8, entryisdir: i32,
rootidentity: *u8, out: *u8, rootidentity: *u8, out: *u8,
objstem: *u8, incs: *u8, lf: *lflags, packageonly: i32, istest: i32, objstem: *u8, incs: *u8, lf: *lflags, packageonly: i32, istest: i32,
rootvariant: i32, testpackage: *u8, emitasm: i32, products: *sepproduct, nproducts: i32, emitasm: i32,
workdir: *u8, scratchout: **u8, workdir: *u8, scratchout: **u8,
graphout: **sepgraph) i32 = { graphout: **sepgraph) i32 = {
if (nproducts < 1 || nproducts > SEP_MAXPRODUCT) { return 1; };
let statusi: i32 = 0;
for (statusi < nproducts) {
if (products[statusi].status != nil) {
let rr: i32 = os.remove(pathstr(products[statusi].status));
if (rr != 0 && rr != -2) { return 1; };
};
statusi += 1;
};
let c6: *u8 = joinpathlit(selfdir, "w6c_ww"); let c6: *u8 = joinpathlit(selfdir, "w6c_ww");
let a6: *u8 = joinpathlit(selfdir, "w6a_ww"); let a6: *u8 = joinpathlit(selfdir, "w6a_ww");
let l6: *u8 = joinpathlit(selfdir, "w6l_ww"); let l6: *u8 = joinpathlit(selfdir, "w6l_ww");
@@ -1776,9 +1836,14 @@ fn buildonesepimpl(selfdir: *u8, src: *u8, entryisdir: i32,
if (graphout != nil) { *graphout = g; }; if (graphout != nil) { *graphout = g; };
let rootpath: *u8 = "\0".ptr; let rootpath: *u8 = "\0".ptr;
if (packageonly != 0 && rootidentity != nil) { rootpath = rootidentity; }; if (packageonly != 0 && rootidentity != nil) { rootpath = rootidentity; };
let root: i32 = sepfindoraddvariant(g, rootpath, src, entryisdir, let producti: i32 = 0;
rootvariant, testpackage); for (producti < nproducts) {
if (root < 0) { return 1; }; products[producti].root = sepfindoraddvariant(g, rootpath, src,
entryisdir, products[producti].variant,
products[producti].testpackage, true);
if (products[producti].root < 0) { return 1; };
producti += 1;
};
let testsupportmodule: str = "test"; let testsupportmodule: str = "test";
// -T generates a dispatcher whose support qualifier is selected by the // -T generates a dispatcher whose support qualifier is selected by the
// command. Represent that compiler-generated requirement as a direct root // command. Represent that compiler-generated requirement as a direct root
@@ -1793,9 +1858,14 @@ fn buildonesepimpl(selfdir: *u8, src: *u8, entryisdir: i32,
let rootissupport: bool = entryisdir != 0 let rootissupport: bool = entryisdir != 0
&& os.samefile(pathstr(tp), pathstr(src)); && os.samefile(pathstr(tp), pathstr(src));
let collision: bool = false; let collision: bool = false;
if (!rootissupport && testpackage != nil) { producti = 0;
collision = cstreqlit(testpackage, "test") for (!rootissupport && producti < nproducts) {
|| cstreqlit(testpackage, "test_test"); let name: *u8 = products[producti].testpackage;
if (name != nil && (cstreqlit(name, "test")
|| cstreqlit(name, "test_test"))) {
collision = true;
};
producti += 1;
}; };
if (!rootissupport && !collision) { if (!rootissupport && !collision) {
let ud: i32 = 0; let ud: i32 = 0;
@@ -1806,10 +1876,17 @@ fn buildonesepimpl(selfdir: *u8, src: *u8, entryisdir: i32,
}; };
}; };
if (collision) { testsupportmodule = SEP_TEST_SUPPORT_MODULE; }; if (collision) { testsupportmodule = SEP_TEST_SUPPORT_MODULE; };
producti = 0;
for (producti < nproducts) {
let root: i32 = products[producti].root;
// A same-test build of the runtime package already owns run // A same-test build of the runtime package already owns run
// and its source imports. An external test still needs the // and its source imports. An external test still needs the
// colocated production node, also its one support dependency. // colocated production node, also its support dependency.
if (!rootissupport || rootvariant == SEP_VARIANT_EXTERNAL) { if (rootissupport
&& products[producti].variant != SEP_VARIANT_EXTERNAL) {
producti += 1;
continue;
};
let ti: i32 = sepfindoradd(g, let ti: i32 = sepfindoradd(g,
testsupportmodule.ptr, tp, td); testsupportmodule.ptr, tp, td);
if (ti < 0) { return 1; }; if (ti < 0) { return 1; };
@@ -1826,34 +1903,96 @@ fn buildonesepimpl(selfdir: *u8, src: *u8, entryisdir: i32,
g.pkg[root].ndeps += 1; g.pkg[root].ndeps += 1;
}; };
}; };
producti += 1;
}; };
}; };
}; };
if (seploadpkg(g, root, searchpath.ptr) < 0) { return 1; }; producti = 0;
if (rootvariant != SEP_VARIANT_PRODUCTION for (producti < nproducts) {
&& (testpackage == nil || !cstreq(g.pkg[root].name, testpackage))) { let root: i32 = products[producti].root;
if (seploadpkg(g, root, searchpath.ptr) < 0) {
g.pkg[root].failed = true;
producti += 1;
continue;
};
if (products[producti].variant != SEP_VARIANT_PRODUCTION
&& (products[producti].testpackage == nil
|| !cstreq(g.pkg[root].name,
products[producti].testpackage))) {
cerr("ww: package-test selector does not match loaded package\n"); cerr("ww: package-test selector does not match loaded package\n");
return 1; g.pkg[root].failed = true;
};
producti += 1;
}; };
let rootpackage: bool = packageonly != 0; let rootpackage: bool = packageonly != 0;
if (rootpackage && cstreqlit(g.pkg[root].name, "main")) { if (rootpackage && !g.pkg[products[0].root].failed
&& cstreqlit(g.pkg[products[0].root].name, "main")) {
cerr("ww: -p requires a non-main package\n"); cerr("ww: -p requires a non-main package\n");
return 1; return 1;
}; };
let ci: i32 = 0; let ci: i32 = 0;
for (ci < g.n) { g.pkg[ci].color = 0; ci += 1; };
let order: []i32 = alloc([], g.n: u64)!; let order: []i32 = alloc([], g.n: u64)!;
order.len = g.n; order.len = g.n;
let stack: []i32 = alloc([], g.n: u64)!; let stack: []i32 = alloc([], g.n: u64)!;
stack.len = g.n; stack.len = g.n;
let norder: i32 = 0; let norder: i32 = 0;
if (septopovisit(g, root, order, &norder, stack, 0) < 0) { return 1; }; // Diagnose cycles per product before constructing the shared union. A
if (!rootpackage) { g.pkg[root].path = "\0".ptr; }; // variant-local cycle must not suppress an independent sibling root.
producti = 0;
for (producti < nproducts) {
let root: i32 = products[producti].root;
if (!g.pkg[root].failed) {
ci = 0;
for (ci < g.n) { g.pkg[ci].color = 0; ci += 1; };
let ignored: i32 = 0;
if (septopovisit(g, root, order,
&ignored, stack, 0) < 0) {
g.pkg[root].failed = true;
};
};
producti += 1;
};
ci = 0;
for (ci < g.n) { g.pkg[ci].color = 0; ci += 1; };
producti = 0;
for (producti < nproducts) {
let root: i32 = products[producti].root;
if (!g.pkg[root].failed) {
if (septopovisit(g, root, order,
&norder, stack, 0) < 0) { return 1; };
};
producti += 1;
};
if (!rootpackage) {
producti = 0;
for (producti < nproducts) {
g.pkg[products[producti].root].path = "\0".ptr;
producti += 1;
};
};
let anyfailed: bool = false;
producti = 0;
for (producti < nproducts) {
if (g.pkg[products[producti].root].failed) { anyfailed = true; };
producti += 1;
};
let oi: i32 = 0; let oi: i32 = 0;
for (oi < norder) { for (oi < norder) {
let pi: i32 = order[oi]; let pi: i32 = order[oi];
let dk: i32 = 0;
for (dk < g.pkg[pi].ndeps) {
if (g.pkg[g.pkg[pi].deps[dk]].failed) {
g.pkg[pi].failed = true;
};
dk += 1;
};
if (g.pkg[pi].failed) {
anyfailed = true;
oi += 1;
continue;
};
let unitf: *u8 = sepfname(g, pi, scratch, ".unit.ww"); let unitf: *u8 = sepfname(g, pi, scratch, ".unit.ww");
let wwi: *u8 = sepfname(g, pi, scratch, ".wwi"); let wwi: *u8 = sepfname(g, pi, scratch, ".wwi");
let asmf: *u8 = sepfname(g, pi, scratch, ".s"); let asmf: *u8 = sepfname(g, pi, scratch, ".s");
@@ -1875,10 +2014,13 @@ fn buildonesepimpl(selfdir: *u8, src: *u8, entryisdir: i32,
cu = unitnew; cw = wwinew; cs = asmnew; cu = unitnew; cw = wwinew; cs = asmnew;
co = objnew; ca = anew; co = objnew; ca = anew;
}; };
let needsexport: bool = (pi != root) || rootpackage; let needsexport: bool = !g.pkg[pi].root || rootpackage;
let needsarchive: bool = (pi != root) || rootpackage; let needsarchive: bool = !g.pkg[pi].root || rootpackage;
if (sepcomposeunit(g, pi, scratch, searchpath.ptr, cu) < 0) { if (sepcomposeunit(g, pi, scratch, searchpath.ptr, cu) < 0) {
return 1; g.pkg[pi].failed = true;
anyfailed = true;
oi += 1;
continue;
}; };
let fresh: bool = false; let fresh: bool = false;
if (warm) { if (warm) {
@@ -1905,7 +2047,8 @@ fn buildonesepimpl(selfdir: *u8, src: *u8, entryisdir: i32,
if (fresh) { if (fresh) {
if (os.remove(pathstr(unitnew)) != 0) { if (os.remove(pathstr(unitnew)) != 0) {
cerrpath("ww: cannot remove ", unitnew, "\n"); cerrpath("ww: cannot remove ", unitnew, "\n");
return 1; g.pkg[pi].failed = true;
anyfailed = true;
}; };
oi += 1; oi += 1;
continue; continue;
@@ -1920,7 +2063,7 @@ fn buildonesepimpl(selfdir: *u8, src: *u8, entryisdir: i32,
// without the -I/wwi pair; root's `.wwi` is unconsumed. // without the -I/wwi pair; root's `.wwi` is unconsumed.
// #79: the root carries -T under `ww test` so w6c // #79: the root carries -T under `ww test` so w6c
// synthesizes the test main; deps never get -T. // synthesizes the test main; deps never get -T.
let roott: bool = (pi == root) && (istest != 0); let roott: bool = g.pkg[pi].root && (istest != 0);
let supportt: bool = g.pkg[pi].testsupport; let supportt: bool = g.pkg[pi].testsupport;
let alen: u64 = 8u64; let alen: u64 = 8u64;
if (!needsexport) { alen = 6u64; if (roott) { alen = 9u64; }; }; if (!needsexport) { alen = 6u64; if (roott) { alen = 9u64; }; };
@@ -1954,7 +2097,10 @@ fn buildonesepimpl(selfdir: *u8, src: *u8, entryisdir: i32,
cerr("ww: execve failed\n"); cerr("ww: execve failed\n");
}; };
cerr("ww: w6c failed\n"); cerr("ww: w6c failed\n");
return 1; g.pkg[pi].failed = true;
anyfailed = true;
oi += 1;
continue;
}; };
}; };
if (emitasm == 0) { if (emitasm == 0) {
@@ -1973,7 +2119,10 @@ fn buildonesepimpl(selfdir: *u8, src: *u8, entryisdir: i32,
cerr("ww: execve failed\n"); cerr("ww: execve failed\n");
}; };
cerr("ww: w6a failed\n"); cerr("ww: w6a failed\n");
return 1; g.pkg[pi].failed = true;
anyfailed = true;
oi += 1;
continue;
}; };
}; };
// Wrap each DEP package's `.o` in its own deterministic `.a` // Wrap each DEP package's `.o` in its own deterministic `.a`
@@ -1984,7 +2133,10 @@ fn buildonesepimpl(selfdir: *u8, src: *u8, entryisdir: i32,
if (emitasm == 0 && needsarchive) { if (emitasm == 0 && needsarchive) {
if (archiveo(co, ca) != 0) { if (archiveo(co, ca) != 0) {
cerr("ww: archive failed\n"); cerr("ww: archive failed\n");
return 1; g.pkg[pi].failed = true;
anyfailed = true;
oi += 1;
continue;
}; };
}; };
// Commit order: artifacts before the unit that vouches for // Commit order: artifacts before the unit that vouches for
@@ -2027,7 +2179,10 @@ fn buildonesepimpl(selfdir: *u8, src: *u8, entryisdir: i32,
} else { } else {
cerr("ww: cannot commit (root)\n"); cerr("ww: cannot commit (root)\n");
}; };
return 1; g.pkg[pi].failed = true;
anyfailed = true;
oi += 1;
continue;
}; };
}; };
oi += 1; oi += 1;
@@ -2035,7 +2190,7 @@ fn buildonesepimpl(selfdir: *u8, src: *u8, entryisdir: i32,
// Tool identity commits only after every package artifact it vouches // Tool identity commits only after every package artifact it vouches
// for is itself committed; a killed pass leaves the old identity and // for is itself committed; a killed pass leaves the old identity and
// forces a full recompile, never a false reuse. // forces a full recompile, never a false reuse.
if (warm) { if (warm && !anyfailed) {
if (!fileequal(toolc, c6)) { if (!fileequal(toolc, c6)) {
if (copyfileatomic(c6, toolc) != 0) { if (copyfileatomic(c6, toolc) != 0) {
cerrpath("ww: cannot record ", toolc, "\n"); cerrpath("ww: cannot record ", toolc, "\n");
@@ -2057,8 +2212,10 @@ fn buildonesepimpl(selfdir: *u8, src: *u8, entryisdir: i32,
}; };
}; };
}; };
if (emitasm != 0) { return 0; }; if (emitasm != 0) { if (anyfailed) { return 1; }; return 0; };
if (rootpackage) { if (rootpackage) {
let root: i32 = products[0].root;
if (g.pkg[root].failed) { return 1; };
let archive: *u8 = sepfname(g, root, scratch, ".a"); let archive: *u8 = sepfname(g, root, scratch, ".a");
let iface: *u8 = sepfname(g, root, scratch, ".wwi"); let iface: *u8 = sepfname(g, root, scratch, ".wwi");
let outiface: *u8 = appendlit(out, ".wwi"); let outiface: *u8 = appendlit(out, ".wwi");
@@ -2070,14 +2227,8 @@ fn buildonesepimpl(selfdir: *u8, src: *u8, entryisdir: i32,
return 0; return 0;
}; };
// Reverse-topo link: root `.o` first (order[norder-1]), dependency `.a` // Each product gets its own reverse-topological link closure. Shared
// files after, then libwwrt.a (which still selectively pulls only the // production actions do not turn variant-local archives into link inputs.
// runtime members a live undef needs). A same-test root already contains
// its production sources; if that production variant is also reached
// through the runtime closure, retain its dependencies but omit its
// duplicate archive.
// argv: 3 fixed (w6l,-o,out) + one root object/archive per package
// + 1 libwwrt + 2*nlibdirs + 2*nlibs + 1 nil.
let nldirs: i32 = 0; let nldirs: i32 = 0;
let nllibs: i32 = 0; let nllibs: i32 = 0;
let ldirs: **u8 = nil; let ldirs: **u8 = nil;
@@ -2088,16 +2239,34 @@ fn buildonesepimpl(selfdir: *u8, src: *u8, entryisdir: i32,
ldirs = lf.libdirs; ldirs = lf.libdirs;
llibs = lf.libs; llibs = lf.libs;
}; };
let total: i32 = 3 + norder + 1 + 2 * nldirs + 2 * nllibs + 1; producti = 0;
for (producti < nproducts) {
let root: i32 = products[producti].root;
if (g.pkg[root].failed) {
anyfailed = true;
producti += 1;
continue;
};
ci = 0;
for (ci < g.n) { g.pkg[ci].color = 0; ci += 1; };
let linkorder: []i32 = alloc([], g.n: u64)!;
linkorder.len = g.n;
let linkstack: []i32 = alloc([], g.n: u64)!;
linkstack.len = g.n;
let nlink: i32 = 0;
if (septopovisit(g, root, linkorder, &nlink,
linkstack, 0) < 0) { return 1; };
// argv: 3 fixed + closure + libwwrt + flags + nil.
let total: i32 = 3 + nlink + 1 + 2 * nldirs + 2 * nllibs + 1;
let largv: []*u8 = alloc([], total: u64)!; let largv: []*u8 = alloc([], total: u64)!;
largv.len = total; largv.len = total;
largv[0] = "w6l\0".ptr; largv[0] = "w6l\0".ptr;
largv[1] = "-o\0".ptr; largv[1] = "-o\0".ptr;
largv[2] = out; largv[2] = products[producti].out;
let pos: i32 = 3; let pos: i32 = 3;
let li: i32 = norder - 1; let li: i32 = nlink - 1;
for (li >= 0) { for (li >= 0) {
let pi: i32 = order[li]; let pi: i32 = linkorder[li];
if (g.pkg[root].variant == SEP_VARIANT_SAME_TEST if (g.pkg[root].variant == SEP_VARIANT_SAME_TEST
&& pi != root && pi != root
&& g.pkg[pi].variant == SEP_VARIANT_PRODUCTION && g.pkg[pi].variant == SEP_VARIANT_PRODUCTION
@@ -2106,7 +2275,6 @@ fn buildonesepimpl(selfdir: *u8, src: *u8, entryisdir: i32,
li -= 1; li -= 1;
continue; continue;
}; };
// root: positional `.o` (force-load); deps: `.a` (selective).
let suf: str = ".a"; let suf: str = ".a";
if (pi == root) { suf = ".o"; }; if (pi == root) { suf = ".o"; };
largv[pos] = sepfname(g, pi, scratch, suf); largv[pos] = sepfname(g, pi, scratch, suf);
@@ -2145,8 +2313,19 @@ fn buildonesepimpl(selfdir: *u8, src: *u8, entryisdir: i32,
cerr("ww: execve failed\n"); cerr("ww: execve failed\n");
}; };
cerr("ww: w6l failed\n"); cerr("ww: w6l failed\n");
return 1; g.pkg[root].failed = true;
anyfailed = true;
producti += 1;
continue;
}; };
if (recordproductstatus(products[producti].status) != 0) {
cerr("ww: cannot record package-test product\n");
g.pkg[root].failed = true;
anyfailed = true;
};
producti += 1;
};
if (anyfailed) { return 1; };
return 0; return 0;
}; };
@@ -2162,9 +2341,15 @@ fn buildonesep(selfdir: *u8, src: *u8, entryisdir: i32,
keepscratch: i32, workdir: *u8) i32 = { keepscratch: i32, workdir: *u8) i32 = {
let scratch: *u8 = nil; let scratch: *u8 = nil;
let g: *sepgraph = nil; let g: *sepgraph = nil;
let product: sepproduct;
product.out = out;
product.testpackage = testpackage;
product.status = nil;
product.variant = rootvariant;
product.root = -1;
let r: i32 = buildonesepimpl(selfdir, src, entryisdir, rootidentity, let r: i32 = buildonesepimpl(selfdir, src, entryisdir, rootidentity,
out, objstem, out, objstem,
incs, lf, packageonly, istest, rootvariant, testpackage, incs, lf, packageonly, istest, &product, 1,
emitasm, workdir, &scratch, &g); emitasm, workdir, &scratch, &g);
sepgraphfree(g); sepgraphfree(g);
if (keepscratch == 0 && scratch != nil) { if (keepscratch == 0 && scratch != nil) {
@@ -2191,6 +2376,22 @@ fn buildonesep(selfdir: *u8, src: *u8, entryisdir: i32,
return r; return r;
}; };
// Build every selected test root for one directory inside one command-owned
// package universe. The first output owns the shared cold sepwork tree.
fn buildpackagetests(selfdir: *u8, src: *u8, incs: *u8, workdir: *u8,
products: *sepproduct, nproducts: i32) i32 = {
let scratch: *u8 = nil;
let g: *sepgraph = nil;
let lf: lflags;
lf.libdirs = nil; lf.nlibdirs = 0;
lf.libs = nil; lf.nlibs = 0;
let r: i32 = buildonesepimpl(selfdir, src, 1, nil,
products[0].out, products[0].out, incs, &lf, 0, 1,
products, nproducts, 0, workdir, &scratch, &g);
sepgraphfree(g);
return r;
};
fn cstrendswithlit(p: *u8, lit: str) bool = { fn cstrendswithlit(p: *u8, lit: str) bool = {
return strings.hassuffix(pathstr(p), lit); return strings.hassuffix(pathstr(p), lit);
}; };
@@ -2817,8 +3018,7 @@ fn dotest(selfdir: *u8, argv: **u8, argc: i32, start: i32) i32 = {
let emitasm: i32 = 0; let emitasm: i32 = 0;
let outstem: *u8 = nil; let outstem: *u8 = nil;
let workdir: *u8 = nil; let workdir: *u8 = nil;
let packagetestkind: *u8 = nil; let products: []sepproduct = alloc([], SEP_MAXPRODUCT: u64)!;
let packagetestname: *u8 = nil;
let packageopts: bool = false; let packageopts: bool = false;
let afterdash: bool = false; let afterdash: bool = false;
let i: i32 = start; let i: i32 = start;
@@ -2830,24 +3030,50 @@ fn dotest(selfdir: *u8, argv: **u8, argc: i32, start: i32) i32 = {
packageopts = true; afterdash = true; i += 1; continue; packageopts = true; afterdash = true; i += 1; continue;
}; };
if (cstreqlit(p, "--ww-package-test")) { if (cstreqlit(p, "--ww-package-test")) {
if (i + 2 >= argc || packagetestkind != nil) { if (i + 4 >= argc || products.len >= SEP_MAXPRODUCT) {
cerr("ww test: --ww-package-test needs kind and package\n"); cerr("ww test: --ww-package-test needs kind, package, output, and status\n");
return 2; return 2;
}; };
packagetestkind = argv[i + 1]; let kind: *u8 = argv[i + 1];
packagetestname = argv[i + 2]; let name: *u8 = argv[i + 2];
let pn: u64 = cstrlen(packagetestname); let output: *u8 = argv[i + 3];
if ((!cstreqlit(packagetestkind, "same") let status: *u8 = argv[i + 4];
&& !cstreqlit(packagetestkind, "external")) let pn: u64 = cstrlen(name);
let variant: i32 = SEP_VARIANT_EXTERNAL;
if (cstreqlit(kind, "same")) {
variant = SEP_VARIANT_SAME_TEST;
};
if ((!cstreqlit(kind, "same")
&& !cstreqlit(kind, "external"))
|| pn == 0u64 || pn >= 256u64 || pn == 0u64 || pn >= 256u64
|| (cstreqlit(packagetestkind, "external") || output[0u64] == 0u8 || status[0u64] == 0u8
|| (cstreqlit(kind, "external")
&& (pn <= 5u64 && (pn <= 5u64
|| !cstrendswithlit(packagetestname, || !cstrendswithlit(name,
"_test")))) { "_test")))) {
cerr("ww test: invalid --ww-package-test variant\n"); cerr("ww test: invalid --ww-package-test variant\n");
return 2; return 2;
}; };
i += 3; let duplicate: bool = false;
let producti: i32 = 0;
for (producti < products.len) {
if (products[producti].variant == variant) {
duplicate = true;
};
producti += 1;
};
if (duplicate) {
cerr("ww test: duplicate --ww-package-test variant\n");
return 2;
};
let product: sepproduct;
product.out = output;
product.testpackage = name;
product.status = status;
product.variant = variant;
product.root = -1;
append(products, product);
i += 5;
continue; continue;
}; };
if (p[1u64] == 73u8) { // '-I' if (p[1u64] == 73u8) { // '-I'
@@ -2929,10 +3155,25 @@ fn dotest(selfdir: *u8, argv: **u8, argc: i32, start: i32) i32 = {
cerr("ww test: -S needs -o\n"); cerr("ww test: -S needs -o\n");
return 2; return 2;
}; };
if (packagetestkind != nil && packageopts) { let producti: i32 = 1;
for (producti < products.len) {
let product: sepproduct = products[producti];
let j: i32 = producti;
for (j > 0 && products[j - 1].variant > product.variant) {
products[j] = products[j - 1];
j -= 1;
};
products[j] = product;
producti += 1;
};
if (products.len != 0 && packageopts) {
cerr("ww test: package-test variant rejects package options\n"); cerr("ww test: package-test variant rejects package options\n");
return 2; return 2;
}; };
if (products.len != 0 && outstem != nil) {
cerr("ww test: package-test products reject -o\n");
return 2;
};
// Go's ./... form: a trailing "..." element is a package-tree // Go's ./... form: a trailing "..." element is a package-tree
// request for the coordinator, never a literal path — recognized // request for the coordinator, never a literal path — recognized
@@ -2946,7 +3187,7 @@ fn dotest(selfdir: *u8, argv: **u8, argc: i32, start: i32) i32 = {
&& target[tlen - 1u64] == '.'; && target[tlen - 1u64] == '.';
}; };
if (istree) { if (istree) {
if (packagetestkind != nil) { if (products.len != 0) {
cerr("ww test: package-test variant needs one directory\n"); cerr("ww test: package-test variant needs one directory\n");
return 2; return 2;
}; };
@@ -2965,7 +3206,7 @@ fn dotest(selfdir: *u8, argv: **u8, argc: i32, start: i32) i32 = {
return 2; return 2;
}; };
// -w forwards: the coordinator keys one persistent driver // -w forwards: the coordinator keys one persistent driver
// workdir per package group under the given root. // workdir per directory plan under the given root.
return execpackagetests(selfdir, argv, argc, start, return execpackagetests(selfdir, argv, argc, start,
targetindex, nil, false); targetindex, nil, false);
}; };
@@ -2991,7 +3232,7 @@ fn dotest(selfdir: *u8, argv: **u8, argc: i32, start: i32) i32 = {
}; };
}; };
if (isdir == 0) { if (isdir == 0) {
if (packagetestkind != nil) { if (products.len != 0) {
cerr("ww test: package-test variant needs one directory\n"); cerr("ww test: package-test variant needs one directory\n");
return 2; return 2;
}; };
@@ -3014,21 +3255,13 @@ fn dotest(selfdir: *u8, argv: **u8, argc: i32, start: i32) i32 = {
cerr("ww test: pattern needs a single test file\n"); cerr("ww test: pattern needs a single test file\n");
return 2; return 2;
}; };
if (packagetestkind != nil) { if (products.len != 0) {
if (compileonly == 0 || outstem == nil) { if (compileonly == 0) {
cerr("ww test: package-test variant needs -c -o\n"); cerr("ww test: package-test products need -c\n");
return 2; return 2;
}; };
let variant: i32 = SEP_VARIANT_EXTERNAL; return buildpackagetests(selfdir, resolved, incs.ptr, workdir,
if (cstreqlit(packagetestkind, "same")) { products.ptr, products.len);
variant = SEP_VARIANT_SAME_TEST;
};
let lf: lflags;
lf.libdirs = nil; lf.nlibdirs = 0;
lf.libs = nil; lf.nlibs = 0;
return buildonesep(selfdir, resolved, 1, nil, outstem, outstem,
incs.ptr, &lf, 0i32, 1i32, variant, packagetestname,
0i32, 1i32, workdir);
}; };
let replacement: *u8 = nil; let replacement: *u8 = nil;
if (resolved != target) { replacement = resolved; }; if (resolved != target) { replacement = resolved; };