Hojun-Cho
7b36f961a9
Per-package build cache for `ww build --sep`: before recompiling a package, recompute a plain-text key manifest (md5sum-hex lines of the package sources, each direct dep's .wwi, the w6c and w6a binaries, plus the compile flags) and reuse the cached .o/.wwi on a byte-for-byte key hit. Both stages shell the same host md5sum (no ww-side md5) so the non-compiler key lines are byte-identical cstage==wwstage; the compiler md5 lines differ per stage BY DESIGN, giving each stage its own cache namespace so a hit can never reuse the other stage's .o and mask a cs!=ww codegen divergence. Cached outputs (.o/.wwi) stay byte-identical across stages. Root package never cached; cache lives under $(OUT) (gitignored, wiped by make clean). Both stages mirrored. Dev-only convenience, off every bootstrap/byte-id gate. Gate 989_pkgcache_run (cold) proves miss-compiles / hit-skips-byte-id / independent key-bust per input class with non-vacuity rehit.
382 lines
12 KiB
C
382 lines
12 KiB
C
/*
|
|
* 989_pkgcache_run — M3-tail commit-5b out/.pkgcache content-keyed package
|
|
* cache gate (#63). COLD, dev-only: the cache is keyed by content, never
|
|
* mtime, and every bootstrap/byte-id gate cold-compiles (the `--sep` scratch
|
|
* is wiped each run), so 5b changes NO gate output. This leg certifies the
|
|
* cache itself, through the REAL `ww` / `ww_ww` drivers.
|
|
*
|
|
* Target: a synthetic 3-level graph root(main) -> mid -> leaf in a private
|
|
* temp tree, so source bytes and the dep interface are fully under test
|
|
* control. The ROOT is never cached (it is the build target); `mid` and
|
|
* `leaf` are the cacheable packages. Each stage uses its OWN WW_PKGCACHE dir
|
|
* (the compiler-binary line of P.key is intentionally stage-specific — it
|
|
* keys CODEGEN identity — so cs and ww keep separate cache namespaces; the
|
|
* cacheable OUTPUTS they reuse stay byte-identical, asserted below).
|
|
*
|
|
* HIT/MISS sentinel: a cache MISS runs w6c, which writes <scratch>/<pkg>.s;
|
|
* a HIT copies the cached .o/.wwi and skips compose+w6c+w6a, so no <pkg>.s
|
|
* is produced. The scratch is wiped each build, so <pkg>.s presence reflects
|
|
* exactly the last build.
|
|
*
|
|
* Legs, per stage (ww, ww_ww):
|
|
* 1. MISS: a cold build compiles (leaf.s AND mid.s present; prog runs == 5).
|
|
* 2. HIT (+ non-vacuity baseline): an unchanged rebuild skips both
|
|
* (leaf.s AND mid.s ABSENT; prog still runs == 5 from cached artifacts).
|
|
* 3. BUST — source edit: perturb mid.ww -> mid MISS, leaf still HIT;
|
|
* restore + reseed -> HIT again.
|
|
* 4. BUST — dep .wwi change: change leaf's EXPORTED interface -> leaf's .wwi
|
|
* md5 changes -> the importer `mid` (own source unchanged) MISSES via its
|
|
* dep line; restore + reseed -> HIT again.
|
|
* 5. BUST — compiler binary md5: corrupt the stored mid/P.key `w6c` line ->
|
|
* the freshly recomputed manifest mismatches -> mid MISS; the rebuild
|
|
* rewrites a correct key -> HIT again. (Models a codegen-changed compiler;
|
|
* a real WW_W6C swap is cstage-only — the wwstage driver hardcodes its
|
|
* toolchain — so a white-box key-line edit proves the class on both.)
|
|
* 6. BUST — flag string: corrupt the stored mid/P.key `flags` line -> mid
|
|
* MISS; rebuild rewrites -> HIT again.
|
|
*
|
|
* Cross-stage (rule 10): the cached mid/leaf P.o AND P.wwi are byte-identical
|
|
* between the cs and ww caches.
|
|
*
|
|
* Light wwstage-driver test (CLAUDE.md rule 14): all build outputs and caches
|
|
* live under a private /tmp tree, so it is parallel-safe and off every
|
|
* byte-id/bootstrap gate. Models 989_sepbuild_run conventions.
|
|
*/
|
|
#include <stdio.h>
|
|
#include <stdlib.h>
|
|
#include <string.h>
|
|
#include <unistd.h>
|
|
#include <sys/wait.h>
|
|
#include <sys/stat.h>
|
|
|
|
#define EXPECT_EXIT 5
|
|
|
|
static int
|
|
runwait(const char *cmd)
|
|
{
|
|
int rc = system(cmd);
|
|
if (rc == -1) return -1;
|
|
if (WIFEXITED(rc)) return WEXITSTATUS(rc);
|
|
return 1;
|
|
}
|
|
|
|
static const char *
|
|
absbin(void)
|
|
{
|
|
const char *b = getenv("BIN");
|
|
if (!b) b = "out/bin";
|
|
if (b[0] == '/') return b;
|
|
static char buf[2048];
|
|
char cwd[1024];
|
|
if (getcwd(cwd, sizeof cwd) == NULL) return NULL;
|
|
snprintf(buf, sizeof buf, "%s/%s", cwd, b);
|
|
return buf;
|
|
}
|
|
|
|
static int
|
|
write_file(const char *path, const char *body)
|
|
{
|
|
FILE *f = fopen(path, "wb");
|
|
if (!f) return -1;
|
|
fputs(body, f);
|
|
fclose(f);
|
|
return 0;
|
|
}
|
|
|
|
static int
|
|
slurp(const char *path, char **outbuf, size_t *outlen)
|
|
{
|
|
FILE *f = fopen(path, "rb");
|
|
if (!f) return -1;
|
|
fseek(f, 0, SEEK_END);
|
|
long n = ftell(f);
|
|
fseek(f, 0, SEEK_SET);
|
|
if (n < 0) { fclose(f); return -1; }
|
|
char *b = malloc((size_t)n + 1);
|
|
if (!b) { fclose(f); return -1; }
|
|
if (fread(b, 1, (size_t)n, f) != (size_t)n) { free(b); fclose(f); return -1; }
|
|
b[n] = '\0';
|
|
fclose(f);
|
|
*outbuf = b;
|
|
*outlen = (size_t)n;
|
|
return 0;
|
|
}
|
|
|
|
static int
|
|
files_eq(const char *a, const char *b)
|
|
{
|
|
char *ba = NULL, *bb = NULL;
|
|
size_t na = 0, nb = 0;
|
|
if (slurp(a, &ba, &na) < 0 || slurp(b, &bb, &nb) < 0) {
|
|
free(ba); free(bb);
|
|
return -1;
|
|
}
|
|
int eq = (na == nb && memcmp(ba, bb, na) == 0);
|
|
free(ba); free(bb);
|
|
return eq ? 0 : 1;
|
|
}
|
|
|
|
/* Flip one byte inside the line of `path` that starts with `prefix` (keeping
|
|
* the line length so the mutation isolates that one input class, not the
|
|
* blob length). Returns 0 on success. */
|
|
static int
|
|
mutate_keyline(const char *path, const char *prefix)
|
|
{
|
|
char *buf = NULL;
|
|
size_t n = 0;
|
|
if (slurp(path, &buf, &n) < 0) return -1;
|
|
size_t pl = strlen(prefix);
|
|
char *hit = NULL;
|
|
for (size_t i = 0; i < n; i++) {
|
|
if ((i == 0 || buf[i-1] == '\n') && i + pl <= n &&
|
|
memcmp(buf + i, prefix, pl) == 0) {
|
|
hit = buf + i + pl;
|
|
break;
|
|
}
|
|
}
|
|
if (!hit || hit >= buf + n || *hit == '\n') { free(buf); return -1; }
|
|
*hit = (*hit == 'a') ? 'b' : 'a';
|
|
FILE *f = fopen(path, "wb");
|
|
if (!f) { free(buf); return -1; }
|
|
fwrite(buf, 1, n, f);
|
|
fclose(f);
|
|
free(buf);
|
|
return 0;
|
|
}
|
|
|
|
/* <scratch>/<pkg>.s present ⇒ that package compiled this build (MISS). */
|
|
static int
|
|
sdot(const char *scratch, const char *pkg)
|
|
{
|
|
char p[1100];
|
|
snprintf(p, sizeof p, "%s/%s.s", scratch, pkg);
|
|
return access(p, 0) == 0;
|
|
}
|
|
|
|
static const char *leaf_v1 =
|
|
"package leaf;\n"
|
|
"export fn base() i32 = { return 3; };\n";
|
|
/* a CHANGED EXPORTED interface (extra export) → leaf's .wwi md5 changes. */
|
|
static const char *leaf_v2 =
|
|
"package leaf;\n"
|
|
"export fn base() i32 = { return 3; };\n"
|
|
"export fn extra() i32 = { return 9; };\n";
|
|
|
|
static const char *mid_v1 =
|
|
"package mid;\n"
|
|
"import leaf;\n"
|
|
"export fn val() i32 = { return leaf.base() + 2; };\n";
|
|
/* mid source perturbed by a comment — busts mid's src md5 without changing
|
|
* codegen (so the program result is preserved). */
|
|
static const char *mid_v2 =
|
|
"package mid;\n"
|
|
"import leaf;\n"
|
|
"// 5b src-edit bust probe\n"
|
|
"export fn val() i32 = { return leaf.base() + 2; };\n";
|
|
|
|
static const char *root_src =
|
|
"package main;\n"
|
|
"import mid;\n"
|
|
"fn main() i32 = { return mid.val(); };\n";
|
|
|
|
int
|
|
main(void)
|
|
{
|
|
const char *bin = absbin();
|
|
if (!bin) return 1;
|
|
int fail = 0;
|
|
char td[64], cmd[8192];
|
|
char rootww[1024], midww[1024], leafww[1024];
|
|
char scratch[1024], prog[1024];
|
|
char cache_cs[1024], cache_ww[1024];
|
|
|
|
snprintf(td, sizeof td, "/tmp/wwpkgc_%d", getpid());
|
|
snprintf(cmd, sizeof cmd, "rm -rf %s", td);
|
|
runwait(cmd);
|
|
snprintf(cmd, sizeof cmd, "mkdir -p %s/mid %s/leaf", td, td);
|
|
runwait(cmd);
|
|
|
|
snprintf(rootww, sizeof rootww, "%s/root.ww", td);
|
|
snprintf(midww, sizeof midww, "%s/mid/mid.ww", td);
|
|
snprintf(leafww, sizeof leafww, "%s/leaf/leaf.ww", td);
|
|
snprintf(prog, sizeof prog, "%s/p", td);
|
|
snprintf(scratch, sizeof scratch, "%s/p.sepwork", td);
|
|
snprintf(cache_cs, sizeof cache_cs, "%s/cache.cs", td);
|
|
snprintf(cache_ww, sizeof cache_ww, "%s/cache.ww", td);
|
|
|
|
if (write_file(rootww, root_src) || write_file(midww, mid_v1) ||
|
|
write_file(leafww, leaf_v1)) { fail++; goto out; }
|
|
|
|
struct { const char *drv, *cache; } stg[] = {
|
|
{ "ww", cache_cs },
|
|
{ "ww_ww", cache_ww },
|
|
};
|
|
|
|
for (int s = 0; s < 2; s++) {
|
|
const char *drv = stg[s].drv;
|
|
const char *cache = stg[s].cache;
|
|
char midkey[1100];
|
|
snprintf(midkey, sizeof midkey, "%s/mid/P.key", cache);
|
|
/* The cstage driver rm -rf's the scratch each build; the wwstage
|
|
* driver only mkdir's it (pre-existing #58(a) asymmetry), so a
|
|
* stale <pkg>.s could persist and defeat the MISS/HIT sentinel.
|
|
* The gate owns its scratch: wipe it before every build so a
|
|
* fresh <pkg>.s reflects exactly the last compile. */
|
|
#define BUILD() do { \
|
|
snprintf(cmd, sizeof cmd, \
|
|
"rm -rf %s; WW_PKGCACHE='%s' timeout 240 %s/%s build --sep " \
|
|
"-o %s %s >/dev/null 2>&1", scratch, cache, bin, drv, prog, \
|
|
rootww); \
|
|
} while (0)
|
|
#define SDOT(pkg) sdot(scratch, (pkg))
|
|
|
|
/* fresh cache for this stage */
|
|
snprintf(cmd, sizeof cmd, "rm -rf %s", cache);
|
|
runwait(cmd);
|
|
/* sources back to v1 (a prior stage's restore may have left them) */
|
|
write_file(midww, mid_v1);
|
|
write_file(leafww, leaf_v1);
|
|
|
|
/* leg 1 — MISS: cold build compiles both, prog runs */
|
|
BUILD();
|
|
if (runwait(cmd) != 0) {
|
|
fprintf(stderr, "pkgcache FAIL[%s]: cold build failed\n", drv);
|
|
fail++; continue;
|
|
}
|
|
if (!SDOT("leaf") || !SDOT("mid")) {
|
|
fprintf(stderr, "pkgcache FAIL[%s]: leg1 MISS — leaf/mid did "
|
|
"not compile\n", drv);
|
|
fail++;
|
|
}
|
|
if (runwait(prog) != EXPECT_EXIT) {
|
|
fprintf(stderr, "pkgcache FAIL[%s]: leg1 prog exit != %d\n",
|
|
drv, EXPECT_EXIT);
|
|
fail++;
|
|
}
|
|
|
|
/* leg 2 — HIT + non-vacuity: unchanged rebuild skips both */
|
|
BUILD();
|
|
runwait(cmd);
|
|
if (SDOT("leaf") || SDOT("mid")) {
|
|
fprintf(stderr, "pkgcache FAIL[%s]: leg2 HIT — recompiled an "
|
|
"unchanged package (cache vacuous)\n", drv);
|
|
fail++;
|
|
}
|
|
if (runwait(prog) != EXPECT_EXIT) {
|
|
fprintf(stderr, "pkgcache FAIL[%s]: leg2 prog exit != %d "
|
|
"(cached artifact unusable)\n", drv, EXPECT_EXIT);
|
|
fail++;
|
|
}
|
|
|
|
/* leg 3 — BUST source edit: mid MISS, leaf still HIT */
|
|
write_file(midww, mid_v2);
|
|
BUILD();
|
|
runwait(cmd);
|
|
if (!SDOT("mid") || SDOT("leaf")) {
|
|
fprintf(stderr, "pkgcache FAIL[%s]: leg3 src-edit — expected "
|
|
"mid MISS + leaf HIT\n", drv);
|
|
fail++;
|
|
}
|
|
/* restore + reseed, then prove HIT returns */
|
|
write_file(midww, mid_v1);
|
|
BUILD(); runwait(cmd); /* reseeds the v1 key (MISS) */
|
|
BUILD(); runwait(cmd); /* must HIT now */
|
|
if (SDOT("mid")) {
|
|
fprintf(stderr, "pkgcache FAIL[%s]: leg3 non-vacuity — mid did "
|
|
"not HIT after restore\n", drv);
|
|
fail++;
|
|
}
|
|
|
|
/* leg 4 — BUST dep .wwi: change leaf interface → importer mid MISSES */
|
|
write_file(leafww, leaf_v2);
|
|
BUILD();
|
|
runwait(cmd);
|
|
if (!SDOT("leaf") || !SDOT("mid")) {
|
|
fprintf(stderr, "pkgcache FAIL[%s]: leg4 dep-.wwi — leaf "
|
|
"interface change did not bust importer mid\n", drv);
|
|
fail++;
|
|
}
|
|
write_file(leafww, leaf_v1);
|
|
BUILD(); runwait(cmd); /* reseed v1 */
|
|
BUILD(); runwait(cmd); /* must HIT now */
|
|
if (SDOT("leaf") || SDOT("mid")) {
|
|
fprintf(stderr, "pkgcache FAIL[%s]: leg4 non-vacuity — did not "
|
|
"HIT after restore\n", drv);
|
|
fail++;
|
|
}
|
|
|
|
/* leg 5 — BUST compiler md5: corrupt the stored mid w6c line */
|
|
if (mutate_keyline(midkey, "w6c ") != 0) {
|
|
fprintf(stderr, "pkgcache FAIL[%s]: leg5 — no w6c line in "
|
|
"mid/P.key\n", drv);
|
|
fail++;
|
|
}
|
|
BUILD();
|
|
runwait(cmd);
|
|
if (!SDOT("mid") || SDOT("leaf")) {
|
|
fprintf(stderr, "pkgcache FAIL[%s]: leg5 compiler-md5 — corrupt "
|
|
"w6c line did not bust mid (only)\n", drv);
|
|
fail++;
|
|
}
|
|
BUILD(); runwait(cmd); /* key rewritten → must HIT */
|
|
if (SDOT("mid")) {
|
|
fprintf(stderr, "pkgcache FAIL[%s]: leg5 non-vacuity — mid did "
|
|
"not HIT after key rewrite\n", drv);
|
|
fail++;
|
|
}
|
|
|
|
/* leg 6 — BUST flag string: corrupt the stored mid flags line */
|
|
if (mutate_keyline(midkey, "flags ") != 0) {
|
|
fprintf(stderr, "pkgcache FAIL[%s]: leg6 — no flags line in "
|
|
"mid/P.key\n", drv);
|
|
fail++;
|
|
}
|
|
BUILD();
|
|
runwait(cmd);
|
|
if (!SDOT("mid")) {
|
|
fprintf(stderr, "pkgcache FAIL[%s]: leg6 flags — corrupt flags "
|
|
"line did not bust mid\n", drv);
|
|
fail++;
|
|
}
|
|
BUILD(); runwait(cmd);
|
|
if (SDOT("mid")) {
|
|
fprintf(stderr, "pkgcache FAIL[%s]: leg6 non-vacuity — mid did "
|
|
"not HIT after key rewrite\n", drv);
|
|
fail++;
|
|
}
|
|
#undef BUILD
|
|
#undef SDOT
|
|
}
|
|
|
|
/* cross-stage byte-id (rule 10): cached OUTPUTS identical cs==ww. The
|
|
* caches end in the v1/HIT state from each stage's leg 6 tail. */
|
|
{
|
|
const char *pkgs[] = { "mid", "leaf" };
|
|
const char *arts[] = { "P.o", "P.wwi" };
|
|
for (int i = 0; i < 2; i++)
|
|
for (int k = 0; k < 2; k++) {
|
|
char a[1100], b[1100];
|
|
snprintf(a, sizeof a, "%s/%s/%s", cache_cs, pkgs[i], arts[k]);
|
|
snprintf(b, sizeof b, "%s/%s/%s", cache_ww, pkgs[i], arts[k]);
|
|
if (files_eq(a, b) != 0) {
|
|
fprintf(stderr, "pkgcache FAIL: cs != ww for cached "
|
|
"%s/%s (rule 10)\n", pkgs[i], arts[k]);
|
|
fail++;
|
|
}
|
|
}
|
|
}
|
|
|
|
out:
|
|
snprintf(cmd, sizeof cmd, "rm -rf %s", td);
|
|
runwait(cmd);
|
|
if (fail) {
|
|
fprintf(stderr, "pkgcache: %d check(s) failed\n", fail);
|
|
return 1;
|
|
}
|
|
printf("pkgcache: 3-level root->mid->leaf via build_one_sep — MISS "
|
|
"compiles, HIT skips (non-vacuous, prog runs), and key busts on each "
|
|
"input class (src / dep .wwi / compiler md5 / flags) — both stages; "
|
|
"cached P.o/P.wwi byte-identical cs==ww\n");
|
|
return 0;
|
|
}
|