From 7b36f961a92433c467a07bc7293018da3c7544d1 Mon Sep 17 00:00:00 2001 From: Hojun-Cho Date: Tue, 16 Jun 2026 15:35:26 +0900 Subject: [PATCH] wcc/ww: out/.pkgcache content-keyed package cache for --sep (M3-tail c5b, #63) Per-package build cache for `ww build --sep`: before recompiling a package, recompute a plain-text key manifest (md5sum-hex lines of the package sources, each direct dep's .wwi, the w6c and w6a binaries, plus the compile flags) and reuse the cached .o/.wwi on a byte-for-byte key hit. Both stages shell the same host md5sum (no ww-side md5) so the non-compiler key lines are byte-identical cstage==wwstage; the compiler md5 lines differ per stage BY DESIGN, giving each stage its own cache namespace so a hit can never reuse the other stage's .o and mask a cs!=ww codegen divergence. Cached outputs (.o/.wwi) stay byte-identical across stages. Root package never cached; cache lives under $(OUT) (gitignored, wiped by make clean). Both stages mirrored. Dev-only convenience, off every bootstrap/byte-id gate. Gate 989_pkgcache_run (cold) proves miss-compiles / hit-skips-byte-id / independent key-bust per input class with non-vacuity rehit. --- Makefile | 12 + cmd/ww/main.c | 211 +++++++++++++++-- selfhost/cmd/ww/main.combined.ww | 228 ++++++++++++++++++ selfhost/cmd/ww/main.ww | 228 ++++++++++++++++++ test/wcc/989_pkgcache_run.c | 381 +++++++++++++++++++++++++++++++ 5 files changed, 1046 insertions(+), 14 deletions(-) create mode 100644 test/wcc/989_pkgcache_run.c diff --git a/Makefile b/Makefile index d6d4fcd8..0dff4b20 100644 --- a/Makefile +++ b/Makefile @@ -566,6 +566,7 @@ TESTS = $(BIN)/test_smoke $(BIN)/test_lex $(BIN)/test_parse $(BIN)/test_check \ $(BIN)/test_sepbuild_run \ $(BIN)/test_sepcycle_dup \ $(BIN)/test_separchive_run \ + $(BIN)/test_pkgcache_run \ $(BIN)/test_floatlit_run \ $(BIN)/test_checked_run \ $(BIN)/test_floatarr_run \ @@ -3129,6 +3130,17 @@ $(BIN)/test_separchive_run: test/wcc/989_separchive_run.c $(BIN)/ww $(BIN)/ww_ww $(BIN)/w6l $(BIN)/w6l_ww $(LIB)/libwwrt.a | $(BIN) $(CC) $(CFLAGS) -o $@ $< +# 989_pkgcache_run — M3-tail commit-5b gate (#46, #63): the out/.pkgcache +# content-keyed package cache. COLD/dev-only (off every byte-id/bootstrap +# gate): MISS compiles, HIT skips + non-vacuously reuses byte-identical +# artifacts, and the key busts on each input class (src / dep .wwi / compiler +# md5 / flags) — both driver stages, with cs==ww cached P.o/P.wwi. Needs both +# driver + both compiler + both linker stages + libwwrt.a. +$(BIN)/test_pkgcache_run: test/wcc/989_pkgcache_run.c $(BIN)/ww $(BIN)/ww_ww \ + $(BIN)/w6c $(BIN)/w6c_ww $(BIN)/w6a $(BIN)/w6a_ww \ + $(BIN)/w6l $(BIN)/w6l_ww $(LIB)/libwwrt.a | $(BIN) + $(CC) $(CFLAGS) -o $@ $< + $(BIN)/test_floatlit_run: test/wcc/989_floatlit_run.c $(BIN)/ww $(BIN)/w6c \ $(BIN)/w6c_ww $(BIN)/w6a $(BIN)/w6l $(LIB)/libwwrt.a | $(BIN) $(CC) $(CFLAGS) -o $@ $< diff --git a/cmd/ww/main.c b/cmd/ww/main.c index 9784014b..fa2fb7e2 100644 --- a/cmd/ww/main.c +++ b/cmd/ww/main.c @@ -942,6 +942,180 @@ archive_o(const char *objpath, const char *apath) return 0; } +/* ---- 5b content-keyed package cache (#63) ---------------------------- + * + * A per-package cache under WW_PKGCACHE (default out/.pkgcache; gitignored, + * make clean wipes $(OUT)). Purely a dev-inner-loop convenience: every + * bootstrap/byte-id gate cold-compiles (--sep scratch is wiped each run), so + * the cache changes NO gate output. Mirrors out/.testcache + Hare get_cache. + * + * rule-10: md5 is NOT reimplemented per stage — both stages shell to the + * host md5sum and assemble an identical text manifest, so a HIT's reused + * P.wwi/P.o stay byte-identical cs==ww by construction. The compiler-binary + * line keys CODEGEN identity, so it is intentionally stage-specific (w6c vs + * w6c_ww) — each stage maintains its own cache namespace; the cacheable + * OUTPUTS it reuses remain byte-identical. */ + +static const char * +pkgcache_root(void) +{ + const char *p = getenv("WW_PKGCACHE"); + if (p && p[0]) return p; + return "out/.pkgcache"; +} + +/* Hex md5 digest of `path` via the host tool. The cache is gate-cold, so a + * host-tool dep is sanctioned (md5sum is already a build dependency). */ +static int +md5_file(const char *path, char *hex, size_t hexsz) +{ + char cmd[1200]; + snprintf(cmd, sizeof cmd, "md5sum '%s' 2>/dev/null", path); + FILE *p = popen(cmd, "r"); + if (p == NULL) return -1; + char line[256]; + char *got = fgets(line, sizeof line, p); + pclose(p); + if (got == NULL) return -1; + size_t i = 0; + while (i + 1 < hexsz && line[i] && line[i] != ' ' + && line[i] != '\t' && line[i] != '\n') i++; + if (i == 0) return -1; + memcpy(hex, line, i); + hex[i] = '\0'; + return 0; +} + +/* Assemble package pi's content-key manifest (D4) into `out` as deterministic + * text: P's own *.ww md5s (sorted by name) | each DIRECT dep's .wwi md5 + * (sorted by dep path) | the w6c/w6a md5s | the flag string. DIRECT deps + * only — reverse-topo folds transitivity bottom-up, since a dep's .wwi + * already folds ITS deps. Returns 0 on success, -1 on any md5/truncation + * failure (caller treats that as non-cacheable → cold compile). */ +static int +sep_manifest(struct sepgraph *g, int pi, const char *scratch, + const char *c6, const char *a6, char *out, size_t outsz) +{ + size_t off = 0; + char hex[64]; + + off += (size_t)snprintf(out + off, outsz - off, "src"); + if (g->pkg[pi].is_dir) { + char **files = NULL; + int n = enumerate_dir_ww(g->pkg[pi].entry, &files); + int rc = 0; + for (int i = 0; i < n; i++) { + char fp[1024]; + snprintf(fp, sizeof fp, "%s/%s", g->pkg[pi].entry, + files[i]); + if (rc == 0 && md5_file(fp, hex, sizeof hex) == 0) + off += (size_t)snprintf(out + off, outsz - off, + " %s", hex); + else + rc = -1; + free(files[i]); + } + free(files); + if (rc != 0) return -1; + } else { + if (md5_file(g->pkg[pi].entry, hex, sizeof hex) != 0) + return -1; + off += (size_t)snprintf(out + off, outsz - off, " %s", hex); + } + off += (size_t)snprintf(out + off, outsz - off, "\n"); + + /* dep lines, sorted by dep path (deps[] are in import-appearance + * order; insertion-sort the indices for a deterministic manifest). */ + int nd = g->pkg[pi].ndeps; + int idx[SEP_MAXPKG]; + for (int i = 0; i < nd; i++) idx[i] = g->pkg[pi].deps[i]; + for (int i = 1; i < nd; i++) { + int v = idx[i], j = i; + while (j > 0 && + strcmp(g->pkg[idx[j-1]].path, g->pkg[v].path) > 0) { + idx[j] = idx[j-1]; + j--; + } + idx[j] = v; + } + for (int i = 0; i < nd; i++) { + int di = idx[i]; + char wwip[1024]; + sep_fname(g, di, scratch, ".wwi", wwip, sizeof wwip); + if (md5_file(wwip, hex, sizeof hex) != 0) return -1; + off += (size_t)snprintf(out + off, outsz - off, + "dep %s %s\n", g->pkg[di].path, hex); + } + + if (md5_file(c6, hex, sizeof hex) != 0) return -1; + off += (size_t)snprintf(out + off, outsz - off, "w6c %s\n", hex); + if (md5_file(a6, hex, sizeof hex) != 0) return -1; + off += (size_t)snprintf(out + off, outsz - off, "w6a %s\n", hex); + off += (size_t)snprintf(out + off, outsz - off, "flags -c -I\n"); + if (off >= outsz) return -1; /* truncated → unusable key */ + return 0; +} + +static void +pkgcache_dir(struct sepgraph *g, int pi, char *out, size_t outsz) +{ + const char *base = g->pkg[pi].path[0] ? g->pkg[pi].path : "__root"; + snprintf(out, outsz, "%s/%s", pkgcache_root(), base); +} + +/* HIT iff a freshly recomputed manifest equals the stored P.key byte-for-byte + * AND both cached artifacts exist; on HIT copy them into the scratch wwi/obj + * paths so the producer loop can skip compose+w6c+w6a. */ +static int +cache_lookup(struct sepgraph *g, int pi, const char *manifest, + const char *wwi, const char *obj) +{ + char dir[1024], keyp[1100], cwwi[1100], cobj[1100], cmd[4096]; + pkgcache_dir(g, pi, dir, sizeof dir); + snprintf(keyp, sizeof keyp, "%s/P.key", dir); + snprintf(cwwi, sizeof cwwi, "%s/P.wwi", dir); + snprintf(cobj, sizeof cobj, "%s/P.o", dir); + + FILE *f = fopen(keyp, "rb"); + if (f == NULL) return 0; + char stored[16384]; + size_t sn = fread(stored, 1, sizeof stored, f); + int more = (fgetc(f) != EOF); + fclose(f); + if (more || sn != strlen(manifest) || memcmp(stored, manifest, sn) != 0) + return 0; + if (access(cwwi, 0) != 0 || access(cobj, 0) != 0) return 0; + snprintf(cmd, sizeof cmd, "cp -f '%s' '%s'", cwwi, wwi); + if (run(cmd) != 0) return 0; + snprintf(cmd, sizeof cmd, "cp -f '%s' '%s'", cobj, obj); + if (run(cmd) != 0) return 0; + return 1; +} + +/* On MISS, persist the freshly compiled artifacts then the manifest. The key + * is written LAST so a crash mid-store never leaves a key whose artifacts are + * absent/partial (the next run simply re-misses). */ +static void +cache_store(struct sepgraph *g, int pi, const char *manifest, + const char *wwi, const char *obj) +{ + char dir[1024], keyp[1100], cwwi[1100], cobj[1100], cmd[4096]; + pkgcache_dir(g, pi, dir, sizeof dir); + snprintf(cmd, sizeof cmd, "mkdir -p '%s'", dir); + if (run(cmd) != 0) return; + snprintf(cwwi, sizeof cwwi, "%s/P.wwi", dir); + snprintf(cobj, sizeof cobj, "%s/P.o", dir); + snprintf(keyp, sizeof keyp, "%s/P.key", dir); + snprintf(cmd, sizeof cmd, "cp -f '%s' '%s'", wwi, cwwi); + if (run(cmd) != 0) return; + snprintf(cmd, sizeof cmd, "cp -f '%s' '%s'", obj, cobj); + if (run(cmd) != 0) return; + FILE *f = fopen(keyp, "wb"); + if (f == NULL) return; + fputs(manifest, f); + fclose(f); +} + /* build_one_sep — the --sep orchestration: discover_deps, reverse_topo, * the transitive producer loop (one `w6c -c -I` per package, dep-first, * each DEP `.o` wrapped in its own deterministic `.a`), then a @@ -1034,20 +1208,29 @@ build_one_sep(const char *src, int entry_is_dir, const char *out, sep_fname(g, pi, scratch, ".wwi", wwi, sizeof wwi); sep_fname(g, pi, scratch, ".s", asmf, sizeof asmf); sep_fname(g, pi, scratch, ".o", obj, sizeof obj); - if (sep_compose_unit(g, pi, scratch, order, norder, srcdir, - unitf) < 0) { free(order); free(g); return 1; } - snprintf(cmd, sizeof cmd, "%s -c -I %s -o %s %s", - c6, wwi, asmf, unitf); - if (run(cmd) != 0) { - fprintf(stderr, "ww --sep: w6c failed for %s\n", - g->pkg[pi].path[0] ? g->pkg[pi].path : "(root)"); - free(order); free(g); return 1; - } - snprintf(cmd, sizeof cmd, "%s -o %s %s", a6, obj, asmf); - if (run(cmd) != 0) { - fprintf(stderr, "ww --sep: w6a failed for %s\n", - g->pkg[pi].path[0] ? g->pkg[pi].path : "(root)"); - free(order); free(g); return 1; + /* 5b: skip compose+w6c+w6a on a content-key HIT. The root is + * never cached — it is the build target, always recompiled. */ + char manifest[16384]; + int cacheable = (pi != root) && sep_manifest(g, pi, scratch, + c6, a6, manifest, sizeof manifest) == 0; + int fresh = cacheable && cache_lookup(g, pi, manifest, wwi, obj); + if (!fresh) { + if (sep_compose_unit(g, pi, scratch, order, norder, srcdir, + unitf) < 0) { free(order); free(g); return 1; } + snprintf(cmd, sizeof cmd, "%s -c -I %s -o %s %s", + c6, wwi, asmf, unitf); + if (run(cmd) != 0) { + fprintf(stderr, "ww --sep: w6c failed for %s\n", + g->pkg[pi].path[0] ? g->pkg[pi].path : "(root)"); + free(order); free(g); return 1; + } + snprintf(cmd, sizeof cmd, "%s -o %s %s", a6, obj, asmf); + if (run(cmd) != 0) { + fprintf(stderr, "ww --sep: w6a failed for %s\n", + g->pkg[pi].path[0] ? g->pkg[pi].path : "(root)"); + free(order); free(g); return 1; + } + if (cacheable) cache_store(g, pi, manifest, wwi, obj); } /* wrap each DEP package's `.o` in its own deterministic `.a` * (5a). The ROOT stays a positional `.o` (force-loaded — it's diff --git a/selfhost/cmd/ww/main.combined.ww b/selfhost/cmd/ww/main.combined.ww index 960cd6d5..47582589 100644 --- a/selfhost/cmd/ww/main.combined.ww +++ b/selfhost/cmd/ww/main.combined.ww @@ -4150,6 +4150,214 @@ fn archiveo(objpath: *u8, apath: *u8) i32 = { return 0; }; +// ---- 5b content-keyed package cache (#63) ---------------------------- +// +// Twin of cstage's pkgcache_root/md5_file/sep_manifest/cache_lookup/ +// cache_store (cmd/ww/main.c). A per-package cache under WW_PKGCACHE +// (default out/.pkgcache; gitignored, make clean wipes $(OUT)). Dev-inner- +// loop convenience only: every gate cold-compiles, so the cache changes no +// gate output. rule-10: md5 is NOT reimplemented here — both stages shell to +// the host md5sum, so a HIT's reused P.wwi/P.o stay byte-identical cs==ww. + +fn pkgcacheroot() *u8 = { + match (os.getenv("WW_PKGCACHE")) { + case let s: str => + if (s.len > 0) { + let buf: []u8 = alloc([], (s.len: u64) + 1u64)!; + let i: i32 = 0; + for (i < s.len) { buf[i] = s[i]; i += 1; }; + buf[s.len] = 0u8; + return buf.ptr; + }; + case void => void; + }; + let d: []u8 = alloc([], 64u64)!; + d.len = 64; + let o: u64 = strinto(d.ptr, 0u64, "out/.pkgcache"); + cstrseal(d.ptr, o); + return d.ptr; +}; + +fn pkgcachedir(cacheroot: *u8, g: *sepgraph, pi: i32) *u8 = { + if (g.pkg[pi].path[0u64] == 0u8) { + return joinpathlit(cacheroot, "__root"); + }; + return joinpathlit(cacheroot, pathstr(g.pkg[pi].path)); +}; + +// md5appendhex — host md5sum of `path`, captured via a shell redirect to +// `tmp` (cache is gate-cold → host-tool dep sanctioned, rule-7). Appends the +// hex digest to `dst` at `off`; returns the new offset, or 0 on failure +// (caller treats 0 as a non-cacheable miss). Mirrors cstage md5_file's +// copy-until-whitespace parse so the digest text is byte-identical. +fn md5appendhex(dst: *u8, off: u64, path: *u8, tmp: *u8) u64 = { + let cmd: []u8 = alloc([], 8192u64)!; + cmd.len = 8192; + let co: u64 = strinto(cmd.ptr, 0u64, "md5sum '"); + co = cstrinto(cmd.ptr, co, path); + co = strinto(cmd.ptr, co, "' > '"); + co = cstrinto(cmd.ptr, co, tmp); + co = strinto(cmd.ptr, co, "'"); + cstrseal(cmd.ptr, co); + let argv: []*u8 = alloc([], 4u64)!; + argv.len = 4; + argv[0] = "sh\0".ptr; + argv[1] = "-c\0".ptr; + argv[2] = cmd.ptr; + argv[3] = nil; + if (procrun("/bin/sh\0".ptr, argv.ptr) != 0) { return 0u64; }; + let (buf, n) = slurp(tmp); + if (buf == nil) { return 0u64; }; + let i: u64 = 0u64; + for (i < n) { + let c: u8 = buf[i]; + if (c == 32u8 || c == 9u8 || c == 10u8) { break; }; + dst[off + i] = c; + i += 1u64; + }; + if (i == 0u64) { return 0u64; }; + return off + i; +}; + +// sepmanifest — assemble package pi's content-key manifest (D4) into `out` +// as deterministic text (see cstage sep_manifest). Returns the manifest +// length, or 0 on any md5 failure (caller → cold compile). The md5 capture +// tmp lives in `scratch` (which exists), not the cache dir (created lazily). +fn sepmanifest(g: *sepgraph, pi: i32, scratch: *u8, c6: *u8, a6: *u8, + out: *u8) u64 = { + let tmp: *u8 = sepfname(g, pi, scratch, ".md5tmp"); + let off: u64 = strinto(out, 0u64, "src"); + if (g.pkg[pi].isdir != 0) { + let (names, n) = enumeratedir(g.pkg[pi].entry); + let i: i32 = 0; + for (i < n) { + let fp: []u8 = alloc([], os.PATH_MAX: u64)!; + fp.len = os.PATH_MAX; + let fo: u64 = cstrinto(fp.ptr, 0u64, g.pkg[pi].entry); + fo = byteinto(fp.ptr, fo, 47u8); // '/' + fo = cstrinto(fp.ptr, fo, names[i]); + cstrseal(fp.ptr, fo); + off = byteinto(out, off, 32u8); // ' ' + let no: u64 = md5appendhex(out, off, fp.ptr, tmp); + if (no == 0u64) { return 0u64; }; + off = no; + i += 1; + }; + } else { + off = byteinto(out, off, 32u8); + let no: u64 = md5appendhex(out, off, g.pkg[pi].entry, tmp); + if (no == 0u64) { return 0u64; }; + off = no; + }; + off = byteinto(out, off, 10u8); // '\n' + + // dep lines, sorted by dep path (insertion sort, mirrors enumeratedir). + let nd: i32 = g.pkg[pi].ndeps; + let idx: []i32 = alloc([], SEP_MAXPKG: u64)!; + idx.len = SEP_MAXPKG; + let i: i32 = 0; + for (i < nd) { idx[i] = g.pkg[pi].deps[i]; i += 1; }; + i = 1; + for (i < nd) { + let j: i32 = i; + for (j > 0) { + let a: *u8 = g.pkg[idx[j - 1]].path; + let b: *u8 = g.pkg[idx[j]].path; + if (bytecmp(a, cstrlen(a), b, cstrlen(b)) <= 0) { j = 0; } + else { + let t: i32 = idx[j]; + idx[j] = idx[j - 1]; + idx[j - 1] = t; + j -= 1; + }; + }; + i += 1; + }; + i = 0; + for (i < nd) { + let di: i32 = idx[i]; + let wwip: *u8 = sepfname(g, di, scratch, ".wwi"); + off = strinto(out, off, "dep "); + off = cstrinto(out, off, g.pkg[di].path); + off = byteinto(out, off, 32u8); + let no: u64 = md5appendhex(out, off, wwip, tmp); + if (no == 0u64) { return 0u64; }; + off = no; + off = byteinto(out, off, 10u8); + i += 1; + }; + + off = strinto(out, off, "w6c "); + let no: u64 = md5appendhex(out, off, c6, tmp); + if (no == 0u64) { return 0u64; }; + off = no; + off = byteinto(out, off, 10u8); + off = strinto(out, off, "w6a "); + no = md5appendhex(out, off, a6, tmp); + if (no == 0u64) { return 0u64; }; + off = no; + off = byteinto(out, off, 10u8); + off = strinto(out, off, "flags -c -I\n"); + return off; +}; + +// copyfile — byte-copy src→dst (TRUNC). Returns 0 on success, -1 on failure. +fn copyfile(src: *u8, dst: *u8) i32 = { + let (buf, n) = slurp(src); + if (buf == nil) { return -1; }; + let fd: i32 = os.open(pathstr(dst), + os.flag.WRONLY | os.flag.CREATE | os.flag.TRUNC, 420i32); // 0o644 + if (fd < 0) { return -1; }; + os.writeall(fd, buf, n); + os.close(fd); + return 0; +}; + +// cachelookup — HIT iff the manifest equals the stored P.key byte-for-byte +// AND both cached artifacts exist; on HIT copy them into the scratch +// wwi/objf so the producer loop can skip compose+w6c+w6a. +fn cachelookup(cacheroot: *u8, g: *sepgraph, pi: i32, manifest: *u8, + mlen: u64, wwi: *u8, objf: *u8) i32 = { + let dir: *u8 = pkgcachedir(cacheroot, g, pi); + let keyp: *u8 = joinpathlit(dir, "P.key"); + let cwwi: *u8 = joinpathlit(dir, "P.wwi"); + let cobj: *u8 = joinpathlit(dir, "P.o"); + let (stored, sn) = slurp(keyp); + if (stored == nil) { return 0; }; + if (sn != mlen) { return 0; }; + let i: u64 = 0u64; + for (i < mlen) { + if (stored[i] != manifest[i]) { return 0; }; + i += 1u64; + }; + if (os.access(pathstr(cwwi), 0i32) != 0) { return 0; }; + if (os.access(pathstr(cobj), 0i32) != 0) { return 0; }; + if (copyfile(cwwi, wwi) != 0) { return 0; }; + if (copyfile(cobj, objf) != 0) { return 0; }; + return 1; +}; + +// cachestore — on MISS persist the artifacts then the key (key LAST: a crash +// mid-store never leaves a key without its artifacts; the next run re-misses). +fn cachestore(cacheroot: *u8, g: *sepgraph, pi: i32, manifest: *u8, + mlen: u64, wwi: *u8, objf: *u8) void = { + let dir: *u8 = pkgcachedir(cacheroot, g, pi); + match (os.mkdirs(pathstr(dir), 493i32)) { // 0o755 + case void => void; + case let e: os.oserror => void; // best-effort; copyfile surfaces a real failure + }; + let cwwi: *u8 = joinpathlit(dir, "P.wwi"); + let cobj: *u8 = joinpathlit(dir, "P.o"); + let keyp: *u8 = joinpathlit(dir, "P.key"); + if (copyfile(wwi, cwwi) != 0) { return; }; + if (copyfile(objf, cobj) != 0) { return; }; + let fd: i32 = os.open(pathstr(keyp), + os.flag.WRONLY | os.flag.CREATE | os.flag.TRUNC, 420i32); // 0o644 + if (fd < 0) { return; }; + os.writeall(fd, manifest, mlen); + os.close(fd); +}; + // buildonesep — the --sep orchestration: discover deps, reverse-topo, // the transitive producer loop (one `w6c -c -I` per package, dep-first, // each `.o` wrapped in its own deterministic per-package `.a`), then a @@ -4265,6 +4473,7 @@ fn buildonesep(selfdir: *u8, src: *u8, entryisdir: i32, out: *u8, if (septopovisit(g, root, order, &norder, stack, 0) < 0) { return 1; }; // Producer loop — dep-first, one `w6c -c -I` per package. + let cacheroot: *u8 = pkgcacheroot(); let oi: i32 = 0; for (oi < norder) { let pi: i32 = order[oi]; @@ -4272,6 +4481,21 @@ fn buildonesep(selfdir: *u8, src: *u8, entryisdir: i32, out: *u8, let wwi: *u8 = sepfname(g, pi, scratch, ".wwi"); let asmf: *u8 = sepfname(g, pi, scratch, ".s"); let objf: *u8 = sepfname(g, pi, scratch, ".o"); + // 5b: skip compose+w6c+w6a on a content-key HIT. The root is + // never cached — it is the build target, always recompiled. + let manbuf: []u8 = alloc([], 16384u64)!; + manbuf.len = 16384; + let mlen: u64 = 0u64; + let cacheable: i32 = 0; + if (pi != root) { + mlen = sepmanifest(g, pi, scratch, c6, a6, manbuf.ptr); + if (mlen > 0u64) { cacheable = 1; }; + }; + let fresh: i32 = 0; + if (cacheable != 0) { + fresh = cachelookup(cacheroot, g, pi, manbuf.ptr, mlen, wwi, objf); + }; + if (fresh == 0) { if (sepcomposeunit(g, pi, scratch, order, norder, searchpath.ptr, unitf) < 0) { return 1; }; @@ -4304,6 +4528,10 @@ fn buildonesep(selfdir: *u8, src: *u8, entryisdir: i32, out: *u8, return 1; }; }; + if (cacheable != 0) { + cachestore(cacheroot, g, pi, manbuf.ptr, mlen, wwi, objf); + }; + }; // Wrap each DEP package's `.o` in its own deterministic `.a` // (5a). The ROOT stays a positional `.o` (force-loaded — it's // the build target), so `main` is defined before any archive is diff --git a/selfhost/cmd/ww/main.ww b/selfhost/cmd/ww/main.ww index df4840be..236ada1f 100644 --- a/selfhost/cmd/ww/main.ww +++ b/selfhost/cmd/ww/main.ww @@ -1276,6 +1276,214 @@ fn archiveo(objpath: *u8, apath: *u8) i32 = { return 0; }; +// ---- 5b content-keyed package cache (#63) ---------------------------- +// +// Twin of cstage's pkgcache_root/md5_file/sep_manifest/cache_lookup/ +// cache_store (cmd/ww/main.c). A per-package cache under WW_PKGCACHE +// (default out/.pkgcache; gitignored, make clean wipes $(OUT)). Dev-inner- +// loop convenience only: every gate cold-compiles, so the cache changes no +// gate output. rule-10: md5 is NOT reimplemented here — both stages shell to +// the host md5sum, so a HIT's reused P.wwi/P.o stay byte-identical cs==ww. + +fn pkgcacheroot() *u8 = { + match (os.getenv("WW_PKGCACHE")) { + case let s: str => + if (s.len > 0) { + let buf: []u8 = alloc([], (s.len: u64) + 1u64)!; + let i: i32 = 0; + for (i < s.len) { buf[i] = s[i]; i += 1; }; + buf[s.len] = 0u8; + return buf.ptr; + }; + case void => void; + }; + let d: []u8 = alloc([], 64u64)!; + d.len = 64; + let o: u64 = strinto(d.ptr, 0u64, "out/.pkgcache"); + cstrseal(d.ptr, o); + return d.ptr; +}; + +fn pkgcachedir(cacheroot: *u8, g: *sepgraph, pi: i32) *u8 = { + if (g.pkg[pi].path[0u64] == 0u8) { + return joinpathlit(cacheroot, "__root"); + }; + return joinpathlit(cacheroot, pathstr(g.pkg[pi].path)); +}; + +// md5appendhex — host md5sum of `path`, captured via a shell redirect to +// `tmp` (cache is gate-cold → host-tool dep sanctioned, rule-7). Appends the +// hex digest to `dst` at `off`; returns the new offset, or 0 on failure +// (caller treats 0 as a non-cacheable miss). Mirrors cstage md5_file's +// copy-until-whitespace parse so the digest text is byte-identical. +fn md5appendhex(dst: *u8, off: u64, path: *u8, tmp: *u8) u64 = { + let cmd: []u8 = alloc([], 8192u64)!; + cmd.len = 8192; + let co: u64 = strinto(cmd.ptr, 0u64, "md5sum '"); + co = cstrinto(cmd.ptr, co, path); + co = strinto(cmd.ptr, co, "' > '"); + co = cstrinto(cmd.ptr, co, tmp); + co = strinto(cmd.ptr, co, "'"); + cstrseal(cmd.ptr, co); + let argv: []*u8 = alloc([], 4u64)!; + argv.len = 4; + argv[0] = "sh\0".ptr; + argv[1] = "-c\0".ptr; + argv[2] = cmd.ptr; + argv[3] = nil; + if (procrun("/bin/sh\0".ptr, argv.ptr) != 0) { return 0u64; }; + let (buf, n) = slurp(tmp); + if (buf == nil) { return 0u64; }; + let i: u64 = 0u64; + for (i < n) { + let c: u8 = buf[i]; + if (c == 32u8 || c == 9u8 || c == 10u8) { break; }; + dst[off + i] = c; + i += 1u64; + }; + if (i == 0u64) { return 0u64; }; + return off + i; +}; + +// sepmanifest — assemble package pi's content-key manifest (D4) into `out` +// as deterministic text (see cstage sep_manifest). Returns the manifest +// length, or 0 on any md5 failure (caller → cold compile). The md5 capture +// tmp lives in `scratch` (which exists), not the cache dir (created lazily). +fn sepmanifest(g: *sepgraph, pi: i32, scratch: *u8, c6: *u8, a6: *u8, + out: *u8) u64 = { + let tmp: *u8 = sepfname(g, pi, scratch, ".md5tmp"); + let off: u64 = strinto(out, 0u64, "src"); + if (g.pkg[pi].isdir != 0) { + let (names, n) = enumeratedir(g.pkg[pi].entry); + let i: i32 = 0; + for (i < n) { + let fp: []u8 = alloc([], os.PATH_MAX: u64)!; + fp.len = os.PATH_MAX; + let fo: u64 = cstrinto(fp.ptr, 0u64, g.pkg[pi].entry); + fo = byteinto(fp.ptr, fo, 47u8); // '/' + fo = cstrinto(fp.ptr, fo, names[i]); + cstrseal(fp.ptr, fo); + off = byteinto(out, off, 32u8); // ' ' + let no: u64 = md5appendhex(out, off, fp.ptr, tmp); + if (no == 0u64) { return 0u64; }; + off = no; + i += 1; + }; + } else { + off = byteinto(out, off, 32u8); + let no: u64 = md5appendhex(out, off, g.pkg[pi].entry, tmp); + if (no == 0u64) { return 0u64; }; + off = no; + }; + off = byteinto(out, off, 10u8); // '\n' + + // dep lines, sorted by dep path (insertion sort, mirrors enumeratedir). + let nd: i32 = g.pkg[pi].ndeps; + let idx: []i32 = alloc([], SEP_MAXPKG: u64)!; + idx.len = SEP_MAXPKG; + let i: i32 = 0; + for (i < nd) { idx[i] = g.pkg[pi].deps[i]; i += 1; }; + i = 1; + for (i < nd) { + let j: i32 = i; + for (j > 0) { + let a: *u8 = g.pkg[idx[j - 1]].path; + let b: *u8 = g.pkg[idx[j]].path; + if (bytecmp(a, cstrlen(a), b, cstrlen(b)) <= 0) { j = 0; } + else { + let t: i32 = idx[j]; + idx[j] = idx[j - 1]; + idx[j - 1] = t; + j -= 1; + }; + }; + i += 1; + }; + i = 0; + for (i < nd) { + let di: i32 = idx[i]; + let wwip: *u8 = sepfname(g, di, scratch, ".wwi"); + off = strinto(out, off, "dep "); + off = cstrinto(out, off, g.pkg[di].path); + off = byteinto(out, off, 32u8); + let no: u64 = md5appendhex(out, off, wwip, tmp); + if (no == 0u64) { return 0u64; }; + off = no; + off = byteinto(out, off, 10u8); + i += 1; + }; + + off = strinto(out, off, "w6c "); + let no: u64 = md5appendhex(out, off, c6, tmp); + if (no == 0u64) { return 0u64; }; + off = no; + off = byteinto(out, off, 10u8); + off = strinto(out, off, "w6a "); + no = md5appendhex(out, off, a6, tmp); + if (no == 0u64) { return 0u64; }; + off = no; + off = byteinto(out, off, 10u8); + off = strinto(out, off, "flags -c -I\n"); + return off; +}; + +// copyfile — byte-copy src→dst (TRUNC). Returns 0 on success, -1 on failure. +fn copyfile(src: *u8, dst: *u8) i32 = { + let (buf, n) = slurp(src); + if (buf == nil) { return -1; }; + let fd: i32 = os.open(pathstr(dst), + os.flag.WRONLY | os.flag.CREATE | os.flag.TRUNC, 420i32); // 0o644 + if (fd < 0) { return -1; }; + os.writeall(fd, buf, n); + os.close(fd); + return 0; +}; + +// cachelookup — HIT iff the manifest equals the stored P.key byte-for-byte +// AND both cached artifacts exist; on HIT copy them into the scratch +// wwi/objf so the producer loop can skip compose+w6c+w6a. +fn cachelookup(cacheroot: *u8, g: *sepgraph, pi: i32, manifest: *u8, + mlen: u64, wwi: *u8, objf: *u8) i32 = { + let dir: *u8 = pkgcachedir(cacheroot, g, pi); + let keyp: *u8 = joinpathlit(dir, "P.key"); + let cwwi: *u8 = joinpathlit(dir, "P.wwi"); + let cobj: *u8 = joinpathlit(dir, "P.o"); + let (stored, sn) = slurp(keyp); + if (stored == nil) { return 0; }; + if (sn != mlen) { return 0; }; + let i: u64 = 0u64; + for (i < mlen) { + if (stored[i] != manifest[i]) { return 0; }; + i += 1u64; + }; + if (os.access(pathstr(cwwi), 0i32) != 0) { return 0; }; + if (os.access(pathstr(cobj), 0i32) != 0) { return 0; }; + if (copyfile(cwwi, wwi) != 0) { return 0; }; + if (copyfile(cobj, objf) != 0) { return 0; }; + return 1; +}; + +// cachestore — on MISS persist the artifacts then the key (key LAST: a crash +// mid-store never leaves a key without its artifacts; the next run re-misses). +fn cachestore(cacheroot: *u8, g: *sepgraph, pi: i32, manifest: *u8, + mlen: u64, wwi: *u8, objf: *u8) void = { + let dir: *u8 = pkgcachedir(cacheroot, g, pi); + match (os.mkdirs(pathstr(dir), 493i32)) { // 0o755 + case void => void; + case let e: os.oserror => void; // best-effort; copyfile surfaces a real failure + }; + let cwwi: *u8 = joinpathlit(dir, "P.wwi"); + let cobj: *u8 = joinpathlit(dir, "P.o"); + let keyp: *u8 = joinpathlit(dir, "P.key"); + if (copyfile(wwi, cwwi) != 0) { return; }; + if (copyfile(objf, cobj) != 0) { return; }; + let fd: i32 = os.open(pathstr(keyp), + os.flag.WRONLY | os.flag.CREATE | os.flag.TRUNC, 420i32); // 0o644 + if (fd < 0) { return; }; + os.writeall(fd, manifest, mlen); + os.close(fd); +}; + // buildonesep — the --sep orchestration: discover deps, reverse-topo, // the transitive producer loop (one `w6c -c -I` per package, dep-first, // each `.o` wrapped in its own deterministic per-package `.a`), then a @@ -1391,6 +1599,7 @@ fn buildonesep(selfdir: *u8, src: *u8, entryisdir: i32, out: *u8, if (septopovisit(g, root, order, &norder, stack, 0) < 0) { return 1; }; // Producer loop — dep-first, one `w6c -c -I` per package. + let cacheroot: *u8 = pkgcacheroot(); let oi: i32 = 0; for (oi < norder) { let pi: i32 = order[oi]; @@ -1398,6 +1607,21 @@ fn buildonesep(selfdir: *u8, src: *u8, entryisdir: i32, out: *u8, let wwi: *u8 = sepfname(g, pi, scratch, ".wwi"); let asmf: *u8 = sepfname(g, pi, scratch, ".s"); let objf: *u8 = sepfname(g, pi, scratch, ".o"); + // 5b: skip compose+w6c+w6a on a content-key HIT. The root is + // never cached — it is the build target, always recompiled. + let manbuf: []u8 = alloc([], 16384u64)!; + manbuf.len = 16384; + let mlen: u64 = 0u64; + let cacheable: i32 = 0; + if (pi != root) { + mlen = sepmanifest(g, pi, scratch, c6, a6, manbuf.ptr); + if (mlen > 0u64) { cacheable = 1; }; + }; + let fresh: i32 = 0; + if (cacheable != 0) { + fresh = cachelookup(cacheroot, g, pi, manbuf.ptr, mlen, wwi, objf); + }; + if (fresh == 0) { if (sepcomposeunit(g, pi, scratch, order, norder, searchpath.ptr, unitf) < 0) { return 1; }; @@ -1430,6 +1654,10 @@ fn buildonesep(selfdir: *u8, src: *u8, entryisdir: i32, out: *u8, return 1; }; }; + if (cacheable != 0) { + cachestore(cacheroot, g, pi, manbuf.ptr, mlen, wwi, objf); + }; + }; // Wrap each DEP package's `.o` in its own deterministic `.a` // (5a). The ROOT stays a positional `.o` (force-loaded — it's // the build target), so `main` is defined before any archive is diff --git a/test/wcc/989_pkgcache_run.c b/test/wcc/989_pkgcache_run.c new file mode 100644 index 00000000..9d7a8843 --- /dev/null +++ b/test/wcc/989_pkgcache_run.c @@ -0,0 +1,381 @@ +/* + * 989_pkgcache_run — M3-tail commit-5b out/.pkgcache content-keyed package + * cache gate (#63). COLD, dev-only: the cache is keyed by content, never + * mtime, and every bootstrap/byte-id gate cold-compiles (the `--sep` scratch + * is wiped each run), so 5b changes NO gate output. This leg certifies the + * cache itself, through the REAL `ww` / `ww_ww` drivers. + * + * Target: a synthetic 3-level graph root(main) -> mid -> leaf in a private + * temp tree, so source bytes and the dep interface are fully under test + * control. The ROOT is never cached (it is the build target); `mid` and + * `leaf` are the cacheable packages. Each stage uses its OWN WW_PKGCACHE dir + * (the compiler-binary line of P.key is intentionally stage-specific — it + * keys CODEGEN identity — so cs and ww keep separate cache namespaces; the + * cacheable OUTPUTS they reuse stay byte-identical, asserted below). + * + * HIT/MISS sentinel: a cache MISS runs w6c, which writes /.s; + * a HIT copies the cached .o/.wwi and skips compose+w6c+w6a, so no .s + * is produced. The scratch is wiped each build, so .s presence reflects + * exactly the last build. + * + * Legs, per stage (ww, ww_ww): + * 1. MISS: a cold build compiles (leaf.s AND mid.s present; prog runs == 5). + * 2. HIT (+ non-vacuity baseline): an unchanged rebuild skips both + * (leaf.s AND mid.s ABSENT; prog still runs == 5 from cached artifacts). + * 3. BUST — source edit: perturb mid.ww -> mid MISS, leaf still HIT; + * restore + reseed -> HIT again. + * 4. BUST — dep .wwi change: change leaf's EXPORTED interface -> leaf's .wwi + * md5 changes -> the importer `mid` (own source unchanged) MISSES via its + * dep line; restore + reseed -> HIT again. + * 5. BUST — compiler binary md5: corrupt the stored mid/P.key `w6c` line -> + * the freshly recomputed manifest mismatches -> mid MISS; the rebuild + * rewrites a correct key -> HIT again. (Models a codegen-changed compiler; + * a real WW_W6C swap is cstage-only — the wwstage driver hardcodes its + * toolchain — so a white-box key-line edit proves the class on both.) + * 6. BUST — flag string: corrupt the stored mid/P.key `flags` line -> mid + * MISS; rebuild rewrites -> HIT again. + * + * Cross-stage (rule 10): the cached mid/leaf P.o AND P.wwi are byte-identical + * between the cs and ww caches. + * + * Light wwstage-driver test (CLAUDE.md rule 14): all build outputs and caches + * live under a private /tmp tree, so it is parallel-safe and off every + * byte-id/bootstrap gate. Models 989_sepbuild_run conventions. + */ +#include +#include +#include +#include +#include +#include + +#define EXPECT_EXIT 5 + +static int +runwait(const char *cmd) +{ + int rc = system(cmd); + if (rc == -1) return -1; + if (WIFEXITED(rc)) return WEXITSTATUS(rc); + return 1; +} + +static const char * +absbin(void) +{ + const char *b = getenv("BIN"); + if (!b) b = "out/bin"; + if (b[0] == '/') return b; + static char buf[2048]; + char cwd[1024]; + if (getcwd(cwd, sizeof cwd) == NULL) return NULL; + snprintf(buf, sizeof buf, "%s/%s", cwd, b); + return buf; +} + +static int +write_file(const char *path, const char *body) +{ + FILE *f = fopen(path, "wb"); + if (!f) return -1; + fputs(body, f); + fclose(f); + return 0; +} + +static int +slurp(const char *path, char **outbuf, size_t *outlen) +{ + FILE *f = fopen(path, "rb"); + if (!f) return -1; + fseek(f, 0, SEEK_END); + long n = ftell(f); + fseek(f, 0, SEEK_SET); + if (n < 0) { fclose(f); return -1; } + char *b = malloc((size_t)n + 1); + if (!b) { fclose(f); return -1; } + if (fread(b, 1, (size_t)n, f) != (size_t)n) { free(b); fclose(f); return -1; } + b[n] = '\0'; + fclose(f); + *outbuf = b; + *outlen = (size_t)n; + return 0; +} + +static int +files_eq(const char *a, const char *b) +{ + char *ba = NULL, *bb = NULL; + size_t na = 0, nb = 0; + if (slurp(a, &ba, &na) < 0 || slurp(b, &bb, &nb) < 0) { + free(ba); free(bb); + return -1; + } + int eq = (na == nb && memcmp(ba, bb, na) == 0); + free(ba); free(bb); + return eq ? 0 : 1; +} + +/* Flip one byte inside the line of `path` that starts with `prefix` (keeping + * the line length so the mutation isolates that one input class, not the + * blob length). Returns 0 on success. */ +static int +mutate_keyline(const char *path, const char *prefix) +{ + char *buf = NULL; + size_t n = 0; + if (slurp(path, &buf, &n) < 0) return -1; + size_t pl = strlen(prefix); + char *hit = NULL; + for (size_t i = 0; i < n; i++) { + if ((i == 0 || buf[i-1] == '\n') && i + pl <= n && + memcmp(buf + i, prefix, pl) == 0) { + hit = buf + i + pl; + break; + } + } + if (!hit || hit >= buf + n || *hit == '\n') { free(buf); return -1; } + *hit = (*hit == 'a') ? 'b' : 'a'; + FILE *f = fopen(path, "wb"); + if (!f) { free(buf); return -1; } + fwrite(buf, 1, n, f); + fclose(f); + free(buf); + return 0; +} + +/* /.s present ⇒ that package compiled this build (MISS). */ +static int +sdot(const char *scratch, const char *pkg) +{ + char p[1100]; + snprintf(p, sizeof p, "%s/%s.s", scratch, pkg); + return access(p, 0) == 0; +} + +static const char *leaf_v1 = + "package leaf;\n" + "export fn base() i32 = { return 3; };\n"; +/* a CHANGED EXPORTED interface (extra export) → leaf's .wwi md5 changes. */ +static const char *leaf_v2 = + "package leaf;\n" + "export fn base() i32 = { return 3; };\n" + "export fn extra() i32 = { return 9; };\n"; + +static const char *mid_v1 = + "package mid;\n" + "import leaf;\n" + "export fn val() i32 = { return leaf.base() + 2; };\n"; +/* mid source perturbed by a comment — busts mid's src md5 without changing + * codegen (so the program result is preserved). */ +static const char *mid_v2 = + "package mid;\n" + "import leaf;\n" + "// 5b src-edit bust probe\n" + "export fn val() i32 = { return leaf.base() + 2; };\n"; + +static const char *root_src = + "package main;\n" + "import mid;\n" + "fn main() i32 = { return mid.val(); };\n"; + +int +main(void) +{ + const char *bin = absbin(); + if (!bin) return 1; + int fail = 0; + char td[64], cmd[8192]; + char rootww[1024], midww[1024], leafww[1024]; + char scratch[1024], prog[1024]; + char cache_cs[1024], cache_ww[1024]; + + snprintf(td, sizeof td, "/tmp/wwpkgc_%d", getpid()); + snprintf(cmd, sizeof cmd, "rm -rf %s", td); + runwait(cmd); + snprintf(cmd, sizeof cmd, "mkdir -p %s/mid %s/leaf", td, td); + runwait(cmd); + + snprintf(rootww, sizeof rootww, "%s/root.ww", td); + snprintf(midww, sizeof midww, "%s/mid/mid.ww", td); + snprintf(leafww, sizeof leafww, "%s/leaf/leaf.ww", td); + snprintf(prog, sizeof prog, "%s/p", td); + snprintf(scratch, sizeof scratch, "%s/p.sepwork", td); + snprintf(cache_cs, sizeof cache_cs, "%s/cache.cs", td); + snprintf(cache_ww, sizeof cache_ww, "%s/cache.ww", td); + + if (write_file(rootww, root_src) || write_file(midww, mid_v1) || + write_file(leafww, leaf_v1)) { fail++; goto out; } + + struct { const char *drv, *cache; } stg[] = { + { "ww", cache_cs }, + { "ww_ww", cache_ww }, + }; + + for (int s = 0; s < 2; s++) { + const char *drv = stg[s].drv; + const char *cache = stg[s].cache; + char midkey[1100]; + snprintf(midkey, sizeof midkey, "%s/mid/P.key", cache); + /* The cstage driver rm -rf's the scratch each build; the wwstage + * driver only mkdir's it (pre-existing #58(a) asymmetry), so a + * stale .s could persist and defeat the MISS/HIT sentinel. + * The gate owns its scratch: wipe it before every build so a + * fresh .s reflects exactly the last compile. */ +#define BUILD() do { \ + snprintf(cmd, sizeof cmd, \ + "rm -rf %s; WW_PKGCACHE='%s' timeout 240 %s/%s build --sep " \ + "-o %s %s >/dev/null 2>&1", scratch, cache, bin, drv, prog, \ + rootww); \ + } while (0) +#define SDOT(pkg) sdot(scratch, (pkg)) + + /* fresh cache for this stage */ + snprintf(cmd, sizeof cmd, "rm -rf %s", cache); + runwait(cmd); + /* sources back to v1 (a prior stage's restore may have left them) */ + write_file(midww, mid_v1); + write_file(leafww, leaf_v1); + + /* leg 1 — MISS: cold build compiles both, prog runs */ + BUILD(); + if (runwait(cmd) != 0) { + fprintf(stderr, "pkgcache FAIL[%s]: cold build failed\n", drv); + fail++; continue; + } + if (!SDOT("leaf") || !SDOT("mid")) { + fprintf(stderr, "pkgcache FAIL[%s]: leg1 MISS — leaf/mid did " + "not compile\n", drv); + fail++; + } + if (runwait(prog) != EXPECT_EXIT) { + fprintf(stderr, "pkgcache FAIL[%s]: leg1 prog exit != %d\n", + drv, EXPECT_EXIT); + fail++; + } + + /* leg 2 — HIT + non-vacuity: unchanged rebuild skips both */ + BUILD(); + runwait(cmd); + if (SDOT("leaf") || SDOT("mid")) { + fprintf(stderr, "pkgcache FAIL[%s]: leg2 HIT — recompiled an " + "unchanged package (cache vacuous)\n", drv); + fail++; + } + if (runwait(prog) != EXPECT_EXIT) { + fprintf(stderr, "pkgcache FAIL[%s]: leg2 prog exit != %d " + "(cached artifact unusable)\n", drv, EXPECT_EXIT); + fail++; + } + + /* leg 3 — BUST source edit: mid MISS, leaf still HIT */ + write_file(midww, mid_v2); + BUILD(); + runwait(cmd); + if (!SDOT("mid") || SDOT("leaf")) { + fprintf(stderr, "pkgcache FAIL[%s]: leg3 src-edit — expected " + "mid MISS + leaf HIT\n", drv); + fail++; + } + /* restore + reseed, then prove HIT returns */ + write_file(midww, mid_v1); + BUILD(); runwait(cmd); /* reseeds the v1 key (MISS) */ + BUILD(); runwait(cmd); /* must HIT now */ + if (SDOT("mid")) { + fprintf(stderr, "pkgcache FAIL[%s]: leg3 non-vacuity — mid did " + "not HIT after restore\n", drv); + fail++; + } + + /* leg 4 — BUST dep .wwi: change leaf interface → importer mid MISSES */ + write_file(leafww, leaf_v2); + BUILD(); + runwait(cmd); + if (!SDOT("leaf") || !SDOT("mid")) { + fprintf(stderr, "pkgcache FAIL[%s]: leg4 dep-.wwi — leaf " + "interface change did not bust importer mid\n", drv); + fail++; + } + write_file(leafww, leaf_v1); + BUILD(); runwait(cmd); /* reseed v1 */ + BUILD(); runwait(cmd); /* must HIT now */ + if (SDOT("leaf") || SDOT("mid")) { + fprintf(stderr, "pkgcache FAIL[%s]: leg4 non-vacuity — did not " + "HIT after restore\n", drv); + fail++; + } + + /* leg 5 — BUST compiler md5: corrupt the stored mid w6c line */ + if (mutate_keyline(midkey, "w6c ") != 0) { + fprintf(stderr, "pkgcache FAIL[%s]: leg5 — no w6c line in " + "mid/P.key\n", drv); + fail++; + } + BUILD(); + runwait(cmd); + if (!SDOT("mid") || SDOT("leaf")) { + fprintf(stderr, "pkgcache FAIL[%s]: leg5 compiler-md5 — corrupt " + "w6c line did not bust mid (only)\n", drv); + fail++; + } + BUILD(); runwait(cmd); /* key rewritten → must HIT */ + if (SDOT("mid")) { + fprintf(stderr, "pkgcache FAIL[%s]: leg5 non-vacuity — mid did " + "not HIT after key rewrite\n", drv); + fail++; + } + + /* leg 6 — BUST flag string: corrupt the stored mid flags line */ + if (mutate_keyline(midkey, "flags ") != 0) { + fprintf(stderr, "pkgcache FAIL[%s]: leg6 — no flags line in " + "mid/P.key\n", drv); + fail++; + } + BUILD(); + runwait(cmd); + if (!SDOT("mid")) { + fprintf(stderr, "pkgcache FAIL[%s]: leg6 flags — corrupt flags " + "line did not bust mid\n", drv); + fail++; + } + BUILD(); runwait(cmd); + if (SDOT("mid")) { + fprintf(stderr, "pkgcache FAIL[%s]: leg6 non-vacuity — mid did " + "not HIT after key rewrite\n", drv); + fail++; + } +#undef BUILD +#undef SDOT + } + + /* cross-stage byte-id (rule 10): cached OUTPUTS identical cs==ww. The + * caches end in the v1/HIT state from each stage's leg 6 tail. */ + { + const char *pkgs[] = { "mid", "leaf" }; + const char *arts[] = { "P.o", "P.wwi" }; + for (int i = 0; i < 2; i++) + for (int k = 0; k < 2; k++) { + char a[1100], b[1100]; + snprintf(a, sizeof a, "%s/%s/%s", cache_cs, pkgs[i], arts[k]); + snprintf(b, sizeof b, "%s/%s/%s", cache_ww, pkgs[i], arts[k]); + if (files_eq(a, b) != 0) { + fprintf(stderr, "pkgcache FAIL: cs != ww for cached " + "%s/%s (rule 10)\n", pkgs[i], arts[k]); + fail++; + } + } + } + +out: + snprintf(cmd, sizeof cmd, "rm -rf %s", td); + runwait(cmd); + if (fail) { + fprintf(stderr, "pkgcache: %d check(s) failed\n", fail); + return 1; + } + printf("pkgcache: 3-level root->mid->leaf via build_one_sep — MISS " + "compiles, HIT skips (non-vacuous, prog runs), and key busts on each " + "input class (src / dep .wwi / compiler md5 / flags) — both stages; " + "cached P.o/P.wwi byte-identical cs==ww\n"); + return 0; +}