Hojun-Cho/ww
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

wcc/ww: out/.pkgcache content-keyed package cache for --sep (M3-tail c5b, #63)

Browse Source 
Per-package build cache for `ww build --sep`: before recompiling a
package, recompute a plain-text key manifest (md5sum-hex lines of the
package sources, each direct dep's .wwi, the w6c and w6a binaries, plus
the compile flags) and reuse the cached .o/.wwi on a byte-for-byte key
hit. Both stages shell the same host md5sum (no ww-side md5) so the
non-compiler key lines are byte-identical cstage==wwstage; the compiler
md5 lines differ per stage BY DESIGN, giving each stage its own cache
namespace so a hit can never reuse the other stage's .o and mask a
cs!=ww codegen divergence. Cached outputs (.o/.wwi) stay byte-identical
across stages. Root package never cached; cache lives under $(OUT)
(gitignored, wiped by make clean). Both stages mirrored.

Dev-only convenience, off every bootstrap/byte-id gate. Gate
989_pkgcache_run (cold) proves miss-compiles / hit-skips-byte-id /
independent key-bust per input class with non-vacuity rehit.
This commit is contained in:
Hojun-Cho
2026-06-16 15:35:26 +09:00
parent 62c7771594
commit 7b36f961a9
5 changed files with 1046 additions and 14 deletions
Download Patch File Download Diff File Expand all files Collapse all files

211
cmd/ww/main.c
View File

@@ -942,6 +942,180 @@ archive_o(const char *objpath, const char *apath)
return 0;
}
/* ---- 5b content-keyed package cache (#63) ----------------------------
*
* A per-package cache under WW_PKGCACHE (default out/.pkgcache; gitignored,
* make clean wipes $(OUT)). Purely a dev-inner-loop convenience: every
* bootstrap/byte-id gate cold-compiles (--sep scratch is wiped each run), so
* the cache changes NO gate output. Mirrors out/.testcache + Hare get_cache.
*
* rule-10: md5 is NOT reimplemented per stage — both stages shell to the
* host md5sum and assemble an identical text manifest, so a HIT's reused
* P.wwi/P.o stay byte-identical cs==ww by construction. The compiler-binary
* line keys CODEGEN identity, so it is intentionally stage-specific (w6c vs
* w6c_ww) — each stage maintains its own cache namespace; the cacheable
* OUTPUTS it reuses remain byte-identical. */
static const char *
pkgcache_root(void)
{
const char *p = getenv("WW_PKGCACHE");
if (p && p[0]) return p;
return "out/.pkgcache";
}
/* Hex md5 digest of `path` via the host tool. The cache is gate-cold, so a
* host-tool dep is sanctioned (md5sum is already a build dependency). */
static int
md5_file(const char *path, char *hex, size_t hexsz)
{
char cmd[1200];
snprintf(cmd, sizeof cmd, "md5sum '%s' 2>/dev/null", path);
FILE *p = popen(cmd, "r");
if (p == NULL) return -1;
char line[256];
char *got = fgets(line, sizeof line, p);
pclose(p);
if (got == NULL) return -1;
size_t i = 0;
while (i + 1 < hexsz && line[i] && line[i] != ' '
&& line[i] != '\t' && line[i] != '\n') i++;
if (i == 0) return -1;
memcpy(hex, line, i);
hex[i] = '\0';
return 0;
}
/* Assemble package pi's content-key manifest (D4) into `out` as deterministic
* text: P's own *.ww md5s (sorted by name) | each DIRECT dep's .wwi md5
* (sorted by dep path) | the w6c/w6a md5s | the flag string. DIRECT deps
* only — reverse-topo folds transitivity bottom-up, since a dep's .wwi
* already folds ITS deps. Returns 0 on success, -1 on any md5/truncation
* failure (caller treats that as non-cacheable → cold compile). */
static int
sep_manifest(struct sepgraph *g, int pi, const char *scratch,
const char *c6, const char *a6, char *out, size_t outsz)
{
size_t off = 0;
char hex[64];
off += (size_t)snprintf(out + off, outsz - off, "src");
if (g->pkg[pi].is_dir) {
char **files = NULL;
int n = enumerate_dir_ww(g->pkg[pi].entry, &files);
int rc = 0;
for (int i = 0; i < n; i++) {
char fp[1024];
snprintf(fp, sizeof fp, "%s/%s", g->pkg[pi].entry,
files[i]);
if (rc == 0 && md5_file(fp, hex, sizeof hex) == 0)
off += (size_t)snprintf(out + off, outsz - off,
" %s", hex);
else
rc = -1;
free(files[i]);
}
free(files);
if (rc != 0) return -1;
} else {
if (md5_file(g->pkg[pi].entry, hex, sizeof hex) != 0)
return -1;
off += (size_t)snprintf(out + off, outsz - off, " %s", hex);
}
off += (size_t)snprintf(out + off, outsz - off, "\n");
/* dep lines, sorted by dep path (deps[] are in import-appearance
* order; insertion-sort the indices for a deterministic manifest). */
int nd = g->pkg[pi].ndeps;
int idx[SEP_MAXPKG];
for (int i = 0; i < nd; i++) idx[i] = g->pkg[pi].deps[i];
for (int i = 1; i < nd; i++) {
int v = idx[i], j = i;
while (j > 0 &&
strcmp(g->pkg[idx[j-1]].path, g->pkg[v].path) > 0) {
idx[j] = idx[j-1];
j--;
}
idx[j] = v;
}
for (int i = 0; i < nd; i++) {
int di = idx[i];
char wwip[1024];
sep_fname(g, di, scratch, ".wwi", wwip, sizeof wwip);
if (md5_file(wwip, hex, sizeof hex) != 0) return -1;
off += (size_t)snprintf(out + off, outsz - off,
"dep %s %s\n", g->pkg[di].path, hex);
}
if (md5_file(c6, hex, sizeof hex) != 0) return -1;
off += (size_t)snprintf(out + off, outsz - off, "w6c %s\n", hex);
if (md5_file(a6, hex, sizeof hex) != 0) return -1;
off += (size_t)snprintf(out + off, outsz - off, "w6a %s\n", hex);
off += (size_t)snprintf(out + off, outsz - off, "flags -c -I\n");
if (off >= outsz) return -1; /* truncated → unusable key */
return 0;
}
static void
pkgcache_dir(struct sepgraph *g, int pi, char *out, size_t outsz)
{
const char *base = g->pkg[pi].path[0] ? g->pkg[pi].path : "__root";
snprintf(out, outsz, "%s/%s", pkgcache_root(), base);
}
/* HIT iff a freshly recomputed manifest equals the stored P.key byte-for-byte
* AND both cached artifacts exist; on HIT copy them into the scratch wwi/obj
* paths so the producer loop can skip compose+w6c+w6a. */
static int
cache_lookup(struct sepgraph *g, int pi, const char *manifest,
const char *wwi, const char *obj)
{
char dir[1024], keyp[1100], cwwi[1100], cobj[1100], cmd[4096];
pkgcache_dir(g, pi, dir, sizeof dir);
snprintf(keyp, sizeof keyp, "%s/P.key", dir);
snprintf(cwwi, sizeof cwwi, "%s/P.wwi", dir);
snprintf(cobj, sizeof cobj, "%s/P.o", dir);
FILE *f = fopen(keyp, "rb");
if (f == NULL) return 0;
char stored[16384];
size_t sn = fread(stored, 1, sizeof stored, f);
int more = (fgetc(f) != EOF);
fclose(f);
if (more || sn != strlen(manifest) || memcmp(stored, manifest, sn) != 0)
return 0;
if (access(cwwi, 0) != 0 || access(cobj, 0) != 0) return 0;
snprintf(cmd, sizeof cmd, "cp -f '%s' '%s'", cwwi, wwi);
if (run(cmd) != 0) return 0;
snprintf(cmd, sizeof cmd, "cp -f '%s' '%s'", cobj, obj);
if (run(cmd) != 0) return 0;
return 1;
}
/* On MISS, persist the freshly compiled artifacts then the manifest. The key
* is written LAST so a crash mid-store never leaves a key whose artifacts are
* absent/partial (the next run simply re-misses). */
static void
cache_store(struct sepgraph *g, int pi, const char *manifest,
const char *wwi, const char *obj)
{
char dir[1024], keyp[1100], cwwi[1100], cobj[1100], cmd[4096];
pkgcache_dir(g, pi, dir, sizeof dir);
snprintf(cmd, sizeof cmd, "mkdir -p '%s'", dir);
if (run(cmd) != 0) return;
snprintf(cwwi, sizeof cwwi, "%s/P.wwi", dir);
snprintf(cobj, sizeof cobj, "%s/P.o", dir);
snprintf(keyp, sizeof keyp, "%s/P.key", dir);
snprintf(cmd, sizeof cmd, "cp -f '%s' '%s'", wwi, cwwi);
if (run(cmd) != 0) return;
snprintf(cmd, sizeof cmd, "cp -f '%s' '%s'", obj, cobj);
if (run(cmd) != 0) return;
FILE *f = fopen(keyp, "wb");
if (f == NULL) return;
fputs(manifest, f);
fclose(f);
}
/* build_one_sep — the --sep orchestration: discover_deps, reverse_topo,
* the transitive producer loop (one `w6c -c -I` per package, dep-first,
* each DEP `.o` wrapped in its own deterministic `.a`), then a
@@ -1034,20 +1208,29 @@ build_one_sep(const char *src, int entry_is_dir, const char *out,
sep_fname(g, pi, scratch, ".wwi", wwi, sizeof wwi);
sep_fname(g, pi, scratch, ".s", asmf, sizeof asmf);
sep_fname(g, pi, scratch, ".o", obj, sizeof obj);
if (sep_compose_unit(g, pi, scratch, order, norder, srcdir,
unitf) < 0) { free(order); free(g); return 1; }
snprintf(cmd, sizeof cmd, "%s -c -I %s -o %s %s",
c6, wwi, asmf, unitf);
if (run(cmd) != 0) {
fprintf(stderr, "ww --sep: w6c failed for %s\n",
g->pkg[pi].path[0] ? g->pkg[pi].path : "(root)");
free(order); free(g); return 1;
}
snprintf(cmd, sizeof cmd, "%s -o %s %s", a6, obj, asmf);
if (run(cmd) != 0) {
fprintf(stderr, "ww --sep: w6a failed for %s\n",
g->pkg[pi].path[0] ? g->pkg[pi].path : "(root)");
free(order); free(g); return 1;
/* 5b: skip compose+w6c+w6a on a content-key HIT. The root is
* never cached — it is the build target, always recompiled. */
char manifest[16384];
int cacheable = (pi != root) && sep_manifest(g, pi, scratch,
c6, a6, manifest, sizeof manifest) == 0;
int fresh = cacheable && cache_lookup(g, pi, manifest, wwi, obj);
if (!fresh) {
if (sep_compose_unit(g, pi, scratch, order, norder, srcdir,
unitf) < 0) { free(order); free(g); return 1; }
snprintf(cmd, sizeof cmd, "%s -c -I %s -o %s %s",
c6, wwi, asmf, unitf);
if (run(cmd) != 0) {
fprintf(stderr, "ww --sep: w6c failed for %s\n",
g->pkg[pi].path[0] ? g->pkg[pi].path : "(root)");
free(order); free(g); return 1;
}
snprintf(cmd, sizeof cmd, "%s -o %s %s", a6, obj, asmf);
if (run(cmd) != 0) {
fprintf(stderr, "ww --sep: w6a failed for %s\n",
g->pkg[pi].path[0] ? g->pkg[pi].path : "(root)");
free(order); free(g); return 1;
}
if (cacheable) cache_store(g, pi, manifest, wwi, obj);
}
/* wrap each DEP package's `.o` in its own deterministic `.a`
* (5a). The ROOT stays a positional `.o` (force-loaded — it's