Hojun-Cho/ww
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

wcc/ww: out/.pkgcache content-keyed package cache for --sep (M3-tail c5b, #63)

Browse Source 
Per-package build cache for `ww build --sep`: before recompiling a
package, recompute a plain-text key manifest (md5sum-hex lines of the
package sources, each direct dep's .wwi, the w6c and w6a binaries, plus
the compile flags) and reuse the cached .o/.wwi on a byte-for-byte key
hit. Both stages shell the same host md5sum (no ww-side md5) so the
non-compiler key lines are byte-identical cstage==wwstage; the compiler
md5 lines differ per stage BY DESIGN, giving each stage its own cache
namespace so a hit can never reuse the other stage's .o and mask a
cs!=ww codegen divergence. Cached outputs (.o/.wwi) stay byte-identical
across stages. Root package never cached; cache lives under $(OUT)
(gitignored, wiped by make clean). Both stages mirrored.

Dev-only convenience, off every bootstrap/byte-id gate. Gate
989_pkgcache_run (cold) proves miss-compiles / hit-skips-byte-id /
independent key-bust per input class with non-vacuity rehit.
This commit is contained in:
Hojun-Cho
2026-06-16 15:35:26 +09:00
parent 62c7771594
commit 7b36f961a9
5 changed files with 1046 additions and 14 deletions
Download Patch File Download Diff File Expand all files Collapse all files

228
selfhost/cmd/ww/main.combined.ww
View File

@@ -4150,6 +4150,214 @@ fn archiveo(objpath: *u8, apath: *u8) i32 = {
return 0;
};
// ---- 5b content-keyed package cache (#63) ----------------------------
//
// Twin of cstage's pkgcache_root/md5_file/sep_manifest/cache_lookup/
// cache_store (cmd/ww/main.c). A per-package cache under WW_PKGCACHE
// (default out/.pkgcache; gitignored, make clean wipes $(OUT)). Dev-inner-
// loop convenience only: every gate cold-compiles, so the cache changes no
// gate output. rule-10: md5 is NOT reimplemented here — both stages shell to
// the host md5sum, so a HIT's reused P.wwi/P.o stay byte-identical cs==ww.
fn pkgcacheroot() *u8 = {
match (os.getenv("WW_PKGCACHE")) {
case let s: str =>
if (s.len > 0) {
let buf: []u8 = alloc([], (s.len: u64) + 1u64)!;
let i: i32 = 0;
for (i < s.len) { buf[i] = s[i]; i += 1; };
buf[s.len] = 0u8;
return buf.ptr;
};
case void => void;
};
let d: []u8 = alloc([], 64u64)!;
d.len = 64;
let o: u64 = strinto(d.ptr, 0u64, "out/.pkgcache");
cstrseal(d.ptr, o);
return d.ptr;
};
fn pkgcachedir(cacheroot: *u8, g: *sepgraph, pi: i32) *u8 = {
if (g.pkg[pi].path[0u64] == 0u8) {
return joinpathlit(cacheroot, "__root");
};
return joinpathlit(cacheroot, pathstr(g.pkg[pi].path));
};
// md5appendhex — host md5sum of `path`, captured via a shell redirect to
// `tmp` (cache is gate-cold → host-tool dep sanctioned, rule-7). Appends the
// hex digest to `dst` at `off`; returns the new offset, or 0 on failure
// (caller treats 0 as a non-cacheable miss). Mirrors cstage md5_file's
// copy-until-whitespace parse so the digest text is byte-identical.
fn md5appendhex(dst: *u8, off: u64, path: *u8, tmp: *u8) u64 = {
let cmd: []u8 = alloc([], 8192u64)!;
cmd.len = 8192;
let co: u64 = strinto(cmd.ptr, 0u64, "md5sum '");
co = cstrinto(cmd.ptr, co, path);
co = strinto(cmd.ptr, co, "' > '");
co = cstrinto(cmd.ptr, co, tmp);
co = strinto(cmd.ptr, co, "'");
cstrseal(cmd.ptr, co);
let argv: []*u8 = alloc([], 4u64)!;
argv.len = 4;
argv[0] = "sh\0".ptr;
argv[1] = "-c\0".ptr;
argv[2] = cmd.ptr;
argv[3] = nil;
if (procrun("/bin/sh\0".ptr, argv.ptr) != 0) { return 0u64; };
let (buf, n) = slurp(tmp);
if (buf == nil) { return 0u64; };
let i: u64 = 0u64;
for (i < n) {
let c: u8 = buf[i];
if (c == 32u8 || c == 9u8 || c == 10u8) { break; };
dst[off + i] = c;
i += 1u64;
};
if (i == 0u64) { return 0u64; };
return off + i;
};
// sepmanifest — assemble package pi's content-key manifest (D4) into `out`
// as deterministic text (see cstage sep_manifest). Returns the manifest
// length, or 0 on any md5 failure (caller → cold compile). The md5 capture
// tmp lives in `scratch` (which exists), not the cache dir (created lazily).
fn sepmanifest(g: *sepgraph, pi: i32, scratch: *u8, c6: *u8, a6: *u8,
out: *u8) u64 = {
let tmp: *u8 = sepfname(g, pi, scratch, ".md5tmp");
let off: u64 = strinto(out, 0u64, "src");
if (g.pkg[pi].isdir != 0) {
let (names, n) = enumeratedir(g.pkg[pi].entry);
let i: i32 = 0;
for (i < n) {
let fp: []u8 = alloc([], os.PATH_MAX: u64)!;
fp.len = os.PATH_MAX;
let fo: u64 = cstrinto(fp.ptr, 0u64, g.pkg[pi].entry);
fo = byteinto(fp.ptr, fo, 47u8); // '/'
fo = cstrinto(fp.ptr, fo, names[i]);
cstrseal(fp.ptr, fo);
off = byteinto(out, off, 32u8); // ' '
let no: u64 = md5appendhex(out, off, fp.ptr, tmp);
if (no == 0u64) { return 0u64; };
off = no;
i += 1;
};
} else {
off = byteinto(out, off, 32u8);
let no: u64 = md5appendhex(out, off, g.pkg[pi].entry, tmp);
if (no == 0u64) { return 0u64; };
off = no;
};
off = byteinto(out, off, 10u8); // '\n'
// dep lines, sorted by dep path (insertion sort, mirrors enumeratedir).
let nd: i32 = g.pkg[pi].ndeps;
let idx: []i32 = alloc([], SEP_MAXPKG: u64)!;
idx.len = SEP_MAXPKG;
let i: i32 = 0;
for (i < nd) { idx[i] = g.pkg[pi].deps[i]; i += 1; };
i = 1;
for (i < nd) {
let j: i32 = i;
for (j > 0) {
let a: *u8 = g.pkg[idx[j - 1]].path;
let b: *u8 = g.pkg[idx[j]].path;
if (bytecmp(a, cstrlen(a), b, cstrlen(b)) <= 0) { j = 0; }
else {
let t: i32 = idx[j];
idx[j] = idx[j - 1];
idx[j - 1] = t;
j -= 1;
};
};
i += 1;
};
i = 0;
for (i < nd) {
let di: i32 = idx[i];
let wwip: *u8 = sepfname(g, di, scratch, ".wwi");
off = strinto(out, off, "dep ");
off = cstrinto(out, off, g.pkg[di].path);
off = byteinto(out, off, 32u8);
let no: u64 = md5appendhex(out, off, wwip, tmp);
if (no == 0u64) { return 0u64; };
off = no;
off = byteinto(out, off, 10u8);
i += 1;
};
off = strinto(out, off, "w6c ");
let no: u64 = md5appendhex(out, off, c6, tmp);
if (no == 0u64) { return 0u64; };
off = no;
off = byteinto(out, off, 10u8);
off = strinto(out, off, "w6a ");
no = md5appendhex(out, off, a6, tmp);
if (no == 0u64) { return 0u64; };
off = no;
off = byteinto(out, off, 10u8);
off = strinto(out, off, "flags -c -I\n");
return off;
};
// copyfile — byte-copy src→dst (TRUNC). Returns 0 on success, -1 on failure.
fn copyfile(src: *u8, dst: *u8) i32 = {
let (buf, n) = slurp(src);
if (buf == nil) { return -1; };
let fd: i32 = os.open(pathstr(dst),
os.flag.WRONLY | os.flag.CREATE | os.flag.TRUNC, 420i32); // 0o644
if (fd < 0) { return -1; };
os.writeall(fd, buf, n);
os.close(fd);
return 0;
};
// cachelookup — HIT iff the manifest equals the stored P.key byte-for-byte
// AND both cached artifacts exist; on HIT copy them into the scratch
// wwi/objf so the producer loop can skip compose+w6c+w6a.
fn cachelookup(cacheroot: *u8, g: *sepgraph, pi: i32, manifest: *u8,
mlen: u64, wwi: *u8, objf: *u8) i32 = {
let dir: *u8 = pkgcachedir(cacheroot, g, pi);
let keyp: *u8 = joinpathlit(dir, "P.key");
let cwwi: *u8 = joinpathlit(dir, "P.wwi");
let cobj: *u8 = joinpathlit(dir, "P.o");
let (stored, sn) = slurp(keyp);
if (stored == nil) { return 0; };
if (sn != mlen) { return 0; };
let i: u64 = 0u64;
for (i < mlen) {
if (stored[i] != manifest[i]) { return 0; };
i += 1u64;
};
if (os.access(pathstr(cwwi), 0i32) != 0) { return 0; };
if (os.access(pathstr(cobj), 0i32) != 0) { return 0; };
if (copyfile(cwwi, wwi) != 0) { return 0; };
if (copyfile(cobj, objf) != 0) { return 0; };
return 1;
};
// cachestore — on MISS persist the artifacts then the key (key LAST: a crash
// mid-store never leaves a key without its artifacts; the next run re-misses).
fn cachestore(cacheroot: *u8, g: *sepgraph, pi: i32, manifest: *u8,
mlen: u64, wwi: *u8, objf: *u8) void = {
let dir: *u8 = pkgcachedir(cacheroot, g, pi);
match (os.mkdirs(pathstr(dir), 493i32)) { // 0o755
case void => void;
case let e: os.oserror => void; // best-effort; copyfile surfaces a real failure
};
let cwwi: *u8 = joinpathlit(dir, "P.wwi");
let cobj: *u8 = joinpathlit(dir, "P.o");
let keyp: *u8 = joinpathlit(dir, "P.key");
if (copyfile(wwi, cwwi) != 0) { return; };
if (copyfile(objf, cobj) != 0) { return; };
let fd: i32 = os.open(pathstr(keyp),
os.flag.WRONLY | os.flag.CREATE | os.flag.TRUNC, 420i32); // 0o644
if (fd < 0) { return; };
os.writeall(fd, manifest, mlen);
os.close(fd);
};
// buildonesep — the --sep orchestration: discover deps, reverse-topo,
// the transitive producer loop (one `w6c -c -I` per package, dep-first,
// each `.o` wrapped in its own deterministic per-package `.a`), then a
@@ -4265,6 +4473,7 @@ fn buildonesep(selfdir: *u8, src: *u8, entryisdir: i32, out: *u8,
if (septopovisit(g, root, order, &norder, stack, 0) < 0) { return 1; };
// Producer loop — dep-first, one `w6c -c -I` per package.
let cacheroot: *u8 = pkgcacheroot();
let oi: i32 = 0;
for (oi < norder) {
let pi: i32 = order[oi];
@@ -4272,6 +4481,21 @@ fn buildonesep(selfdir: *u8, src: *u8, entryisdir: i32, out: *u8,
let wwi: *u8 = sepfname(g, pi, scratch, ".wwi");
let asmf: *u8 = sepfname(g, pi, scratch, ".s");
let objf: *u8 = sepfname(g, pi, scratch, ".o");
// 5b: skip compose+w6c+w6a on a content-key HIT. The root is
// never cached — it is the build target, always recompiled.
let manbuf: []u8 = alloc([], 16384u64)!;
manbuf.len = 16384;
let mlen: u64 = 0u64;
let cacheable: i32 = 0;
if (pi != root) {
mlen = sepmanifest(g, pi, scratch, c6, a6, manbuf.ptr);
if (mlen > 0u64) { cacheable = 1; };
};
let fresh: i32 = 0;
if (cacheable != 0) {
fresh = cachelookup(cacheroot, g, pi, manbuf.ptr, mlen, wwi, objf);
};
if (fresh == 0) {
if (sepcomposeunit(g, pi, scratch, order, norder, searchpath.ptr, unitf) < 0) {
return 1;
};
@@ -4304,6 +4528,10 @@ fn buildonesep(selfdir: *u8, src: *u8, entryisdir: i32, out: *u8,
return 1;
};
};
if (cacheable != 0) {
cachestore(cacheroot, g, pi, manbuf.ptr, mlen, wwi, objf);
};
};
// Wrap each DEP package's `.o` in its own deterministic `.a`
// (5a). The ROOT stays a positional `.o` (force-loaded — it's
// the build target), so `main` is defined before any archive is

228
selfhost/cmd/ww/main.ww
View File

@@ -1276,6 +1276,214 @@ fn archiveo(objpath: *u8, apath: *u8) i32 = {
return 0;
};
// ---- 5b content-keyed package cache (#63) ----------------------------
//
// Twin of cstage's pkgcache_root/md5_file/sep_manifest/cache_lookup/
// cache_store (cmd/ww/main.c). A per-package cache under WW_PKGCACHE
// (default out/.pkgcache; gitignored, make clean wipes $(OUT)). Dev-inner-
// loop convenience only: every gate cold-compiles, so the cache changes no
// gate output. rule-10: md5 is NOT reimplemented here — both stages shell to
// the host md5sum, so a HIT's reused P.wwi/P.o stay byte-identical cs==ww.
fn pkgcacheroot() *u8 = {
match (os.getenv("WW_PKGCACHE")) {
case let s: str =>
if (s.len > 0) {
let buf: []u8 = alloc([], (s.len: u64) + 1u64)!;
let i: i32 = 0;
for (i < s.len) { buf[i] = s[i]; i += 1; };
buf[s.len] = 0u8;
return buf.ptr;
};
case void => void;
};
let d: []u8 = alloc([], 64u64)!;
d.len = 64;
let o: u64 = strinto(d.ptr, 0u64, "out/.pkgcache");
cstrseal(d.ptr, o);
return d.ptr;
};
fn pkgcachedir(cacheroot: *u8, g: *sepgraph, pi: i32) *u8 = {
if (g.pkg[pi].path[0u64] == 0u8) {
return joinpathlit(cacheroot, "__root");
};
return joinpathlit(cacheroot, pathstr(g.pkg[pi].path));
};
// md5appendhex — host md5sum of `path`, captured via a shell redirect to
// `tmp` (cache is gate-cold → host-tool dep sanctioned, rule-7). Appends the
// hex digest to `dst` at `off`; returns the new offset, or 0 on failure
// (caller treats 0 as a non-cacheable miss). Mirrors cstage md5_file's
// copy-until-whitespace parse so the digest text is byte-identical.
fn md5appendhex(dst: *u8, off: u64, path: *u8, tmp: *u8) u64 = {
let cmd: []u8 = alloc([], 8192u64)!;
cmd.len = 8192;
let co: u64 = strinto(cmd.ptr, 0u64, "md5sum '");
co = cstrinto(cmd.ptr, co, path);
co = strinto(cmd.ptr, co, "' > '");
co = cstrinto(cmd.ptr, co, tmp);
co = strinto(cmd.ptr, co, "'");
cstrseal(cmd.ptr, co);
let argv: []*u8 = alloc([], 4u64)!;
argv.len = 4;
argv[0] = "sh\0".ptr;
argv[1] = "-c\0".ptr;
argv[2] = cmd.ptr;
argv[3] = nil;
if (procrun("/bin/sh\0".ptr, argv.ptr) != 0) { return 0u64; };
let (buf, n) = slurp(tmp);
if (buf == nil) { return 0u64; };
let i: u64 = 0u64;
for (i < n) {
let c: u8 = buf[i];
if (c == 32u8 || c == 9u8 || c == 10u8) { break; };
dst[off + i] = c;
i += 1u64;
};
if (i == 0u64) { return 0u64; };
return off + i;
};
// sepmanifest — assemble package pi's content-key manifest (D4) into `out`
// as deterministic text (see cstage sep_manifest). Returns the manifest
// length, or 0 on any md5 failure (caller → cold compile). The md5 capture
// tmp lives in `scratch` (which exists), not the cache dir (created lazily).
fn sepmanifest(g: *sepgraph, pi: i32, scratch: *u8, c6: *u8, a6: *u8,
out: *u8) u64 = {
let tmp: *u8 = sepfname(g, pi, scratch, ".md5tmp");
let off: u64 = strinto(out, 0u64, "src");
if (g.pkg[pi].isdir != 0) {
let (names, n) = enumeratedir(g.pkg[pi].entry);
let i: i32 = 0;
for (i < n) {
let fp: []u8 = alloc([], os.PATH_MAX: u64)!;
fp.len = os.PATH_MAX;
let fo: u64 = cstrinto(fp.ptr, 0u64, g.pkg[pi].entry);
fo = byteinto(fp.ptr, fo, 47u8); // '/'
fo = cstrinto(fp.ptr, fo, names[i]);
cstrseal(fp.ptr, fo);
off = byteinto(out, off, 32u8); // ' '
let no: u64 = md5appendhex(out, off, fp.ptr, tmp);
if (no == 0u64) { return 0u64; };
off = no;
i += 1;
};
} else {
off = byteinto(out, off, 32u8);
let no: u64 = md5appendhex(out, off, g.pkg[pi].entry, tmp);
if (no == 0u64) { return 0u64; };
off = no;
};
off = byteinto(out, off, 10u8); // '\n'
// dep lines, sorted by dep path (insertion sort, mirrors enumeratedir).
let nd: i32 = g.pkg[pi].ndeps;
let idx: []i32 = alloc([], SEP_MAXPKG: u64)!;
idx.len = SEP_MAXPKG;
let i: i32 = 0;
for (i < nd) { idx[i] = g.pkg[pi].deps[i]; i += 1; };
i = 1;
for (i < nd) {
let j: i32 = i;
for (j > 0) {
let a: *u8 = g.pkg[idx[j - 1]].path;
let b: *u8 = g.pkg[idx[j]].path;
if (bytecmp(a, cstrlen(a), b, cstrlen(b)) <= 0) { j = 0; }
else {
let t: i32 = idx[j];
idx[j] = idx[j - 1];
idx[j - 1] = t;
j -= 1;
};
};
i += 1;
};
i = 0;
for (i < nd) {
let di: i32 = idx[i];
let wwip: *u8 = sepfname(g, di, scratch, ".wwi");
off = strinto(out, off, "dep ");
off = cstrinto(out, off, g.pkg[di].path);
off = byteinto(out, off, 32u8);
let no: u64 = md5appendhex(out, off, wwip, tmp);
if (no == 0u64) { return 0u64; };
off = no;
off = byteinto(out, off, 10u8);
i += 1;
};
off = strinto(out, off, "w6c ");
let no: u64 = md5appendhex(out, off, c6, tmp);
if (no == 0u64) { return 0u64; };
off = no;
off = byteinto(out, off, 10u8);
off = strinto(out, off, "w6a ");
no = md5appendhex(out, off, a6, tmp);
if (no == 0u64) { return 0u64; };
off = no;
off = byteinto(out, off, 10u8);
off = strinto(out, off, "flags -c -I\n");
return off;
};
// copyfile — byte-copy src→dst (TRUNC). Returns 0 on success, -1 on failure.
fn copyfile(src: *u8, dst: *u8) i32 = {
let (buf, n) = slurp(src);
if (buf == nil) { return -1; };
let fd: i32 = os.open(pathstr(dst),
os.flag.WRONLY | os.flag.CREATE | os.flag.TRUNC, 420i32); // 0o644
if (fd < 0) { return -1; };
os.writeall(fd, buf, n);
os.close(fd);
return 0;
};
// cachelookup — HIT iff the manifest equals the stored P.key byte-for-byte
// AND both cached artifacts exist; on HIT copy them into the scratch
// wwi/objf so the producer loop can skip compose+w6c+w6a.
fn cachelookup(cacheroot: *u8, g: *sepgraph, pi: i32, manifest: *u8,
mlen: u64, wwi: *u8, objf: *u8) i32 = {
let dir: *u8 = pkgcachedir(cacheroot, g, pi);
let keyp: *u8 = joinpathlit(dir, "P.key");
let cwwi: *u8 = joinpathlit(dir, "P.wwi");
let cobj: *u8 = joinpathlit(dir, "P.o");
let (stored, sn) = slurp(keyp);
if (stored == nil) { return 0; };
if (sn != mlen) { return 0; };
let i: u64 = 0u64;
for (i < mlen) {
if (stored[i] != manifest[i]) { return 0; };
i += 1u64;
};
if (os.access(pathstr(cwwi), 0i32) != 0) { return 0; };
if (os.access(pathstr(cobj), 0i32) != 0) { return 0; };
if (copyfile(cwwi, wwi) != 0) { return 0; };
if (copyfile(cobj, objf) != 0) { return 0; };
return 1;
};
// cachestore — on MISS persist the artifacts then the key (key LAST: a crash
// mid-store never leaves a key without its artifacts; the next run re-misses).
fn cachestore(cacheroot: *u8, g: *sepgraph, pi: i32, manifest: *u8,
mlen: u64, wwi: *u8, objf: *u8) void = {
let dir: *u8 = pkgcachedir(cacheroot, g, pi);
match (os.mkdirs(pathstr(dir), 493i32)) { // 0o755
case void => void;
case let e: os.oserror => void; // best-effort; copyfile surfaces a real failure
};
let cwwi: *u8 = joinpathlit(dir, "P.wwi");
let cobj: *u8 = joinpathlit(dir, "P.o");
let keyp: *u8 = joinpathlit(dir, "P.key");
if (copyfile(wwi, cwwi) != 0) { return; };
if (copyfile(objf, cobj) != 0) { return; };
let fd: i32 = os.open(pathstr(keyp),
os.flag.WRONLY | os.flag.CREATE | os.flag.TRUNC, 420i32); // 0o644
if (fd < 0) { return; };
os.writeall(fd, manifest, mlen);
os.close(fd);
};
// buildonesep — the --sep orchestration: discover deps, reverse-topo,
// the transitive producer loop (one `w6c -c -I` per package, dep-first,
// each `.o` wrapped in its own deterministic per-package `.a`), then a
@@ -1391,6 +1599,7 @@ fn buildonesep(selfdir: *u8, src: *u8, entryisdir: i32, out: *u8,
if (septopovisit(g, root, order, &norder, stack, 0) < 0) { return 1; };
// Producer loop — dep-first, one `w6c -c -I` per package.
let cacheroot: *u8 = pkgcacheroot();
let oi: i32 = 0;
for (oi < norder) {
let pi: i32 = order[oi];
@@ -1398,6 +1607,21 @@ fn buildonesep(selfdir: *u8, src: *u8, entryisdir: i32, out: *u8,
let wwi: *u8 = sepfname(g, pi, scratch, ".wwi");
let asmf: *u8 = sepfname(g, pi, scratch, ".s");
let objf: *u8 = sepfname(g, pi, scratch, ".o");
// 5b: skip compose+w6c+w6a on a content-key HIT. The root is
// never cached — it is the build target, always recompiled.
let manbuf: []u8 = alloc([], 16384u64)!;
manbuf.len = 16384;
let mlen: u64 = 0u64;
let cacheable: i32 = 0;
if (pi != root) {
mlen = sepmanifest(g, pi, scratch, c6, a6, manbuf.ptr);
if (mlen > 0u64) { cacheable = 1; };
};
let fresh: i32 = 0;
if (cacheable != 0) {
fresh = cachelookup(cacheroot, g, pi, manbuf.ptr, mlen, wwi, objf);
};
if (fresh == 0) {
if (sepcomposeunit(g, pi, scratch, order, norder, searchpath.ptr, unitf) < 0) {
return 1;
};
@@ -1430,6 +1654,10 @@ fn buildonesep(selfdir: *u8, src: *u8, entryisdir: i32, out: *u8,
return 1;
};
};
if (cacheable != 0) {
cachestore(cacheroot, g, pi, manbuf.ptr, mlen, wwi, objf);
};
};
// Wrap each DEP package's `.o` in its own deterministic `.a`
// (5a). The ROOT stays a positional `.o` (force-loaded — it's
// the build target), so `main` is defined before any archive is