Hojun-Cho/ww
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

pkgcache: reject 0-byte artifacts on store and lookup, self-heal torn writes (#10)

Browse Source 
A torn producer write (e.g. disk-full mid-copy) could leave a 0-byte P.wwi or
P.o in out/.pkgcache under a self-consistent key; cache_lookup checked only
existence, so every later build HIT and served the empty artifact forever
(silent serve-wrong). Reject size==0 on both sides, symmetric across stages:
store refuses to commit a 0-byte temp before the key write, lookup treats a
0-byte cached artifact as a MISS so existing poison self-heals on re-derive.
A valid .wwi/.o is never 0 bytes, so the guard cannot misfire.

Regression: test/wcc/989_pkgcache_poison_run.c, table-driven over
{poison P.wwi | P.o | both}, non-vacuity proven by guard-neuter.
This commit is contained in:
Hojun-Cho
2026-06-22 20:52:11 +09:00
parent c0383274d1
commit a1484aef28
4 changed files with 291 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
Makefile
View File

@@ -535,6 +535,7 @@ TESTS = $(BIN)/test_smoke $(BIN)/test_lex $(BIN)/test_parse $(BIN)/test_check \
$(BIN)/test_separchive_run \
$(BIN)/test_pkgcache_run \
$(BIN)/test_pkgcache_concurrent_run \
$(BIN)/test_pkgcache_poison_run \
$(BIN)/test_c6soak_run \
$(BIN)/test_septest_run \
$(BIN)/test_coloimport_sep \
@@ -2951,6 +2952,16 @@ $(BIN)/test_pkgcache_concurrent_run: test/wcc/989_pkgcache_concurrent_run.c $(BI
$(BIN)/w6l $(BIN)/w6l_ww $(LIB)/libwwrt.a | $(BIN)
$(CC) $(CFLAGS) -o $@ $<
# 989_pkgcache_poison_run — #10 (BUG-B): a 0-byte cached P.wwi/P.o under a valid
# P.key (torn producer write) must self-heal — rejected on both lookup (MISS,
# re-derive) and store (never commit a 0-byte). Table-driven over which artifact
# is poisoned {wwi/o/both}, both driver stages. COLD/dev-only (off every byte-id/
# bootstrap gate). Needs both driver + compiler + asm + linker stages + libwwrt.a.
$(BIN)/test_pkgcache_poison_run: test/wcc/989_pkgcache_poison_run.c $(BIN)/ww $(BIN)/ww_ww \
$(BIN)/w6c $(BIN)/w6c_ww $(BIN)/w6a $(BIN)/w6a_ww \
$(BIN)/w6l $(BIN)/w6l_ww $(LIB)/libwwrt.a | $(BIN)
$(CC) $(CFLAGS) -o $@ $<
# 989_c6soak_run — M3-tail commit-6 dual-path soak gate, phase-1 leg (#46
# c6). Builds real lib chains (encoding.utf8 dotted + strings/bytes same-leaf
# `contains`) BOTH combined and --sep, asserts combined==sep run-equivalence

13
cmd/ww/main.c
View File

@@ -780,6 +780,17 @@ pkgcache_dir(struct sepgraph *g, int pi, char *out, size_t outsz)
snprintf(out, outsz, "%s/%s", pkgcache_root(), base);
}
/* A torn producer write (e.g. disk-full mid-copy) can leave a 0-byte P.wwi/P.o
* under a self-consistent key; size==0 is unambiguous poison (wwi_emit always
* writes >=1 line, a valid .o is never empty), so both store and lookup reject
* it — self-healing, Go-build-cache style (#10). */
static int
filenonempty(const char *p)
{
struct stat st;
return stat(p, &st) == 0 && st.st_size > 0;
}
/* HIT iff a freshly recomputed manifest equals the stored P.key byte-for-byte
* AND both cached artifacts exist; on HIT copy them into the scratch wwi/obj
* paths so the producer loop can skip compose+w6c+w6a. */
@@ -802,6 +813,7 @@ cache_lookup(struct sepgraph *g, int pi, const char *manifest,
if (more || sn != strlen(manifest) || memcmp(stored, manifest, sn) != 0)
return 0;
if (access(cwwi, 0) != 0 || access(cobj, 0) != 0) return 0;
if (!filenonempty(cwwi) || !filenonempty(cobj)) return 0;
snprintf(cmd, sizeof cmd, "cp -f '%s' '%s'", cwwi, wwi);
if (run(cmd) != 0) return 0;
snprintf(cmd, sizeof cmd, "cp -f '%s' '%s'", cobj, obj);
@@ -847,6 +859,7 @@ cache_store(struct sepgraph *g, int pi, const char *manifest,
if (run(cmd) != 0) goto cleanup;
snprintf(cmd, sizeof cmd, "cp -f '%s' '%s'", obj, tobj);
if (run(cmd) != 0) goto cleanup;
if (!filenonempty(twwi) || !filenonempty(tobj)) goto cleanup;
f = fopen(tkey, "wb");
if (f == NULL) goto cleanup;
fputs(manifest, f);

20
selfhost/cmd/ww/main.ww
View File

@@ -1241,6 +1241,22 @@ fn copyfile(src: *u8, dst: *u8) i32 = {
return 0;
};
// cachefilesize — open+filesize+close; -1 on any error. A torn producer write
// (e.g. disk-full mid-copy) can leave a 0-byte P.wwi/P.o under a self-consistent
// key; size==0 is unambiguous poison (wwi_emit always writes >=1 line, a valid
// .o is never empty), so both cachelookup and cachestore reject it — self-healing
// (#10). Twin of cstage filenonempty.
fn cachefilesize(path: *u8) i64 = {
let fd: i32 = os.open(pathstr(path), os.flag.RDONLY, 0i32);
if (fd < 0) { return -1i64; };
let szr: (i64 | os.oserror) = os.filesize(fd);
os.close(fd);
match (szr) {
case let v: i64 => return v;
case let e: os.oserror => return -1i64;
};
};
// cachelookup — HIT iff the manifest equals the stored P.key byte-for-byte
// AND both cached artifacts exist; on HIT copy them into the scratch
// wwi/objf so the producer loop can skip compose+w6c+w6a.
@@ -1260,6 +1276,8 @@ fn cachelookup(cacheroot: *u8, g: *sepgraph, pi: i32, manifest: *u8,
};
if (os.access(pathstr(cwwi), 0i32) != 0) { return 0; };
if (os.access(pathstr(cobj), 0i32) != 0) { return 0; };
if (cachefilesize(cwwi) <= 0i64) { return 0; };
if (cachefilesize(cobj) <= 0i64) { return 0; };
if (copyfile(cwwi, wwi) != 0) { return 0; };
if (copyfile(cobj, objf) != 0) { return 0; };
return 1;
@@ -1340,6 +1358,8 @@ fn cachestore(cacheroot: *u8, g: *sepgraph, pi: i32, manifest: *u8,
let tkey: *u8 = cachetmp(dir, "P.key", pid);
if (copyfile(wwi, twwi) != 0) { cachermtmp(twwi, tobj, tkey); return; };
if (copyfile(objf, tobj) != 0) { cachermtmp(twwi, tobj, tkey); return; };
if (cachefilesize(twwi) <= 0i64) { cachermtmp(twwi, tobj, tkey); return; };
if (cachefilesize(tobj) <= 0i64) { cachermtmp(twwi, tobj, tkey); return; };
let fd: i32 = os.open(pathstr(tkey),
os.flag.WRONLY | os.flag.CREATE | os.flag.TRUNC, 420i32); // 0o644
if (fd < 0) { cachermtmp(twwi, tobj, tkey); return; };

247
test/wcc/989_pkgcache_poison_run.c Normal file
View File

@@ -0,0 +1,247 @@
/*
* 989_pkgcache_poison_run — out/.pkgcache 0-byte-artifact self-heal gate (#10,
* BUG-B). A torn producer write (e.g. disk-full mid-copy) can leave a 0-byte
* P.wwi/P.o under a SELF-CONSISTENT P.key. Pre-fix the cache served that empty
* artifact forever (the key matched and access() saw the file), so every later
* build link-failed or produced a wrong binary — a SILENT serve-wrong that
* never surfaces on its own. The fix rejects size==0 on BOTH the lookup (read)
* and the store (write) side: a poisoned entry self-heals (treated as MISS →
* re-derived), and a torn store never commits the key. The guard is symmetric
* cstage (filenonempty, stat) / wwstage (cachefilesize, open+filesize) — the
* hit/miss DECISION must match (rule 10).
*
* Target: a 2-level graph root(main) -> leaf in a private temp tree. ROOT is
* never cached (it is the build target); `leaf` is the cacheable package. We
* seed a warm cache, POISON the cached artifact to 0 bytes, rebuild, and assert
* the build self-heals: leaf re-compiles (MISS sentinel) AND the program runs
* to its correct value (not an empty/link-failed binary) AND the cache is
* re-stored non-empty.
*
* HIT/MISS sentinel (same as 989_pkgcache_run): a MISS runs w6c → writes
* <scratch>/leaf.s; a HIT copies the cached .o/.wwi and skips compose+w6c+w6a,
* so no leaf.s. The scratch is wiped before each build so leaf.s presence
* reflects exactly the last build.
*
* Table-driven over WHICH artifact is poisoned: { P.wwi | P.o | both }. Each
* row, per stage (ww, ww_ww):
* - warm baseline: an unchanged rebuild HITs (leaf.s ABSENT, prog runs) —
* proves the cache is genuinely warm before we poison it (non-vacuity);
* - POISON the row's artifact(s) to 0 bytes;
* - heal rebuild: leaf re-compiles (leaf.s PRESENT = MISS, not served poison)
* AND prog runs == EXPECT_EXIT AND both cached artifacts are non-empty again
* (re-stored). Pre-fix this row served the 0-byte artifact and the assert
* on leaf.s (and on prog exit) fails.
*
* Light wwstage-driver test (CLAUDE.md rule 14): all build outputs and caches
* live under a private /tmp tree, wiped on exit, so it is parallel-safe and off
* every byte-id/bootstrap gate. Models 989_pkgcache_run conventions.
*/
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <unistd.h>
#include <sys/wait.h>
#include <sys/stat.h>
#define EXPECT_EXIT 7
static int
runwait(const char *cmd)
{
int rc = system(cmd);
if (rc == -1) return -1;
if (WIFEXITED(rc)) return WEXITSTATUS(rc);
return 1;
}
static const char *
absbin(void)
{
const char *b = getenv("BIN");
if (!b) b = "out/bin";
if (b[0] == '/') return b;
static char buf[2048];
char cwd[1024];
if (getcwd(cwd, sizeof cwd) == NULL) return NULL;
snprintf(buf, sizeof buf, "%s/%s", cwd, b);
return buf;
}
static int
write_file(const char *path, const char *body)
{
FILE *f = fopen(path, "wb");
if (!f) return -1;
fputs(body, f);
fclose(f);
return 0;
}
/* Truncate `path` to 0 bytes (models a torn producer write under a valid key);
* "wb" opens-and-truncates, leaving the file present but empty. */
static int
truncate0(const char *path)
{
FILE *f = fopen(path, "wb");
if (!f) return -1;
fclose(f);
return 0;
}
static long
fsize(const char *path)
{
struct stat st;
if (stat(path, &st) != 0) return -1;
return (long)st.st_size;
}
/* <scratch>/<pkg>.s present ⇒ that package compiled this build (MISS). */
static int
sdot(const char *scratch, const char *pkg)
{
char p[1100];
snprintf(p, sizeof p, "%s/%s.s", scratch, pkg);
return access(p, 0) == 0;
}
static const char *leaf_src =
"package leaf;\n"
"export fn base() i32 = { return 7; };\n";
static const char *root_src =
"package main;\n"
"import leaf;\n"
"fn main() i32 = { return leaf.base(); };\n";
int
main(void)
{
const char *bin = absbin();
if (!bin) return 1;
int fail = 0;
char td[64], cmd[8192];
char rootww[1024], leafww[1024];
char scratch[1024], prog[1024];
snprintf(td, sizeof td, "/tmp/wwpkgpoison_%d", getpid());
snprintf(cmd, sizeof cmd, "rm -rf %s", td);
runwait(cmd);
snprintf(cmd, sizeof cmd, "mkdir -p %s/leaf", td);
runwait(cmd);
snprintf(rootww, sizeof rootww, "%s/root.ww", td);
snprintf(leafww, sizeof leafww, "%s/leaf/leaf.ww", td);
snprintf(prog, sizeof prog, "%s/p", td);
snprintf(scratch, sizeof scratch, "%s/p.sepwork", td);
if (write_file(rootww, root_src) || write_file(leafww, leaf_src)) {
fail++; goto out;
}
/* Which cached artifact(s) the row poisons. */
struct { const char *tag; int wwi, obj; } rows[] = {
{ "P.wwi", 1, 0 },
{ "P.o", 0, 1 },
{ "both", 1, 1 },
};
struct { const char *drv; } stg[] = { { "ww" }, { "ww_ww" } };
for (int s = 0; s < 2; s++) {
const char *drv = stg[s].drv;
char cache[1024], cwwi[1100], cobj[1100];
snprintf(cache, sizeof cache, "%s/cache.%d", td, s);
snprintf(cwwi, sizeof cwwi, "%s/leaf/P.wwi", cache);
snprintf(cobj, sizeof cobj, "%s/leaf/P.o", cache);
/* The cstage driver rm -rf's the scratch each build; the wwstage
* driver only mkdir's it (#58(a) asymmetry), so a stale leaf.s could
* persist and defeat the MISS/HIT sentinel. The gate owns its scratch:
* wipe it before every build so a fresh leaf.s reflects the last build. */
#define BUILD() do { \
snprintf(cmd, sizeof cmd, \
"rm -rf %s; WW_PKGCACHE='%s' timeout 240 %s/%s build --sep " \
"-o %s %s >/dev/null 2>&1", scratch, cache, bin, drv, prog, \
rootww); \
} while (0)
/* fresh cache for this stage, cold-seed it (MISS) */
snprintf(cmd, sizeof cmd, "rm -rf %s", cache);
runwait(cmd);
BUILD();
if (runwait(cmd) != 0) {
fprintf(stderr, "poison FAIL[%s]: cold seed build failed\n", drv);
fail++; continue;
}
if (!sdot(scratch, "leaf")) {
fprintf(stderr, "poison FAIL[%s]: cold seed — leaf did not "
"compile (MISS expected)\n", drv);
fail++;
}
if (runwait(prog) != EXPECT_EXIT) {
fprintf(stderr, "poison FAIL[%s]: cold seed prog exit != %d\n",
drv, EXPECT_EXIT);
fail++;
}
for (size_t r = 0; r < sizeof rows / sizeof rows[0]; r++) {
/* warm baseline — the cache is warm (heals re-store it), so an
* unchanged rebuild must HIT; proves we poison a LIVE entry. */
BUILD(); runwait(cmd);
if (sdot(scratch, "leaf")) {
fprintf(stderr, "poison FAIL[%s/%s]: warm baseline — leaf "
"recompiled (cache not warm before poison)\n",
drv, rows[r].tag);
fail++;
}
/* poison the row's artifact(s) to 0 bytes, key stays valid */
if (rows[r].wwi && truncate0(cwwi) != 0) {
fprintf(stderr, "poison FAIL[%s/%s]: cannot truncate P.wwi\n",
drv, rows[r].tag);
fail++;
}
if (rows[r].obj && truncate0(cobj) != 0) {
fprintf(stderr, "poison FAIL[%s/%s]: cannot truncate P.o\n",
drv, rows[r].tag);
fail++;
}
/* heal rebuild — a 0-byte artifact under a valid key must be
* treated as MISS (re-derived), NOT served. */
BUILD(); runwait(cmd);
if (!sdot(scratch, "leaf")) {
fprintf(stderr, "poison FAIL[%s/%s]: served poison — leaf did "
"NOT re-derive after 0-byte artifact (silent serve-wrong)\n",
drv, rows[r].tag);
fail++;
}
if (runwait(prog) != EXPECT_EXIT) {
fprintf(stderr, "poison FAIL[%s/%s]: heal prog exit != %d "
"(empty/link-failed binary)\n", drv, rows[r].tag, EXPECT_EXIT);
fail++;
}
/* re-store must leave both artifacts non-empty (write guard let a
* good store through, AND a torn store never commits a 0-byte). */
if (fsize(cwwi) <= 0 || fsize(cobj) <= 0) {
fprintf(stderr, "poison FAIL[%s/%s]: cached artifact still "
"0-byte after heal (re-store did not repair)\n",
drv, rows[r].tag);
fail++;
}
}
#undef BUILD
}
out:
snprintf(cmd, sizeof cmd, "rm -rf %s", td);
runwait(cmd);
if (fail) {
fprintf(stderr, "poison: %d check(s) failed\n", fail);
return 1;
}
printf("poison: root->leaf via build_one_sep — a 0-byte cached P.wwi/P.o/both "
"under a valid key self-heals (treated as MISS, re-derived to correct "
"value, re-stored non-empty) — both stages\n");
return 0;
}