ww/test/wcc/989_lib_byteid.c

/*
 * 989_lib_byteid — cstage-vs-wwstage byte-identity gate over the lib/
 * surface that the embedded combined.ww set does NOT cover.
 *
 * The 990-997 gates byte-id only the modules the selfhost tools import
 * (ascii/bytes/errors/io/math/memio/os/rt/strconv/strings/time/types/
 * utf8 + lex/parse/wcc/ww). Every other lib/ module compiles through
 * the CSTAGE driver alone (the 9xx _run tests), so a cs≠ww divergence
 * there ships gate-green — regex.finish was cs≠ww for weeks this way
 * (task #21, FC0). This gate closes the class: each lib test fixture is
 * sep-built through BOTH driver stages and the per-package asm compared.
 *
 * #94 sep layout: the gate drives `ww --sep` (cstage) and `ww_ww --sep`
 * (wwstage) instead of feeding a single .combined.ww to raw w6c/w6c_ww.
 * Under sep EACH package compiles to its OWN <stem>.sepwork/<pkg>.s — the
 * lib body lands in <pkg>.s, NOT in __root.s (which holds only the test
 * fixture / probe main). So the byte-id MUST concat EVERY <pkg>.s (the
 * imported lib body + lib/test's auto-bundled runner included), never
 * just __root.s — comparing the root alone would byte-id the harness
 * wrapper while the engine ships uncovered (the green-covering-nothing
 * trap this gate exists to close).
 *
 * Three pinned outcomes, loud over blind (task #21 ruling):
 *   ID       — byte-identical across all per-package .s, zero tolerance.
 *   DIVERGE  — known cs≠ww divergence, documented in task #59 with the
 *              first-diff symbol. Both stages must still compile and
 *              the concatenated asm must still DIFFER: when a fix lands,
 *              the entry fails loudly demanding graduation to ID rather
 *              than silently widening or narrowing coverage.
 *   WWREJECT — cstage --sep compiles, wwstage --sep errors (front-end
 *              gap, task #59). Graduation is pinned the same way.
 *
 * Modules with no test fixture and no embedding (sort/path/endian/net/
 * hash/fnv/crypto.math/c.libc) are covered via import-probe stubs:
 * `import` drags the whole module into its own sep package, so both
 * cgens chew the full module body. The driver silently skips an
 * unresolvable import, so each probe carries a sentinel proving the
 * body landed in a per-package .unit.ww; and a corpus-completeness scan
 * fails loudly on any lib/ dir not enrolled here, so new modules can't
 * ship uncovered.
 *
 * Compile+cmp only (--sep -o redirects every artifact to a getpid-keyed
 * /tmp dir; WW_PKGCACHE pinned there so out/.pkgcache is untouched), so
 * this stays phase-1 parallel-safe next to the 9xx _run tests.
 *
 * 9xx is full; shares the 989 prefix per the 989_sha256 precedent (the
 * `short` name keys the binary).
 */
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <unistd.h>
#include <sys/wait.h>
#include <sys/stat.h>

enum { M_ID, M_DIVERGE, M_WWREJECT, NENT_EXPECT = 42 };

struct ent {
	const char *fixture;	/* repo-relative .ww; NULL → probe entry */
	const char *probe;	/* inline import-probe source */
	const char *inc;	/* extra -I dir beyond dirname(fixture) */
	int mode;
	const char *cite;	/* task #59 entry for non-ID modes */
	/* probe-only: the driver SILENTLY SKIPS an unresolvable import
	 * (cmd/ww/main.c locate_import), so a probe whose module fails
	 * to resolve shrinks to an empty main and byte-id's trivially —
	 * green while covering nothing. sentinel must appear in the
	 * resolved unit to prove the module body actually landed.
	 * (Fixtures self-protect: they call module symbols, so a drop
	 * is a loud w6c reject.) moddir feeds the completeness check. */
	const char *sentinel;
	const char *moddir;
};

static const struct ent ents[] = {
	/* -------- byte-identical: zero tolerance ----------------- */
	{ .fixture = "lib/bytes/bytestest.ww",             .mode = M_ID },
	{ .fixture = "lib/dirs/dirstest.ww",               .mode = M_ID },
	{ .fixture = "lib/encoding/base32/base32_test.ww", .mode = M_ID },
	/* graduated from #59.4 DIVERGE by the #22a N_DOT tagged-arg
	 * cursor arm (ww widened t.N args with a clamped tag 0 — the
	 * base .s diff is exactly that PUSHQ pair, 2 sites);
	 * runtime-correct per 979_hex_run */
	{ .fixture = "lib/encoding/hex/hextest.ww",        .mode = M_ID },
	{ .fixture = "lib/encoding/utf8/utf8test.ww",      .mode = M_ID },
	{ .fixture = "lib/getopt/getopttest.ww",           .mode = M_ID },
	{ .fixture = "lib/hash/adler32/adler32_test.ww",   .mode = M_ID },
	{ .fixture = "lib/hash/crc16/crc16_test.ww",       .mode = M_ID },
	{ .fixture = "lib/hash/crc32/crc32_test.ww",       .mode = M_ID },
	{ .fixture = "lib/hash/crc64/crc64_test.ww",       .mode = M_ID },
	/* graduated from #59.7 DIVERGE by the #61 *[N]T index-stride fix */
	{ .fixture = "lib/hash/siphash/siphash_test.ww",   .mode = M_ID },
	{ .fixture = "lib/math/checked/checked_test.ww",   .mode = M_ID },
	{ .fixture = "lib/math/random/random_test.ww",     .mode = M_ID },
	{ .fixture = "lib/memio/memiotest.ww",             .mode = M_ID },
	{ .fixture = "lib/os/ostest.ww",                   .mode = M_ID },
	{ .fixture = "lib/regex/regex_test.ww",            .mode = M_ID },
	{ .fixture = "lib/strconv/test/ftostest.ww",       .mode = M_ID },
	/* graduated from #59.10 DIVERGE by the #62 float-literal fold fix
	 * (wwstage lexer now folds through strconv.stof64, matching
	 * cstage's strtod bit-for-bit) */
	{ .fixture = "lib/strconv/test/stoftest.ww",       .mode = M_ID },
	{ .fixture = "lib/strconv/test/inttest.ww",        .mode = M_ID },
	{ .fixture = "lib/strings/stringstest.ww",         .mode = M_ID },
	{ .fixture = "lib/temp/temptest.ww",               .mode = M_ID },
	{ .fixture = "lib/time/timetest.ww",               .mode = M_ID },
	/* graduated from #59.2 DIVERGE by the #129 sretretsize fixes
	 * (same-module alias priority + callsretsize callee-context swap;
	 * bufio imports io so cross-module io.stream returns were spuriously
	 * sret-classified, producing divergent caller prologues) */
	{ .fixture = "lib/bufio/bufiotest.ww",             .mode = M_ID },
	/* graduated from #59.6 DIVERGE by the #129 same fix (fmt imports
	 * io transitively through bufio; sretretsize curmod swap closed
	 * the getopt.printusage spurious LEAQ DI caller divergence) */
	{ .fixture = "lib/fmt/fmttest.ww",                 .mode = M_ID },
	/* fixtureless modules, import-probe shape */
	{ .probe = "package main;\nimport sort;\nfn main() i32 = { return 0; };\n",
	  .mode = M_ID, .sentinel = "package sort;", .moddir = "lib/sort" },
	/* c2-stack path::buffer realignment: w6c compiles, w6c_ww rejects
	 * path's module-global slice consts (const dot/dotdot: []u8) = the
	 * #120/#29 acceptance divergence (module-level composite globals);
	 * + the #148 twin #151 (global []u8 by-value arg). Both → #125
	 * batch. cstage runtime is covered by 989_path_run. M_WWREJECT
	 * self-graduates: flips RED to M_ID the day wwstage accepts this. */
	{ .probe = "package main;\nimport path;\nfn main() i32 = { return 0; };\n",
	  .mode = M_WWREJECT, .cite = "#120/#29", .sentinel = "package path;", .moddir = "lib/path" },
	{ .probe = "package main;\nimport endian;\nfn main() i32 = { return 0; };\n",
	  .mode = M_ID, .sentinel = "package endian;", .moddir = "lib/endian" },
	{ .probe = "package main;\nimport net;\nfn main() i32 = { return 0; };\n",
	  .mode = M_ID, .sentinel = "package net;", .moddir = "lib/net" },
	{ .probe = "package main;\nimport hash;\nfn main() i32 = { return 0; };\n",
	  .mode = M_ID, .sentinel = "package hash;", .moddir = "lib/hash" },
	/* fnv lives off the driver's default root (lib/hash/fnv) */
	{ .probe = "package main;\nimport fnv;\nfn main() i32 = { return 0; };\n",
	  .inc = "lib/hash/fnv",
	  .mode = M_ID, .sentinel = "package fnv;", .moddir = "lib/hash/fnv" },
	/* sentinel is a fn, not `package math;` — lib/math is also
	 * package math, so a silent fallback there would still match */
	{ .probe = "package main;\nimport crypto.math;\nfn main() i32 = { return 0; };\n",
	  .mode = M_ID, .sentinel = "fn rotl32", .moddir = "lib/crypto/math" },
	{ .probe = "package main;\nimport c.libc;\nfn main() i32 = { return 0; };\n",
	  .mode = M_ID, .sentinel = "package libc;", .moddir = "lib/c/libc" },
	/* -------- documented-allowed cs≠ww (task #59) ------------ */
	{ .fixture = "lib/ascii/asciitest.ww",
	  .mode = M_ID, .cite = "#59.1 graduated by #146 str==" },
	/* #59.2 bufio graduated to M_ID above (#129 fix) */
	{ .fixture = "lib/encoding/base64/base64_test.ww",
	  .mode = M_DIVERGE, .cite = "#59.3" },
	/* #59.4 hextest graduated to M_ID above (#22a reviewer fixes) */
	{ .fixture = "lib/errors/errnotest.ww",
	  .mode = M_DIVERGE, .cite = "#59.5" },
	/* #59.6 fmt graduated to M_ID above (#129 fix) */
	/* #59.7 siphash graduated to M_ID above (#61 fix) */
	{ .fixture = "lib/log/logtest.ww",
	  .mode = M_DIVERGE, .cite = "#59.8" },
	{ .fixture = "lib/os/stattest.ww",
	  .mode = M_DIVERGE, .cite = "#59.9" },
	/* #59.10 stoftest graduated to M_ID above (#62 fix) */
	{ .fixture = "lib/ww/syntax/toktest.ww", .inc = "lib/ww",
	  .mode = M_ID, .cite = "#59.11 graduated by #146 str==" },
	/* toktest/asttest resolve `import syntax` via -I lib/ww, cf 905 */
	{ .fixture = "lib/ww/syntax/asttest.ww", .inc = "lib/ww",
	  .mode = M_ID, .cite = "#59.12 graduated by #146 str==" },
	/* -------- wwstage front-end gaps (task #59) -------------- */
	{ .fixture = "lib/crypto/sha256/sha256_test.ww",
	  .mode = M_WWREJECT, .cite = "#59.13" },
	/* #59.14 graduated to M_ID by #27: the #27b enum-as-int cgen fix
	 * (value passthrough, not a tagged assertion) + the fnmatch.ww:126
	 * `'\\': u8` explicit cast (the rune-lit→u8 narrow w6c_ww rejects at
	 * a checked position) make w6c_ww compile fnmatch byte-identically. */
	{ .fixture = "lib/fnmatch/fnmatchtest.ww",
	  .mode = M_ID, .cite = "#59.14 graduated by #27" },
	/* #59.15 shlex graduated to M_ID by B-full Layer 1 (#14 nominal
	 * typeeqast): the cross-module bare-vs-qualified variant forward
	 * w6c_ww over-rejected now compiles byte-identically. */
	{ .fixture = "lib/shlex/shlextest.ww",
	  .mode = M_ID, .cite = "#59.15 graduated by #14 nominal typeeqast" },
	{ .fixture = NULL },
};

static int
runwait(const char *cmd)
{
	int rc = system(cmd);
	if (rc == -1) return -1;
	if (WIFEXITED(rc)) return WEXITSTATUS(rc);
	return 1;
}

static const char *
absbin(void)
{
	const char *b = getenv("BIN");
	if (!b) b = "out/bin";
	if (b[0] == '/') return b;
	static char buf[2048];
	char cwd[1024];
	if (getcwd(cwd, sizeof cwd) == NULL) return NULL;
	snprintf(buf, sizeof buf, "%s/%s", cwd, b);
	return buf;
}

static int
slurp(const char *path, char **outbuf, size_t *outlen)
{
	FILE *f = fopen(path, "rb");
	if (!f) return -1;
	fseek(f, 0, SEEK_END);
	long n = ftell(f);
	fseek(f, 0, SEEK_SET);
	if (n < 0) { fclose(f); return -1; }
	char *b = malloc((size_t)n + 1);
	if (!b) { fclose(f); return -1; }
	if (fread(b, 1, (size_t)n, f) != (size_t)n) { free(b); fclose(f); return -1; }
	b[n] = '\0';
	fclose(f);
	*outbuf = b;
	*outlen = (size_t)n;
	return 0;
}

static int
check_one(const char *bin, const char *cwd, const struct ent *e, int idx)
{
	const char *label = e->fixture ? e->fixture : "import-probe";
	char td[64], cmd[8192];
	int rc = -1;

	snprintf(td, sizeof td, "/tmp/wwlbi_%d_%d", getpid(), idx);
	snprintf(cmd, sizeof cmd, "rm -rf %s", td);
	runwait(cmd);
	mkdir(td, 0755);

	char base[256];
	if (e->fixture) {
		const char *b = strrchr(e->fixture, '/');
		snprintf(base, sizeof base, "%s", b ? b + 1 : e->fixture);
		snprintf(cmd, sizeof cmd, "cp %s/%s %s/%s", cwd, e->fixture,
		    td, base);
		if (runwait(cmd) != 0) {
			fprintf(stderr, "lib_byteid FAIL: %s — cp\n", label);
			goto out;
		}
	} else {
		snprintf(base, sizeof base, "probe.ww");
		char p[512];
		snprintf(p, sizeof p, "%s/probe.ww", td);
		FILE *f = fopen(p, "wb");
		if (!f) goto out;
		fputs(e->probe, f);
		fclose(f);
		label = e->moddir;
	}

	/* dirname(fixture) leads the search path so bare same-module
	 * imports (`import base32` from lib/encoding/base32) resolve as
	 * they do under the in-tree driver run. */
	char incs[2048] = "";
	if (e->fixture) {
		const char *slash = strrchr(e->fixture, '/');
		int n = (int)(slash - e->fixture);
		snprintf(incs, sizeof incs, "-I %s/%.*s", cwd, n, e->fixture);
	}
	if (e->inc) {
		size_t n = strlen(incs);
		snprintf(incs + n, sizeof incs - n, " -I %s/%s", cwd, e->inc);
	}

	char stem[512];
	snprintf(stem, sizeof stem, "%s", base);
	char *sd = strrchr(stem, '.');
	if (sd) *sd = '\0';

	/* Two sep driver builds. @test fixtures are main-less, so `test -c
	 * --sep` carries -T (synthesizes the entry + keeps @test fns + auto-
	 * bundles lib/test's run() callee) and emits per-package asm without
	 * a harness run; import-probes carry their own `fn main()`, which -T
	 * loud-rejects (910), so they `build --sep` (non-T). Each stage's
	 * artifacts land under its own -o stem so out/ and the source tree
	 * stay clean; WW_PKGCACHE is pinned to td. */
	const char *sub = e->fixture ? "test -c --sep" : "build --sep";
	char stemc[600], stemw[600];
	snprintf(stemc, sizeof stemc, "%s/c_%s", td, stem);
	snprintf(stemw, sizeof stemw, "%s/w_%s", td, stem);

	snprintf(cmd, sizeof cmd,
	    "cd %s && WW_PKGCACHE=%s/pc timeout 180 %s/ww %s %s -o %s %s "
	    ">/dev/null 2>&1", td, td, bin, sub, incs, stemc, base);
	int ce = runwait(cmd);
	snprintf(cmd, sizeof cmd,
	    "cd %s && WW_PKGCACHE=%s/pw timeout 180 %s/ww_ww %s %s -o %s %s "
	    ">/dev/null 2>&1", td, td, bin, sub, incs, stemw, base);
	int we = runwait(cmd);

	/* the cstage --sep build resolves the units (root + each package);
	 * its __root.unit.ww proves resolution ran. WWREJECT still requires
	 * cstage to compile, so this holds for every mode. */
	char rootu[760];
	snprintf(rootu, sizeof rootu, "%s.sepwork/__root.unit.ww", stemc);
	if (ce != 0 || access(rootu, 0) != 0) {
		fprintf(stderr, "lib_byteid FAIL: %s — cstage --sep produced "
		    "no resolved unit\n", label);
		goto out;
	}

	/* probe coverage: the driver SILENTLY SKIPS an unresolvable import
	 * (cmd/ww/main.c locate_import), shrinking the probe to an empty main
	 * that byte-ids trivially. The sentinel must appear in some per-
	 * package .unit.ww (the lib body's own sep unit, not __root) to prove
	 * the module landed. Fixtures self-protect by calling module symbols. */
	if (e->sentinel) {
		snprintf(cmd, sizeof cmd,
		    "grep -Fq -- '%s' %s.sepwork/*.unit.ww >/dev/null 2>&1",
		    e->sentinel, stemc);
		if (runwait(cmd) != 0) {
			fprintf(stderr, "lib_byteid FAIL: %s — import "
			    "silently dropped (no \"%s\" in any per-package "
			    "unit); probe covers nothing\n", label, e->sentinel);
			goto out;
		}
	}

	if (e->mode == M_WWREJECT) {
		if (we == 0) {
			fprintf(stderr, "lib_byteid FAIL: %s — wwstage --sep now "
			    "compiles this; graduate the %s pin to ID or "
			    "DIVERGE\n", label, e->cite);
			goto out;
		}
		rc = 0;
		goto out;
	}
	if (we != 0) {
		fprintf(stderr, "lib_byteid FAIL: %s — wwstage --sep rejected\n",
		    label);
		goto out;
	}

	/* byte-id over EVERY per-package .s (lib body + lib/test included),
	 * never just __root.s — see the header note. Identical package sets
	 * both stages, so the *.s glob (shell-sorted) concatenates in the
	 * same order on each side. */
	char cs[800], ws[800];
	snprintf(cs, sizeof cs, "%s/all_c.s", td);
	snprintf(ws, sizeof ws, "%s/all_w.s", td);
	snprintf(cmd, sizeof cmd, "cat %s.sepwork/*.s > %s 2>/dev/null",
	    stemc, cs);
	if (system(cmd)) {}
	snprintf(cmd, sizeof cmd, "cat %s.sepwork/*.s > %s 2>/dev/null",
	    stemw, ws);
	if (system(cmd)) {}

	char *bc = NULL, *bw = NULL;
	size_t nc = 0, nw = 0;
	if (slurp(cs, &bc, &nc) < 0 || slurp(ws, &bw, &nw) < 0) {
		fprintf(stderr, "lib_byteid FAIL: %s — cannot read .s\n", label);
		free(bc); free(bw);
		goto out;
	}
	int same = (nc == nw && memcmp(bc, bw, nc) == 0);
	free(bc); free(bw);

	if (e->mode == M_ID && !same) {
		fprintf(stderr, "lib_byteid FAIL: %s — cs %zu vs ww %zu bytes "
		    "(byte-id broken)\n", label, nc, nw);
		goto out;
	}
	if (e->mode == M_DIVERGE && same) {
		fprintf(stderr, "lib_byteid FAIL: %s — now byte-identical; "
		    "graduate the %s pin to ID\n", label, e->cite);
		goto out;
	}
	rc = 0;
out:
	snprintf(cmd, sizeof cmd, "rm -rf %s", td);
	runwait(cmd);
	return rc;
}

/* every lib/ dir holding .ww source must be accounted for — enrolled
 * in ents[] (fixture dirname or probe moddir) or on the covered[]
 * list below; otherwise a new module ships with zero byte-id coverage
 * and the corpus rots silently as lib/ grows. */
static const char *covered[] = {
	/* selfhost-embedded: byte-id'd by the 990-997 gates */
	"lib/io", "lib/math", "lib/rt", "lib/types",
	/* module body dragged into the lib/strconv/test fixtures */
	"lib/strconv",
	/* #17: the @test runner is AUTO-BUNDLED into every -T sep build (its
	 * own test.sepwork/test.s), so every @test fixture above byte-ids it
	 * cs/ww; 911_attest_record also compares it directly. It has no
	 * _test.ww of its own. */
	"lib/test",
	NULL,
};

static int
accounted(const char *dir)
{
	for (int i = 0; ents[i].fixture || ents[i].probe; i++) {
		if (ents[i].fixture) {
			const char *s = strrchr(ents[i].fixture, '/');
			size_t n = (size_t)(s - ents[i].fixture);
			if (strlen(dir) == n &&
			    strncmp(dir, ents[i].fixture, n) == 0)
				return 1;
		} else if (ents[i].moddir && strcmp(dir, ents[i].moddir) == 0)
			return 1;
	}
	for (int i = 0; covered[i]; i++)
		if (strcmp(dir, covered[i]) == 0)
			return 1;
	return 0;
}

static int
corpus_complete(const char *cwd)
{
	char cmd[2048];
	/* E3-C1: the flip routes lib @test builds through sep, which drops
	 * <stem>.sepwork/<pkg>.unit.ww scratch next to source; exclude it so
	 * the scan enrols real source dirs only, not generated work dirs. */
	snprintf(cmd, sizeof cmd, "cd %s && find lib -name '*.ww' "
	    "! -name '*.combined.ww' ! -path '*.sepwork/*' "
	    "| sed 's|/[^/]*$||' | sort -u", cwd);
	FILE *p = popen(cmd, "r");
	if (!p) return -1;
	char dir[512];
	int bad = 0, seen = 0;
	while (fgets(dir, sizeof dir, p)) {
		char *nl = strchr(dir, '\n');
		if (nl) *nl = '\0';
		seen++;
		if (!accounted(dir)) {
			fprintf(stderr, "lib_byteid FAIL: %s — un-enrolled "
			    "lib module; add it to the 989 corpus\n", dir);
			bad++;
		}
	}
	pclose(p);
	/* an empty scan means find itself broke — never pass on that */
	if (seen == 0) {
		fprintf(stderr, "lib_byteid FAIL: lib/ scan found nothing\n");
		return -1;
	}
	return bad ? -1 : 0;
}

int
main(void)
{
	const char *bin = absbin();
	if (!bin) return 1;
	char cwd[1024];
	if (getcwd(cwd, sizeof cwd) == NULL) return 1;
	int nent = (int)(sizeof ents / sizeof ents[0]) - 1;
	if (nent != NENT_EXPECT) {
		fprintf(stderr, "lib_byteid FAIL: corpus has %d entries, want %d\n",
		    nent, NENT_EXPECT);
		return 1;
	}

	int fail = 0, n = 0, nid = 0, ndiv = 0, nrej = 0;
	if (corpus_complete(cwd) != 0) fail++;
	for (int i = 0; ents[i].fixture || ents[i].probe; i++) {
		if (check_one(bin, cwd, &ents[i], i) != 0) fail++;
		else if (ents[i].mode == M_ID) nid++;
		else if (ents[i].mode == M_DIVERGE) ndiv++;
		else nrej++;
		n++;
	}
	if (fail) {
		fprintf(stderr, "lib_byteid: %d/%d entr(ies) failed\n", fail, n);
		return 1;
	}
	printf("lib_byteid: %d byte-identical, %d pinned-divergent (#59), "
	    "%d pinned-wwreject (#59) across %d lib units\n",
	    nid, ndiv, nrej, n);
	return 0;
}