ww/test/runww.ww

// runww — ww-native behavior-test harness, the analog of Go's
// test/run.go. Each case file under test/wcc/data/<name>/case.ww carries
// a leading `//ww:` directive describing how to test it; runww builds (and
// optionally runs) the case through the cstage `ww` driver and checks the
// directive holds. This replaces the per-test C harnesses that embedded ww
// source as C string literals and duplicated fork/exec/grep boilerplate.
//
// This is T1: the routine behavioral-correctness gate, cstage-only for the
// RUN/RUNEXIT/COMPILE arms. The wwstage-behavior run is deliberately NOT here
// — per the rob/USER tier+stage split, a wwstage miscompile is already caught
// by T2 (cstage.s vs wwstage.s byte-id over the same case.ww corpus, strictly
// more sensitive than re-running behavior) or by T1's own cstage run (the
// both-wrong-identical blind spot is cstage being wrong, which only a
// behavioral check on cstage catches). So rule-10 stage-symmetry is enforced
// by T2 (a separate pre-push tool), not by runww; routine runs stay cstage-only.
//
// The ERROR arm is the one exception: it runs BOTH stages. A rejected program
// emits no `.s`, so T2's `.s`-cmp gate structurally CANNOT cover wwstage-reject
// (drew-fold3-spec §1a). The retired C reject twins (948/949/944) asserted both
// w6c AND w6c_ww reject with the same diagnostic body, so runww's ERROR arm runs
// w6c_ww directly on the same case.ww and asserts it ALSO fails with the
// substring present (#20 non-vacuity) — the coverage-equivalence safety net for
// reject-row migration.
//
// Directives (first `//ww:` line of the case):
//   //ww:run              compile + execute, expect exit 0
//   //ww:run-exit N       compile + execute, expect exit N
//   //ww:error "<sub>"    compilation must FAIL with <sub> on stderr
//   //ww:compile          compile must succeed (don't run)
//
// Invoke (pilot): `BIN=out/bin out/bin/ww run test/runww.ww <case.ww>...`.
// Full `ww test` integration is a later fold.
//
// Spawn + stderr capture are lifted from the driver's procrun
// (selfhost/cmd/ww/main.ww:154: fork+execve+wait4+real-exit-code); the
// only addition is the child-side dup2(efd, 2) redirect, mirroring the C
// harnesses' `2>errf` so the parent can grep the diagnostic. #20
// non-vacuity: a `//ww:error` row passes ONLY if the build failed AND the
// diagnostic substring is present — a crash (no diagnostic) FAILS the row.
//
// Env: the harness execve's the child with a nil envp, so a case that
// `import`s a lib module (needing WW_SRCLIB/WW_LIB) is out of pilot scope
// until os exposes an envp() forwarder; the pilot cases are self-contained.

package main;

import os;

type dkind = enum i32 {
	NONE    = 0,
	RUN     = 1,
	RUNEXIT = 2,
	ERROR   = 3,
	COMPILE = 4,
};

type dirv = struct {
	kind: dkind,
	code: i32,	// expected exit for RUNEXIT
	subp: *u8,	// diagnostic substring (view into the case buffer)
	subn: i32,
};

fn pr(s: str) void = {
	os.write(1i32, s.ptr, s.len: u64);
};

fn mkstr(p: *u8, n: i32) str = {
	let s: str;
	s.ptr = p;
	s.len = n;
	return s;
};

// cstr — NUL-terminated heap copy for an execve argv element.
fn cstr(s: str) *u8 = {
	let b: []u8 = alloc([], (s.len: u64) + 1u64)!;
	b.len = s.len + 1;
	let i: i32 = 0;
	for (i < s.len) { b[i] = s[i]; i += 1; };
	b[s.len] = 0u8;
	return b.ptr;
};

fn joinp(dir: str, name: str) str = {
	let n: i32 = dir.len + name.len;
	let b: []u8 = alloc([], n: u64)!;
	b.len = n;
	let i: i32 = 0;
	for (i < dir.len) { b[i] = dir[i]; i += 1; };
	let j: i32 = 0;
	for (j < name.len) { b[dir.len + j] = name[j]; j += 1; };
	return mkstr(b.ptr, n);
};

fn findsub(hp: *u8, hn: i32, np: *u8, nn: i32) i32 = {
	if (nn == 0) { return 0; };
	let i: i32 = 0;
	for (i + nn <= hn) {
		let j: i32 = 0;
		let ok: bool = true;
		for (j < nn) {
			if (hp[i + j] != np[j]) { ok = false; break; };
			j += 1;
		};
		if (ok) { return i; };
		i += 1;
	};
	return -1;
};

fn contains(hp: *u8, hn: i32, needle: str) bool = {
	return findsub(hp, hn, needle.ptr, needle.len) >= 0;
};

fn matchat(hp: *u8, hn: i32, off: i32, lit: str) bool = {
	if (off + lit.len > hn) { return false; };
	let i: i32 = 0;
	for (i < lit.len) {
		if (hp[off + i] != lit[i]) { return false; };
		i += 1;
	};
	return true;
};

// readfile — slurp `path` into `buf`, returning bytes read or -1.
fn readfile(path: str, buf: []u8) i64 = {
	let fd: i32 = os.open(path, os.flag.RDONLY, 0i32);
	if (fd < 0) { return -1i64; };
	let total: u64 = 0u64;
	for (true) {
		let r: i64 = os.read(fd, buf.ptr + total, (buf.len: u64) - total);
		if (r <= 0i64) { break; };
		total += r: u64;
		if (total >= buf.len: u64) { break; };
	};
	os.close(fd);
	return total: i64;
};

fn parsedir(cp: *u8, cn: i32, out: *dirv) void = {
	out.kind = dkind.NONE;
	out.code = 0;
	out.subp = nil: *u8;
	out.subn = 0;

	let i: i32 = findsub(cp, cn, "//ww:".ptr, 5i32);
	if (i < 0) { return; };
	let d: i32 = i + 5;	// first byte after "//ww:"

	let le: i32 = d;		// end of the directive line
	for (le < cn) {
		if (cp[le] == 10u8) { break; };	// '\n'
		le += 1;
	};

	if (matchat(cp, cn, d, "run-exit")) {
		out.kind = dkind.RUNEXIT;
		let k: i32 = d + 8;
		for (k < le) { if (cp[k] != 32u8) { break; }; k += 1; };	// skip ' '
		let v: i32 = 0;
		for (k < le) {
			let c: u8 = cp[k];
			if (c < 48u8) { break; };
			if (c > 57u8) { break; };
			v = v * 10 + (c - 48u8): i32;
			k += 1;
		};
		out.code = v;
		return;
	};
	if (matchat(cp, cn, d, "run")) { out.kind = dkind.RUN; return; };
	if (matchat(cp, cn, d, "compile")) { out.kind = dkind.COMPILE; return; };
	if (matchat(cp, cn, d, "error")) {
		out.kind = dkind.ERROR;
		// substring between the two double-quote bytes (34)
		let q1: i32 = -1;
		let k: i32 = d;
		for (k < le) { if (cp[k] == 34u8) { q1 = k; break; }; k += 1; };
		// #20 non-vacuity: an absent or empty substring would collapse the
		// ERROR check to a bare rc!=0, which a crash satisfies — the exact
		// hole this harness closes. Reject the directive loudly instead.
		if (q1 < 0) { out.kind = dkind.NONE; return; };
		let q2: i32 = -1;
		k = q1 + 1;
		for (k < le) { if (cp[k] == 34u8) { q2 = k; break; }; k += 1; };
		if (q2 < 0) { out.kind = dkind.NONE; return; };
		if (q2 == q1 + 1) { out.kind = dkind.NONE; return; };
		out.subp = cp + (q1 + 1): u64;
		out.subn = q2 - (q1 + 1);
		return;
	};
};

// runcap — fork+execve `drv` with `argv`, child stderr redirected into
// errpath. Returns the child's real exit code (1 on signal, -1 on
// fork/wait failure). Lifted from selfhost/cmd/ww/main.ww:154 procrun.
fn runcap(drv: str, argv: []*u8, errpath: str) i32 = {
	let pid: i32 = os.fork();
	if (pid < 0) { return -1; };
	if (pid == 0) {
		let efd: i32 = os.open(errpath,
		    os.flag.WRONLY | os.flag.CREATE | os.flag.TRUNC, 420i32);	// 0o644
		if (efd >= 0) { os.dup2(efd, 2i32); };
		os.execve(drv, argv.ptr, nil: **u8);
		os.exit(127);
	};
	let status: i32 = 0;
	let r: i32 = os.wait4(pid, &status, 0i32, nil: *void);
	if (r < 0) { return -1; };
	if ((status & 127i32) != 0) { return 1; };
	return (status >> 8i32) & 255i32;
};

// runcase — build (and maybe run) one case through `drv`; return whether
// the directive held.
fn runcase(drv: str, casepath: str, d: *dirv, errpath: str, tmpout: str) bool = {
	let argv: []*u8 = alloc([], 6u64)!;
	if (d.kind == dkind.COMPILE) {
		argv.len = 6;
		argv[0] = cstr(drv);
		argv[1] = cstr("build");
		argv[2] = cstr("-o");
		argv[3] = cstr(tmpout);
		argv[4] = cstr(casepath);
		argv[5] = nil: *u8;
	} else {
		// RUN / RUNEXIT / ERROR all go through `ww run`: on a reject the
		// compile fails before execution, so the diagnostic still lands on
		// stderr and nothing is left to clean up.
		argv.len = 4;
		argv[0] = cstr(drv);
		argv[1] = cstr("run");
		argv[2] = cstr(casepath);
		argv[3] = nil: *u8;
	};

	let rc: i32 = runcap(drv, argv, errpath);

	if (d.kind == dkind.ERROR) {
		if (rc == 0) { return false; };	// built+ran ok → not a reject
		let eb: []u8 = alloc([], 65536u64)!;
		eb.len = 65536;
		let en: i64 = readfile(errpath, eb);
		if (en < 0i64) { return false; };
		// #20: rc!=0 AND the diagnostic body present (a crash has no body).
		return contains(eb.ptr, en: i32, mkstr(d.subp, d.subn));
	};
	if (d.kind == dkind.RUNEXIT) { return rc == d.code; };
	if (d.kind == dkind.RUN) { return rc == 0; };
	if (d.kind == dkind.COMPILE) { return rc == 0; };
	return false;
};

// wwerror — the ERROR arm's wwstage leg: invoke w6c_ww directly on the same
// case.ww (mirroring the retired C twins' `$BIN/w6c_ww -o <out> <src>` direct
// invocation, not the `ww` driver — w6c_ww is the bare frontend compiler) and
// assert it ALSO hard-rejects with the shared diagnostic body present. #20
// non-vacuity: passes ONLY if w6c_ww failed AND the substring is on stderr.
fn wwerror(wdrv: str, casepath: str, d: *dirv, errpath: str, tmpout: str) bool = {
	let argv: []*u8 = alloc([], 5u64)!;
	argv.len = 5;
	argv[0] = cstr(wdrv);
	argv[1] = cstr("-o");
	argv[2] = cstr(tmpout);
	argv[3] = cstr(casepath);
	argv[4] = nil: *u8;

	let rc: i32 = runcap(wdrv, argv, errpath);
	if (rc == 0) { return false; };	// w6c_ww accepted → wwstage-reject dropped
	let eb: []u8 = alloc([], 65536u64)!;
	eb.len = 65536;
	let en: i64 = readfile(errpath, eb);
	if (en < 0i64) { return false; };
	return contains(eb.ptr, en: i32, mkstr(d.subp, d.subn));
};

export fn main() int = {
	let a: []str = os.args();
	if (a.len < 2) {
		pr("runww: usage: runww <case.ww>...\n");
		return 2;
	};

	let bin: str = "out/bin";
	match (os.getenv("BIN")) {
	case let v: str => { bin = v; };
	case void       => { };
	};
	let cdrv: str = joinp(bin, "/ww");
	// ERROR arm goes dual-stage: w6c_ww (the bare wwstage frontend) is
	// resolved off the SAME $BIN the C reject twins used ($BIN/w6c_ww).
	let wdrv: str = joinp(bin, "/w6c_ww");

	// Fixed scratch paths: the pilot drives cases sequentially in one
	// process, so reuse is safe; per-pid uniqueness for parallel invocation
	// is a later wiring fold (task #29).
	let errpath: str = "/tmp/runww.err";
	let tmpout: str = "/tmp/runww.bin";

	let total: i32 = 0;
	let fail: i32 = 0;

	let ci: i32 = 1;
	for (ci < a.len) {
		let casepath: str = a[ci];

		let cbuf: []u8 = alloc([], 65536u64)!;
		cbuf.len = 65536;
		let cn: i64 = readfile(casepath, cbuf);
		if (cn < 0i64) {
			pr("FAIL read "); pr(casepath); pr("\n");
			fail += 1; total += 1; ci += 1;
			continue;
		};

		let d: dirv;
		parsedir(cbuf.ptr, cn: i32, &d);
		if (d.kind == dkind.NONE) {
			pr("FAIL no-directive "); pr(casepath); pr("\n");
			fail += 1; total += 1; ci += 1;
			continue;
		};

		total += 1;
		let pass: bool = runcase(cdrv, casepath, &d, errpath, tmpout);
		if (!pass) {
			pr("FAIL cstage  "); pr(casepath); pr("\n");
			fail += 1;
		} else if (d.kind == dkind.ERROR) {
			// cstage rejected; now demand wwstage rejects identically.
			if (wwerror(wdrv, casepath, &d, errpath, tmpout)) {
				pr("PASS dual    "); pr(casepath); pr("\n");
			} else {
				pr("FAIL wwstage "); pr(casepath); pr("\n");
				fail += 1;
			};
		} else {
			pr("PASS cstage  "); pr(casepath); pr("\n");
		};

		ci += 1;
	};

	if (fail > 0) {
		pr("runww: FAIL\n");
		return 1;
	};
	pr("runww: all ok\n");
	return 0;
};