ww/internal/wwpackage/package.ww

package wwpackage;

import os;
import os.exec;
import strconv;
import strings;
import temp;
import time;

type pkgsource = struct {
	path: str,
	dir: str,
	pkg: str,
	test: bool,
	attest: bool,
};

type pkgfolder = struct {
	path: str,
	start: i32,
	end: i32,
	prodpkg: str,
	hastests: bool,
};

type pkggroup = struct {
	dir: str,
	pkg: str,
	start: i32,
	end: i32,
	prodpkg: str,
	external: bool,
	root: str,
	combined: str,
	bin: str,
	buildout: str,
	builderr: str,
	runout: str,
	runerr: str,
};

type pkgdiscover = struct {
	paths: []str,
	errors: i32,
};

// Preserve the caller's toolchain environment while pinning the locale and
// temporary directory used by the current package group.
fn toolenv(tmpdir: str) []str = {
	let inherited: []str = os.getenvs();
	let env: []str = alloc([], (inherited.len + 2): u64)!;
	let i: i32 = 0;
	for (i < inherited.len) {
		if (!strings.hasprefix(inherited[i], "TMPDIR=")
		    && !strings.hasprefix(inherited[i], "LC_ALL=")) {
			append(env, inherited[i]);
		};
		i += 1;
	};
	append(env, "LC_ALL=C");
	append(env, strings.concat("TMPDIR=", tmpdir));
	return env;
};

fn pkgwrite(fd: i32, s: str) bool = {
	let r: (i64 | os.oserror) = os.writeall(fd, s.ptr, s.len: u64);
	match (r) {
	case let n: i64 => return n == s.len: i64;
	case let e: os.oserror => return false;
	};
};

fn pkgput(fd: i32, s: str) void = {
	pkgwrite(fd, s);
};

fn pkgputln(fd: i32, s: str) void = {
	pkgput(fd, s);
	pkgput(fd, "\n");
};

fn pkgfailpath(path: str, reason: str) void = {
	pkgput(os.STDERR_FILENO, "wwtest package: ");
	pkgput(os.STDERR_FILENO, path);
	pkgput(os.STDERR_FILENO, ": ");
	pkgputln(os.STDERR_FILENO, reason);
};

fn pkgusage() void = {
	let s: str = strings.concat(
		"usage: wwtest package [-c] [-list] [-j N] [-I DIR] [-run|-filter GLOB] [-timeout-ms=N] [DIR | DIR/...] [-- GLOB ...]\n",
		"  *_test.ww is canonical; noncanonical files require an actual @test declaration\n",
		"  -c retains the compiled package binaries; -j is reserved by sequential v1\n");
	pkgput(os.STDERR_FILENO, s);
};

fn pkgread(path: str, out: *str) bool = {
	let fd: i32 = os.open(path, os.flag.RDONLY, 0i32);
	if (fd < 0) { return false; };
	let sr: (i64 | os.oserror) = os.filesize(fd);
	let n: i64 = -1i64;
	match (sr) {
	case let v: i64 => n = v;
	case let e: os.oserror => { os.close(fd); return false; };
	};
	if (n < 0i64) { os.close(fd); return false; };
	let b: []u8 = alloc([], (n + 1i64): u64)!;
	b.len = (n + 1i64): i32;
	let rr: (i64 | os.oserror) = os.readall(fd, b.ptr, n: u64);
	os.close(fd);
	let got: i64 = -1i64;
	match (rr) {
	case let v: i64 => got = v;
	case let e: os.oserror => return false;
	};
	if (got != n) { return false; };
	let ni: i32 = n: i32;
	b[ni] = 0u8;
	out.ptr = b.ptr;
	out.len = n: i32;
	return true;
};

fn pkgidentfirst(c: u8) bool = {
	if (c >= 'a' && c <= 'z') { return true; };
	if (c >= 'A' && c <= 'Z') { return true; };
	return c == '_';
};

fn pkgident(c: u8) bool = {
	if (pkgidentfirst(c)) { return true; };
	return c >= '0' && c <= '9';
};

fn pkgskipspace(src: str, start: i32) i32 = {
	let i: i32 = start;
	for (i < src.len) {
		let c: u8 = src[i];
		if (c == ' ' || c == '\t' || c == '\n' || c == '\r') {
			i += 1;
			continue;
		};
		if (c == '/' && i + 1 < src.len && src[i + 1] == '/') {
			i += 2;
			for (i < src.len && src[i] != '\n') { i += 1; };
			continue;
		};
		break;
	};
	return i;
};

fn pkgclause(src: str, out: *str) bool = {
	let i: i32 = pkgskipspace(src, 0);
	let word: str = "package";
	if (i + word.len >= src.len) { return false; };
	let j: i32 = 0;
	for (j < word.len) {
		if (src[i + j] != word[j]) { return false; };
		j += 1;
	};
	i += word.len;
	if (i >= src.len) { return false; };
	if (!(src[i] == ' ' || src[i] == '\t' || src[i] == '\n' || src[i] == '\r')) {
		return false;
	};
	i = pkgskipspace(src, i);
	if (i >= src.len || !pkgidentfirst(src[i])) { return false; };
	let begin: i32 = i;
	i += 1;
	for (i < src.len && pkgident(src[i])) { i += 1; };
	let end: i32 = i;
	i = pkgskipspace(src, i);
	if (i >= src.len || src[i] != ';') { return false; };
	out.ptr = src.ptr + (begin: u64);
	out.len = end - begin;
	return true;
};

fn pkgattest(src: str) bool = {
	let i: i32 = 0;
	for (i < src.len) {
		for (i < src.len && (src[i] == ' ' || src[i] == '\t' ||
		    src[i] == '\r')) { i += 1; };
		if (i + 5 < src.len && src[i] == '@' && src[i + 1] == 't' &&
		    src[i + 2] == 'e' && src[i + 3] == 's' && src[i + 4] == 't' &&
		    (src[i + 5] == ' ' || src[i + 5] == '\t')) {
			return true;
		};
		for (i < src.len && src[i] != '\n') { i += 1; };
		if (i < src.len) { i += 1; };
	};
	return false;
};

fn pkgdirname(path: str) str = {
	let i: i32 = path.len - 1;
	for (i >= 0) {
		if (path[i] == '/') {
			if (i == 0) { return "/"; };
			let r: str;
			r.ptr = path.ptr;
			r.len = i;
			return r;
		};
		i -= 1;
	};
	return ".";
};

fn pkgbase(path: str) str = {
	let i: i32 = path.len - 1;
	for (i >= 0) {
		if (path[i] == '/') {
			let r: str;
			r.ptr = path.ptr + ((i + 1): u64);
			r.len = path.len - i - 1;
			return r;
		};
		i -= 1;
	};
	return path;
};

fn pkgmodeis(m: os.mode, want: os.mode) bool = {
	return (((m: u32) & 61440u32) == (want: u32));
};

fn pkgisdir(path: str) bool = {
	let fi: os.filestat;
	match (os.lstat(&fi, path)) {
	case void => return pkgmodeis(fi.mode, os.mode.DIR);
	case let e: os.oserror => return false;
	};
};

fn pkgkeepfile(name: str) bool = {
	if (!strings.hassuffix(name, ".ww")) { return false; };
	if (strings.hassuffix(name, ".combined.ww")) { return false; };
	return true;
};

// One explicit package directory; with recurse (the DIR/... form) every
// subdirectory whose name does not begin with '.' or '_' is descended as
// well, Go's ./... convention.  lstat is used for the root and each
// candidate so a symlink cannot be used to cross the boundary (including
// DT_UNKNOWN files); a symlinked subdirectory is skipped, not followed.
fn pkgdiscoverdir(path: str, st: *pkgdiscover, recurse: bool) void = {
	let rootstat: os.filestat;
	match (os.lstat(&rootstat, path)) {
	case void => void;
	case let e: os.oserror => {
		pkgfailpath(path, "cannot stat package directory");
		st.errors += 1;
		return;
	};
	};
	if (pkgmodeis(rootstat.mode, os.mode.LINK)) {
		pkgfailpath(path, "symlink traversal is not allowed");
		st.errors += 1;
		return;
	};
	if (!pkgmodeis(rootstat.mode, os.mode.DIR)) {
		pkgfailpath(path, "expected one package directory");
		st.errors += 1;
		return;
	};
	let fd: i32 = os.open(path, os.flag.RDONLY, 0i32);
	if (fd < 0) {
		pkgfailpath(path, "cannot open directory");
		st.errors += 1;
		return;
	};
	let buf: []u8 = alloc([], 8192u64)!;
	buf.len = 8192;
	let n: i64 = os.getdents64(fd, buf.ptr, 8192u64);
	for (n > 0i64) {
		let off: u64 = 0u64;
		for (off < n: u64) {
			let reclen: u64 = (buf[off + 16u64]: u64)
				+ (buf[off + 17u64]: u64) * 256u64;
			if (reclen < 20u64 || off + reclen > n: u64) {
				pkgfailpath(path, "malformed directory entry");
				st.errors += 1;
				off = n: u64;
				continue;
			};
			let np: *u8 = buf.ptr + off + 19u64;
			let nl: i32 = 0;
			for (nl: u64 + 19u64 < reclen && np[nl] != 0u8) { nl += 1; };
			let name: str;
			name.ptr = np;
			name.len = nl;
			if (pkgkeepfile(name)) {
				let child: str = strings.concat(path, "/", name);
				let fi: os.filestat;
				match (os.lstat(&fi, child)) {
				case void => {
					if (pkgmodeis(fi.mode, os.mode.LINK)) {
						pkgfailpath(child,
							"symlink source is not allowed");
						st.errors += 1;
					} else if (pkgmodeis(fi.mode, os.mode.REG)) {
						append(st.paths, child);
					};
				};
				case let e: os.oserror => {
					pkgfailpath(child, "cannot stat source");
					st.errors += 1;
				};
				};
			} else if (recurse && name.len != 0
				&& name[0] != '.' && name[0] != '_') {
				let child: str = strings.concat(path, "/", name);
				let fi: os.filestat;
				match (os.lstat(&fi, child)) {
				case void => {
					if (pkgmodeis(fi.mode, os.mode.DIR)) {
						pkgdiscoverdir(child, st, true);
					};
				};
				case let e: os.oserror => {
					pkgfailpath(child,
						"cannot stat directory entry");
					st.errors += 1;
				};
				};
			};
			off += reclen;
		};
		n = os.getdents64(fd, buf.ptr, 8192u64);
	};
	if (n < 0i64) {
		pkgfailpath(path, "directory read failed");
		st.errors += 1;
	};
	os.close(fd);
};

fn pkgsort(ss: []str) void = {
	let i: i32 = 1;
	for (i < ss.len) {
		let j: i32 = i;
		for (j > 0 && strings.compare(ss[j - 1], ss[j]) > 0) {
			let t: str = ss[j];
			ss[j] = ss[j - 1];
			ss[j - 1] = t;
			j -= 1;
		};
		i += 1;
	};
};

fn pkgsortgroups(gs: []pkggroup) void = {
	let i: i32 = 1;
	for (i < gs.len) {
		let j: i32 = i;
		for (j > 0) {
			let c: int = strings.compare(gs[j - 1].dir, gs[j].dir);
			if (c == 0) { c = strings.compare(gs[j - 1].pkg, gs[j].pkg); };
			if (c <= 0) { break; };
			let t: pkggroup = gs[j];
			gs[j] = gs[j - 1];
			gs[j - 1] = t;
			j -= 1;
		};
		i += 1;
	};
};

fn pkgdedup(ss: []str) []str = {
	if (ss.len == 0) { return ss; };
	let out: []str = alloc([], ss.len: u64)!;
	let i: i32 = 0;
	for (i < ss.len) {
		if (i == 0 || strings.compare(ss[i - 1], ss[i]) != 0) {
			append(out, ss[i]);
		};
		i += 1;
	};
	return out;
};

fn pkgparsedec(s: str, max: i64) bool = {
	if (s.len == 0) { return false; };
	let i: i32 = 0;
	let n: i64 = 0i64;
	for (i < s.len) {
		if (s[i] < '0' || s[i] > '9') { return false; };
		let digit: i64 = (s[i] - '0'): i64;
		if (n > (max - digit) / 10i64) { return false; };
		n = n * 10i64 + digit;
		i += 1;
	};
	return n > 0i64;
};

fn pkgdefaultbuilder() str = {
	let av: []str = os.args();
	if (av.len == 0 || av[0].len == 0) { return "ww"; };
	return strings.concat(pkgdirname(av[0]), "/ww");
};

fn pkgmakedir(path: str) bool = {
	return os.mkdir(path, 448i32) == 0;
};

fn pkgremoveall(path: str) bool = {
	if (!pkgisdir(path)) { return os.remove(path) == 0; };
	let fd: i32 = os.open(path, os.flag.RDONLY, 0i32);
	if (fd < 0) { return false; };
	let ok: bool = true;
	let buf: []u8 = alloc([], 8192u64)!;
	buf.len = 8192;
	let n: i64 = os.getdents64(fd, buf.ptr, 8192u64);
	for (n > 0i64) {
		let off: u64 = 0u64;
		for (off < n: u64) {
			let reclen: u64 = (buf[off + 16u64]: u64)
				+ (buf[off + 17u64]: u64) * 256u64;
			if (reclen < 20u64 || off + reclen > n: u64) {
				ok = false;
				off = n: u64;
				continue;
			};
			let np: *u8 = buf.ptr + off + 19u64;
			let nl: i32 = 0;
			for (nl: u64 + 19u64 < reclen && np[nl] != 0u8) { nl += 1; };
			let name: str;
			name.ptr = np;
			name.len = nl;
			if (!(strings.compare(name, ".") == 0 || strings.compare(name, "..") == 0)) {
				let child: str = strings.concat(path, "/", name);
				let dtype: u8 = buf[off + 18u64];
				if (dtype == 4u8 || (dtype == 0u8 && pkgisdir(child))) {
					if (!pkgremoveall(child)) { ok = false; };
				} else if (os.remove(child) != 0) { ok = false; };
			};
			off += reclen;
		};
		n = os.getdents64(fd, buf.ptr, 8192u64);
	};
	if (n < 0i64) { ok = false; };
	os.close(fd);
	if (os.rmdir(path) != 0) { ok = false; };
	return ok;
};

fn pkgemitfile(path: str, fd: i32) bool = {
	let s: str;
	if (!pkgread(path, &s)) { return false; };
	if (s.len != 0) {
		pkgput(fd, s);
		if (s[s.len - 1] != '\n') { pkgput(fd, "\n"); };
	};
	return true;
};

fn pkgcombined(g: *pkggroup, srcs: []pkgsource) bool = {
	let fd: i32 = os.open(g.combined,
		os.flag.WRONLY | os.flag.CREATE | os.flag.EXCL, 384i32);
	if (fd < 0) { return false; };
	let ok: bool = true;
	if (!g.external) {
		let i: i32 = g.start;
		for (i < g.end) {
			if (!srcs[i].test) {
				if (!pkgwrite(fd, "//ww:module-reset\n")) { ok = false; };
				let s: str;
				if (!pkgread(srcs[i].path, &s) || !pkgwrite(fd, s)
					|| !pkgwrite(fd, "\n")) { ok = false; };
			};
			i += 1;
		};
	};
	let i: i32 = g.start;
	for (i < g.end) {
		if (srcs[i].test && strings.compare(srcs[i].pkg, g.pkg) == 0) {
			if (!pkgwrite(fd, "//ww:module-reset\n")) { ok = false; };
			let s: str;
			if (!pkgread(srcs[i].path, &s) || !pkgwrite(fd, s)
				|| !pkgwrite(fd, "\n")) { ok = false; };
		};
		i += 1;
	};
	os.close(fd);
	return ok;
};

fn pkgsetpaths(g: *pkggroup, root: str, index: i32,
	compileonly: bool) bool = {
	let num: str = strconv.i32tos(index, strconv.base.DEC);
	g.root = strings.concat(root, "/group-", num);
	if (!pkgmakedir(g.root)) { return false; };
	g.combined = strings.concat(g.root, "/package.ww");
	if (compileonly) {
		g.bin = strings.concat(g.dir, "/", g.pkg, ".test");
	} else {
		g.bin = strings.concat(g.root, "/package.test");
	};
	g.buildout = strings.concat(g.root, "/build.stdout");
	g.builderr = strings.concat(g.root, "/build.stderr");
	g.runout = strings.concat(g.root, "/test.stdout");
	g.runerr = strings.concat(g.root, "/test.stderr");
	return true;
};

fn pkglabel(g: *pkggroup) void = {
	pkgput(os.STDOUT_FILENO, g.dir);
	pkgput(os.STDOUT_FILENO, " [");
	pkgput(os.STDOUT_FILENO, g.pkg);
	if (g.external) { pkgput(os.STDOUT_FILENO, ", external"); }
	else { pkgput(os.STDOUT_FILENO, ", same-package"); };
	pkgput(os.STDOUT_FILENO, "]");
};

fn pkgreportcommand(kind: str, g: *pkggroup, r: *exec.result) void = {
	pkgput(os.STDERR_FILENO, "FAIL ");
	pkgput(os.STDERR_FILENO, g.dir);
	pkgput(os.STDERR_FILENO, " [");
	pkgput(os.STDERR_FILENO, g.pkg);
	pkgput(os.STDERR_FILENO, "] (");
	pkgput(os.STDERR_FILENO, kind);
	if (r.errno != 0 || r.cleanuperrno != 0
	    || r.termination == exec.termination.ERROR) {
		pkgput(os.STDERR_FILENO, " harness error ");
		let code: i32 = r.errno;
		if (code == 0) { code = r.cleanuperrno; };
		pkgput(os.STDERR_FILENO, strconv.i32tos(code, strconv.base.DEC));
	} else if (r.termination == exec.termination.SIGNAL) {
		pkgput(os.STDERR_FILENO, " signal ");
		pkgput(os.STDERR_FILENO, strconv.i32tos(r.code, strconv.base.DEC));
	} else {
		pkgput(os.STDERR_FILENO, " exit ");
		pkgput(os.STDERR_FILENO, strconv.i32tos(r.code, strconv.base.DEC));
	};
	pkgputln(os.STDERR_FILENO, ")");
};

fn pkgrungroup(g: *pkggroup, srcs: []pkgsource, builder: str,
	filters: []str, includes: []str, timeoutarg: str,
	list: bool, compileonly: bool) bool = {
	if (!pkgcombined(g, srcs)) {
		pkgfailpath(g.combined, "cannot compose package test source");
		return false;
	};
	let ba: []str = alloc([], (9 + includes.len * 2): u64)!;
	append(ba, builder);
	append(ba, "test");
	append(ba, "-c");
	append(ba, "-o");
	append(ba, g.bin);
	append(ba, "-I");
	append(ba, g.dir);
	let ii: i32 = 0;
	for (ii < includes.len) {
		append(ba, "-I");
		append(ba, includes[ii]);
		ii += 1;
	};
	append(ba, g.combined);
	let env: []str = toolenv(g.root);
	let bcmd: exec.command;
	bcmd.path = builder;
	bcmd.argv = ba;
	bcmd.env = env;
	bcmd.dir = "";
	bcmd.stdoutpath = g.buildout;
	bcmd.stderrpath = g.builderr;
	bcmd.deadline.sec = 0i64;
	bcmd.deadline.nsec = 0i64;
	bcmd.grace = 0i64: time.duration;
	let br: exec.result;
	exec.run(&bcmd, &br);
	let buildcaptures: bool = pkgemitfile(g.buildout, os.STDOUT_FILENO);
	buildcaptures = pkgemitfile(g.builderr, os.STDERR_FILENO) && buildcaptures;
	if (!buildcaptures) {
		pkgfailpath(g.root, "cannot read build capture");
		return false;
	};
	if (br.errno != 0 || br.cleanuperrno != 0
	    || br.termination != exec.termination.EXIT || br.code != 0) {
		pkgreportcommand("build", g, &br);
		return false;
	};
	if (compileonly) {
		pkgput(os.STDOUT_FILENO, "built ");
		pkglabel(g);
		pkgput(os.STDOUT_FILENO, " -> ");
		pkgputln(os.STDOUT_FILENO, g.bin);
		return true;
	};
	let ra: []str = alloc([], (filters.len + 4): u64)!;
	append(ra, g.bin);
	append(ra, strings.concat("-package=", g.pkg));
	if (list) { append(ra, "-list"); };
	if (timeoutarg.len != 0) { append(ra, timeoutarg); };
	let i: i32 = 0;
	for (i < filters.len) { append(ra, filters[i]); i += 1; };
	let rcmd: exec.command;
	rcmd.path = g.bin;
	rcmd.argv = ra;
	rcmd.env = env;
	rcmd.dir = "";
	rcmd.stdoutpath = g.runout;
	rcmd.stderrpath = g.runerr;
	rcmd.deadline.sec = 0i64;
	rcmd.deadline.nsec = 0i64;
	rcmd.grace = 0i64: time.duration;
	let rr: exec.result;
	exec.run(&rcmd, &rr);
	let runcaptures: bool = pkgemitfile(g.runout, os.STDOUT_FILENO);
	runcaptures = pkgemitfile(g.runerr, os.STDERR_FILENO) && runcaptures;
	if (!runcaptures) {
		pkgfailpath(g.root, "cannot read test capture");
		return false;
	};
	if (rr.errno != 0 || rr.cleanuperrno != 0
	    || rr.termination != exec.termination.EXIT || rr.code != 0) {
		pkgreportcommand("test", g, &rr);
		return false;
	};
	pkgput(os.STDOUT_FILENO, "ok   ");
	pkglabel(g);
	pkgput(os.STDOUT_FILENO, "\n");
	return true;
};

export fn packagecommand(args: []str) int = {
	let compileonly: bool = false;
	let list: bool = false;
	let sawj: bool = false;
	let afterdash: bool = false;
	let roots: []str = alloc([], 2u64)!;
	let filters: []str = alloc([], (args.len + 1): u64)!;
	let includes: []str = alloc([], (args.len + 1): u64)!;
	let timeoutarg: str = "";
	let builder: str = pkgdefaultbuilder();
	let i: i32 = 0;
	for (i < args.len) {
		let a: str = args[i];
		if (afterdash) { append(filters, a); i += 1; continue; };
		if (strings.compare(a, "--") == 0) { afterdash = true; i += 1; continue; };
		if (strings.compare(a, "-c") == 0) { compileonly = true; i += 1; continue; };
		if (strings.compare(a, "-list") == 0) { list = true; i += 1; continue; };
		if (strings.compare(a, "-I") == 0) {
			if (i + 1 >= args.len) { pkgusage(); return 2; };
			append(includes, args[i + 1]);
			i += 2;
			continue;
		};
		if (strings.hasprefix(a, "-I") && a.len > 2) {
			append(includes, a[2:a.len]);
			i += 1;
			continue;
		};
		if (strings.hasprefix(a, "-timeout-ms=")) {
			if (timeoutarg.len != 0 || a.len == 12
				|| !pkgparsedec(a[12:a.len], 3600000i64)) {
				pkgusage();
				return 2;
			};
			timeoutarg = a;
			i += 1;
			continue;
		};
		if (strings.compare(a, "-j") == 0) {
			if (i + 1 >= args.len
				|| !pkgparsedec(args[i + 1], 2147483647i64)) {
				pkgusage();
				return 2;
			};
			sawj = true;
			i += 2;
			continue;
		};
		if (strings.compare(a, "-filter") == 0 || strings.compare(a, "-run") == 0) {
			if (i + 1 >= args.len) { pkgusage(); return 2; };
			append(filters, args[i + 1]);
			i += 2;
			continue;
		};
		if (strings.compare(a, "--ww-driver") == 0) {
			if (i + 1 >= args.len || args[i + 1].len == 0) {
				pkgusage();
				return 2;
			};
			builder = args[i + 1];
			i += 2;
			continue;
		};
		if (a.len != 0 && a[0] == '-') { pkgusage(); return 2; };
		append(roots, a);
		i += 1;
	};
	if (roots.len == 0) { append(roots, "."); };
	if (roots.len != 1) {
		pkgputln(os.STDERR_FILENO,
			"wwtest package: exactly one package directory is supported");
		return 2;
	};
	if (compileonly && (list || filters.len != 0 || timeoutarg.len != 0)) {
		pkgusage();
		return 2;
	};
	if (sawj) {
		pkgputln(os.STDERR_FILENO,
			"wwtest package: -j is reserved; sequential package scheduling is active");
	};

	// Go's ./... form: a trailing "..." path element walks the tree
	// rooted at the prefix instead of one explicit directory.
	let discoverroot: str = roots[0];
	let recurse: bool = false;
	if (strings.compare(discoverroot, "...") == 0) {
		discoverroot = ".";
		recurse = true;
	} else if (strings.hassuffix(discoverroot, "/...")) {
		discoverroot = discoverroot[0:discoverroot.len - 4];
		if (discoverroot.len == 0) { discoverroot = "/"; };
		recurse = true;
	};

	let ds: pkgdiscover;
	let discoveredpaths: []str = alloc([], 64u64)!;
	ds.paths = discoveredpaths;
	ds.errors = 0;
	pkgdiscoverdir(discoverroot, &ds, recurse);
	if (ds.errors != 0) { return 1; };
	pkgsort(ds.paths);
	if (ds.paths.len == 0) {
		pkgfailpath(discoverroot, "directory contains no WW package sources");
		return 1;
	};

	let srcs: []pkgsource = alloc([], ds.paths.len: u64)!;
	i = 0;
	for (i < ds.paths.len) {
		let body: str;
		let pn: str;
		if (!pkgread(ds.paths[i], &body) || !pkgclause(body, &pn)) {
			pkgfailpath(ds.paths[i], "invalid or missing package clause");
			return 1;
		};
		let s: pkgsource;
		s.path = ds.paths[i];
		s.dir = strings.dup(pkgdirname(ds.paths[i]));
		s.pkg = strings.dup(pn);
		s.attest = pkgattest(body);
		s.test = strings.hassuffix(pkgbase(ds.paths[i]), "_test.ww")
			|| s.attest;
		append(srcs, s);
		i += 1;
	};

	let folders: []pkgfolder = alloc([], srcs.len: u64)!;
	i = 0;
	for (i < srcs.len) {
		let f: pkgfolder;
		f.path = srcs[i].dir;
		f.start = i;
		f.end = i;
		f.prodpkg = "";
		f.hastests = false;
		for (f.end < srcs.len && strings.compare(srcs[f.end].dir, f.path) == 0) {
			if (srcs[f.end].test) {
				if (srcs[f.end].attest) { f.hastests = true; };
			} else if (f.prodpkg.len == 0) {
				f.prodpkg = srcs[f.end].pkg;
			} else if (strings.compare(f.prodpkg, srcs[f.end].pkg) != 0) {
				pkgfailpath(f.path, "production sources declare conflicting packages");
				return 1;
			};
			f.end += 1;
		};
		append(folders, f);
		i = f.end;
	};

	let groups: []pkggroup = alloc([], srcs.len: u64)!;
	i = 0;
	for (i < folders.len) {
		let f: pkgfolder = folders[i];
		if (!f.hastests) {
			pkgput(os.STDOUT_FILENO, "?    ");
			pkgput(os.STDOUT_FILENO, f.path);
			pkgputln(os.STDOUT_FILENO, " [no tests]");
			i += 1;
			continue;
		};
		let externalpkg: str = "";
		if (f.prodpkg.len != 0) {
			externalpkg = strings.concat(f.prodpkg, "_test");
		};
		let standalonepkg: str = "";
		let j: i32 = f.start;
		for (j < f.end) {
			if (srcs[j].test) {
				if (f.prodpkg.len != 0
					&& strings.compare(srcs[j].pkg, f.prodpkg) != 0
					&& strings.compare(srcs[j].pkg, externalpkg) != 0) {
					pkgfailpath(srcs[j].path,
						"test package must match production package or <package>_test");
					return 1;
				};
				if (f.prodpkg.len == 0) {
					if (standalonepkg.len == 0) {
						standalonepkg = srcs[j].pkg;
					} else if (strings.compare(standalonepkg,
						srcs[j].pkg) != 0) {
						pkgfailpath(f.path,
							"test-only directory declares conflicting packages");
						return 1;
					};
				};
				if (!srcs[j].attest) { j += 1; continue; };
				let found: bool = false;
				let k: i32 = 0;
				for (k < groups.len) {
					if (strings.compare(groups[k].dir, f.path) == 0
						&& strings.compare(groups[k].pkg, srcs[j].pkg) == 0) {
						found = true;
						break;
					};
					k += 1;
				};
				if (!found) {
					let g: pkggroup;
					g.dir = f.path;
					g.pkg = srcs[j].pkg;
					g.start = f.start;
					g.end = f.end;
					g.prodpkg = f.prodpkg;
					g.external = f.prodpkg.len != 0
						&& strings.compare(f.prodpkg, g.pkg) != 0;
					append(groups, g);
				};
			};
			j += 1;
		};
		i += 1;
	};
	if (groups.len == 0) { return 0; };
	pkgsortgroups(groups);

	let borrowed: str = temp.dir();
	let tmproot: str = strings.dup(borrowed);
	let failed: i32 = 0;
	i = 0;
	for (i < groups.len) {
		if (!pkgsetpaths(&groups[i], tmproot, i, compileonly)) {
			pkgfailpath(tmproot, "cannot create group temporary directory");
			failed += 1;
			break;
		};
		if (!pkgrungroup(&groups[i], srcs, builder, filters, includes,
			timeoutarg, list, compileonly)) {
			failed += 1;
		};
		i += 1;
	};
	if (!pkgremoveall(tmproot)) {
		pkgput(os.STDERR_FILENO, "wwtest package: cleanup failed; retained ");
		pkgputln(os.STDERR_FILENO, tmproot);
		failed += 1;
	};
	if (failed != 0) { return 1; };
	return 0;
};