ww: grow package universes dynamically

internal/wwpackage/package.ww

@@ -35,6 +35,7 @@ type pkggroup = struct {
 	runout: str,
 	runerr: str,
 	state: i32,
+	runstartfailed: bool,
 	runres: exec.result,
 };
 
@@ -52,6 +53,44 @@ type pkgplan = struct {
 	buildres: exec.result,
 };
 
+def PKG_COUNT_MAX: i32 = 2147483647;
+def PKG_INITIAL_CAP: i32 = 8;
+
+fn pkgallocstrs(cap: i32) ([]str | nomem) = {
+	let value: []str = alloc([], cap: u64)?;
+	return value;
+};
+
+fn pkgallocbytes(cap: i32) ([]u8 | nomem) = {
+	let value: []u8 = alloc([], cap: u64)?;
+	return value;
+};
+
+fn pkgallocsources(cap: i32) ([]pkgsource | nomem) = {
+	let value: []pkgsource = alloc([], cap: u64)?;
+	return value;
+};
+
+fn pkgallocfolders(cap: i32) ([]pkgfolder | nomem) = {
+	let value: []pkgfolder = alloc([], cap: u64)?;
+	return value;
+};
+
+fn pkgallocgroups(cap: i32) ([]pkggroup | nomem) = {
+	let value: []pkggroup = alloc([], cap: u64)?;
+	return value;
+};
+
+fn pkgallocplans(cap: i32) ([]pkgplan | nomem) = {
+	let value: []pkgplan = alloc([], cap: u64)?;
+	return value;
+};
+
+fn pkgallocprocesses(cap: i32) ([]exec.process | nomem) = {
+	let value: []exec.process = alloc([], cap: u64)?;
+	return value;
+};
+
 // Product and directory-plan process states for the bounded coordinator.
 def PKGQUEUED: i32 = 0;
 def PKGBUILDING: i32 = 1;
@@ -63,13 +102,88 @@ def pkgpoll: time.duration = 1000000i64: time.duration;
 type pkgdiscover = struct {
 	paths: []str,
 	errors: i32,
+	fatal: bool,
+};
+
+fn pkggrowcap(current: i32, need: i32) i32 = {
+	if (need < 0) { return -1; };
+	if (need <= current) { return current; };
+	let cap: i32 = current;
+	if (cap == 0) { cap = PKG_INITIAL_CAP; };
+	for (cap < need) {
+		if (cap > PKG_COUNT_MAX / 2) {
+			cap = PKG_COUNT_MAX;
+			break;
+		};
+		cap *= 2;
+	};
+	if (cap < need) { return -1; };
+	return cap;
+};
+
+fn pkgappenddiscovered(st: *pkgdiscover, path: str) bool = {
+	if (st.paths.len == PKG_COUNT_MAX) {
+		pkgputln(os.STDERR_FILENO,
+			"wwtest package: package graph is too large");
+		st.errors += 1;
+		st.fatal = true;
+		return false;
+	};
+	let need: i32 = st.paths.len + 1;
+	if (need > st.paths.cap) {
+		let cap: i32 = pkggrowcap(st.paths.cap, need);
+		if (cap < 0) {
+			pkgputln(os.STDERR_FILENO,
+				"wwtest package: package graph is too large");
+			st.errors += 1;
+			st.fatal = true;
+			return false;
+		};
+		let allocation: ([]str | nomem) = pkgallocstrs(cap);
+		let next: []str;
+		match (allocation) {
+		case let value: []str => next = value;
+		case nomem => {
+			pkgputln(os.STDERR_FILENO,
+				"wwtest package: out of memory");
+			st.errors += 1;
+			st.fatal = true;
+			return false;
+		};
+		};
+		let n: i32 = st.paths.len;
+		next.len = cap;
+		let i: i32 = 0;
+		for (i < n) { next[i] = st.paths[i]; i += 1; };
+		next.len = n;
+		if (st.paths.ptr != nil) {
+			os.free(st.paths.ptr: *void,
+				(st.paths.cap: u64) * (size(str): u64));
+		};
+		st.paths = next;
+	};
+	append(st.paths, path);
+	return true;
 };
 
 // Preserve the caller's toolchain environment while pinning the locale and
 // temporary directory used by the current build plan or test product.
-fn toolenv(tmpdir: str) []str = {
+fn toolenv(tmpdir: str, out: *[]str) bool = {
 	let inherited: []str = os.getenvs();
-	let env: []str = alloc([], (inherited.len + 2): u64)!;
+	if (inherited.len > PKG_COUNT_MAX - 2) {
+		pkgputln(os.STDERR_FILENO,
+			"wwtest package: package graph is too large");
+		return false;
+	};
+	let allocation: ([]str | nomem) = pkgallocstrs(inherited.len + 2);
+	let env: []str;
+	match (allocation) {
+	case let value: []str => env = value;
+	case nomem => {
+		pkgputln(os.STDERR_FILENO, "wwtest package: out of memory");
+		return false;
+	};
+	};
 	let i: i32 = 0;
 	for (i < inherited.len) {
 		if (!strings.hasprefix(inherited[i], "TMPDIR=")
@@ -80,7 +194,8 @@ fn toolenv(tmpdir: str) []str = {
 	};
 	append(env, "LC_ALL=C");
 	append(env, strings.concat("TMPDIR=", tmpdir));
-	return env;
+	*out = env;
+	return true;
 };
 
 fn pkgwrite(fd: i32, s: str) bool = {
@@ -125,8 +240,24 @@ fn pkgread(path: str, out: *str) bool = {
 	case let v: i64 => n = v;
 	case let e: os.oserror => { os.close(fd); return false; };
 	};
-	if (n < 0i64) { os.close(fd); return false; };
+	if (n < 0i64 || n >= PKG_COUNT_MAX: i64) {
-	let b: []u8 = alloc([], (n + 1i64): u64)!;
+		os.close(fd);
+		if (n >= PKG_COUNT_MAX: i64) {
+			pkgputln(os.STDERR_FILENO,
+				"wwtest package: package graph is too large");
+		};
+		return false;
+	};
+	let allocation: ([]u8 | nomem) = pkgallocbytes((n + 1i64): i32);
+	let b: []u8;
+	match (allocation) {
+	case let value: []u8 => b = value;
+	case nomem => {
+		os.close(fd);
+		pkgputln(os.STDERR_FILENO, "wwtest package: out of memory");
+		return false;
+	};
+	};
 	b.len = (n + 1i64): i32;
 	let rr: (i64 | os.oserror) = os.readall(fd, b.ptr, n: u64);
 	os.close(fd);
@@ -275,6 +406,7 @@ fn pkgkeepfile(name: str) bool = {
 // canonical planning coalesces it with the target directory. Recursive child
 // symlinks remain skipped and symlink source files remain rejected.
 fn pkgdiscoverdir(path: str, st: *pkgdiscover, recurse: bool) void = {
+	if (st.fatal) { return; };
 	let rootstat: os.filestat;
 	match (os.lstat(&rootstat, path)) {
 	case void => void;
@@ -305,7 +437,18 @@ fn pkgdiscoverdir(path: str, st: *pkgdiscover, recurse: bool) void = {
 		st.errors += 1;
 		return;
 	};
-	let buf: []u8 = alloc([], 8192u64)!;
+	let bufallocation: ([]u8 | nomem) = pkgallocbytes(8192);
+	let buf: []u8;
+	match (bufallocation) {
+	case let value: []u8 => buf = value;
+	case nomem => {
+		os.close(fd);
+		pkgputln(os.STDERR_FILENO, "wwtest package: out of memory");
+		st.errors += 1;
+		st.fatal = true;
+		return;
+	};
+	};
 	buf.len = 8192;
 	let n: i64 = os.getdents64(fd, buf.ptr, 8192u64);
 	for (n > 0i64) {
@@ -335,7 +478,10 @@ fn pkgdiscoverdir(path: str, st: *pkgdiscover, recurse: bool) void = {
 							"symlink source is not allowed");
 						st.errors += 1;
 					} else if (pkgmodeis(fi.mode, os.mode.REG)) {
-						append(st.paths, child);
+						if (!pkgappenddiscovered(st, child)) {
+							os.close(fd);
+							return;
+						};
 					};
 				};
 				case let e: os.oserror => {
@@ -351,6 +497,7 @@ fn pkgdiscoverdir(path: str, st: *pkgdiscover, recurse: bool) void = {
 				case void => {
 					if (pkgmodeis(fi.mode, os.mode.DIR)) {
 						pkgdiscoverdir(child, st, true);
+						if (st.fatal) { os.close(fd); return; };
 					};
 				};
 				case let e: os.oserror => {
@@ -484,7 +631,12 @@ fn pkgremoveall(path: str) bool = {
 	if (!pkgmodeis(opened.mode, os.mode.DIR)
 	    || opened.inode != st.inode) { os.close(fd); return false; };
 	let ok: bool = true;
-	let buf: []u8 = alloc([], 8192u64)!;
+	let allocation: ([]u8 | nomem) = pkgallocbytes(8192);
+	let buf: []u8;
+	match (allocation) {
+	case let value: []u8 => buf = value;
+	case nomem => { os.close(fd); return false; };
+	};
 	buf.len = 8192;
 	let n: i64 = os.getdents64(fd, buf.ptr, 8192u64);
 	for (n > 0i64) {
@@ -643,9 +795,28 @@ fn pkgreportcommand(kind: str, g: *pkggroup, r: *exec.result) void = {
 };
 
 fn pkgstartbuild(p: *pkgplan, groups: []pkggroup, builder: str, includes: []str,
-	h: *exec.process) void = {
+	h: *exec.process) bool = {
-	let ba: []str = alloc([],
+	let nproducts: i32 = p.end - p.start;
-		(12 + (p.end - p.start) * 6 + includes.len * 2): u64)!;
+	let capacity: i32 = 12;
+	if (nproducts < 0 || nproducts > (PKG_COUNT_MAX - capacity) / 6) {
+		pkgputln(os.STDERR_FILENO, "wwtest package: package graph is too large");
+		return false;
+	};
+	capacity += nproducts * 6;
+	if (includes.len > (PKG_COUNT_MAX - capacity) / 2) {
+		pkgputln(os.STDERR_FILENO, "wwtest package: package graph is too large");
+		return false;
+	};
+	capacity += includes.len * 2;
+	let allocation: ([]str | nomem) = pkgallocstrs(capacity);
+	let ba: []str;
+	match (allocation) {
+	case let value: []str => ba = value;
+	case nomem => {
+		pkgputln(os.STDERR_FILENO, "wwtest package: out of memory");
+		return false;
+	};
+	};
 	append(ba, builder);
 	append(ba, "test");
 	append(ba, "-c");
@@ -682,10 +853,12 @@ fn pkgstartbuild(p: *pkgplan, groups: []pkggroup, builder: str, includes: []str,
 		append(ba, p.workdir);
 	};
 	append(ba, p.dir);
+	let env: []str;
+	if (!toolenv(p.root, &env)) { return false; };
 	let bcmd: exec.command;
 	bcmd.path = builder;
 	bcmd.argv = ba;
-	bcmd.env = toolenv(p.root);
+	bcmd.env = env;
 	bcmd.dir = "";
 	bcmd.stdoutpath = p.buildout;
 	bcmd.stderrpath = p.builderr;
@@ -693,21 +866,37 @@ fn pkgstartbuild(p: *pkgplan, groups: []pkggroup, builder: str, includes: []str,
 	bcmd.deadline.nsec = 0i64;
 	bcmd.grace = 0i64: time.duration;
 	exec.start(h, &bcmd);
+	return true;
 };
 
 fn pkgstartrun(g: *pkggroup, filters: []str, timeoutarg: str,
-	list: bool, h: *exec.process) void = {
+	list: bool, h: *exec.process) bool = {
-	let ra: []str = alloc([], (filters.len + 4): u64)!;
+	if (filters.len > PKG_COUNT_MAX - 4) {
+		pkgputln(os.STDERR_FILENO,
+			"wwtest package: package graph is too large");
+		return false;
+	};
+	let allocation: ([]str | nomem) = pkgallocstrs(filters.len + 4);
+	let ra: []str;
+	match (allocation) {
+	case let value: []str => ra = value;
+	case nomem => {
+		pkgputln(os.STDERR_FILENO, "wwtest package: out of memory");
+		return false;
+	};
+	};
 	append(ra, g.bin);
 	append(ra, strings.concat("-package=", g.pkg));
 	if (list) { append(ra, "-list"); };
 	if (timeoutarg.len != 0) { append(ra, timeoutarg); };
 	let i: i32 = 0;
 	for (i < filters.len) { append(ra, filters[i]); i += 1; };
+	let env: []str;
+	if (!toolenv(g.root, &env)) { return false; };
 	let rcmd: exec.command;
 	rcmd.path = g.bin;
 	rcmd.argv = ra;
-	rcmd.env = toolenv(g.root);
+	rcmd.env = env;
 	rcmd.dir = "";
 	rcmd.stdoutpath = g.runout;
 	rcmd.stderrpath = g.runerr;
@@ -715,6 +904,7 @@ fn pkgstartrun(g: *pkggroup, filters: []str, timeoutarg: str,
 	rcmd.deadline.nsec = 0i64;
 	rcmd.grace = 0i64: time.duration;
 	exec.start(h, &rcmd);
+	return true;
 };
 
 fn pkgproductbuilt(g: *pkggroup) bool = {
@@ -735,6 +925,7 @@ fn pkgemitgroup(g: *pkggroup, compileonly: bool) bool = {
 		pkgputln(os.STDOUT_FILENO, g.bin);
 		return true;
 	};
+	if (g.runstartfailed) { return false; };
 	let runstdout: str;
 	let runstderr: str;
 	if (!pkgread(g.runout, &runstdout) || !pkgread(g.runerr, &runstderr)) {
@@ -791,9 +982,39 @@ export fn packagecommand(args: []str) int = {
 	let list: bool = false;
 	let jobs: i32 = 1;
 	let afterdash: bool = false;
-	let roots: []str = alloc([], 2u64)!;
+	if (args.len == PKG_COUNT_MAX) {
-	let filters: []str = alloc([], (args.len + 1): u64)!;
+		pkgputln(os.STDERR_FILENO,
-	let includes: []str = alloc([], (args.len + 1): u64)!;
+			"wwtest package: package graph is too large");
+		return 1;
+	};
+	let argcapacity: i32 = args.len + 1;
+	let rootallocation: ([]str | nomem) = pkgallocstrs(argcapacity);
+	let roots: []str;
+	match (rootallocation) {
+	case let value: []str => roots = value;
+	case nomem => {
+		pkgputln(os.STDERR_FILENO, "wwtest package: out of memory");
+		return 1;
+	};
+	};
+	let filterallocation: ([]str | nomem) = pkgallocstrs(argcapacity);
+	let filters: []str;
+	match (filterallocation) {
+	case let value: []str => filters = value;
+	case nomem => {
+		pkgputln(os.STDERR_FILENO, "wwtest package: out of memory");
+		return 1;
+	};
+	};
+	let includeallocation: ([]str | nomem) = pkgallocstrs(argcapacity);
+	let includes: []str;
+	match (includeallocation) {
+	case let value: []str => includes = value;
+	case nomem => {
+		pkgputln(os.STDERR_FILENO, "wwtest package: out of memory");
+		return 1;
+	};
+	};
 	let timeoutarg: str = "";
 	let outname: str = "";
 	let workroot: str = "";
@@ -925,9 +1146,10 @@ export fn packagecommand(args: []str) int = {
 	};
 
 	let ds: pkgdiscover;
-	let discoveredpaths: []str = alloc([], 64u64)!;
+	let emptypaths: []str;
-	ds.paths = discoveredpaths;
+	ds.paths = emptypaths;
 	ds.errors = 0;
+	ds.fatal = false;
 	pkgdiscoverdir(discoverroot, &ds, recurse);
 	if (ds.errors != 0) { return 1; };
 	pkgsort(ds.paths);
@@ -941,7 +1163,16 @@ export fn packagecommand(args: []str) int = {
 		return 1;
 	};
 
-	let srcs: []pkgsource = alloc([], ds.paths.len: u64)!;
+	let sourceallocation: ([]pkgsource | nomem) =
+		pkgallocsources(ds.paths.len);
+	let srcs: []pkgsource;
+	match (sourceallocation) {
+	case let value: []pkgsource => srcs = value;
+	case nomem => {
+		pkgputln(os.STDERR_FILENO, "wwtest package: out of memory");
+		return 1;
+	};
+	};
 	i = 0;
 	for (i < ds.paths.len) {
 		let body: str;
@@ -959,7 +1190,16 @@ export fn packagecommand(args: []str) int = {
 		i += 1;
 	};
 
-	let folders: []pkgfolder = alloc([], srcs.len: u64)!;
+	let folderallocation: ([]pkgfolder | nomem) =
+		pkgallocfolders(srcs.len);
+	let folders: []pkgfolder;
+	match (folderallocation) {
+	case let value: []pkgfolder => folders = value;
+	case nomem => {
+		pkgputln(os.STDERR_FILENO, "wwtest package: out of memory");
+		return 1;
+	};
+	};
 	i = 0;
 	for (i < srcs.len) {
 		let f: pkgfolder;
@@ -977,7 +1217,16 @@ export fn packagecommand(args: []str) int = {
 		i = f.end;
 	};
 
-	let groups: []pkggroup = alloc([], srcs.len: u64)!;
+	let groupallocation: ([]pkggroup | nomem) =
+		pkgallocgroups(srcs.len);
+	let groups: []pkggroup;
+	match (groupallocation) {
+	case let value: []pkggroup => groups = value;
+	case nomem => {
+		pkgputln(os.STDERR_FILENO, "wwtest package: out of memory");
+		return 1;
+	};
+	};
 	i = 0;
 	for (i < folders.len) {
 		let f: pkgfolder = folders[i];
@@ -1050,7 +1299,15 @@ export fn packagecommand(args: []str) int = {
 			"wwtest package: cannot use -o with multiple packages");
 		return 2;
 	};
-	let plans: []pkgplan = alloc([], 1u64)!;
+	let planallocation: ([]pkgplan | nomem) = pkgallocplans(1);
+	let plans: []pkgplan;
+	match (planallocation) {
+	case let value: []pkgplan => plans = value;
+	case nomem => {
+		pkgputln(os.STDERR_FILENO, "wwtest package: out of memory");
+		return 1;
+	};
+	};
 	let plan: pkgplan;
 	plan.dir = discoverroot;
 	plan.sourceroot = canonicalroot;
@@ -1081,19 +1338,48 @@ export fn packagecommand(args: []str) int = {
 	// Build the complete request in one command-owned package universe. Once
 	// the union build completes, successful products run under the same global
 	// -j bound. Emission stays in byte-sorted group order below.
-	let handles: []exec.process = alloc([], plans.len: u64)!;
+	let handleallocation: ([]exec.process | nomem) =
+		pkgallocprocesses(plans.len);
+	let handles: []exec.process;
+	match (handleallocation) {
+	case let value: []exec.process => handles = value;
+	case nomem => {
+		pkgputln(os.STDERR_FILENO, "wwtest package: out of memory");
+		if (!pkgremoveall(tmproot)) {
+			pkgput(os.STDERR_FILENO,
+				"wwtest package: cleanup failed; retained ");
+			pkgputln(os.STDERR_FILENO, tmproot);
+		};
+		return 1;
+	};
+	};
 	i = 0;
 	for (i < plans.len) {
 		let h: exec.process;
 		append(handles, h);
 		i += 1;
 	};
-	let runhandles: []exec.process = alloc([], groups.len: u64)!;
+	let runhandleallocation: ([]exec.process | nomem) =
+		pkgallocprocesses(groups.len);
+	let runhandles: []exec.process;
+	match (runhandleallocation) {
+	case let value: []exec.process => runhandles = value;
+	case nomem => {
+		pkgputln(os.STDERR_FILENO, "wwtest package: out of memory");
+		if (!pkgremoveall(tmproot)) {
+			pkgput(os.STDERR_FILENO,
+				"wwtest package: cleanup failed; retained ");
+			pkgputln(os.STDERR_FILENO, tmproot);
+		};
+		return 1;
+	};
+	};
 	i = 0;
 	for (i < groups.len) {
 		let h: exec.process;
 		append(runhandles, h);
 		groups[i].state = PKGQUEUED;
+		groups[i].runstartfailed = false;
 		i += 1;
 	};
 	let planlaunched: i32 = 0;
@@ -1118,10 +1404,15 @@ export fn packagecommand(args: []str) int = {
 							g.state = PKGDONE;
 							productcompleted += 1;
 						} else {
-							pkgstartrun(g, filters, timeoutarg, list,
+							if (!pkgstartrun(g, filters, timeoutarg, list,
-								&runhandles[gi]);
+							    &runhandles[gi])) {
-							g.state = PKGRUNNING;
+								g.runstartfailed = true;
-							active += 1;
+								g.state = PKGDONE;
+								productcompleted += 1;
+							} else {
+								g.state = PKGRUNNING;
+								active += 1;
+							};
 						};
 						filling = true;
 						break;
@@ -1130,8 +1421,15 @@ export fn packagecommand(args: []str) int = {
 				};
 			};
 			if (!filling && planlaunched < plans.len) {
-				pkgstartbuild(&plans[planlaunched], groups, builder, includes,
+				if (!pkgstartbuild(&plans[planlaunched], groups, builder,
-					&handles[planlaunched]);
+					includes, &handles[planlaunched])) {
+					if (!pkgremoveall(tmproot)) {
+						pkgput(os.STDERR_FILENO,
+							"wwtest package: cleanup failed; retained ");
+						pkgputln(os.STDERR_FILENO, tmproot);
+					};
+					return 1;
+				};
 				plans[planlaunched].state = PKGBUILDING;
 				active += 1;
 				planlaunched += 1;