Hojun-Cho/ww
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

ww: grow package universes dynamically

Browse Source
This commit is contained in:
Hojun-Cho
2026-08-13 08:08:49 +09:00
parent f45844ffa8
commit 00903b986b
3 changed files with 2016 additions and 493 deletions
Download Patch File Download Diff File Expand all files Collapse all files

711
cmd/ww/main.c
View File

File diff suppressed because it is too large Load Diff

358
internal/wwpackage/package.ww
View File

@@ -35,6 +35,7 @@ type pkggroup = struct {
runout: str,
runerr: str,
state: i32,
runstartfailed: bool,
runres: exec.result,
};
@@ -52,6 +53,44 @@ type pkgplan = struct {
buildres: exec.result,
};
def PKG_COUNT_MAX: i32 = 2147483647;
def PKG_INITIAL_CAP: i32 = 8;
fn pkgallocstrs(cap: i32) ([]str | nomem) = {
let value: []str = alloc([], cap: u64)?;
return value;
};
fn pkgallocbytes(cap: i32) ([]u8 | nomem) = {
let value: []u8 = alloc([], cap: u64)?;
return value;
};
fn pkgallocsources(cap: i32) ([]pkgsource | nomem) = {
let value: []pkgsource = alloc([], cap: u64)?;
return value;
};
fn pkgallocfolders(cap: i32) ([]pkgfolder | nomem) = {
let value: []pkgfolder = alloc([], cap: u64)?;
return value;
};
fn pkgallocgroups(cap: i32) ([]pkggroup | nomem) = {
let value: []pkggroup = alloc([], cap: u64)?;
return value;
};
fn pkgallocplans(cap: i32) ([]pkgplan | nomem) = {
let value: []pkgplan = alloc([], cap: u64)?;
return value;
};
fn pkgallocprocesses(cap: i32) ([]exec.process | nomem) = {
let value: []exec.process = alloc([], cap: u64)?;
return value;
};
// Product and directory-plan process states for the bounded coordinator.
def PKGQUEUED: i32 = 0;
def PKGBUILDING: i32 = 1;
@@ -63,13 +102,88 @@ def pkgpoll: time.duration = 1000000i64: time.duration;
type pkgdiscover = struct {
paths: []str,
errors: i32,
fatal: bool,
};
fn pkggrowcap(current: i32, need: i32) i32 = {
if (need < 0) { return -1; };
if (need <= current) { return current; };
let cap: i32 = current;
if (cap == 0) { cap = PKG_INITIAL_CAP; };
for (cap < need) {
if (cap > PKG_COUNT_MAX / 2) {
cap = PKG_COUNT_MAX;
break;
};
cap *= 2;
};
if (cap < need) { return -1; };
return cap;
};
fn pkgappenddiscovered(st: *pkgdiscover, path: str) bool = {
if (st.paths.len == PKG_COUNT_MAX) {
pkgputln(os.STDERR_FILENO,
"wwtest package: package graph is too large");
st.errors += 1;
st.fatal = true;
return false;
};
let need: i32 = st.paths.len + 1;
if (need > st.paths.cap) {
let cap: i32 = pkggrowcap(st.paths.cap, need);
if (cap < 0) {
pkgputln(os.STDERR_FILENO,
"wwtest package: package graph is too large");
st.errors += 1;
st.fatal = true;
return false;
};
let allocation: ([]str | nomem) = pkgallocstrs(cap);
let next: []str;
match (allocation) {
case let value: []str => next = value;
case nomem => {
pkgputln(os.STDERR_FILENO,
"wwtest package: out of memory");
st.errors += 1;
st.fatal = true;
return false;
};
};
let n: i32 = st.paths.len;
next.len = cap;
let i: i32 = 0;
for (i < n) { next[i] = st.paths[i]; i += 1; };
next.len = n;
if (st.paths.ptr != nil) {
os.free(st.paths.ptr: *void,
(st.paths.cap: u64) * (size(str): u64));
};
st.paths = next;
};
append(st.paths, path);
return true;
};
// Preserve the caller's toolchain environment while pinning the locale and
// temporary directory used by the current build plan or test product.
fn toolenv(tmpdir: str) []str = {
fn toolenv(tmpdir: str, out: *[]str) bool = {
let inherited: []str = os.getenvs();
let env: []str = alloc([], (inherited.len + 2): u64)!;
if (inherited.len > PKG_COUNT_MAX - 2) {
pkgputln(os.STDERR_FILENO,
"wwtest package: package graph is too large");
return false;
};
let allocation: ([]str | nomem) = pkgallocstrs(inherited.len + 2);
let env: []str;
match (allocation) {
case let value: []str => env = value;
case nomem => {
pkgputln(os.STDERR_FILENO, "wwtest package: out of memory");
return false;
};
};
let i: i32 = 0;
for (i < inherited.len) {
if (!strings.hasprefix(inherited[i], "TMPDIR=")
@@ -80,7 +194,8 @@ fn toolenv(tmpdir: str) []str = {
};
append(env, "LC_ALL=C");
append(env, strings.concat("TMPDIR=", tmpdir));
return env;
*out = env;
return true;
};
fn pkgwrite(fd: i32, s: str) bool = {
@@ -125,8 +240,24 @@ fn pkgread(path: str, out: *str) bool = {
case let v: i64 => n = v;
case let e: os.oserror => { os.close(fd); return false; };
};
if (n < 0i64) { os.close(fd); return false; };
let b: []u8 = alloc([], (n + 1i64): u64)!;
if (n < 0i64 || n >= PKG_COUNT_MAX: i64) {
os.close(fd);
if (n >= PKG_COUNT_MAX: i64) {
pkgputln(os.STDERR_FILENO,
"wwtest package: package graph is too large");
};
return false;
};
let allocation: ([]u8 | nomem) = pkgallocbytes((n + 1i64): i32);
let b: []u8;
match (allocation) {
case let value: []u8 => b = value;
case nomem => {
os.close(fd);
pkgputln(os.STDERR_FILENO, "wwtest package: out of memory");
return false;
};
};
b.len = (n + 1i64): i32;
let rr: (i64 | os.oserror) = os.readall(fd, b.ptr, n: u64);
os.close(fd);
@@ -275,6 +406,7 @@ fn pkgkeepfile(name: str) bool = {
// canonical planning coalesces it with the target directory. Recursive child
// symlinks remain skipped and symlink source files remain rejected.
fn pkgdiscoverdir(path: str, st: *pkgdiscover, recurse: bool) void = {
if (st.fatal) { return; };
let rootstat: os.filestat;
match (os.lstat(&rootstat, path)) {
case void => void;
@@ -305,7 +437,18 @@ fn pkgdiscoverdir(path: str, st: *pkgdiscover, recurse: bool) void = {
st.errors += 1;
return;
};
let buf: []u8 = alloc([], 8192u64)!;
let bufallocation: ([]u8 | nomem) = pkgallocbytes(8192);
let buf: []u8;
match (bufallocation) {
case let value: []u8 => buf = value;
case nomem => {
os.close(fd);
pkgputln(os.STDERR_FILENO, "wwtest package: out of memory");
st.errors += 1;
st.fatal = true;
return;
};
};
buf.len = 8192;
let n: i64 = os.getdents64(fd, buf.ptr, 8192u64);
for (n > 0i64) {
@@ -335,7 +478,10 @@ fn pkgdiscoverdir(path: str, st: *pkgdiscover, recurse: bool) void = {
"symlink source is not allowed");
st.errors += 1;
} else if (pkgmodeis(fi.mode, os.mode.REG)) {
append(st.paths, child);
if (!pkgappenddiscovered(st, child)) {
os.close(fd);
return;
};
};
};
case let e: os.oserror => {
@@ -351,6 +497,7 @@ fn pkgdiscoverdir(path: str, st: *pkgdiscover, recurse: bool) void = {
case void => {
if (pkgmodeis(fi.mode, os.mode.DIR)) {
pkgdiscoverdir(child, st, true);
if (st.fatal) { os.close(fd); return; };
};
};
case let e: os.oserror => {
@@ -484,7 +631,12 @@ fn pkgremoveall(path: str) bool = {
if (!pkgmodeis(opened.mode, os.mode.DIR)
|| opened.inode != st.inode) { os.close(fd); return false; };
let ok: bool = true;
let buf: []u8 = alloc([], 8192u64)!;
let allocation: ([]u8 | nomem) = pkgallocbytes(8192);
let buf: []u8;
match (allocation) {
case let value: []u8 => buf = value;
case nomem => { os.close(fd); return false; };
};
buf.len = 8192;
let n: i64 = os.getdents64(fd, buf.ptr, 8192u64);
for (n > 0i64) {
@@ -643,9 +795,28 @@ fn pkgreportcommand(kind: str, g: *pkggroup, r: *exec.result) void = {
};
fn pkgstartbuild(p: *pkgplan, groups: []pkggroup, builder: str, includes: []str,
h: *exec.process) void = {
let ba: []str = alloc([],
(12 + (p.end - p.start) * 6 + includes.len * 2): u64)!;
h: *exec.process) bool = {
let nproducts: i32 = p.end - p.start;
let capacity: i32 = 12;
if (nproducts < 0 || nproducts > (PKG_COUNT_MAX - capacity) / 6) {
pkgputln(os.STDERR_FILENO, "wwtest package: package graph is too large");
return false;
};
capacity += nproducts * 6;
if (includes.len > (PKG_COUNT_MAX - capacity) / 2) {
pkgputln(os.STDERR_FILENO, "wwtest package: package graph is too large");
return false;
};
capacity += includes.len * 2;
let allocation: ([]str | nomem) = pkgallocstrs(capacity);
let ba: []str;
match (allocation) {
case let value: []str => ba = value;
case nomem => {
pkgputln(os.STDERR_FILENO, "wwtest package: out of memory");
return false;
};
};
append(ba, builder);
append(ba, "test");
append(ba, "-c");
@@ -682,10 +853,12 @@ fn pkgstartbuild(p: *pkgplan, groups: []pkggroup, builder: str, includes: []str,
append(ba, p.workdir);
};
append(ba, p.dir);
let env: []str;
if (!toolenv(p.root, &env)) { return false; };
let bcmd: exec.command;
bcmd.path = builder;
bcmd.argv = ba;
bcmd.env = toolenv(p.root);
bcmd.env = env;
bcmd.dir = "";
bcmd.stdoutpath = p.buildout;
bcmd.stderrpath = p.builderr;
@@ -693,21 +866,37 @@ fn pkgstartbuild(p: *pkgplan, groups: []pkggroup, builder: str, includes: []str,
bcmd.deadline.nsec = 0i64;
bcmd.grace = 0i64: time.duration;
exec.start(h, &bcmd);
return true;
};
fn pkgstartrun(g: *pkggroup, filters: []str, timeoutarg: str,
list: bool, h: *exec.process) void = {
let ra: []str = alloc([], (filters.len + 4): u64)!;
list: bool, h: *exec.process) bool = {
if (filters.len > PKG_COUNT_MAX - 4) {
pkgputln(os.STDERR_FILENO,
"wwtest package: package graph is too large");
return false;
};
let allocation: ([]str | nomem) = pkgallocstrs(filters.len + 4);
let ra: []str;
match (allocation) {
case let value: []str => ra = value;
case nomem => {
pkgputln(os.STDERR_FILENO, "wwtest package: out of memory");
return false;
};
};
append(ra, g.bin);
append(ra, strings.concat("-package=", g.pkg));
if (list) { append(ra, "-list"); };
if (timeoutarg.len != 0) { append(ra, timeoutarg); };
let i: i32 = 0;
for (i < filters.len) { append(ra, filters[i]); i += 1; };
let env: []str;
if (!toolenv(g.root, &env)) { return false; };
let rcmd: exec.command;
rcmd.path = g.bin;
rcmd.argv = ra;
rcmd.env = toolenv(g.root);
rcmd.env = env;
rcmd.dir = "";
rcmd.stdoutpath = g.runout;
rcmd.stderrpath = g.runerr;
@@ -715,6 +904,7 @@ fn pkgstartrun(g: *pkggroup, filters: []str, timeoutarg: str,
rcmd.deadline.nsec = 0i64;
rcmd.grace = 0i64: time.duration;
exec.start(h, &rcmd);
return true;
};
fn pkgproductbuilt(g: *pkggroup) bool = {
@@ -735,6 +925,7 @@ fn pkgemitgroup(g: *pkggroup, compileonly: bool) bool = {
pkgputln(os.STDOUT_FILENO, g.bin);
return true;
};
if (g.runstartfailed) { return false; };
let runstdout: str;
let runstderr: str;
if (!pkgread(g.runout, &runstdout) || !pkgread(g.runerr, &runstderr)) {
@@ -791,9 +982,39 @@ export fn packagecommand(args: []str) int = {
let list: bool = false;
let jobs: i32 = 1;
let afterdash: bool = false;
let roots: []str = alloc([], 2u64)!;
let filters: []str = alloc([], (args.len + 1): u64)!;
let includes: []str = alloc([], (args.len + 1): u64)!;
if (args.len == PKG_COUNT_MAX) {
pkgputln(os.STDERR_FILENO,
"wwtest package: package graph is too large");
return 1;
};
let argcapacity: i32 = args.len + 1;
let rootallocation: ([]str | nomem) = pkgallocstrs(argcapacity);
let roots: []str;
match (rootallocation) {
case let value: []str => roots = value;
case nomem => {
pkgputln(os.STDERR_FILENO, "wwtest package: out of memory");
return 1;
};
};
let filterallocation: ([]str | nomem) = pkgallocstrs(argcapacity);
let filters: []str;
match (filterallocation) {
case let value: []str => filters = value;
case nomem => {
pkgputln(os.STDERR_FILENO, "wwtest package: out of memory");
return 1;
};
};
let includeallocation: ([]str | nomem) = pkgallocstrs(argcapacity);
let includes: []str;
match (includeallocation) {
case let value: []str => includes = value;
case nomem => {
pkgputln(os.STDERR_FILENO, "wwtest package: out of memory");
return 1;
};
};
let timeoutarg: str = "";
let outname: str = "";
let workroot: str = "";
@@ -925,9 +1146,10 @@ export fn packagecommand(args: []str) int = {
};
let ds: pkgdiscover;
let discoveredpaths: []str = alloc([], 64u64)!;
ds.paths = discoveredpaths;
let emptypaths: []str;
ds.paths = emptypaths;
ds.errors = 0;
ds.fatal = false;
pkgdiscoverdir(discoverroot, &ds, recurse);
if (ds.errors != 0) { return 1; };
pkgsort(ds.paths);
@@ -941,7 +1163,16 @@ export fn packagecommand(args: []str) int = {
return 1;
};
let srcs: []pkgsource = alloc([], ds.paths.len: u64)!;
let sourceallocation: ([]pkgsource | nomem) =
pkgallocsources(ds.paths.len);
let srcs: []pkgsource;
match (sourceallocation) {
case let value: []pkgsource => srcs = value;
case nomem => {
pkgputln(os.STDERR_FILENO, "wwtest package: out of memory");
return 1;
};
};
i = 0;
for (i < ds.paths.len) {
let body: str;
@@ -959,7 +1190,16 @@ export fn packagecommand(args: []str) int = {
i += 1;
};
let folders: []pkgfolder = alloc([], srcs.len: u64)!;
let folderallocation: ([]pkgfolder | nomem) =
pkgallocfolders(srcs.len);
let folders: []pkgfolder;
match (folderallocation) {
case let value: []pkgfolder => folders = value;
case nomem => {
pkgputln(os.STDERR_FILENO, "wwtest package: out of memory");
return 1;
};
};
i = 0;
for (i < srcs.len) {
let f: pkgfolder;
@@ -977,7 +1217,16 @@ export fn packagecommand(args: []str) int = {
i = f.end;
};
let groups: []pkggroup = alloc([], srcs.len: u64)!;
let groupallocation: ([]pkggroup | nomem) =
pkgallocgroups(srcs.len);
let groups: []pkggroup;
match (groupallocation) {
case let value: []pkggroup => groups = value;
case nomem => {
pkgputln(os.STDERR_FILENO, "wwtest package: out of memory");
return 1;
};
};
i = 0;
for (i < folders.len) {
let f: pkgfolder = folders[i];
@@ -1050,7 +1299,15 @@ export fn packagecommand(args: []str) int = {
"wwtest package: cannot use -o with multiple packages");
return 2;
};
let plans: []pkgplan = alloc([], 1u64)!;
let planallocation: ([]pkgplan | nomem) = pkgallocplans(1);
let plans: []pkgplan;
match (planallocation) {
case let value: []pkgplan => plans = value;
case nomem => {
pkgputln(os.STDERR_FILENO, "wwtest package: out of memory");
return 1;
};
};
let plan: pkgplan;
plan.dir = discoverroot;
plan.sourceroot = canonicalroot;
@@ -1081,19 +1338,48 @@ export fn packagecommand(args: []str) int = {
// Build the complete request in one command-owned package universe. Once
// the union build completes, successful products run under the same global
// -j bound. Emission stays in byte-sorted group order below.
let handles: []exec.process = alloc([], plans.len: u64)!;
let handleallocation: ([]exec.process | nomem) =
pkgallocprocesses(plans.len);
let handles: []exec.process;
match (handleallocation) {
case let value: []exec.process => handles = value;
case nomem => {
pkgputln(os.STDERR_FILENO, "wwtest package: out of memory");
if (!pkgremoveall(tmproot)) {
pkgput(os.STDERR_FILENO,
"wwtest package: cleanup failed; retained ");
pkgputln(os.STDERR_FILENO, tmproot);
};
return 1;
};
};
i = 0;
for (i < plans.len) {
let h: exec.process;
append(handles, h);
i += 1;
};
let runhandles: []exec.process = alloc([], groups.len: u64)!;
let runhandleallocation: ([]exec.process | nomem) =
pkgallocprocesses(groups.len);
let runhandles: []exec.process;
match (runhandleallocation) {
case let value: []exec.process => runhandles = value;
case nomem => {
pkgputln(os.STDERR_FILENO, "wwtest package: out of memory");
if (!pkgremoveall(tmproot)) {
pkgput(os.STDERR_FILENO,
"wwtest package: cleanup failed; retained ");
pkgputln(os.STDERR_FILENO, tmproot);
};
return 1;
};
};
i = 0;
for (i < groups.len) {
let h: exec.process;
append(runhandles, h);
groups[i].state = PKGQUEUED;
groups[i].runstartfailed = false;
i += 1;
};
let planlaunched: i32 = 0;
@@ -1118,11 +1404,16 @@ export fn packagecommand(args: []str) int = {
g.state = PKGDONE;
productcompleted += 1;
} else {
pkgstartrun(g, filters, timeoutarg, list,
&runhandles[gi]);
if (!pkgstartrun(g, filters, timeoutarg, list,
&runhandles[gi])) {
g.runstartfailed = true;
g.state = PKGDONE;
productcompleted += 1;
} else {
g.state = PKGRUNNING;
active += 1;
};
};
filling = true;
break;
};
@@ -1130,8 +1421,15 @@ export fn packagecommand(args: []str) int = {
};
};
if (!filling && planlaunched < plans.len) {
pkgstartbuild(&plans[planlaunched], groups, builder, includes,
&handles[planlaunched]);
if (!pkgstartbuild(&plans[planlaunched], groups, builder,
includes, &handles[planlaunched])) {
if (!pkgremoveall(tmproot)) {
pkgput(os.STDERR_FILENO,
"wwtest package: cleanup failed; retained ");
pkgputln(os.STDERR_FILENO, tmproot);
};
return 1;
};
plans[planlaunched].state = PKGBUILDING;
active += 1;
planlaunched += 1;

1400
selfhost/cmd/ww/main.ww
View File

File diff suppressed because it is too large Load Diff