From 0aff801def79579434979c2ad92b5c4a3eb52429 Mon Sep 17 00:00:00 2001 From: Hojun-Cho Date: Thu, 13 Aug 2026 02:48:12 +0900 Subject: [PATCH] ww: canonicalize directory package identities --- cmd/ww/main.c | 322 ++++++++++++++++++++++----- docs/build-system.md | 156 +++++++++---- internal/wwpackage/package.ww | 59 ++++- selfhost/cmd/ww/main.ww | 407 +++++++++++++++++++++++++++++----- 4 files changed, 781 insertions(+), 163 deletions(-) diff --git a/cmd/ww/main.c b/cmd/ww/main.c index a81b17d7..f34e491c 100644 --- a/cmd/ww/main.c +++ b/cmd/ww/main.c @@ -107,14 +107,14 @@ run_test_bin(const char *bin, const char *pattern) * prevents delegation recursion. */ static int exec_package_tests(int argc, char **argv, const char *target, - const char *resolved, int add_dot) + const char *resolved, const char *root_identity, int add_dot) { const char *override = getenv("WW_WWTEST"); char fallback[1024]; const char *prog = override && override[0] ? override : fallback; if (prog == fallback) snprintf(fallback, sizeof fallback, "%s/wwtest", self_dir); - char **xargv = calloc((size_t)argc + 6, sizeof *xargv); + char **xargv = calloc((size_t)argc + 8, sizeof *xargv); if (xargv == NULL) { fputs("ww test: cannot allocate package coordinator arguments\n", stderr); @@ -125,6 +125,10 @@ exec_package_tests(int argc, char **argv, const char *target, xargv[n++] = "package"; xargv[n++] = "--ww-driver"; xargv[n++] = (char *)self_path; + if (root_identity != NULL) { + xargv[n++] = "--ww-root-identity"; + xargv[n++] = (char *)root_identity; + } for (int i = 0; i < argc; i++) { if (add_dot && !dotted && strcmp(argv[i], "--") == 0) { xargv[n++] = "."; @@ -148,6 +152,8 @@ struct ImportSet { int n, cap; }; +#define SEP_LOCAL_IMPORT_PREFIX "__wwlocal" + static int import_seen(struct ImportSet *s, const char *path) { @@ -178,6 +184,14 @@ import_path_form(const char *name, char *out, size_t outsz) out[i] = '\0'; } +static int +reserved_import_path(const char *name) +{ + size_t n = strlen(SEP_LOCAL_IMPORT_PREFIX); + return strncmp(name, SEP_LOCAL_IMPORT_PREFIX, n) == 0 + && (name[n] == '\0' || name[n] == '.'); +} + /* An import path names one directory package. There is deliberately no * /.ww branch here: literal or searched single-file roots are a * CLI compatibility concern handled by locate_module, never an import edge. */ @@ -554,6 +568,7 @@ struct sepgraph { struct sepproduct { const char *dir; const char *out; + const char *identity; /* explicit canonical lookup identity, if any */ const char *test_package; const char *status; char artifact[64]; @@ -607,6 +622,22 @@ sep_diag_directory_identities(const char *entry, const char *a, const char *b) entry, a, b); } +static int sep_import_component(const char *, size_t); + +static int +sep_import_base_valid(const char *path) +{ + const char *p = path; + while (*p != '\0') { + const char *dot = strchr(p, '.'); + size_t n = dot != NULL ? (size_t)(dot - p) : strlen(p); + if (!sep_import_component(p, n)) return 0; + if (dot == NULL) return 1; + p = dot + 1; + } + return 0; +} + /* Bind the canonical ordinary import identity of one provisional directory * action. The action's compiler path is derived from that base and its semantic * variant; neither requested-root state nor artifact naming participates. */ @@ -620,6 +651,10 @@ sep_bind_import_base(struct sepgraph *g, int pi, const char *base) SEP_IMPORT_PATH_MAX - 1); return -1; } + if (!reserved_import_path(base) && !sep_import_base_valid(base)) { + fprintf(stderr, "ww: invalid package path %s\n", base); + return -1; + } if (p->import_base[0] != '\0') { if (strcmp(p->import_base, base) == 0) return 0; g->identity_failed = 1; @@ -996,12 +1031,6 @@ sep_external_production_name(const struct seppkg *pkg, const char *path, && strcmp(pkg->test_package + n, "_test") == 0; } -static int -sep_external_production_import(const struct seppkg *pkg, const char *path) -{ - return sep_external_production_name(pkg, path, 0); -} - /* Canonical bindings make a shared package independent of which selected * root reaches it first. Directory bindings and the raw-file inline * compatibility binding are part of the package action's source meaning. */ @@ -1126,6 +1155,11 @@ sep_scan_file(struct sepgraph *g, int pi, const char *file, const char *name = u->usepath ? u->usepath : u->str; if (previous && strcmp(previous, name) == 0) continue; previous = name; + if (reserved_import_path(name)) { + errorf(u->pos, "package path %s is reserved", name); + rc = -1; + break; + } if (strlen(name) >= sizeof g->pkg[0].path) { errorf(u->pos, "import path is too long (limit %zu bytes)", sizeof g->pkg[0].path - 1); @@ -1140,16 +1174,21 @@ sep_scan_file(struct sepgraph *g, int pi, const char *file, break; } char ipath[1024]; - int external_production = sep_external_production_import( - &g->pkg[pi], name); - int located = 0; - if (external_production) { - snprintf(ipath, sizeof ipath, "%s", g->pkg[pi].entry); - located = 1; - } else { + int external_production = 0; + /* A selected external logical root is already an exact resolved + * path/directory pair. Its source import of that same full ordinary + * identity reuses the colocated production action. No declaration leaf + * or unrelated cached package can override normal context lookup. */ + const char *bound = g->pkg[pi].variant == SEP_VARIANT_EXTERNAL + && g->pkg[pi].import_base[0] != '\0' + && strcmp(g->pkg[pi].import_base, name) == 0 + ? g->pkg[pi].canon : NULL; + int located = bound != NULL; + if (located) + snprintf(ipath, sizeof ipath, "%s", bound); + else located = locate_import(searchpath, path_form, ipath, sizeof ipath); - } if (!located) { const char *dot = strrchr(name, '.'); const char *leaf = dot ? dot + 1 : name; @@ -1188,7 +1227,7 @@ sep_scan_file(struct sepgraph *g, int pi, const char *file, external_production = 1; if (self && !external_production) { const char *owner = g->pkg[pi].path[0] - ? g->pkg[pi].path : g->pkg[pi].name; + ? g->pkg[pi].path : g->pkg[pi].canon; errorf(u->pos, "self-import: package '%s' cannot import itself", owner[0] ? owner : "(root)"); rc = -1; @@ -1399,7 +1438,7 @@ sep_load_pkg(struct sepgraph *g, int pi, int context) if (rc < 0) { fprintf(stderr, "ww: package %s resolves imports differently in %s and %s\n", - g->pkg[pi].path[0] ? g->pkg[pi].path : g->pkg[pi].name, + g->pkg[pi].path[0] ? g->pkg[pi].path : g->pkg[pi].canon, g->context[g->pkg[pi].emit_context].root, g->context[context].root); } @@ -1430,13 +1469,177 @@ sep_load_pkg(struct sepgraph *g, int pi, int context) return 0; } -/* Finalize every provisional directory root after source discovery but before - * generated-main construction or compilation. A source import binding wins; - * otherwise a manifest-free literal root uses its validated declared package - * name (external p_test roots bind the ordinary base p). */ +static int +sep_import_component(const char *s, size_t n) +{ + if (n == 0 || !((s[0] >= 'a' && s[0] <= 'z') + || (s[0] >= 'A' && s[0] <= 'Z') || s[0] == '_')) + return 0; + for (size_t i = 1; i < n; i++) + if (!((s[i] >= 'a' && s[i] <= 'z') + || (s[i] >= 'A' && s[i] <= 'Z') + || (s[i] >= '0' && s[i] <= '9') || s[i] == '_')) + return 0; + return kwlookup(s, (u64)n) == TK_NONE; +} + +static int +sep_import_path_from_relative(const char *rel, char *out, size_t outsz) +{ + size_t off = 0; + const char *p = rel; + while (*p != '\0') { + const char *slash = strchr(p, '/'); + size_t n = slash ? (size_t)(slash - p) : strlen(p); + if (!sep_import_component(p, n)) return 0; + if (off + n + (slash != NULL) + 1 > outsz) return -1; + memcpy(out + off, p, n); + off += n; + if (slash == NULL) break; + out[off++] = '.'; + p = slash + 1; + } + out[off] = '\0'; + return off != 0; +} + +/* A reverse candidate is authoritative only when the normal ordered forward + * lookup selects this exact canonical directory. This prevents a later or + * nested source root from manufacturing an alias shadowed by an earlier root. */ +static int +sep_reverse_import_base(const struct sepgraph *g, const struct seppkg *pkg, + int context, char *out, size_t outsz) +{ + const char *searchpath = g->context[context].searchpath; + const char *p = searchpath; + while (*p != '\0') { + const char *e = strchr(p, ':'); + size_t n = e ? (size_t)(e - p) : strlen(p); + char *root = malloc(n + 1); + if (root == NULL) return -1; + memcpy(root, p, n); + root[n] = '\0'; + char *canon = n == 0 ? NULL : realpath(root, NULL); + free(root); + if (canon != NULL) { + size_t rn = strlen(canon); + const char *rel = NULL; + if (rn == 1 && canon[0] == '/' && pkg->canon[0] == '/' + && pkg->canon[1] != '\0') + rel = pkg->canon + 1; + else if (strncmp(pkg->canon, canon, rn) == 0 + && pkg->canon[rn] == '/' && pkg->canon[rn + 1] != '\0') + rel = pkg->canon + rn + 1; + if (rel != NULL) { + int ir = sep_import_path_from_relative(rel, out, outsz); + if (ir < 0) { + fprintf(stderr, + "ww: package path is too long (limit %d bytes)\n", + SEP_IMPORT_PATH_MAX - 1); + free(canon); + return -1; + } + if (ir > 0 && reserved_import_path(out)) ir = 0; + if (ir > 0) { + char located[1024]; + if (locate_import(searchpath, rel, located, + sizeof located)) { + char *selected = realpath(located, NULL); + int same = selected != NULL + && strcmp(selected, pkg->canon) == 0; + free(selected); + if (same) { free(canon); return 1; } + } + } + } + free(canon); + } + if (!e) break; + p = e + 1; + } + return 0; +} + +static int +sep_ordinary_declared_name(const struct seppkg *p, char *out, size_t outsz) +{ + if (p->name[0] == '\0') return -1; + size_t n = strlen(p->name); + if (p->variant == SEP_VARIANT_EXTERNAL) { + if (n <= 5 || strcmp(p->name + n - 5, "_test") != 0) { + fprintf(stderr, + "ww: package-test selector does not name an external package\n"); + return -1; + } + n -= 5; + } + if (n + 1 > outsz) return -1; + memcpy(out, p->name, n); + out[n] = '\0'; + return 0; +} + +/* The reserved local namespace is reversible, so filesystem identity never + * depends on a hash, request order, output name, or another selected package. */ +static int +sep_local_import_base(const struct seppkg *p, char *out, size_t outsz) +{ + char leaf[sizeof p->name]; + if (sep_ordinary_declared_name(p, leaf, sizeof leaf) < 0) return -1; + size_t off = 0; + int n = snprintf(out, outsz, "%s.p", SEP_LOCAL_IMPORT_PREFIX); + if (n < 0 || (size_t)n >= outsz) return -1; + off = (size_t)n; + static const char hex[] = "0123456789abcdef"; + for (const unsigned char *s = (const unsigned char *)p->canon; + *s != '\0'; s++) { + unsigned char c = *s; + if ((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') + || (c >= '0' && c <= '9')) { + if (off + 1 >= outsz) return -1; + out[off++] = (char)c; + } else if (c == '_' || c == '/') { + if (off + 2 >= outsz) return -1; + out[off++] = '_'; + out[off++] = c == '_' ? 'u' : 's'; + } else { + if (off + 4 >= outsz) return -1; + out[off++] = '_'; + out[off++] = 'x'; + out[off++] = hex[c >> 4]; + out[off++] = hex[c & 15]; + } + } + size_t ln = strlen(leaf); + if (off + 1 + ln + 1 > outsz) return -1; + out[off++] = '.'; + memcpy(out + off, leaf, ln + 1); + return 0; +} + +/* Finalization verifies every reached context before generated-main creation. + * Source bindings and explicit lookup identities remain authoritative; a + * literal root either round-trips through an active root or receives the + * reserved reversible local identity. */ static int sep_finalize_directory_identities(struct sepgraph *g) { + for (int pi = 0; pi < g->n; pi++) { + struct seppkg *p = &g->pkg[pi]; + if (!p->is_dir || p->generated_main || p->failed || !p->loaded + || p->role == SEP_ROLE_TEST_SUPPORT + || p->import_base[0] != '\0') + continue; + for (int ci = 0; ci < g->ncontext; ci++) { + if (p->context_state[ci] != 2) continue; + char candidate[SEP_IMPORT_PATH_MAX]; + int found = sep_reverse_import_base(g, p, ci, candidate, + sizeof candidate); + if (found < 0) return -1; + if (found > 0 && sep_bind_import_base(g, pi, candidate) < 0) + return -1; + } + } for (int pi = 0; pi < g->n; pi++) { struct seppkg *p = &g->pkg[pi]; if (!p->is_dir || p->generated_main || p->failed || !p->loaded @@ -1453,27 +1656,15 @@ sep_finalize_directory_identities(struct sepgraph *g) base = g->pkg[i].import_base; break; } - char fallback[SEP_IMPORT_PATH_MAX]; + char local[SEP_IMPORT_PATH_MAX]; if (base == NULL) { - if (p->name[0] == '\0') { + if (sep_local_import_base(p, local, sizeof local) < 0) { fprintf(stderr, - "ww: package directory %s has no canonical import identity\n", - p->entry); + "ww: local package identity is too long (limit %d bytes)\n", + SEP_IMPORT_PATH_MAX - 1); return -1; } - if (p->variant == SEP_VARIANT_EXTERNAL) { - size_t n = strlen(p->name); - if (n <= 5 || strcmp(p->name + n - 5, "_test") != 0) { - fprintf(stderr, - "ww: package-test selector does not name an external package\n"); - return -1; - } - memcpy(fallback, p->name, n - 5); - fallback[n - 5] = '\0'; - } else { - snprintf(fallback, sizeof fallback, "%s", p->name); - } - base = fallback; + base = local; } if (sep_bind_import_base(g, pi, base) < 0) return -1; } @@ -1816,7 +2007,7 @@ static void workdir_stamp_text(char *buf, size_t bufsz, int is_test, int emit_asm) { snprintf(buf, bufsz, "ww workdir fmt %d mode %s asm %d\n", - is_test ? 8 : 7, is_test ? "test" : "build", emit_asm); + is_test ? 9 : 8, is_test ? "test" : "build", emit_asm); } /* A stale global builder identity invalidates every committed unit voucher in @@ -1860,8 +2051,7 @@ invalidate_workdir_units(const char *scratch) * `-w` workdir with content-identity package reuse. */ static int build_one_sep_impl(const char *src, int entry_is_dir, - const char *root_identity, const char *out, - const char *objstem, const char *extra_includes, + const char *out, const char *objstem, const char *extra_includes, const struct seplinkflags *linkflags, int package_only, int is_test, struct sepproduct *products, int nproducts, int emit_asm, const char *workdir, char *scratchout, size_t scratchoutsz, @@ -1974,8 +2164,6 @@ build_one_sep_impl(const char *src, int entry_is_dir, if (g == NULL) return 1; g->support_context = -1; if (graphout) *graphout = g; - const char *rootpath = package_only && root_identity - ? root_identity : ""; int support_for[SEP_MAXPRODUCT]; for (int i = 0; i < nproducts; i++) support_for[i] = -1; for (int i = 0; i < nproducts; i++) { @@ -2000,6 +2188,8 @@ build_one_sep_impl(const char *src, int entry_is_dir, if (products[i].context < 0) return 1; const char *selector = products[i].variant == SEP_VARIANT_PRODUCTION ? NULL : products[i].test_package; + const char *rootpath = products[i].identity != NULL + ? products[i].identity : ""; products[i].root = sep_find_or_add_variant(g, rootpath, entry, entry_is_dir, products[i].variant, selector, SEP_ROLE_NORMAL, products[i].artifact, 1); @@ -2477,6 +2667,7 @@ build_one_sep(const char *src, int entry_is_dir, const char *root_identity, struct sepproduct product = { .dir = src, .out = out, + .identity = root_identity, .test_package = test_package, .status = NULL, .artifact = {0}, @@ -2486,7 +2677,7 @@ build_one_sep(const char *src, int entry_is_dir, const char *root_identity, }; if (!package_only && !entry_is_dir) snprintf(product.artifact, sizeof product.artifact, "__root"); - int r = build_one_sep_impl(src, entry_is_dir, root_identity, out, objstem, + int r = build_one_sep_impl(src, entry_is_dir, out, objstem, extra_includes, linkflags, package_only, is_test, &product, 1, emit_asm, workdir, scratch, sizeof scratch, &g); @@ -2520,12 +2711,15 @@ build_one_sep(const char *src, int entry_is_dir, const char *root_identity, * request. Its first output owns the shared cold sepwork tree; every product * remains an independent root compile and link inside that tree. */ static int -build_package_tests(const char *src, const char *extra_includes, - const char *workdir, struct sepproduct *products, int nproducts) +build_package_tests(const char *src, const char *root_identity, + const char *extra_includes, const char *workdir, + struct sepproduct *products, int nproducts) { char scratch[1100] = {0}; struct sepgraph *g = NULL; - int r = build_one_sep_impl(src, 1, NULL, products[0].out, + for (int i = 0; i < nproducts; i++) + products[i].identity = root_identity; + int r = build_one_sep_impl(src, 1, products[0].out, products[0].out, extra_includes, NULL, 0, 1, products, nproducts, 0, workdir, scratch, sizeof scratch, &g); sep_graph_free(g); @@ -2592,6 +2786,7 @@ resolve_module(const char *name, const char *incs, char *out, size_t outsz, return 1; } } + if (reserved_import_path(name)) return 0; char sp[4096]; search_path(incs, sp, sizeof sp); char path_form[256]; @@ -2763,7 +2958,7 @@ do_build(int argc, char **argv) } else { basename_no_ext(resolved, out, sizeof out); } - const char *root_identity = package_only && !literal ? src : NULL; + const char *root_identity = !literal && is_dir ? src : NULL; return build_one_sep(resolved, is_dir, root_identity, out, objstem, incs, &linkflags, package_only, 0, SEP_VARIANT_PRODUCTION, NULL, emit_asm, 1, workdir); @@ -2781,6 +2976,8 @@ do_run(int argc, char **argv) outflag, sizeof outflag, NULL, 0, &src, NULL, NULL); if (next < 0) return 2; if (src == NULL) src = "."; + struct stat requested; + int literal = stat(src, &requested) == 0; char resolved[1024]; int is_dir = 0; if (!resolve_module(src, incs, resolved, sizeof resolved, &is_dir)) { @@ -2796,7 +2993,9 @@ do_run(int argc, char **argv) snprintf(tmp, sizeof tmp, "%s/main", tmpdir); /* The freshly acquired directory owns both the executable and the * adjacent main.sepwork tree. Nothing outside it is adopted or removed. */ - if (build_one_sep(resolved, is_dir, NULL, tmp, tmp, incs, &linkflags, + const char *root_identity = !literal && is_dir ? src : NULL; + if (build_one_sep(resolved, is_dir, root_identity, tmp, tmp, incs, + &linkflags, 0, 0, SEP_VARIANT_PRODUCTION, NULL, 0, 0, NULL) != 0) { if (unlink(tmp) != 0 && errno != ENOENT) fputs("ww: cannot remove temporary output\n", stderr); @@ -2860,6 +3059,7 @@ do_test(int argc, char **argv) char workdir[1024] = {0}; int packageopts = 0; int afterdash = 0; + const char *request_identity = NULL; /* #17: an optional second positional after the target is a fnmatch * name-filter pattern, forwarded to the test binary as argv[1]. Only * meaningful for a single test file/module — rejected in dir mode. */ @@ -2889,6 +3089,15 @@ do_test(int argc, char **argv) "%s%s", n ? ":" : "", dir); } else if (strcmp(argv[i], "-c") == 0) { compileonly = 1; + } else if (strcmp(argv[i], "--ww-root-identity") == 0) { + if (i + 1 >= argc || request_identity != NULL + || argv[i + 1][0] == '\0' + || reserved_import_path(argv[i + 1])) { + fprintf(stderr, + "ww test: invalid --ww-root-identity\n"); + return 2; + } + request_identity = argv[++i]; } else if (strcmp(argv[i], "--ww-package-test") == 0) { if (i + 5 >= argc || nproducts >= SEP_MAXPRODUCT) { fprintf(stderr, @@ -3041,7 +3250,7 @@ do_test(int argc, char **argv) } /* -w forwards: the coordinator keys one persistent driver * workdir for the complete selected test request. */ - return exec_package_tests(argc, argv, src, NULL, 0); + return exec_package_tests(argc, argv, src, NULL, NULL, 0); } struct stat st; if (stat(target, &st) != 0) { @@ -3076,10 +3285,12 @@ do_test(int argc, char **argv) "ww test: package-test products need -c\n"); return 2; } - return build_package_tests(resolved, incs, workdir, + return build_package_tests(resolved, request_identity, + incs, workdir, products, nproducts); } - return exec_package_tests(argc, argv, src, resolved, 0); + return exec_package_tests(argc, argv, src, resolved, + request_identity != NULL ? request_identity : target, 0); } if (packageopts) { fprintf(stderr, @@ -3232,10 +3443,11 @@ do_test(int argc, char **argv) "ww test: package-test products need -c\n"); return 2; } - return build_package_tests(target, incs, workdir, + return build_package_tests(target, request_identity, incs, workdir, products, nproducts); } - return exec_package_tests(argc, argv, src, NULL, src == NULL); + return exec_package_tests(argc, argv, src, NULL, request_identity, + src == NULL); } int diff --git a/docs/build-system.md b/docs/build-system.md index 07d56df2..58c380a6 100644 --- a/docs/build-system.md +++ b/docs/build-system.md @@ -2837,24 +2837,28 @@ binary `.wwe` format described above, but the separate direct-input ownership boundary is live in production Cstage and WWstage compilers and drivers. An ordinary executable directory root is one normal package action. Its -declared package identity tags its owner-only unit; it receives only direct -exports, emits `.wwi`, `.o`, and a deterministic `.a`, and is compiled exactly -once. The narrow compiler `--entry` flag controls only bare `main` codegen and is -independent of export production. The linker receives that root archive first, -then the complete reachable package-archive closure and runtime archive; it -never receives `.wwi`. The linkers seed `main` before archive selection, so the -existing WWAR member protocol needs no special root object or format change. +finalized canonical import identity tags its owner-only unit; it receives only +direct exports, emits `.wwi`, `.o`, and a deterministic `.a`, and is compiled +exactly once. The declared package name only validates the last component of +that identity. The narrow compiler `--entry` flag controls only bare `main` +codegen and is independent of export production. The linker receives that root +archive first, then the complete reachable package-archive closure and runtime +archive; it never receives `.wwi`. The linkers seed `main` before archive +selection, so the existing WWAR member protocol needs no special root object or +format change. `ww build -p -o lib.a DIR` explicitly requests a non-main package product: it emits a deterministic archive at `lib.a` and its compiler interface at `lib.a.wwi`, without invoking the linker. A logical target retains its full identity (`ww build -p -I ROOT -o bar.a foo.bar` emits `foo.bar.*` symbols), -while a literal directory uses its declared leaf package. Package output -requires a directory and `-p` cannot be combined with assembly-only `-S`. Two -cold builds with identical inputs are required to produce byte-identical -requested products. Compiler intrinsics keep their package-mode runtime ABI -independent of transitive source interfaces (for example, `alloc` lowers to the -runtime allocator without requiring an `rt.wwi` compiler input). +while a literal directory is reverse-resolved through the active source roots +or receives the deterministic local identity described below. Its declared +leaf can never invent or truncate that identity. Package output requires a +directory and `-p` cannot be combined with assembly-only `-S`. Two cold builds +with identical inputs are required to produce byte-identical requested +products. Compiler intrinsics keep their package-mode runtime ABI independent +of transitive source interfaces (for example, `alloc` lowers to the runtime +allocator without requiring an `rt.wwi` compiler input). ### 11.7 Implemented directory package-test slice @@ -2904,22 +2908,63 @@ state, discovery role, product ordinal, output path, persistent artifact key, and discovery order never enter the triple. A literal directory root may enter the interner before its full import spelling -is known. It is provisionally interned by canonical directory and variant, -binds immediately if a source import reaches it, and otherwise binds after its -sources establish the validated manifest-free package name. All binding is -finished before generated-main construction or compilation. One bound import -path mapping to two directories and one directory acquiring two incompatible -ordinary import paths are both command-global deterministic errors before any -compiler, assembler, archiver, or linker ambiguity. The same check spans -variants: an external action with ordinary base `p` cannot hide a different -directory's production `p` behind its derived compiler path `p_test`. +is known. It is provisionally interned by canonical directory and variant, and +a later source import of that directory binds and reuses the provisional action. +After all source discovery, but before generated-main construction or any tool +invocation, each still-unbound directory is finalized by this exact algorithm: + +1. For every import-resolution context that reached the directory, walk that + context's roots in its normal forward precedence: the selected package's + directory, explicit `-I` roots in command order, then `WW_SRCLIB` or the + selected toolchain source root. Recursive package-test requests privately + insert their symlink-resolved discovery root before user `-I` roots, so + descendants retain their complete relative identity. +2. Canonicalize each candidate root and require the package directory to be a + strict descendant. Every relative path component must be a non-keyword WW + identifier. Convert separators to dots, then resolve that relative spelling + again through the complete ordered context. Accept it only if ordinary + forward lookup selects the same canonical directory. Thus an earlier shadow + invalidates a name inferred from a later or nested root. +3. Bind the first precedence-valid candidate from each reaching context through + the command-global bidirectional interner. An identity supplied by successful + logical package lookup, such as `-p encoding.utf8`, is already bound and is + preserved exactly. That forward-selected identity is authoritative: reverse + derivation applies only to still-unbound literal roots, so a nested active + root cannot rename an explicitly resolved package. +4. If no active root can represent the directory, bind the reserved, + non-source-importable identity + `__wwlocal.p.`. The + escape is injective and reversible over path bytes: ASCII letters and digits + are copied, `_` becomes `_u`, `/` becomes `_s`, and every other byte becomes + `_xHH` with lowercase hexadecimal. Source imports of `__wwlocal` or any of + its children are rejected, so this command-local identity creates no alias. + +The selected full identity's final component is then validated against the +ordinary declared package name (or against the ordinary leaf obtained by +removing `_test` for the external variant). There is no fallback from an empty +import path to a declaration name. Relative, absolute, and symlink spellings +converge through the canonical directory; two unrelated local directories with +the same declaration therefore remain distinct. One bound import path mapping +to two directories and one directory acquiring two incompatible ordinary +import paths are command-global deterministic errors before any compiler, +assembler, archiver, or linker ambiguity. The same check spans variants: an +external action cannot hide a different directory's production package behind +its derived `_test` compiler path. + +The derivation and diagnostics are implemented symmetrically in +`cmd/ww/main.c` and `selfhost/cmd/ww/main.ww`. The package coordinator in +`internal/wwpackage/package.ww` supplies the canonical recursive discovery root, +preserves an explicitly resolved logical request identity, and keys a persistent +request workdir only by the canonical discovery directory. `w6c` and `wcc` +continue to consume and validate the finalized dotted identity; neither tool +performs directory lookup or introduces a package registry. Artifact publication follows the semantic action instead of product order: -production uses `p`, internal uses `p-internal-test`, external uses -`p_test-external-test`, and their generated mains append `-main`. Generated -package identities are likewise variant-derived, for example -`__wwtestmain.p.internal.main` and -`__wwtestmain.p_test.external.main`. Equivalent products therefore reuse an +production uses the full finalized ordinary identity, internal appends +`-internal-test`, external appends `_test-external-test`, and their generated +mains append `-main`. Generated package identities are likewise derived from +the full variant identity, for example `__wwtestmain.a.foo.internal.main` and +`__wwtestmain.b.foo_test.external.main`. Equivalent products therefore reuse an already-interned directory action and generated-main action and publish through the same persistent-workdir slots regardless of discovery or product order. The narrow raw single-file compatibility path alone retains `__root`. @@ -3004,7 +3049,10 @@ arguments accepted by its native linker, while Cstage preserves the equivalent split forms. Generated artifact paths are bounds-checked before any unit is opened, so distinct root keys cannot alias by truncation. Recursive discovery groups by physical directory before sorting filenames, and -one stable escaped request key names the persistent command work directory. +one stable escape of the canonical discovery directory names the persistent +command work directory. It contains neither a declared package leaf nor a +product ordinal, so equivalent path spellings and reordered products select the +same request state. Every `*_test.ww` package variant is built even when a file only declares helpers and contains no `@test`, so its package clause and imports are still checked by the shared loader. Every built variant is run, and a successful @@ -3128,7 +3176,9 @@ stamp. A warm invocation byte-compares all applicable live executables before considering any committed unit reusable. A missing or changed driver copy invalidates every `.unit.ww` voucher before compilation; old artifacts may remain recoverable, but none can be reused without a freshly committed unit. -The workdir format revisions are 6 for ordinary builds and 7 for tests. +The workdir format revisions are 8 for ordinary builds and 9 for tests. The +identity-finalization change bumps both formats so no leaf-keyed unit voucher +can be reused as a full-path package action. This closes a real hidden-input boundary. The driver, rather than `w6c`, owns canonical directory interning, source-derived graph construction, owner-only @@ -3231,6 +3281,19 @@ export with its committed predecessor before allowing a direct importer to reuse owner-identical artifacts, retaining correctness without a new cache schema or identity record. +The canonical-root regressions additionally prove the following in both +stages: a literal root under one import root publishes its complete dotted +identity; logical, literal absolute, equivalent, relative, and symlink routes +emit byte-identical `.unit.ww`, `.wwi`, and `.a`; root-only and combined +root/import requests emit those same bytes; dependency-first and root-first +discovery each compile the shared production action once; recursive `a/foo` +and `b/foo` directories declaring the same `package foo` publish distinct +`a.foo` and `b.foo` variants; and two outside-root `package foo` directories +coexist in one command under distinct reversible local identities. Equivalent +recursive spellings reuse the same persistent request directory without new +compilation, while source imports of the reserved local namespace reject before +tool invocation. + The exact-argv regression uses the real diamond `base -> {left,right} -> root`. It proves one compile per node; no input for `base`; only `base.wwi` for each middle node; only sorted `left.wwi` and @@ -3242,12 +3305,11 @@ Cstage builds and two clean WWstage builds. The existing directory-package variant regression checks separate production, internal, external, and generated-main actions, exact generated-main direct variant/support exports, canonical production reuse across ordinary and test products, and archive-only -link closures. It also reverses equivalent product descriptors, proves a -selected ordinary root imported by another selected product compiles once, -and changes a shared direct export in persistent workdirs to prove propagation -through direct importers stops at the first byte-identical regenerated export. -Both stages compile and run those actions with owner-only units and -byte-identical artifacts. +link closures. It also reverses equivalent product descriptors and compares +exact compiler and linker trace bytes, then changes a shared direct export in +persistent workdirs to prove propagation through direct importers stops at the +first byte-identical regenerated export. Both stages compile and run those +actions with owner-only units and byte-identical artifacts. The pinned official Go 1.26.5 tag (commit `c19862e5f8415b4f24b189d065ed739517c548ba`) supplies the design boundary: @@ -3255,25 +3317,39 @@ The pinned official Go 1.26.5 tag (commit - `go/build` represents one selected directory package with its import path, package name, ordinary files, internal-test files, external-test files, and their imports ([`build.go`, lines 436–493](https://github.com/golang/go/blob/c19862e5f8415b4f24b189d065ed739517c548ba/src/go/build/build.go#L436-L493)). + Those fields remain separate in Go; WW's final-component/name equality is its + existing language validation layered on the canonical identity, not a claim + that Go conflates `Name` with `ImportPath`. Its directory reader is required to return name-sorted entries ([lines 108–111](https://github.com/golang/go/blob/c19862e5f8415b4f24b189d065ed739517c548ba/src/go/build/build.go#L108-L111)), - import lookup selects one directory in search order + and local directory loading reverse-derives a complete import path by checking + `GOROOT/src` first and then `GOPATH` roots in order. A candidate under a later + root is rejected when the same relative path resolves through an earlier root + to another directory; an outside-root directory remains without an ordinary + import path + ([lines 612–665](https://github.com/golang/go/blob/c19862e5f8415b4f24b189d065ed739517c548ba/src/go/build/build.go#L612-L665)). + Forward import lookup selects one directory in search order ([lines 725–767](https://github.com/golang/go/blob/c19862e5f8415b4f24b189d065ed739517c548ba/src/go/build/build.go#L725-L767)), and the selected directory alone is scanned ([lines 859–913](https://github.com/golang/go/blob/c19862e5f8415b4f24b189d065ed739517c548ba/src/go/build/build.go#L859-L913)). The sorted scan assigns each accepted source to that package's ordinary, internal-test, or external-test list ([lines 948–1036](https://github.com/golang/go/blob/c19862e5f8415b4f24b189d065ed739517c548ba/src/go/build/build.go#L948-L1036)). -- `cmd/go/internal/load` expands source imports before recording their - canonical paths +- `cmd/go/internal/load` derives an outside-root local directory's deterministic + pseudo-import path from its slash-form absolute directory and establishes the + package-data cache/promise boundary around that resolved key + ([`pkg.go`, lines 633–647](https://github.com/golang/go/blob/c19862e5f8415b4f24b189d065ed739517c548ba/src/cmd/go/internal/load/pkg.go#L633-L647)). + WW uses the same reserved full-directory principle but a reversible byte + escape, strengthening it so two canonical directory spellings cannot collapse + merely through character sanitization. The Go loader expands source imports + before recording their canonical paths ([`pkg.go`, lines 658–669](https://github.com/golang/go/blob/c19862e5f8415b4f24b189d065ed739517c548ba/src/cmd/go/internal/load/pkg.go#L658-L669)). It resolves canonical path and directory before consulting the package-data cache ([lines 833–842](https://github.com/golang/go/blob/c19862e5f8415b4f24b189d065ed739517c548ba/src/cmd/go/internal/load/pkg.go#L833-L842), [lines 863–911](https://github.com/golang/go/blob/c19862e5f8415b4f24b189d065ed739517c548ba/src/cmd/go/internal/load/pkg.go#L863-L911)), and the command-global package cache returns the existing package pointer for a later root or import of the resolved identity - ([lines 633–647](https://github.com/golang/go/blob/c19862e5f8415b4f24b189d065ed739517c548ba/src/cmd/go/internal/load/pkg.go#L633-L647), - [lines 757–775](https://github.com/golang/go/blob/c19862e5f8415b4f24b189d065ed739517c548ba/src/cmd/go/internal/load/pkg.go#L757-L775)). + ([lines 757–775](https://github.com/golang/go/blob/c19862e5f8415b4f24b189d065ed739517c548ba/src/cmd/go/internal/load/pkg.go#L757-L775)). The package's parsed import list becomes its direct package dependencies, rather than a transitive flattening ([lines 433–440](https://github.com/golang/go/blob/c19862e5f8415b4f24b189d065ed739517c548ba/src/cmd/go/internal/load/pkg.go#L433-L440), diff --git a/internal/wwpackage/package.ww b/internal/wwpackage/package.ww index 68c83ab2..1f1fe6d1 100644 --- a/internal/wwpackage/package.ww +++ b/internal/wwpackage/package.ww @@ -38,6 +38,8 @@ type pkggroup = struct { type pkgplan = struct { dir: str, + sourceroot: str, + identity: str, root: str, workdir: str, buildout: str, @@ -438,6 +440,24 @@ fn pkgmakedir(path: str) bool = { return os.mkdir(path, 448i32) == 0; }; +// Resolve a directory to the kernel's symlink-free absolute spelling. The +// coordinator is single-threaded while planning, so the temporary cwd change +// cannot race a child process launch. +fn pkgcanonicaldir(path: str, out: *str) bool = { + let before: []u8 = alloc([], os.PATH_MAX: u64)!; + before.len = os.PATH_MAX; + let bn: i64 = os.getcwd(before.ptr, before.len: u64); + if (bn <= 1i64 || bn > before.len: i64) { return false; }; + if (os.chdir(path) != 0) { return false; }; + let after: []u8 = alloc([], os.PATH_MAX: u64)!; + after.len = os.PATH_MAX; + let an: i64 = os.getcwd(after.ptr, after.len: u64); + let restored: i32 = os.chdir(strings.frombytes(before[0:(bn - 1i64): i32])); + if (restored != 0 || an <= 1i64 || an > after.len: i64) { return false; }; + *out = strings.dup(strings.frombytes(after[0:(an - 1i64): i32])); + return true; +}; + // The coordinator removes only its minted temp root; validating the opened // inode keeps a replaced path from carrying cleanup outside that ownership. fn pkgremoveall(path: str) bool = { @@ -519,9 +539,8 @@ fn pkgworkescape(s: str) str = { return strings.frombytes(out); }; -fn pkgworkkey(dir: str, pkg: str) str = { - return strings.concat("d_", pkgworkescape(dir), "_p_", - pkgworkescape(pkg)); +fn pkgworkkey(dir: str) str = { + return strings.concat("d_", pkgworkescape(dir)); }; fn pkgsetplanpaths(p: *pkgplan, groups: []pkggroup, root: str, index: i32, @@ -531,10 +550,10 @@ fn pkgsetplanpaths(p: *pkgplan, groups: []pkggroup, root: str, index: i32, if (!pkgmakedir(p.root)) { return false; }; p.workdir = ""; if (workroot.len != 0) { - // The request root and first byte-sorted product name make one stable - // escaped key for the command-scoped driver workdir. + // Canonical request-directory identity makes equivalent spellings and + // product reorderings select one command-scoped driver workdir. p.workdir = strings.concat(workroot, "/", - pkgworkkey(p.dir, groups[p.start].pkg)); + pkgworkkey(p.sourceroot)); match (os.mkdirs(p.workdir, 448)) { case void => void; case let e: os.oserror => return false; @@ -601,10 +620,14 @@ fn pkgreportcommand(kind: str, g: *pkggroup, r: *exec.result) void = { fn pkgstartbuild(p: *pkgplan, groups: []pkggroup, builder: str, includes: []str, h: *exec.process) void = { let ba: []str = alloc([], - (8 + (p.end - p.start) * 6 + includes.len * 2): u64)!; + (12 + (p.end - p.start) * 6 + includes.len * 2): u64)!; append(ba, builder); append(ba, "test"); append(ba, "-c"); + if (p.identity.len != 0) { + append(ba, "--ww-root-identity"); + append(ba, p.identity); + }; let i: i32 = p.start; for (i < p.end) { let g: *pkggroup = &groups[i]; @@ -619,6 +642,10 @@ fn pkgstartbuild(p: *pkgplan, groups: []pkggroup, builder: str, includes: []str, append(ba, g.buildok); i += 1; }; + // Recursive descendants derive full identities relative to the canonical + // discovery root using the driver's ordinary ordered import lookup. + append(ba, "-I"); + append(ba, p.sourceroot); let ii: i32 = 0; for (ii < includes.len) { append(ba, "-I"); @@ -745,6 +772,7 @@ export fn packagecommand(args: []str) int = { let timeoutarg: str = ""; let outname: str = ""; let workroot: str = ""; + let requestidentity: str = ""; let builder: str = pkgdefaultbuilder(); let i: i32 = 0; for (i < args.len) { @@ -819,6 +847,16 @@ export fn packagecommand(args: []str) int = { i += 2; continue; }; + if (strings.compare(a, "--ww-root-identity") == 0) { + if (i + 1 >= args.len || args[i + 1].len == 0 + || requestidentity.len != 0) { + pkgusage(); + return 2; + }; + requestidentity = args[i + 1]; + i += 2; + continue; + }; if (a.len != 0 && a[0] == '-') { pkgusage(); return 2; }; append(roots, a); i += 1; @@ -872,6 +910,11 @@ export fn packagecommand(args: []str) int = { pkgfailpath(discoverroot, "directory contains no WW package sources"); return 1; }; + let canonicalroot: str; + if (!pkgcanonicaldir(discoverroot, &canonicalroot)) { + pkgfailpath(discoverroot, "cannot canonicalize package directory"); + return 1; + }; let srcs: []pkgsource = alloc([], ds.paths.len: u64)!; i = 0; @@ -985,6 +1028,8 @@ export fn packagecommand(args: []str) int = { let plans: []pkgplan = alloc([], 1u64)!; let plan: pkgplan; plan.dir = discoverroot; + plan.sourceroot = canonicalroot; + plan.identity = requestidentity; plan.start = 0; plan.end = groups.len; plan.state = PKGQUEUED; diff --git a/selfhost/cmd/ww/main.ww b/selfhost/cmd/ww/main.ww index d4e1f117..3687755f 100644 --- a/selfhost/cmd/ww/main.ww +++ b/selfhost/cmd/ww/main.ww @@ -24,6 +24,7 @@ import syntax; // (os.PATH_MAX: i32 = 4096) — the duplicate u64 def was dropped to close // the cgen #127 mod-mangle attribution bug consumer per rule-7. def CMD_MAX: u64 = 8192u64; +def SEP_LOCAL_IMPORT_PREFIX: str = "__wwlocal"; let selfpath: *u8; @@ -164,6 +165,42 @@ fn owncstr(s: str) *u8 = { return b.ptr; }; +fn reservedimport(s: str) bool = { + let prefix: str = SEP_LOCAL_IMPORT_PREFIX; + if (s.len < prefix.len) { return false; }; + let i: i32 = 0; + for (i < prefix.len) { + if (s[i] != prefix[i]) { return false; }; + i += 1; + }; + return s.len == prefix.len || s[prefix.len] == '.': u8; +}; + +fn reservedimportpath(p: *u8) bool = { + return reservedimport(pathstr(p)); +}; + +// The driver is single-threaded while constructing a graph. Saving and +// restoring cwd gives WWstage the same symlink-resolved absolute directory +// spelling that Cstage obtains from realpath, without adding a library API. +fn canonicaldir(path: str) *u8 = { + let before: []u8 = alloc([], os.PATH_MAX: u64)!; + before.len = os.PATH_MAX; + let bn: i64 = os.getcwd(before.ptr, before.len: u64); + if (bn <= 1i64 || bn > before.len: i64) { return nil; }; + if (os.chdir(path) != 0) { return nil; }; + let after: []u8 = alloc([], os.PATH_MAX: u64)!; + after.len = os.PATH_MAX; + let an: i64 = os.getcwd(after.ptr, after.len: u64); + let restored: i32 = os.chdir(pathstr(before.ptr)); + if (restored != 0) { + cerr("ww: cannot restore current directory\n"); + return nil; + }; + if (an <= 1i64 || an > after.len: i64) { return nil; }; + return arenadupcstr(after.ptr, (an - 1i64): u64); +}; + fn envpath(name: str) *u8 = { match (os.getenv(name)) { case let p: str => { @@ -185,7 +222,7 @@ fn toolpath(selfdir: *u8, envvar: str, name: str) *u8 = { // package.ww roots stay on runsingletest, which is the recursion boundary when // wwtest builds its already-aggregated package binaries. fn execpackagetests(selfdir: *u8, argv: **u8, argc: i32, start: i32, - targetindex: i32, resolved: *u8, adddot: bool) i32 = { + targetindex: i32, resolved: *u8, rootidentity: *u8, adddot: bool) i32 = { let prog: *u8 = joinpathlit(selfdir, "wwtest"); match (os.getenv("WW_WWTEST")) { case let p: str => { @@ -194,7 +231,7 @@ fn execpackagetests(selfdir: *u8, argv: **u8, argc: i32, start: i32, case void => void; }; - let cap: i32 = argc - start + 6; + let cap: i32 = argc - start + 8; let execargv: []*u8 = alloc([], cap: u64)!; execargv.len = cap; let n: i32 = 0; @@ -202,6 +239,10 @@ fn execpackagetests(selfdir: *u8, argv: **u8, argc: i32, start: i32, execargv[n] = "package".ptr; n += 1; execargv[n] = "--ww-driver".ptr; n += 1; execargv[n] = selfpath; n += 1; + if (rootidentity != nil) { + execargv[n] = "--ww-root-identity".ptr; n += 1; + execargv[n] = rootidentity; n += 1; + }; let dotted: bool = false; let i: i32 = start; for (i < argc) { @@ -761,6 +802,7 @@ type seppkg = struct { path: *u8, // compiler/import identity derived from importbase importbase: *u8, // canonical ordinary directory import identity entry: *u8, // resolved package dir (or file, file root), NUL-term + canon: *u8, // canonical location; never package identity artifact: *u8, // stable non-importable variant artifact key name: *u8, // validated declared name; directory packages only testpackage: *u8, @@ -801,6 +843,7 @@ type sepgraph = struct { type sepproduct = struct { dir: *u8, out: *u8, + identity: *u8, testpackage: *u8, status: *u8, artifact: *u8, @@ -843,6 +886,19 @@ fn sepdiagdiridentities(entry: *u8, a: *u8, b: *u8) void = { cerr(" and "); cerr(pathstr(second)); cerr("\n"); }; +fn sepimportbasevalid(base: *u8) bool = { + let total: u64 = cstrlen(base); + let pos: u64 = 0u64; + for (pos < total) { + let end: u64 = pos; + for (end < total && base[end] != '.': u8) { end += 1u64; }; + if (!sepimportcomponent(base + pos, end - pos)) { return false; }; + if (end == total) { return true; }; + pos = end + 1u64; + }; + return false; +}; + // Bind a provisional directory action to its canonical ordinary import // identity. The compiler path is derived from that base and the semantic // variant; root/product/artifact state never participates. @@ -851,6 +907,10 @@ fn sepbindimportbase(g: *sepgraph, pi: i32, base: *u8) i32 = { cerr("ww: package path is too long (limit 255 bytes)\n"); return -1; }; + if (!reservedimportpath(base) && !sepimportbasevalid(base)) { + cerr("ww: invalid package path "); cerr(pathstr(base)); cerr("\n"); + return -1; + }; let p: *seppkg = &g.pkg[pi]; if (p.importbase != nil) { if (cstreq(p.importbase, base)) { return 0; }; @@ -862,8 +922,7 @@ fn sepbindimportbase(g: *sepgraph, pi: i32, base: *u8) i32 = { let i: i32 = 0; for (i < g.n) { if (i != pi && g.pkg[i].isdir != 0 && !g.pkg[i].generatedmain) { - let samelocation: bool = os.samefile(pathstr(g.pkg[i].entry), - pathstr(p.entry)); + let samelocation: bool = cstreq(g.pkg[i].canon, p.canon); let supportalias: bool = p.role == SEP_ROLE_TEST_SUPPORT || g.pkg[i].role == SEP_ROLE_TEST_SUPPORT; if (!supportalias && !samelocation @@ -910,14 +969,33 @@ fn sepfindoraddvariant(g: *sepgraph, path: *u8, entry: *u8, cerr("ww: package path is too long (limit 255 bytes)\n"); return -1; }; + let canon: *u8 = nil; + if (isdir != 0) { + canon = canonicaldir(pathstr(entry)); + if (canon == nil) { + cerr("ww: cannot canonicalize package "); + cerr(pathstr(entry)); cerr("\n"); + return -1; + }; + if (cstrlen(canon) >= 1024u64) { + cerr("ww: canonical package path is too long\n"); + return -1; + }; + }; let incoming: *u8 = nil; if (isdir != 0 && path[0u64] != 0u8) { incoming = sepvariantpath(variant, path); }; let i: i32 = 0; for (i < g.n) { - let samelocation: bool = os.samefile(pathstr(g.pkg[i].entry), - pathstr(entry)); + let samelocation: bool = false; + if (isdir != 0 && g.pkg[i].isdir != 0 && canon != nil + && g.pkg[i].canon != nil) { + samelocation = cstreq(g.pkg[i].canon, canon); + } else { + samelocation = os.samefile(pathstr(g.pkg[i].entry), + pathstr(entry)); + }; if (isdir != 0 && g.pkg[i].isdir != 0 && !g.pkg[i].generatedmain) { let supportalias: bool = role == SEP_ROLE_TEST_SUPPORT @@ -987,6 +1065,7 @@ fn sepfindoraddvariant(g: *sepgraph, path: *u8, entry: *u8, g.pkg[g.n].path = arenadupcstr(path, plen); g.pkg[g.n].importbase = nil; g.pkg[g.n].entry = arenadupcstr(entry, elen); + g.pkg[g.n].canon = canon; g.pkg[g.n].artifact = nil; g.pkg[g.n].name = nil; g.pkg[g.n].testpackage = nil; @@ -1025,7 +1104,7 @@ fn sepfindoraddvariant(g: *sepgraph, path: *u8, entry: *u8, if (g.pkg[i].isdir != 0 && !g.pkg[i].generatedmain && g.pkg[i].role != SEP_ROLE_TEST_SUPPORT && role != SEP_ROLE_TEST_SUPPORT - && os.samefile(pathstr(g.pkg[i].entry), pathstr(entry)) + && cstreq(g.pkg[i].canon, canon) && g.pkg[i].importbase != nil) { inherited = g.pkg[i].importbase; break; @@ -1197,10 +1276,6 @@ fn sepexternalname(pkg: *seppkg, path: *u8, n: u64, && pkg.testpackage[leafn + 4u64] == 't'; }; -fn sepexternalproduction(pkg: *seppkg, path: *u8, n: u64) bool = { - return sepexternalname(pkg, path, n, false); -}; - fn sepbindadd(bindings: *[]sepbind, kind: u8, name: str, target: *u8) void = { let i: i32 = 0; @@ -1357,17 +1432,26 @@ fn sepscanfile(g: *sepgraph, pi: i32, file: *u8, searchpath: *u8, previous = u.usepath; let idp: *u8 = u.usepath.ptr; let idn: u64 = u.usepath.len: u64; + if (reservedimport(u.usepath)) { + cerrpos(u.file, u.line, u.col); + cerr(": error: package path "); + cerr(u.usepath); cerr(" is reserved\n"); + return -1; + }; if (idn >= 256u64) { cerrpos(u.file, u.line, u.col); cerr(": error: import path is too long (limit 255 bytes)\n"); return -1; }; - let externalproduction: bool = sepexternalproduction( - &g.pkg[pi], idp, idn); + let externalproduction: bool = false; let ipath: *u8 = nil; - if (externalproduction) { - ipath = g.pkg[pi].entry; - } else { + if (g.pkg[pi].variant == SEP_VARIANT_EXTERNAL + && g.pkg[pi].importbase != nil + && cstrlen(g.pkg[pi].importbase) == idn + && bytecmp(g.pkg[pi].importbase, idn, idp, idn) == 0) { + ipath = g.pkg[pi].canon; + }; + if (ipath == nil) { ipath = locateimport(searchpath, idp, idn); }; if (ipath != nil) { @@ -1382,7 +1466,7 @@ fn sepscanfile(g: *sepgraph, pi: i32, file: *u8, searchpath: *u8, cerr(": error: self-import: package '"); if (g.pkg[pi].path[0u64] != 0u8) { cerr(pathstr(g.pkg[pi].path)); - } else { cerr(pathstr(g.pkg[pi].name)); }; + } else { cerr(pathstr(g.pkg[pi].canon)); }; cerr("' cannot import itself\n"); return -1; }; @@ -1447,8 +1531,11 @@ fn sepdepcmp(g: *sepgraph, a: i32, b: i32) i32 = { if (g.pkg[a].variant > g.pkg[b].variant) { return 1; }; if (g.pkg[a].role < g.pkg[b].role) { return -1; }; if (g.pkg[a].role > g.pkg[b].role) { return 1; }; - return strings.compare(pathstr(g.pkg[a].entry), - pathstr(g.pkg[b].entry)): i32; + if (g.pkg[a].canon != nil && g.pkg[b].canon != nil) { + return strings.compare(pathstr(g.pkg[a].canon), + pathstr(g.pkg[b].canon)): i32; + }; + return strings.compare(pathstr(g.pkg[a].entry), pathstr(g.pkg[b].entry)): i32; }; fn generatedmainkind(variant: i32) str = { @@ -1516,6 +1603,9 @@ fn sepaddgeneratedmain(g: *sepgraph, product: *sepproduct, ordinal: i32, p.path = mainpath; p.importbase = nil; p.entry = g.pkg[variant].entry; + let canonkind: *u8 = appendlit(g.pkg[variant].canon, "#"); + canonkind = appendlit(canonkind, kind); + p.canon = appendlit(canonkind, "-test-main"); let variantartifact: *u8 = g.pkg[variant].artifact; if (variantartifact == nil) { variantartifact = g.pkg[variant].path; }; p.artifact = appendlit(variantartifact, "-main"); @@ -1649,7 +1739,7 @@ fn seploadpkg(g: *sepgraph, pi: i32, context: i32) i32 = { cerr("ww: package "); if (g.pkg[pi].path[0u64] != 0u8) { cerr(pathstr(g.pkg[pi].path)); - } else { cerr(pathstr(g.pkg[pi].name)); }; + } else { cerr(pathstr(g.pkg[pi].canon)); }; cerr(" resolves imports differently in "); cerr(pathstr(g.context[g.pkg[pi].emitcontext].root)); cerr(" and "); cerr(pathstr(g.context[context].root)); cerr("\n"); @@ -1684,12 +1774,186 @@ fn seploadpkg(g: *sepgraph, pi: i32, context: i32) i32 = { return 0; }; -// Finalize provisional directory identities after source discovery and before -// generated-main construction or compilation. Source-import bindings win; -// otherwise a literal manifest-free root falls back to its validated package -// name (external p_test roots bind the ordinary base p). +fn sepimportcomponent(s: *u8, n: u64) bool = { + if (n == 0u64) { return false; }; + let first: u8 = s[0u64]; + if (!((first >= 'a': u8 && first <= 'z': u8) + || (first >= 'A': u8 && first <= 'Z': u8) + || first == '_': u8)) { return false; }; + let i: u64 = 1u64; + for (i < n) { + let c: u8 = s[i]; + if (!((c >= 'a': u8 && c <= 'z': u8) + || (c >= 'A': u8 && c <= 'Z': u8) + || (c >= '0': u8 && c <= '9': u8) + || c == '_': u8)) { return false; }; + i += 1u64; + }; + return syntax.kwlookup(s, n: i32) == syntax.tkind.TK_NONE; +}; + +fn sepimportpathfromrelative(rel: *u8, out: *u8, outsz: u64) i32 = { + let off: u64 = 0u64; + let p: u64 = 0u64; + let total: u64 = cstrlen(rel); + for (p < total) { + let q: u64 = p; + for (q < total && rel[q] != '/': u8) { q += 1u64; }; + let n: u64 = q - p; + if (!sepimportcomponent(rel + p, n)) { return 0; }; + let more: bool = q < total; + let separator: u64 = 0u64; + if (more) { separator = 1u64; }; + if (off + n + separator + 1u64 > outsz) { + return -1; + }; + bytecpy(out + off, rel + p, n); + off += n; + if (more) { out[off] = '.': u8; off += 1u64; }; + p = q + 1u64; + }; + out[off] = 0u8; + if (off == 0u64) { return 0; }; + return 1; +}; + +// A reverse candidate is authoritative only when the ordinary ordered lookup +// selects this exact canonical directory. Later or nested roots therefore +// cannot manufacture an alias hidden by an earlier source root. +fn sepreverseimportbase(g: *sepgraph, p: *seppkg, context: i32, + out: *u8, outsz: u64) i32 = { + let searchpath: *u8 = g.context[context].searchpath; + let total: u64 = cstrlen(searchpath); + let pos: u64 = 0u64; + for (pos < total) { + let end: u64 = pos; + for (end < total && searchpath[end] != ':': u8) { end += 1u64; }; + if (end > pos) { + let root: str; + root.ptr = searchpath + pos; + root.len = (end - pos): i32; + let canonroot: *u8 = canonicaldir(root); + if (canonroot != nil) { + let rn: u64 = cstrlen(canonroot); + let dn: u64 = cstrlen(p.canon); + let rel: *u8 = nil; + if (rn == 1u64 && canonroot[0u64] == '/': u8 + && dn > 1u64 && p.canon[0u64] == '/': u8) { + rel = p.canon + 1u64; + } else { if (dn > rn + 1u64 + && bytecmp(p.canon, rn, canonroot, rn) == 0 + && p.canon[rn] == '/': u8) { + rel = p.canon + rn + 1u64; + }; }; + if (rel != nil) { + let converted: i32 = sepimportpathfromrelative(rel, + out, outsz); + if (converted < 0) { + cerr("ww: package path is too long (limit 255 bytes)\n"); + return -1; + }; + if (converted > 0 && reservedimportpath(out)) { + converted = 0; + }; + if (converted > 0) { + let selected: *u8 = locateimport(searchpath, rel, + cstrlen(rel)); + if (selected != nil) { + let selectedcanon: *u8 = canonicaldir(pathstr(selected)); + if (selectedcanon != nil + && cstreq(selectedcanon, p.canon)) { + return 1; + }; + }; + }; + }; + }; + }; + pos = end + 1u64; + }; + return 0; +}; + +fn sepordinarydeclaredname(p: *seppkg) *u8 = { + if (p.name == nil || p.name[0u64] == 0u8) { return nil; }; + let n: u64 = cstrlen(p.name); + if (p.variant == SEP_VARIANT_EXTERNAL) { + if (n <= 5u64 || !cstrendswithlit(p.name, "_test")) { + cerr("ww: package-test selector does not name an external package\n"); + return nil; + }; + n -= 5u64; + }; + return arenadupcstr(p.name, n); +}; + +// The reserved local namespace is reversible, so filesystem identity never +// depends on a hash, request order, output name, or another selected package. +fn seplocalimportbase(p: *seppkg) *u8 = { + let leaf: *u8 = sepordinarydeclaredname(p); + if (leaf == nil) { return nil; }; + let need: u64 = SEP_LOCAL_IMPORT_PREFIX.len: u64 + 2u64 + + cstrlen(p.canon) * 4u64 + 1u64 + cstrlen(leaf) + 1u64; + let out: []u8 = alloc([], need)!; + out.len = need: i32; + let off: u64 = strinto(out.ptr, 0u64, SEP_LOCAL_IMPORT_PREFIX); + off = byteinto(out.ptr, off, '.': u8); + off = byteinto(out.ptr, off, 'p': u8); + let hex: str = "0123456789abcdef"; + let i: u64 = 0u64; + for (p.canon[i] != 0u8) { + let c: u8 = p.canon[i]; + if ((c >= 'a': u8 && c <= 'z': u8) + || (c >= 'A': u8 && c <= 'Z': u8) + || (c >= '0': u8 && c <= '9': u8)) { + off = byteinto(out.ptr, off, c); + } else { if (c == '_': u8 || c == '/': u8) { + off = byteinto(out.ptr, off, '_': u8); + let escaped: u8 = 's': u8; + if (c == '_': u8) { escaped = 'u': u8; }; + off = byteinto(out.ptr, off, escaped); + } else { + off = byteinto(out.ptr, off, '_': u8); + off = byteinto(out.ptr, off, 'x': u8); + let high: i32 = (c / 16u8): i32; + let low: i32 = (c % 16u8): i32; + off = byteinto(out.ptr, off, hex[high]); + off = byteinto(out.ptr, off, hex[low]); + }; }; + i += 1u64; + }; + off = byteinto(out.ptr, off, '.': u8); + off = cstrinto(out.ptr, off, leaf); + cstrseal(out.ptr, off); + return out.ptr; +}; + +// Finalization verifies every reached context before generated-main creation. +// Source bindings and explicit lookup identities remain authoritative; a +// literal root either round-trips through an active root or receives the +// reserved reversible local identity. fn sepfinalizedirectoryidentities(g: *sepgraph) i32 = { let pi: i32 = 0; + for (pi < g.n) { + let p: *seppkg = &g.pkg[pi]; + if (p.isdir != 0 && !p.generatedmain && !p.failed && p.loaded + && p.role != SEP_ROLE_TEST_SUPPORT && p.importbase == nil) { + let ci: i32 = 0; + for (ci < g.ncontext) { + if (p.contextstate[ci] == 2u8) { + let candidate: [256]u8; + let found: i32 = sepreverseimportbase(g, p, ci, + &candidate[0], 256u64); + if (found < 0) { return -1; }; + if (found > 0 && sepbindimportbase(g, pi, + &candidate[0]) < 0) { return -1; }; + }; + ci += 1; + }; + }; + pi += 1; + }; + pi = 0; for (pi < g.n) { let p: *seppkg = &g.pkg[pi]; if (p.isdir != 0 && !p.generatedmain && !p.failed && p.loaded @@ -1701,7 +1965,7 @@ fn sepfinalizedirectoryidentities(g: *sepgraph) i32 = { && !g.pkg[i].generatedmain && g.pkg[i].role != SEP_ROLE_TEST_SUPPORT && p.role != SEP_ROLE_TEST_SUPPORT - && os.samefile(pathstr(g.pkg[i].entry), pathstr(p.entry)) + && cstreq(g.pkg[i].canon, p.canon) && g.pkg[i].importbase != nil) { base = g.pkg[i].importbase; break; @@ -1709,26 +1973,11 @@ fn sepfinalizedirectoryidentities(g: *sepgraph) i32 = { i += 1; }; if (base == nil) { - if (p.name == nil || p.name[0u64] == 0u8) { - cerr("ww: package directory "); cerr(pathstr(p.entry)); - cerr(" has no canonical import identity\n"); + base = seplocalimportbase(p); + if (base == nil || cstrlen(base) >= 256u64) { + cerr("ww: local package identity is too long (limit 255 bytes)\n"); return -1; }; - if (p.variant == SEP_VARIANT_EXTERNAL) { - let n: u64 = cstrlen(p.name); - if (n <= 5u64 || !cstrendswithlit(p.name, "_test")) { - cerr("ww: package-test selector does not name an external package\n"); - return -1; - }; - let fallback: []u8 = alloc([], n - 5u64 + 1u64)!; - fallback.len = (n - 5u64 + 1u64): i32; - let k: u64 = 0u64; - for (k < n - 5u64) { fallback[k] = p.name[k]; k += 1u64; }; - fallback[n - 5u64] = 0u8; - base = fallback.ptr; - } else { - base = p.name; - }; }; if (sepbindimportbase(g, pi, base) < 0) { return -1; }; }; @@ -2133,14 +2382,14 @@ fn copyfileatomic(src: *u8, dst: *u8) i32 = { fn workdirstamptext(istest: i32, emitasm: i32) str = { if (istest != 0) { if (emitasm != 0) { - return "ww workdir fmt 8 mode test asm 1\n"; + return "ww workdir fmt 9 mode test asm 1\n"; }; - return "ww workdir fmt 8 mode test asm 0\n"; + return "ww workdir fmt 9 mode test asm 0\n"; }; if (emitasm != 0) { - return "ww workdir fmt 7 mode build asm 1\n"; + return "ww workdir fmt 8 mode build asm 1\n"; }; - return "ww workdir fmt 7 mode build asm 0\n"; + return "ww workdir fmt 8 mode build asm 0\n"; }; fn stampmatches(path: *u8, want: str) bool = { @@ -2244,8 +2493,8 @@ fn cerrpath(head: str, path: *u8, tail: str) void = { }; fn buildonesepimpl(selfdir: *u8, src: *u8, entryisdir: i32, - rootidentity: *u8, out: *u8, - objstem: *u8, incs: *u8, lf: *lflags, packageonly: i32, istest: i32, + out: *u8, objstem: *u8, incs: *u8, lf: *lflags, + packageonly: i32, istest: i32, products: *sepproduct, nproducts: i32, emitasm: i32, workdir: *u8, scratchout: **u8, graphout: **sepgraph) i32 = { @@ -2418,8 +2667,6 @@ fn buildonesepimpl(selfdir: *u8, src: *u8, entryisdir: i32, identityfailed = false, })!; if (graphout != nil) { *graphout = g; }; - let rootpath: *u8 = "\0".ptr; - if (packageonly != 0 && rootidentity != nil) { rootpath = rootidentity; }; let supportfor: []i32 = alloc([], nproducts: u64)!; supportfor.len = nproducts; let producti: i32 = 0; @@ -2439,6 +2686,10 @@ fn buildonesepimpl(selfdir: *u8, src: *u8, entryisdir: i32, if (products[producti].variant == SEP_VARIANT_PRODUCTION) { selector = nil; }; + let rootpath: *u8 = "\0".ptr; + if (products[producti].identity != nil) { + rootpath = products[producti].identity; + }; products[producti].root = sepfindoraddvariant(g, rootpath, entry, entryisdir, products[producti].variant, selector, @@ -3044,6 +3295,7 @@ fn buildonesep(selfdir: *u8, src: *u8, entryisdir: i32, let product: sepproduct; product.dir = src; product.out = out; + product.identity = rootidentity; product.testpackage = testpackage; product.status = nil; product.artifact = nil; @@ -3053,8 +3305,7 @@ fn buildonesep(selfdir: *u8, src: *u8, entryisdir: i32, product.variant = rootvariant; product.root = -1; product.variantroot = -1; - let r: i32 = buildonesepimpl(selfdir, src, entryisdir, rootidentity, - out, objstem, + let r: i32 = buildonesepimpl(selfdir, src, entryisdir, out, objstem, incs, lf, packageonly, istest, &product, 1, emitasm, workdir, &scratch, &g); sepgraphfree(g); @@ -3084,14 +3335,19 @@ fn buildonesep(selfdir: *u8, src: *u8, entryisdir: i32, // Build every selected directory/variant root inside one command-owned package // universe. The first output owns the shared cold sepwork tree. -fn buildpackagetests(selfdir: *u8, src: *u8, incs: *u8, workdir: *u8, - products: *sepproduct, nproducts: i32) i32 = { +fn buildpackagetests(selfdir: *u8, src: *u8, rootidentity: *u8, + incs: *u8, workdir: *u8, products: *sepproduct, nproducts: i32) i32 = { let scratch: *u8 = nil; let g: *sepgraph = nil; let lf: lflags; lf.libdirs = nil; lf.nlibdirs = 0; lf.libs = nil; lf.nlibs = 0; - let r: i32 = buildonesepimpl(selfdir, src, 1, nil, + let i: i32 = 0; + for (i < nproducts) { + products[i].identity = rootidentity; + i += 1; + }; + let r: i32 = buildonesepimpl(selfdir, src, 1, products[0].out, products[0].out, incs, &lf, 0, 1, products, nproducts, 0, workdir, &scratch, &g); sepgraphfree(g); @@ -3182,6 +3438,7 @@ fn resolvemodule(selfdir: *u8, name: *u8, incs: *u8, isdir: *i32) *u8 = { *isdir = foundisdir; return arenadupcstr(name, nlen); }; + if (reservedimportpath(name)) { return nil; }; let search: *u8 = buildsearchpath(selfdir, incs); return locatemodule(search, name, nlen, isdir); @@ -3394,7 +3651,7 @@ fn dobuild(selfdir: *u8, argv: **u8, argc: i32, start: i32) i32 = { lf.libs = libs.ptr; lf.nlibs = nlibs; let rootidentity: *u8 = nil; - if (packageonly != 0 && !requestedliteral) { rootidentity = src; }; + if (!requestedliteral && isdir != 0) { rootidentity = src; }; return buildonesep(selfdir, resolved, isdir, rootidentity, out, objstem, incs.ptr, &lf, packageonly, 0i32, SEP_VARIANT_PRODUCTION, nil, @@ -3541,6 +3798,12 @@ fn dorun(selfdir: *u8, argv: **u8, argc: i32, start: i32) i32 = { let dot: [2]u8 = ['.': u8, 0u8]; src = &dot[0]; }; + let requestedliteral: bool = false; + let requestedstat: os.filestat; + match (os.stat(&requestedstat, pathstr(src))) { + case void => requestedliteral = true; + case let e: os.oserror => void; + }; let isdir: i32 = 0; let resolved: *u8 = resolvemodule(selfdir, src, incs.ptr, &isdir); if (resolved == nil) { @@ -3562,7 +3825,10 @@ fn dorun(selfdir: *u8, argv: **u8, argc: i32, start: i32) i32 = { lf.libs = libs.ptr; lf.nlibs = nlibs; // The freshly acquired directory owns both main and main.sepwork. - if (buildonesep(selfdir, resolved, isdir, nil, outp, outp, incs.ptr, &lf, + let rootidentity: *u8 = nil; + if (!requestedliteral && isdir != 0) { rootidentity = src; }; + if (buildonesep(selfdir, resolved, isdir, rootidentity, + outp, outp, incs.ptr, &lf, 0i32, 0i32, SEP_VARIANT_PRODUCTION, nil, 0i32, 0i32, nil) != 0) { let cleanrc: i32 = os.remove(pathstr(outp)); if (cleanrc != 0 && cleanrc != -2i32) { @@ -3740,6 +4006,7 @@ fn dotest(selfdir: *u8, argv: **u8, argc: i32, start: i32) i32 = { let emitasm: i32 = 0; let outstem: *u8 = nil; let workdir: *u8 = nil; + let requestidentity: *u8 = nil; let products: []sepproduct = alloc([], SEP_MAXPRODUCT: u64)!; let packageopts: bool = false; let afterdash: bool = false; @@ -3751,6 +4018,17 @@ fn dotest(selfdir: *u8, argv: **u8, argc: i32, start: i32) i32 = { if (cstreqlit(p, "--")) { packageopts = true; afterdash = true; i += 1; continue; }; + if (cstreqlit(p, "--ww-root-identity")) { + if (i + 1 >= argc || requestidentity != nil + || argv[i + 1][0u64] == 0u8 + || reservedimportpath(argv[i + 1])) { + cerr("ww test: invalid --ww-root-identity\n"); + return 2; + }; + requestidentity = argv[i + 1]; + i += 2; + continue; + }; if (cstreqlit(p, "--ww-package-test")) { if (i + 5 >= argc || products.len >= SEP_MAXPRODUCT) { cerr("ww test: --ww-package-test needs kind, package, directory, output, and status\n"); @@ -3784,6 +4062,7 @@ fn dotest(selfdir: *u8, argv: **u8, argc: i32, start: i32) i32 = { let product: sepproduct; product.dir = dir; product.out = output; + product.identity = nil; product.testpackage = name; product.status = status; product.artifact = nil; @@ -3943,7 +4222,7 @@ fn dotest(selfdir: *u8, argv: **u8, argc: i32, start: i32) i32 = { // -w forwards: the coordinator keys one persistent driver workdir // for the complete selected test request. return execpackagetests(selfdir, argv, argc, start, - targetindex, nil, false); + targetindex, nil, nil, false); }; let resolved: *u8 = target; @@ -3958,6 +4237,7 @@ fn dotest(selfdir: *u8, argv: **u8, argc: i32, start: i32) i32 = { }; case let e: os.oserror => void; }; + let requestedliteral: bool = found; if (!found) { resolved = resolvemodule(selfdir, target, incs.ptr, &isdir); if (resolved == nil) { @@ -3966,6 +4246,10 @@ fn dotest(selfdir: *u8, argv: **u8, argc: i32, start: i32) i32 = { return 1; }; }; + let rootidentity: *u8 = requestidentity; + if (!requestedliteral && rootidentity == nil && isdir != 0) { + rootidentity = target; + }; if (isdir == 0) { if (products.len != 0) { cerr("ww test: package-test variant needs one directory\n"); @@ -3995,13 +4279,14 @@ fn dotest(selfdir: *u8, argv: **u8, argc: i32, start: i32) i32 = { cerr("ww test: package-test products need -c\n"); return 2; }; - return buildpackagetests(selfdir, resolved, incs.ptr, workdir, + return buildpackagetests(selfdir, resolved, rootidentity, + incs.ptr, workdir, products.ptr, products.len); }; let replacement: *u8 = nil; if (resolved != target) { replacement = resolved; }; return execpackagetests(selfdir, argv, argc, start, targetindex, - replacement, targetindex < 0); + replacement, rootidentity, targetindex < 0); }; export fn main(argc: i32, argv: **u8) i32 = {