diff --git a/Makefile b/Makefile index 3639f07a..ce22c38d 100644 --- a/Makefile +++ b/Makefile @@ -65,7 +65,12 @@ WWTEST_SRC = cmd/wwtest/wwtest.ww \ lib/strconv/decimal.ww lib/strconv/ftos.ww lib/strconv/ftos_data.ww \ lib/strconv/stof.ww lib/strconv/stof_data.ww lib/strconv/strconv.ww \ lib/strings/strings.ww lib/temp/temp.ww lib/time/time.ww \ - lib/types/types.ww + lib/types/types.ww \ + lib/ww/syntax/ast.ww lib/ww/syntax/decl.ww \ + lib/ww/syntax/expr.ww lib/ww/syntax/lex.ww \ + lib/ww/syntax/parse.ww lib/ww/syntax/stmt.ww \ + lib/ww/syntax/sym.ww lib/ww/syntax/tok.ww \ + lib/ww/syntax/typ.ww WWFIXTURE_BIN = $(BIN)/wwfixture WWFIXTURE_SRC = cmd/wwfixture/wwfixture.ww \ diff --git a/cmd/ww/main.c b/cmd/ww/main.c index a78edc1c..51032152 100644 --- a/cmd/ww/main.c +++ b/cmd/ww/main.c @@ -42,9 +42,15 @@ static const char *usage = static char *self_dir; static const char *self_path; +static struct { + char *path; + char *buf; + u64 len; +} source_documentation_preload; static char *sep_sprintf(const char *, ...); static int sep_reserve(void **, int *, int, size_t); static int sep_fail_size(void); +static void source_operand_no_sources(const char *); static const char * envpath(const char *name) @@ -536,11 +542,8 @@ static int sep_slurp(const char*, char**, u64*); * rather than a textual attribute scan, whether a production source contains * @test; otherwise valid whitespace/comments could silently drop a test. */ static int -source_has_test_decl(const char *path) +source_has_test_decl(const char *path, char *buf, u64 len) { - char *buf; - u64 len; - if (sep_slurp(path, &buf, &len) < 0) return -1; /* Keep directory-loader package-clause diagnostics stable. The full * parser reports its language-level "missing package clause" first; * the imports-only pass owns the package-loader wording and also avoids @@ -553,14 +556,12 @@ source_has_test_decl(const char *path) Node *imports = parseimports(&ip); if (il.errs || ip.errs) { freearena(ia); - free(buf); return -1; } if (imports->module == NULL) { Pos pp = { path, 1, 1 }; errorf(pp, "invalid or missing package clause"); freearena(ia); - free(buf); return -1; } freearena(ia); @@ -572,7 +573,6 @@ source_has_test_decl(const char *path) Node *file = parsefile(&p); if (l.errs || p.errs) { freearena(a); - free(buf); return -1; } int found = 0; @@ -584,7 +584,6 @@ source_has_test_decl(const char *path) break; } freearena(a); - free(buf); return found; } @@ -655,14 +654,8 @@ sep_reserve(void **buf, int *cap, int need, size_t elemsz) * The coordinator chooses variants, but the command owns which real source * paths enter a package compilation. */ static int -source_package_name(const char *path, char **out) +source_package_name(const char *path, char *buf, u64 len, char **out) { - char *buf; - u64 len; - if (sep_slurp(path, &buf, &len) < 0) { - fprintf(stderr, "ww: cannot read %s\n", path); - return -1; - } Arena *a = newarena(); Lex l; Parser p; @@ -671,25 +664,21 @@ source_package_name(const char *path, char **out) Node *imports = parseimports(&p); if (l.errs || p.errs) { freearena(a); - free(buf); return -1; } if (imports->module == NULL) { Pos pp = { path, 1, 1 }; errorf(pp, "invalid or missing package clause"); freearena(a); - free(buf); return -1; } *out = strdup(imports->module); if (*out == NULL) { sep_fail_nomem(); freearena(a); - free(buf); return -1; } freearena(a); - free(buf); return 0; } @@ -798,6 +787,205 @@ source_build_header(const char *path) return 0; } +/* Go 1.26.5 reserves the exact package name "documentation" for files that + * are ignored by the package loader. Avoid parsing ordinary sources on this + * early selection pass: only a quiet, exact package-clause candidate enters + * the narrow package/import-header parser. */ +static int +source_documentation_skip_trivia(const char *src, u64 len, u64 *at) +{ + u64 i = *at; + for (;;) { + while (i < len && (src[i] == ' ' || src[i] == '\t' + || src[i] == '\r' || src[i] == '\n')) + i++; + if (i + 1 < len && src[i] == '/' && src[i + 1] == '/') { + i += 2; + while (i < len && src[i] != '\n') i++; + continue; + } + if (i + 1 < len && src[i] == '/' && src[i + 1] == '*') { + i += 2; + while (i + 1 < len + && !(src[i] == '*' && src[i + 1] == '/')) + i++; + if (i + 1 >= len) return 0; + i += 2; + continue; + } + break; + } + *at = i; + return 1; +} + +static int +source_documentation_ident(unsigned char c) +{ + return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') + || (c >= '0' && c <= '9') || c == '_'; +} + +static int +source_documentation_word(const char *src, u64 len, u64 *at, + const char *word) +{ + u64 n = (u64)strlen(word), i = *at; + if (i > len || len - i < n || memcmp(src + i, word, (size_t)n) != 0) + return 0; + if (len - i > n + && source_documentation_ident((unsigned char)src[i + n])) + return 0; + *at = i + n; + return 1; +} + +static int +source_documentation_candidate(const char *src, u64 len) +{ + u64 at = 0; + if (len >= 3 && (unsigned char)src[0] == 0xef + && (unsigned char)src[1] == 0xbb + && (unsigned char)src[2] == 0xbf) + at = 3; + if (!source_documentation_skip_trivia(src, len, &at) + || !source_documentation_word(src, len, &at, "package") + || !source_documentation_skip_trivia(src, len, &at) + || !source_documentation_word(src, len, &at, "documentation") + || !source_documentation_skip_trivia(src, len, &at)) + return 0; + return at < len && src[at] == ';'; +} + +/* -1: diagnosed header error; 0: ordinary source; 1: documentation source. + * The caller owns BUF. The raw-'i' arm mirrors readGoInfo's attempt to read + * another import before it knows whether the complete keyword follows. Exact + * import tokens have already been consumed (or diagnosed) by the parser. */ +static int +source_documentation_buffer(const char *path, char *buf, u64 len) +{ + if (!source_documentation_candidate(buf, len)) return 0; + Arena *a = newarena(); + Lex l; + Parser p; + lexinit(&l, a, path, buf, len); + parserinit(&p, a, &l); + Node *header = parsepackageheader(&p); + if (l.errs || p.errs) { + freearena(a); + return -1; + } + if (header->module == NULL) { + Pos pp = { path, 1, 1 }; + errorf(pp, "invalid or missing package clause"); + freearena(a); + return -1; + } + if (l.pos < len && buf[l.pos] == 'i') { + Pos ip = { path, l.line, l.col }; + errorf(ip, "expected top-level decl"); + freearena(a); + return -1; + } + int documentation = strcmp(header->module, "documentation") == 0; + freearena(a); + return documentation; +} + +static void +source_documentation_preload_clear(void) +{ + free(source_documentation_preload.path); + free(source_documentation_preload.buf); + memset(&source_documentation_preload, 0, + sizeof source_documentation_preload); +} + +/* A regular named root is read exactly once. An ordinary result carries the + * same bytes into sep_scan_file; a documentation result is rejected before a + * run/test scratch directory can be acquired. Nonregular sources never enter + * this preflight and retain their established stream/error route. */ +static int +source_preload_documentation(const char *path) +{ + source_documentation_preload_clear(); + char *buf; + u64 len; + if (sep_slurp(path, &buf, &len) < 0) { + fprintf(stderr, "ww: cannot read %s\n", path); + return -1; + } + int documentation = source_documentation_buffer(path, buf, len); + if (documentation != 0) { + free(buf); + return documentation; + } + char *copy = strdup(path); + if (copy == NULL) { + sep_fail_nomem(); + free(buf); + return -1; + } + source_documentation_preload.path = copy; + source_documentation_preload.buf = buf; + source_documentation_preload.len = len; + return 0; +} + +static int +source_regular_file(const char *path) +{ + struct stat st; + return stat(path, &st) == 0 && S_ISREG(st.st_mode); +} + +struct sepsource { + char *path; + char *buf; + u64 len; +}; + +struct sepdirobs_source { + char *name; /* one eligible directory member name */ + char *path; /* first request spelling, for diagnostics */ + char *buf; /* exact regular-file bytes, once observed */ + u64 len; + char *package; /* test source's once-parsed package name */ + int is_test; + int observed; /* stat/read/classification has run */ + int result; /* -2 diagnosed, 0 ignored, 1 ordinary */ +}; + +struct sepdirobs { + char *canon; /* canonical directory cache key */ + char *entry; /* first request spelling */ + struct sepdirobs_source *source; + int nsource; + int sourcecap; + int enumerated; + int enum_result; /* -1 unreadable directory, 0 success */ +}; + +static int +source_snapshot_add(struct sepsource **list, int *n, int *cap, + const char *path, const char *buf, u64 len) +{ + if (*n == INT_MAX) return sep_fail_size(); + if (sep_reserve((void **)list, cap, *n + 1, sizeof **list) < 0) + return -1; + char *pathcopy = strdup(path); + char *bufcopy = malloc((size_t)len + 1); + if (pathcopy == NULL || bufcopy == NULL) { + free(pathcopy); + free(bufcopy); + return sep_fail_nomem(); + } + memcpy(bufcopy, buf, (size_t)len + 1); + (*list)[*n] = (struct sepsource){ pathcopy, bufcopy, len }; + (*n)++; + return 0; +} + static int source_list_add(char ***list, int *n, int *cap, const char *path) { @@ -823,15 +1011,14 @@ source_list_free(char **list, int n) * byte-sorted so compiler test discovery is deterministic without generated * package amalgamation. */ static int -enumerate_dir_ww(const char *dirpath, int variant, const char *test_package, - char ***out_files) +sep_dir_membership(struct sepdirobs *obs) { - DIR *d = opendir(dirpath); - if (d == NULL) { *out_files = NULL; return -1; } + if (obs->enumerated) return obs->enum_result; + obs->enumerated = 1; + DIR *d = opendir(obs->entry); + if (d == NULL) return obs->enum_result = -1; char **names = NULL; int nnames = 0, capnames = 0; - char **prod = NULL, **tests = NULL; - int nprod = 0, capprod = 0, ntests = 0, captests = 0; struct dirent *ent = NULL; for (;;) { errno = 0; @@ -842,147 +1029,104 @@ enumerate_dir_ww(const char *dirpath, int variant, const char *test_package, if (nl <= 3) continue; if (nm[0] == '.' || nm[0] == '_') continue; if (strcmp(nm + nl - 3, ".ww") != 0) continue; + if (!sep_source_matches_target(nm)) continue; if (source_list_add(&names, &nnames, &capnames, nm) < 0) { source_list_free(names, nnames); closedir(d); - *out_files = NULL; - return -2; + return obs->enum_result = -2; } } int direrr = errno; closedir(d); if (direrr != 0) { source_list_free(names, nnames); - *out_files = NULL; - return -1; + return obs->enum_result = -1; } if (nnames > 1) qsort(names, (size_t)nnames, sizeof *names, strs_cmp); for (int ni = 0; ni < nnames; ni++) { const char *nm = names[ni]; size_t nl = strlen(nm); - if (!sep_source_matches_target(nm)) continue; - int is_test = nl >= 8 && strcmp(nm + nl - 8, "_test.ww") == 0; - if (variant == SEP_VARIANT_PRODUCTION && is_test) continue; - if (variant == SEP_VARIANT_EXTERNAL && !is_test) continue; - char path[PATH_MAX]; - int pn = snprintf(path, sizeof path, "%s/%s", dirpath, nm); - if (pn < 0 || (size_t)pn >= sizeof path) { - fprintf(stderr, "ww: package source path is too long\n"); + if (obs->nsource == INT_MAX + || sep_reserve((void **)&obs->source, &obs->sourcecap, + obs->nsource + 1, sizeof *obs->source) < 0) { source_list_free(names, nnames); - source_list_free(prod, nprod); - source_list_free(tests, ntests); - *out_files = NULL; - return -2; + return obs->enum_result = -2; } - struct stat st; - if (lstat(path, &st) != 0) { - fprintf(stderr, - "ww: %s: package source is not a regular file\n", path); + struct sepdirobs_source *s = &obs->source[obs->nsource]; + memset(s, 0, sizeof *s); + obs->nsource++; + s->name = strdup(nm); + if (s->name == NULL) { + sep_fail_nomem(); source_list_free(names, nnames); - source_list_free(prod, nprod); - source_list_free(tests, ntests); - *out_files = NULL; - return -2; - } - if (S_ISLNK(st.st_mode)) { - if (stat(path, &st) != 0) { - fprintf(stderr, - "ww: %s: package source is not a regular file\n", - path); - source_list_free(names, nnames); - source_list_free(prod, nprod); - source_list_free(tests, ntests); - *out_files = NULL; - return -2; - } - /* go/build ignores a source-shaped symlink to a directory. */ - if (S_ISDIR(st.st_mode)) continue; - } - if (!S_ISREG(st.st_mode)) { - fprintf(stderr, - "ww: %s: package source is not a regular file\n", path); - source_list_free(names, nnames); - source_list_free(prod, nprod); - source_list_free(tests, ntests); - *out_files = NULL; - return -2; - } - int has_test = !is_test ? source_has_test_decl(path) : 0; - if (has_test < 0) { - source_list_free(names, nnames); - source_list_free(prod, nprod); - source_list_free(tests, ntests); - *out_files = NULL; - return -2; - } - if (has_test > 0) { - fprintf(stderr, - "ww: %s: @test declaration outside *_test.ww\n", - path); - source_list_free(names, nnames); - source_list_free(prod, nprod); - source_list_free(tests, ntests); - *out_files = NULL; - return -2; - } - if (is_test) { - char *package = NULL; - if (source_package_name(path, &package) < 0) { - source_list_free(names, nnames); - source_list_free(prod, nprod); - source_list_free(tests, ntests); - *out_files = NULL; - return -2; - } - if (test_package == NULL || strcmp(package, test_package) != 0) { - free(package); - continue; - } - free(package); - } - char ***list = is_test ? &tests : ∏ - int *n = is_test ? &ntests : &nprod; - int *cap = is_test ? &captests : &capprod; - if (source_list_add(list, n, cap, path) < 0) { - source_list_free(names, nnames); - source_list_free(prod, nprod); - source_list_free(tests, ntests); - *out_files = NULL; - return -2; + return obs->enum_result = -2; } + s->is_test = nl >= 8 && strcmp(nm + nl - 8, "_test.ww") == 0; } source_list_free(names, nnames); - if (nprod > 1) qsort(prod, (size_t)nprod, sizeof *prod, strs_cmp); - if (ntests > 1) qsort(tests, (size_t)ntests, sizeof *tests, strs_cmp); - if (nprod > INT_MAX - ntests) { - sep_fail_size(); - source_list_free(prod, nprod); - source_list_free(tests, ntests); - *out_files = NULL; - return -2; + return obs->enum_result = 0; +} + +/* Observe a member lazily, after variant eligibility. The cached result is + * authoritative for the rest of this request, including documentation and + * diagnosed failures, so later variants cannot restat or reread the path. */ +static int +sep_dir_observe_source(struct sepdirobs *obs, struct sepdirobs_source *s) +{ + if (s->observed) return s->result; + s->observed = 1; + char path[PATH_MAX]; + int pn = snprintf(path, sizeof path, "%s/%s", obs->entry, s->name); + if (pn < 0 || (size_t)pn >= sizeof path) { + fprintf(stderr, "ww: package source path is too long\n"); + return s->result = -2; } - int total = nprod + ntests; - if ((size_t)total > (size_t)-1 / sizeof *prod) { - sep_fail_size(); - source_list_free(prod, nprod); - source_list_free(tests, ntests); - *out_files = NULL; - return -2; - } - char **all = total ? malloc((size_t)total * sizeof *all) : NULL; - if (total && all == NULL) { + s->path = strdup(path); + if (s->path == NULL) { sep_fail_nomem(); - source_list_free(prod, nprod); - source_list_free(tests, ntests); - *out_files = NULL; - return -2; + return s->result = -2; } - for (int i = 0; i < nprod; i++) all[i] = prod[i]; - for (int i = 0; i < ntests; i++) all[nprod + i] = tests[i]; - free(prod); - free(tests); - *out_files = all; - return total; + struct stat st; + if (lstat(s->path, &st) != 0) { + fprintf(stderr, + "ww: %s: package source is not a regular file\n", s->path); + return s->result = -2; + } + if (S_ISLNK(st.st_mode)) { + if (stat(s->path, &st) != 0) { + fprintf(stderr, + "ww: %s: package source is not a regular file\n", s->path); + return s->result = -2; + } + /* go/build ignores a source-shaped symlink to a directory. */ + if (S_ISDIR(st.st_mode)) return s->result = 0; + } + if (!S_ISREG(st.st_mode)) { + fprintf(stderr, + "ww: %s: package source is not a regular file\n", s->path); + return s->result = -2; + } + if (sep_slurp(s->path, &s->buf, &s->len) < 0) { + fprintf(stderr, "ww: cannot read %s\n", s->path); + return s->result = -2; + } + int documentation = source_documentation_buffer(s->path, s->buf, s->len); + if (documentation < 0) return s->result = -2; + if (documentation > 0) return s->result = 0; + if (s->is_test) { + if (source_package_name(s->path, s->buf, s->len, + &s->package) < 0) + return s->result = -2; + } else { + int has_test = source_has_test_decl(s->path, s->buf, s->len); + if (has_test < 0) return s->result = -2; + if (has_test > 0) { + fprintf(stderr, + "ww: %s: @test declaration outside *_test.ww\n", s->path); + return s->result = -2; + } + } + return s->result = 1; } /* ww build — separate-compilation driver (task #46/c3). @@ -1433,8 +1577,10 @@ struct seppkg { char *name; /* validated declared name; directory packages only */ char *test_package; /* selected test package; root variants only */ char *for_test; /* directory product owning a recompiled action */ - char **sources; /* owned, byte-sorted selected paths; dirs only */ + struct sepsource *sources; /* owned byte-sorted path+byte snapshots; dirs */ int nsources; + char *entry_buf; /* request snapshot for a regular one-file root */ + u64 entry_len; int is_dir; int variant; /* SEP_VARIANT_*; dependencies are production */ int role; /* normal, reserved test support, or generated main */ @@ -1473,6 +1619,9 @@ struct sepgraph { struct seppkg *pkg; int n; int pkgcap; + struct sepdirobs *dirobs; /* request-owned canonical observations */ + int ndirobs; + int dirobscap; struct sepcontext *context; int ncontext; int contextcap; @@ -1521,6 +1670,78 @@ sep_reserve_packages(struct sepgraph *g, int need) sizeof *g->pkg); } +static struct sepdirobs * +sep_graph_dir_observation(struct sepgraph *g, const struct seppkg *p) +{ + for (int i = 0; i < g->ndirobs; i++) + if (strcmp(g->dirobs[i].canon, p->canon) == 0) + return &g->dirobs[i]; + if (g->ndirobs == INT_MAX + || sep_reserve((void **)&g->dirobs, &g->dirobscap, + g->ndirobs + 1, sizeof *g->dirobs) < 0) + return NULL; + struct sepdirobs *obs = &g->dirobs[g->ndirobs]; + memset(obs, 0, sizeof *obs); + obs->canon = strdup(p->canon); + obs->entry = strdup(p->entry); + if (obs->canon == NULL || obs->entry == NULL) { + sep_fail_nomem(); + free(obs->canon); + free(obs->entry); + memset(obs, 0, sizeof *obs); + return NULL; + } + g->ndirobs++; + return obs; +} + +/* Select one action's ordered source set from request-owned observations. + * Production members precede same-package tests. Crucially, variant filters + * run before sep_dir_observe_source, so a production-only request merely sees + * test names in readdir data and never stats or opens their paths. */ +static int +sep_select_dir_sources(struct sepgraph *g, struct seppkg *p) +{ + struct sepdirobs *obs = sep_graph_dir_observation(g, p); + if (obs == NULL) return -2; + int er = sep_dir_membership(obs); + if (er != 0) return er; + /* Diagnose and cache observations in the directory's one global byte + * order. Source assembly below retains the language's production-before- + * test order without changing filesystem diagnostic precedence. */ + for (int i = 0; i < obs->nsource; i++) { + struct sepdirobs_source *s = &obs->source[i]; + if (p->variant == SEP_VARIANT_PRODUCTION && s->is_test) + continue; + if (p->variant == SEP_VARIANT_EXTERNAL && !s->is_test) + continue; + if (sep_dir_observe_source(obs, s) < 0) return -2; + } + int cap = 0; + for (int partition = 0; partition < 2; partition++) { + for (int i = 0; i < obs->nsource; i++) { + struct sepdirobs_source *s = &obs->source[i]; + if ((partition == 0 && s->is_test) + || (partition == 1 && !s->is_test)) + continue; + if (p->variant == SEP_VARIANT_PRODUCTION && s->is_test) + continue; + if (p->variant == SEP_VARIANT_EXTERNAL && !s->is_test) + continue; + int observed = s->result; + if (observed == 0) continue; + if (s->is_test + && (p->test_package == NULL + || strcmp(s->package, p->test_package) != 0)) + continue; + if (source_snapshot_add(&p->sources, &p->nsources, &cap, + s->path, s->buf, s->len) < 0) + return -2; + } + } + return p->nsources; +} + static int sep_reserve_folds(struct sepfoldset *set, int need) { @@ -2628,8 +2849,12 @@ sep_find_or_add_role(struct sepgraph *g, const char *path, const char *entry, static void sep_pkg_free_fields(struct seppkg *p) { - for (int j = 0; j < p->nsources; j++) free(p->sources[j]); + for (int j = 0; j < p->nsources; j++) { + free(p->sources[j].path); + free(p->sources[j].buf); + } free(p->sources); + free(p->entry_buf); for (int j = 0; j < p->bindings.n; j++) { free(p->bindings.v[j].name); free(p->bindings.v[j].source); @@ -2651,8 +2876,8 @@ sep_pkg_free_fields(struct seppkg *p) memset(p, 0, sizeof *p); } -/* Release the one package-owned directory-membership list. Every graph exit - * funnels through this function; regular-file nodes own no source list. */ +/* Release variant-owned byte copies and request-owned canonical-directory + * memberships/observations. Every graph exit funnels through this function. */ static void sep_graph_free(struct sepgraph *g) { @@ -2674,8 +2899,21 @@ sep_graph_free(struct sepgraph *g) free(g->context[i].route); free(g->context[i].source_root); } + for (int i = 0; i < g->ndirobs; i++) { + struct sepdirobs *obs = &g->dirobs[i]; + for (int j = 0; j < obs->nsource; j++) { + free(obs->source[j].name); + free(obs->source[j].path); + free(obs->source[j].buf); + free(obs->source[j].package); + } + free(obs->source); + free(obs->canon); + free(obs->entry); + } free(g->package_folds.v); free(g->file_folds.v); + free(g->dirobs); free(g->context); free(g->pkg); free(g); @@ -3507,16 +3745,65 @@ static int sep_scan_file(struct sepgraph *g, int pi, const char *file, int context, struct ImportSet *filevisit, struct sepbindset *bindings, struct sepchildren *children, - int owned_source) + int owned_source, const char *selected_buf, u64 selected_len) { if (import_seen(filevisit, file)) return 0; if (import_add(filevisit, file) < 0) return -1; char *buf; u64 len; - if (sep_slurp(file, &buf, &len) < 0) { + int documentation_prechecked = 0; + if (selected_buf != NULL) { + buf = malloc((size_t)selected_len + 1); + if (buf == NULL) return sep_fail_nomem(); + memcpy(buf, selected_buf, (size_t)selected_len + 1); + len = selected_len; + documentation_prechecked = 1; + } else if (!owned_source && g->pkg[pi].entry_buf != NULL + && strcmp(g->pkg[pi].entry, file) == 0) { + buf = malloc((size_t)g->pkg[pi].entry_len + 1); + if (buf == NULL) return sep_fail_nomem(); + memcpy(buf, g->pkg[pi].entry_buf, + (size_t)g->pkg[pi].entry_len + 1); + len = g->pkg[pi].entry_len; + documentation_prechecked = 1; + } else if (!owned_source && source_documentation_preload.path != NULL + && strcmp(source_documentation_preload.path, file) == 0) { + buf = source_documentation_preload.buf; + len = source_documentation_preload.len; + free(source_documentation_preload.path); + source_documentation_preload.path = NULL; + source_documentation_preload.buf = NULL; + source_documentation_preload.len = 0; + documentation_prechecked = 1; + } else if (sep_slurp(file, &buf, &len) < 0) { fprintf(stderr, "ww: cannot read %s\n", file); return -1; } + if (!owned_source && g->pkg[pi].entry_buf == NULL + && strcmp(g->pkg[pi].entry, file) == 0 + && (documentation_prechecked || source_regular_file(file))) { + g->pkg[pi].entry_buf = malloc((size_t)len + 1); + if (g->pkg[pi].entry_buf == NULL) { + sep_fail_nomem(); + free(buf); + return -1; + } + memcpy(g->pkg[pi].entry_buf, buf, (size_t)len + 1); + g->pkg[pi].entry_len = len; + } + if (!owned_source && !documentation_prechecked + && source_regular_file(file)) { + int documentation = source_documentation_buffer(file, buf, len); + if (documentation < 0) { + free(buf); + return -1; + } + if (documentation > 0) { + source_operand_no_sources(file); + free(buf); + return -1; + } + } Arena *a = newarena(); Lex l; Parser p; @@ -4052,10 +4339,21 @@ sep_clone_for_test(struct sepgraph *g, int original, const char *owner, } int sourcecap = 0; for (int i = 0; i < src->nsources; i++) { - if (source_list_add(&p->sources, &p->nsources, &sourcecap, - src->sources[i]) < 0) + if (source_snapshot_add(&p->sources, &p->nsources, &sourcecap, + src->sources[i].path, src->sources[i].buf, + src->sources[i].len) < 0) goto fail; } + if (src->entry_buf != NULL) { + p->entry_buf = malloc((size_t)src->entry_len + 1); + if (p->entry_buf == NULL) { + sep_fail_nomem(); + goto fail; + } + memcpy(p->entry_buf, src->entry_buf, + (size_t)src->entry_len + 1); + p->entry_len = src->entry_len; + } for (int i = 0; i < src->bindings.n; i++) { struct sepbind *b = &src->bindings.v[i]; Pos pos = { b->source, b->line, b->col }; @@ -4161,17 +4459,14 @@ sep_prepare_pkg_context(struct sepgraph *g, int pi, int context, if (!g->pkg[pi].loaded) { g->pkg[pi].loaded = 1; if (g->pkg[pi].is_dir) { - const char *test_package = g->pkg[pi].test_package; - g->pkg[pi].nsources = enumerate_dir_ww(g->pkg[pi].entry, - g->pkg[pi].variant, test_package, - &g->pkg[pi].sources); - if (g->pkg[pi].nsources == -2) { - rc = -1; /* diagnosed in enumerate_dir_ww */ - } else if (g->pkg[pi].nsources < 0) { + int selected = sep_select_dir_sources(g, &g->pkg[pi]); + if (selected == -2) { + rc = -1; /* diagnosed during source observation */ + } else if (selected < 0) { fprintf(stderr, "ww: cannot read directory %s\n", g->pkg[pi].entry); rc = -1; - } else if (g->pkg[pi].nsources == 0) { + } else if (selected == 0) { fprintf(stderr, "ww: %s: directory contains no WW package sources\n", g->pkg[pi].entry); @@ -4179,16 +4474,18 @@ sep_prepare_pkg_context(struct sepgraph *g, int pi, int context, } for (int i = 0; i < g->pkg[pi].nsources && rc == 0; i++) rc = sep_register_file_fold(g, g->pkg[pi].canon, - g->pkg[pi].sources[i]); + g->pkg[pi].sources[i].path); } } if (g->pkg[pi].is_dir) { for (int i = 0; i < g->pkg[pi].nsources && rc == 0; i++) - rc = sep_scan_file(g, pi, g->pkg[pi].sources[i], - context, &fv, &bindings, children, 1); + rc = sep_scan_file(g, pi, g->pkg[pi].sources[i].path, + context, &fv, &bindings, children, 1, + g->pkg[pi].sources[i].buf, + g->pkg[pi].sources[i].len); } else if (rc == 0) { rc = sep_scan_file(g, pi, g->pkg[pi].entry, context, - &fv, &bindings, children, 0); + &fv, &bindings, children, 0, NULL, 0); } sep_import_set_free(&fv); if (bindings.n > 1) @@ -4827,13 +5124,18 @@ sep_validate_module_closure(struct sepgraph *g, const int *order, int n, * //ww:module-reset primary boundary. No source or export outside pi's * sorted owned-source set may enter this unit. */ static int -sep_emit_body(FILE *out, const char *path, const char *modpath) +sep_emit_body(FILE *out, const char *path, const char *modpath, + const char *selected_buf, u64 selected_len) { - char *buf; - u64 len; - if (sep_slurp(path, &buf, &len) < 0) { - fprintf(stderr, "ww: cannot read %s\n", path); - return -1; + char *owned = NULL; + const char *buf = selected_buf; + u64 len = selected_len; + if (buf == NULL) { + if (sep_slurp(path, &owned, &len) < 0) { + fprintf(stderr, "ww: cannot read %s\n", path); + return -1; + } + buf = owned; } /* #57: tag the primary body by its full dotted import path so the * definer mangles == the importer reference; a root build (path "") @@ -4859,7 +5161,7 @@ sep_emit_body(FILE *out, const char *path, const char *modpath) != (size_t)(len - off) || fputc('\n', out) == EOF) bad = 1; - free(buf); + free(owned); if (bad) { fprintf(stderr, "ww: cannot write package unit\n"); return -1; @@ -4931,10 +5233,12 @@ sep_compose_unit(struct sepgraph *g, int pi, const char *scratch, bodyrc = -1; } else if (g->pkg[pi].is_dir) { for (int i = 0; i < g->pkg[pi].nsources && bodyrc == 0; i++) - bodyrc = sep_emit_body(u, g->pkg[pi].sources[i], - g->pkg[pi].path); + bodyrc = sep_emit_body(u, g->pkg[pi].sources[i].path, + g->pkg[pi].path, g->pkg[pi].sources[i].buf, + g->pkg[pi].sources[i].len); } else { - bodyrc = sep_emit_body(u, g->pkg[pi].entry, g->pkg[pi].path); + bodyrc = sep_emit_body(u, g->pkg[pi].entry, g->pkg[pi].path, + g->pkg[pi].entry_buf, g->pkg[pi].entry_len); } const char *own_suffix; size_t own_parents; @@ -7169,6 +7473,7 @@ build_one_sep(const char *src, int entry_is_dir, const char *root_identity, } } } + source_documentation_preload_clear(); return r; } @@ -7619,6 +7924,18 @@ do_build(int argc, char **argv) free(incs); return 1; } + if (!is_dir && source_regular_file(resolved)) { + int documentation = source_preload_documentation(resolved); + if (documentation < 0) { + free(incs); + return 1; + } + if (documentation > 0) { + source_operand_no_sources(resolved); + free(incs); + return 1; + } + } char out[PATH_MAX]; const char *objstem = NULL; int discard_output = strcmp(outflag, "/dev/null") == 0; @@ -7736,6 +8053,18 @@ do_run(int argc, char **argv) free(incs); return 1; } + if (!is_dir && source_regular_file(resolved)) { + int documentation = source_preload_documentation(resolved); + if (documentation < 0) { + free(incs); + return 1; + } + if (documentation > 0) { + source_operand_no_sources(resolved); + free(incs); + return 1; + } + } char tmpdir[PATH_MAX], tmp[PATH_MAX]; snprintf(tmpdir, sizeof tmpdir, "/tmp/ww_run_%d", getpid()); if (mkdir(tmpdir, 0700) != 0) { @@ -7797,6 +8126,17 @@ do_run(int argc, char **argv) return rc; } +static int +source_test_preflight(const char *path, int compileonly, int emit_asm) +{ + if (!source_regular_file(path)) return 0; + int documentation = source_preload_documentation(path); + if (documentation > 0) source_operand_no_sources(path); + if (documentation == 0) return 0; + if (!compileonly && !emit_asm) fputs("FAIL\n", stdout); + return 1; +} + static int do_test(int argc, char **argv) { @@ -8293,6 +8633,8 @@ do_test(int argc, char **argv) "ww test: package-test variant needs one directory\n"); return 2; } + if (source_test_preflight(resolved, compileonly, emit_asm)) + return 1; char tmpdir[PATH_MAX] = {0}, tmp[PATH_MAX]; const char *outp; int retain_output = outstem[0] && !discard_output; @@ -8375,6 +8717,8 @@ do_test(int argc, char **argv) "ww test: package options need a directory\n"); return 2; } + if (source_test_preflight(target, compileonly, emit_asm)) + return 1; char tmpdir[PATH_MAX] = {0}, tmp[PATH_MAX]; const char *outp; int retain_output = outstem[0] && !discard_output; diff --git a/docs/build-system.md b/docs/build-system.md index 24ccbf35..f09ab8d8 100644 --- a/docs/build-system.md +++ b/docs/build-system.md @@ -10646,8 +10646,9 @@ suffix. - **behavior derived from the pinned implementation** — the rule does not complete the remaining suffix-first run front, multiple named sources, finite `.ww` FIFO capture, shared test-process state/failure topology, - Go-compatible `-run` regular expressions, or `package documentation` - suppression. Existing `.ww` directory slices, hidden-source exclusion, + or Go-compatible `-run` regular expressions. Exact `package documentation` + suppression was still open when section 11.54 closed and is completed in + section 11.55. Existing `.ww` directory slices, hidden-source exclusion, named `_test.ww` build omission, recursive/multiple-root coordination, package syntax, and import syntax remain intact. @@ -10655,6 +10656,202 @@ Build workdir format remains `18`, test workdir format remains `19`, and semantic storage format remains `3`. No schema, action descriptor, cache/result record, manifest, transaction protocol, or lock changes. +### 11.55 Implemented exact `package documentation` source suppression + +A selected source whose successfully parsed package name is exactly +`documentation` is a documentation source, not a semantic package source. It +is removed by loading before source-family folding, import-edge construction, +or any build, run, or test action. `documentation_test`, `documentationx`, and +every other package name remain ordinary. + +#### Pinned authority, header boundary, and applicability + +- **behavior directly implemented or asserted by pinned Go** — the sole + authority is official Go 1.26.5 commit + `c19862e5f8415b4f24b189d065ed739517c548ba`. Its `go/build` loader records a + package/import-header error before excluding an exact parsed package name + `documentation`, and does so before `_test.go` or package-family + classification. Named Go files pass through the same rule. Public Go command + help also reserves the name and says such files are ignored. +- **behavior directly implemented or asserted by pinned Go** — the pinned + loader reads the package clause and contiguous import section, not an + arbitrary body. A successful ordinary-body stop removes its one-byte + lookahead. It first probes a raw following `i` as a possible `import`, + however, so an `i` that does not form that exact keyword is a malformed + attempted import rather than a successful body boundary. Exact `import` + followed by malformed import syntax retains the ordinary import-parser + diagnostic. +- **behavior directly implemented or asserted by pinned Go** — official + `src/go/build/read_test.go` tests the ordinary-body stop and malformed-import + recovery, but no test or testdata in the pinned tree directly names + `package documentation`. `cmd/go/testdata/script/mod_doc.txt` concerns module + documentation and is not evidence for this rule. This absence is explicit; + host-Go observations do not fill it. +- **directly measured WW behavior** — before suppression, both driver stages + incorrectly treated documentation files as ordinary sources: doc-only roots + produced semantic artifacts, mixed roots conflicted, imported providers + created edges and actions, and test routes created test products. Valid + malformed bodies also exposed stage-specific recovery diagnostics, while + malformed package and contiguous-import headers already failed before an + action. +- **behavior derived from the pinned implementation** — the quiet candidate + recognizer admits one leading BOM, leading whitespace, comments, internal + line directives, and trivia around `package`, exact identifier + `documentation`, and `;`. It emits no diagnostic and sends only a plausible + exact candidate through the existing package/import-header parser, so + ordinary sources acquire no new early diagnostics. Malformed candidate + headers retain that parser's positioned errors. +- **behavior derived from the pinned implementation** — after a valid exact + header, a non-`i` first raw byte at the body boundary ends loading and every + later byte is ignored. If that first raw byte is `i` but does not form the + exact `import` token, both stage drivers and the shared coordinator emit + exactly `::: error: expected top-level decl\n`, owned by + that byte. A malformed exact import retains its existing diagnostic. A + malformed package clause, malformed contiguous import, reached header NUL, + or unterminated header-trivia comment also remains an error. No no-source + diagnostic follows any such header error. + +#### Selection, package identity, and imports + +- **behavior derived from the pinned implementation** — existing CLI and + operand-shape errors, logical resolution and file-kind checks, and suffix, + hidden-prefix, target, and test-role eligibility retain precedence. The rule + then applies to selected directory members, literal and logical named roots, + raw tests, direct and recursive coordinator discovery, and every dotted + dependency resolving to a directory. Source imports remain directory-only; + a logical one-file provider is only a CLI-root compatibility route. +- **behavior derived from the pinned implementation** — named-file + documentation preflight is entered only after symlink-following `stat` + classifies the selected source as regular. A symlink to a regular file is + therefore included. The preflight owns one exact read buffer: a suppressed + documentation source is discarded from that buffer, while an ordinary source + carries the same bytes into graph loading instead of being reopened. FIFO and + other nonregular named-source routes retain their existing handling and are + neither classified nor otherwise changed by this preflight. +- **behavior derived from the pinned implementation** — coordinator directory + discovery collects metadata only. It canonicalizes and deduplicates selected + paths before reading source, classifies each unique source exactly once, and + passes the retained ordinary-source buffer to its existing source validation. + Direct-root errors and recursive pattern/group accounting are computed per + request from those classification results; a directory member is not + reclassified for each spelling or pattern that found it. The delegated stage + driver owns a separate request-graph observation: it enumerates each reached + canonical directory once, lazily opens only role-eligible files, and reuses + each observed path, classification, package-name/`@test` attestation, and byte + snapshot across production, same-package-test, external-test, and copied test + actions. A production-only request therefore still leaves `*_test.ww` + unopened. The coordinator and delegated driver are distinct existing process + boundaries; this slice does not add a cross-process atomic snapshot for a + source concurrently rewritten between those observations. +- **behavior derived from the pinned implementation** — the three test-source + routes remain distinct. A visible literal named `_test.ww` build first + validates its package/import header and then omits it by the already completed + test-only rule; documentation classification, including the raw-`i` check, + does not run. Directory production keeps eligible `*_test.ww` entries + unopened. Raw `ww test` and selected directory test variants do run the + documentation classifier. A logical request whose resolved file merely has + an `_test.ww` physical basename is not the literal named-build special case. +- **behavior derived from the pinned implementation** — an omitted source + contributes no package member, declared-family or test-family candidate, + import occurrence, binding or edge, qualifier, initializer, declaration, + symbol, graph action, unit, `.wwi`, or persistence identity. Its physical + pathname, parent, and symlink information remain loader observations only. + A logical one-file CLI source receives `__root` identity only when retained + as ordinary. A doc-only dotted directory provider has no package sources and + cannot satisfy an import; a same-named `.ww` file remains an import decoy + under the pre-existing directory-only import rule. +- **behavior derived from the pinned implementation** — a mixed directory + retains exactly the canonical local or dotted identity and source graph of + the same tree with the documentation file absent. Imports appearing in the + valid contiguous documentation header are checked only to establish the + header; they never become dependency edges. Imports, declarations, + initializers, tests, aborts, nonzero mains, missing dependencies, and syntax + after a successful non-`i` body boundary are unobserved. + +#### Build, run, and test empty selections + +- **behavior derived from the pinned implementation** — a direct + non-coordinator doc-only `ww build DIR` exits 1 with empty stdout and exact + stderr `ww: DIR: directory contains no WW package sources\n`. A selected + named or logical non-test source uses its physical containing directory in + the same diagnostic, and `-S`, output, and `/dev/null` modes do not displace + it. A direct root already routed through the coordinator, including an + output-directory request, instead uses exact stderr + `wwtest package: DIR: directory contains no WW package sources\n` with the + same status and stdout. +- **behavior derived from the pinned implementation** — the literal visible + named `_test.ww` build retains section 11.51's outcome even when its valid + declared name is `documentation`: no effective `-o` (including an + assembly-only request), or exact `/dev/null`, succeeds silently; a + non-directory output exits 1 with + `ww: no packages to build\n`; an output-directory request exits 1 with + `ww: no main packages to build\n`. Header errors still precede those + outcomes. +- **behavior derived from the pinned implementation** — `ww run` on a selected + doc-only physical or logical directory exits 1 with empty stdout and + `ww: DIR: directory contains no WW package sources\n`. The existing private + `/tmp/ww_run_` is acquired before directory enumeration and then + removed. A named or logical one-file run uses its physical parent's + no-source diagnostic before acquiring run scratch. The separately open + suffix-first run behavior may select a visible `_test.ww`; when it does, this + rule suppresses that source. A `.ww`-spelled directory keeps the earlier + stat-first run rejection. +- **behavior derived from the pinned implementation** — direct doc-only + `ww test DIR` exits 1 with stdout `FAIL\n` and the coordinator no-source + stderr; `ww test -c DIR` exits 1 with empty stdout and the same stderr. + Directory `test -S` without the required `-o` retains status 2 and exact + `ww test: -S needs -o\n`; after a valid `-o`, the directory retains status 2 + and exact `ww test: -S needs a single test file\n`. Both checks precede + source classification. Named or logical raw `ww test FILE` exits 1 with + `FAIL\n` and the physical parent's driver no-source stderr; raw `-c` and + `-S` exit 1 with empty stdout and the same stderr. No requested output or + work state changes. +- **behavior derived from the pinned implementation** — a recursive build + pattern retaining only doc-only directories exits 0 with empty stdout and + one `ww: warning: "PATTERN" matched no packages\n`; recursive test exits 1 + with empty stdout, that warning, then + `ww test: no packages to test\n`. A pattern also containing ordinary + directories builds or tests only those groups without warning. Multiple + patterns warn once for each no-match pattern. Any directly named doc-only + sibling root fails the complete request during discovery before any ordinary + group starts. +- **behavior derived from the pinned implementation** — the positioned raw-`i` + header error has empty stdout for build, run, `test -c`, and raw `test -S`. + An explicit running raw or directory/recursive test adds exactly `FAIL\n`. + Literal named `_test.ww` build and directory `test -S` retain their earlier + precedence and never reach this classifier. + +#### Actions, lifecycle, parity, formats, and scope + +- **behavior derived from the pinned implementation** — a doc-only source + causes no compiler, assembler, archiver, linker, generated test support/main, + test child, initializer, or program runtime. It creates no unit, `.wwi`, + assembly, object, archive, executable, output directory, sidecar, workdir + stamp, transaction, capture, or retained result. Mixed outputs and comparable + semantic artifacts are byte-identical to the source-absent control. +- **behavior derived from the pinned implementation** — a documentation source + has no publication destination or persistent key. Adding, removing, or + changing only an ignored documentation body does not invalidate, refresh, + replace, or become a reuse input for committed ordinary actions. Doc-only + cold failure publishes nothing; a warm no-source or header failure preserves + prior public and semantic bytes. Ordinary sibling publication, producer + failure, transaction rollback, and invalidation remain unchanged. +- **behavior derived from the pinned implementation** — classification state + is ephemeral and request-owned: the named-root preload is process-local and + cleared at the build boundary, while directory observations live only in the + request graph. Neither is persisted or shared across requests, and no lock, + schema, cache, or cross-request identity is added. Normal and controlled + failure use the existing reader and request-private cleanup. The verified + direct-SIGTERM fixed-`.new` poisoning gap remains open and is neither reached + nor repaired by this slice. +- **behavior derived from the pinned implementation** — Cstage and WWstage + must agree on selected status, stdout, stderr, source membership, graphs, + actions, semantic artifacts, and lifecycle. The producer-provenance + `.wwtool.ww` remains intentionally stage-specific. Build workdir format + remains `18`, test workdir format remains `19`, and semantic storage format + remains `3`; no action descriptor, cache/result record, transaction marker, + manifest, database, or lock is added. + ## 12. Candidate architectures and hard-gate decision Five candidates were developed as coherent systems, not as feature bins. diff --git a/docs/spec.md b/docs/spec.md index 4b5e3fb5..399ba896 100644 --- a/docs/spec.md +++ b/docs/spec.md @@ -319,6 +319,87 @@ ImportPath = ident { "." ident } . delegated graph-import scan or tools. An ordinary build compares production names only; one test product compares its production, same-package test, and external-test selections without merging their units. +- After existing filename, target, kind, and test-role eligibility, a selected + source whose package/import header successfully declares the exact name + `documentation` is omitted before semantic source membership and import + scanning. The loader accepts the ordinary leading BOM, whitespace, comments, + internal line directives, and package-clause trivia when recognizing this + exact candidate. `documentation_test`, `documentationx`, and all other names + remain ordinary sources. + + Candidate recognition is silent. A malformed package clause, malformed + contiguous import, reached header NUL, or unterminated header-trivia comment + retains its existing positioned load error before omission. After a valid + exact package/import prefix, a non-`i` first raw byte establishes the body + boundary and the remaining bytes are not parsed. A first raw `i` that does + not form the exact `import` token is instead diagnosed at that byte as + `expected top-level decl`; exact `import` followed by malformed import syntax + retains the ordinary import-parser diagnostic. Thus syntax, types, late + imports, imports missing only in the body, declarations, initializers, + tests, and runtime behavior after a successful non-`i` boundary are not + observed. + + An omitted documentation source creates no package member or family, + source-file import occurrence, binding or graph edge, qualifier, + initializer, declaration, symbol, `.wwi`, action, publication destination, + or persistence identity. A mixed package is semantically identical to the + same source set without that file. A doc-only dotted directory provider has + no package sources and cannot satisfy an import. A same-named `.ww` file is + still not an import provider under the directory-only import rule; when such + a file is selected as a logical CLI root, it receives the established + `__root` identity only if retained as ordinary. The rule applies equally to + selected literal and logical roots, directory members, raw tests, recursive + discovery, and dotted directory providers. + + Named-file documentation preflight applies only after symlink-following + `stat` classifies the literal or resolved source as regular, including a + symlink to a regular file. It retains one exact read buffer: documentation + bytes are suppressed from that buffer, while ordinary bytes from the same + preflight are reused by graph loading rather than reopened. FIFO and other + nonregular named-source routes keep their prior behavior and are not touched + by this preflight. + + Coordinator directory discovery records metadata only. Selected paths are + canonicalized and deduplicated before source reads; each unique source is + then classified exactly once. For an ordinary result, the same buffered bytes + feed existing coordinator source validation. Direct-root failure and + recursive pattern/group accounting are performed per request after + classification, without rereading or reclassifying a source reached by more + than one spelling or pattern. Independently, one delegated driver request + enumerates each canonical directory once and lazily snapshots each + role-eligible regular source once; that observation supplies classification, + package/test attestation, import scanning, copied test actions, and unit + emission across every variant in the request. A production-only request does + not open excluded `*_test.ww` entries. Coordinator and delegated-driver + observations remain separate process boundaries; concurrent rewriting between + them gains no new atomic-snapshot guarantee. + + Test-role precedence remains route-specific. A visible literal named + `_test.ww` passed to `ww build` validates its package/import header and then + follows the test-only empty-selection rule below without documentation + classification. A production directory build leaves `*_test.ww` unopened. + Raw `ww test` and selected directory test variants do classify documentation + sources. A logical request whose resolved provider merely has an `_test.ww` + physical basename is not the literal named-build special case. Directory + `ww test -S` without its required `-o` rejects as + `ww test: -S needs -o`; with a valid `-o`, a directory rejects as + `ww test: -S needs a single test file`. Both checks precede classification. + + A doc-only direct build or run root takes its route's existing + `directory contains no WW package sources` result; recursive doc-only + directories do not become packages, while mixed roots are exactly the + source-absent control. No compiler, assembler, archiver, linker, generated + test support/main, test child, initializer, or program runtime is attributable + to the omitted source, and it creates no unit, interface, assembly, object, + archive, executable, output, transaction, capture, or retained result. + Changing only an ignored documentation body cannot invalidate or replace a + committed semantic generation. Cold failure publishes nothing; warm + no-source or header failure preserves prior public and semantic bytes. + Classification state is ephemeral: a process-local named-root preload is + cleared at the build boundary, and directory observations die with the + request graph. No state is shared or persisted across requests, and no lock, + schema, cache, or serialized identity is added. Build workdir format remains + 18, test workdir format remains 19, and semantic storage format remains 3. - A single existing raw `.ww` operand is also subject to the unconditional leading-name rule: if its final requested basename begins `.` or `_`, it is ignored before the source is opened. Named raw sources otherwise retain their @@ -983,6 +1064,26 @@ support, generated main, link, binary, captured runtime result, or process. The coordinator reports that successful validation exactly as `? [no test files]\n`. +An exact `package documentation` source that reaches a test selection is +suppressed by the package rule in §4 before production, same-package, or +external-test family construction. It contributes no `@test`, dependency, +initializer, support action, generated-main input, binary, process, captured +result, or retained output. In a mixed directory the test product is exactly +the product of the ordinary sources alone; a doc-only directory is not the +successful production-with-no-tests case. A direct running directory request +instead fails with `FAIL\n` and the coordinator's +`directory contains no WW package sources` diagnostic, while `-c` omits +`FAIL`. A selected named or logical raw running request likewise emits `FAIL` +and its physical parent's driver no-source diagnostic; raw `-c` and `-S` omit +the marker. Directory `-S` without the required `-o` retains +`ww test: -S needs -o`; after a valid `-o`, it retains +`ww test: -S needs a single test file`. Both precede source classification. +Recursive doc-only matches are omitted as packages: running test reports the +per-pattern no-match warning followed by `ww test: no packages to test`, while +mixed recursive selections run only ordinary groups. A documentation-header +error retains precedence over every no-source result; an explicit running test +adds `FAIL`, but compile-only and assembly-only routes do not. + Every test-bearing directory product links one request-private runnable. The test output-option name is exactly `o`, with the accepted forms `-o VALUE`, `--o VALUE`, `-o=VALUE`, and `--o=VALUE`. Equals forms split only at their diff --git a/docs/test-system-v2.md b/docs/test-system-v2.md index 9b015631..5b8a1e16 100644 --- a/docs/test-system-v2.md +++ b/docs/test-system-v2.md @@ -265,6 +265,80 @@ directory/recursive selection, package/import identities, graph/action identities, persistence formats (build 18, test 19, semantic 3), and test process topology are explicit non-effects. +Exact `package documentation` is a separate source-suppression rule after the +completed filename, target, file-kind, and test-role gates. Official Go 1.26.5 +`go/build` directly implements the exclusion after package/import-header +loading and before `_test.go` or package-family classification. Its +`read_test.go` directly tests the ordinary-body stop and malformed-import +recovery, but the pinned official test/testdata tree has no test that directly +names `package documentation`; `mod_doc.txt` is unrelated. That test absence is +recorded rather than replaced with host-Go behavior. + +The focused package observer is therefore WW-native dual-stage proof. It +covers one leading BOM, whitespace/comments/internal line directives and +package-clause trivia; exact-name controls; malformed package and contiguous +import headers; the raw body-boundary `i` that does not form exact `import` and +must diagnose `expected top-level decl`; and ignored non-`i` bodies containing +late imports, missing imports, declarations, initializers, `@test`, aborts, or +nonzero mains. It also covers literal and logical single files, direct and +recursive directories, the unchanged one-file import-decoy boundary, dotted +directory providers, mixed and doc-only source sets, and exact Cstage/WWstage +status, stream, source-set, diagnostic, and semantic-artifact parity. + +Named-file documentation preflight is restricted to sources whose +symlink-following `stat` result is regular, including symlinks to regular +files. It retains one exact read buffer: documentation sources are suppressed +from it, while ordinary sources carry those same bytes into graph loading. +FIFO and other nonregular named-source routes are untouched. Coordinator +directory discovery is metadata-only; after canonicalization and deduplication, +each unique selected source is classified exactly once. Buffered ordinary +bytes are then reused for coordinator source validation, and per-request +direct/recursive error, match, and group accounting follows classification. +The delegated driver separately owns a request-graph directory observation: +one canonical membership list and one lazy regular-source snapshot feed every +reached production/test variant, including package-name/`@test` checks, import +scanning, copied test actions, and unit emission. Production-only selection +still does not open excluded `*_test.ww` files. The coordinator and delegated +driver retain their separate process observations; the slice does not promise +an atomic source snapshot across a concurrent rewrite between them. + +The test route matrix remains intentional. A visible literal named +`_test.ww` build validates only its package/import header and then follows the +existing test-only empty-selection rule; it does not run documentation +classification or the synthetic raw-`i` check. Directory production leaves +`*_test.ww` unopened. Raw `ww test` and selected directory test variants do +classify exact documentation sources, while a logical request whose resolved +provider merely has an `_test.ww` physical basename is not the literal named +build special case. Directory `test -S` without its required `-o` first rejects +with `ww test: -S needs -o`; after a valid `-o`, a directory rejects with +`ww test: -S needs a single test file`. Both branches precede documentation +classification. + +A doc-only direct running directory test exits 1 with exact `FAIL\n` stdout and +the coordinator's no-source stderr; `-c` has the same error and empty stdout. +A doc-only named or logical raw running test likewise emits `FAIL\n` plus the +driver's physical-parent no-source diagnostic; raw `-c` and `-S` have empty +stdout. A documentation-header error precedes no-source, with `FAIL` added only +by an explicit running raw or directory/recursive test. Recursive doc-only +patterns become no matches: running test emits the per-pattern warning then +`ww test: no packages to test`, while mixed patterns retain and run only +ordinary groups without warning. A directly named doc-only sibling root fails +discovery for the complete request before any group starts. + +Suppression happens before package folders, internal/external families, +products, graph edges, support, generated main, or execution. The documentation +source creates no compiler, assembler, archiver, linker, test child, capture, +result, retained binary, public output, work-state artifact, transaction, or +persistence key. Mixed products and semantic artifacts equal the source-absent +control byte for byte; changing only an ignored documentation body cannot +invalidate or replace a committed generation. Cold failure publishes nothing, +warm header/no-source failure preserves prior public and semantic bytes, and +normal request-private cleanup and concurrency isolation remain unchanged. +The known external-driver fixed-`.new` interruption poisoning remains open. +Build workdir format remains 18, test workdir format remains 19, and semantic +storage format remains 3; no test-result cache, schema, action descriptor, +transaction marker, or lock is introduced. + An existing local directory whose requested build basename ends `.ww` (including a visible `_test.ww` symlink to a directory) remains a directory package, not a raw named test source. WWstage `ww build` now uses the same diff --git a/internal/wwpackage/package.ww b/internal/wwpackage/package.ww index eb368f94..2f398c2b 100644 --- a/internal/wwpackage/package.ww +++ b/internal/wwpackage/package.ww @@ -6,6 +6,7 @@ import strconv; import strings; import temp; import time; +import ww.syntax; type pkgsource = struct { path: str, @@ -165,6 +166,20 @@ type pkgdiscover = struct { fatal: bool, }; +type pkgrequestselection = struct { + requested: str, + discoverroot: str, + start: i32, + end: i32, + recurse: bool, + haderrors: bool, +}; + +fn pkgallocrequests(cap: i32) ([]pkgrequestselection | nomem) = { + let value: []pkgrequestselection = alloc([], cap: u64)?; + return value; +}; + fn pkggrowcap(current: i32, need: i32) i32 = { if (need < 0) { return -1; }; if (need <= current) { return current; }; @@ -765,6 +780,60 @@ fn pkgskipspace(src: str, start: i32) i32 = { return i; }; +fn pkgdocumentationword(src: str, at: *i32, word: str) bool = { + let i: i32 = *at; + if (i < 0 || i > src.len || src.len - i < word.len) { return false; }; + let j: i32 = 0; + for (j < word.len) { + if (src[i + j] != word[j]) { return false; }; + j += 1; + }; + if (src.len - i > word.len && pkgident(src[i + word.len])) { + return false; + }; + *at = i + word.len; + return true; +}; + +// This is deliberately a quiet exact-name recognizer. Only a plausible +// reserved package clause enters the narrow syntax header parser, so ordinary +// sources retain their existing full-body diagnostic timing. +fn pkgdocumentationcandidate(src: str) bool = { + let at: i32 = 0; + if (src.len >= 3 && src[0] == 0xefu8 && src[1] == 0xbbu8 + && src[2] == 0xbfu8) { at = 3; }; + at = pkgskipspace(src, at); + if (!pkgdocumentationword(src, &at, "package")) { return false; }; + at = pkgskipspace(src, at); + if (!pkgdocumentationword(src, &at, "documentation")) { return false; }; + at = pkgskipspace(src, at); + return at < src.len && src[at] == ';'; +}; + +// -1 is a diagnosed package/import-header error, 0 an ordinary source, and +// 1 an exact documentation source. A remaining raw 'i' reproduces pinned +// readGoInfo's failed-import-keyword branch; an exact import token was already +// consumed or diagnosed by parsepackageheader. +fn pkgdocumentation(path: str, src: str) i32 = { + if (!pkgdocumentationcandidate(src)) { return 0; }; + let l: syntax.lex; + syntax.lexinit(&l, path, src.ptr, src.len: u64); + let ps: syntax.parser; + syntax.parserinit(&ps, &l); + let header: *syntax.node = syntax.parsepackageheader(&ps); + if (l.errs > 0 || ps.errs > 0) { return -1; }; + if (header.nmod.len == 0) { + pkgfailsource(path, 1, 1, "invalid or missing package clause"); + return -1; + }; + if (l.lpos < src.len: u64 && src[l.lpos] == 'i') { + pkgfailsource(path, l.line, l.col, "expected top-level decl"); + return -1; + }; + if (strings.compare(header.nmod, "documentation") == 0) { return 1; }; + return 0; +}; + fn pkgclause(src: str, out: *str) bool = { // Match the compiler readers: one UTF-8 BOM is invisible only at the // first raw source position. The full stage driver owns later-BOM errors. @@ -1412,6 +1481,19 @@ fn pkgdedup(ss: []str) []str = { return ss; }; +fn pkgindexselected(ss: []str, value: str) i32 = { + let i: i32 = 0; + for (i < ss.len) { + if (strings.compare(ss[i], value) == 0) { return i; }; + i += 1; + }; + return -1; +}; + +fn pkgcontainsselected(ss: []str, value: str) bool = { + return pkgindexselected(ss, value) >= 0; +}; + fn pkgparsedec(s: str, max: i64) i64 = { if (s.len == 0) { return -1i64; }; let i: i32 = 0; @@ -2182,6 +2264,16 @@ export fn packagecommand(args: []str) int = { directroots.paths = emptydirect; directroots.errors = 0; directroots.fatal = false; + let requestallocation: ([]pkgrequestselection | nomem) = + pkgallocrequests(roots.len); + let requests: []pkgrequestselection; + match (requestallocation) { + case let value: []pkgrequestselection => requests = value; + case nomem => { + pkgputln(os.STDERR_FILENO, "wwtest package: out of memory"); + return 1; + }; + }; let anyrecurse: bool = false; i = 0; for (i < roots.len) { @@ -2231,14 +2323,32 @@ export fn packagecommand(args: []str) int = { }; pkgfreematcher(&matcher); }; - if (recurse && ds.paths.len == before && ds.errors == errorsbefore) { - pkgput(os.STDERR_FILENO, "ww: warning: "); - pkgputquoted(os.STDERR_FILENO, requested); - pkgputln(os.STDERR_FILENO, " matched no packages"); - } else if (!recurse && ds.paths.len == before - && ds.errors == errorsbefore) { - pkgfailpath(discoverroot, "directory contains no WW package sources"); - ds.errors += 1; + let request: pkgrequestselection; + request.requested = requested; + request.discoverroot = discoverroot; + request.start = before; + request.end = ds.paths.len; + request.recurse = recurse; + request.haderrors = ds.errors != errorsbefore; + append(requests, request); + i += 1; + }; + // Preserve established empty-discovery diagnostics before any new source + // read. Requests with raw candidates are accounted again only after exact + // documentation sources have been removed from the canonical unique set. + i = 0; + for (i < requests.len) { + if (requests[i].start == requests[i].end + && !requests[i].haderrors) { + if (requests[i].recurse) { + pkgput(os.STDERR_FILENO, "ww: warning: "); + pkgputquoted(os.STDERR_FILENO, requests[i].requested); + pkgputln(os.STDERR_FILENO, " matched no packages"); + } else { + pkgfailpath(requests[i].discoverroot, + "directory contains no WW package sources"); + ds.errors += 1; + }; }; i += 1; }; @@ -2264,8 +2374,112 @@ export fn packagecommand(args: []str) int = { ds.paths[i] = canonicalsource; i += 1; }; - pkgsort(ds.paths); - ds.paths = pkgdedup(ds.paths); + // Keep the canonical occurrence array in discovery order for per-request + // membership. Classification reads each canonical path exactly once after + // the established directory/source ordering and deduplication, so overlap + // cannot duplicate a header diagnostic, pre-existing diagnostic precedence + // is retained, and ordinary bytes can feed source validation unchanged. + let occurrencepaths: []str = ds.paths; + let selectedallocation: ([]str | nomem) = pkgallocstrs(ds.paths.len); + let selected: []str; + match (selectedallocation) { + case let value: []str => selected = value; + case nomem => { + pkgputln(os.STDERR_FILENO, "wwtest package: out of memory"); + return 1; + }; + }; + i = 0; + for (i < ds.paths.len) { append(selected, ds.paths[i]); i += 1; }; + pkgsort(selected); + selected = pkgdedup(selected); + let bodyallocation: ([]str | nomem) = pkgallocstrs(selected.len); + let bodies: []str; + match (bodyallocation) { + case let value: []str => bodies = value; + case nomem => { + pkgputln(os.STDERR_FILENO, "wwtest package: out of memory"); + return 1; + }; + }; + let readallocation: ([]i32 | nomem) = pkgalloci32(selected.len); + let readstates: []i32; + match (readallocation) { + case let value: []i32 => readstates = value; + case nomem => { + pkgputln(os.STDERR_FILENO, "wwtest package: out of memory"); + return 1; + }; + }; + let retained: i32 = 0; + i = 0; + for (i < selected.len) { + let path: str = selected[i]; + let body: str = ""; + let readstate: i32 = 0; + let documentation: i32 = 0; + if (buildonly && strings.hassuffix(pkgbase(path), "_test.ww")) { + readstate = 2; + } else if (pkgread(path, &body)) { + readstate = 1; + documentation = pkgdocumentation(path, body); + if (documentation < 0) { + return pkgteststatusfail(explicitstatus, compileonly); + }; + }; + if (documentation == 0) { + selected[retained] = path; + append(bodies, body); + append(readstates, readstate); + retained += 1; + }; + i += 1; + }; + selected.len = retained; + // A raw nonempty request can become empty only through this suppression. + // Each recursive spelling owns one warning; a direct empty root is a + // request error that prevents every retained sibling from launching. + i = 0; + for (i < requests.len) { + if (requests[i].start != requests[i].end + && !requests[i].haderrors) { + let found: bool = false; + let oi: i32 = requests[i].start; + for (oi < requests[i].end && !found) { + found = pkgcontainsselected(selected, occurrencepaths[oi]); + oi += 1; + }; + if (!found) { + if (requests[i].recurse) { + pkgput(os.STDERR_FILENO, "ww: warning: "); + pkgputquoted(os.STDERR_FILENO, + requests[i].requested); + pkgputln(os.STDERR_FILENO, " matched no packages"); + } else { + pkgfailpath(requests[i].discoverroot, + "directory contains no WW package sources"); + ds.errors += 1; + }; + }; + }; + i += 1; + }; + if (ds.errors != 0) { + return pkgteststatusfail(explicitstatus, compileonly); + }; + let sortedallocation: ([]str | nomem) = pkgallocstrs(selected.len); + let sortedpaths: []str; + match (sortedallocation) { + case let value: []str => sortedpaths = value; + case nomem => { + pkgputln(os.STDERR_FILENO, "wwtest package: out of memory"); + return 1; + }; + }; + i = 0; + for (i < selected.len) { append(sortedpaths, selected[i]); i += 1; }; + pkgsort(sortedpaths); + ds.paths = sortedpaths; if (ds.paths.len == 0) { if (buildonly) { if (explicitout && !buildnull && pkgoutputdir(outname)) { @@ -2304,9 +2518,15 @@ export fn packagecommand(args: []str) int = { i += 1; continue; }; - let body: str; + let selectedindex: i32 = pkgindexselected(selected, ds.paths[i]); + if (selectedindex < 0) { + pkgputln(os.STDERR_FILENO, + "wwtest package: package graph is inconsistent"); + return 1; + }; + let body: str = bodies[selectedindex]; let pn: str; - if (!pkgread(ds.paths[i], &body)) { + if (readstates[selectedindex] != 1) { pkgfailpath(ds.paths[i], "invalid or missing package clause"); return pkgteststatusfail(explicitstatus, compileonly); }; diff --git a/selfhost/cmd/ww/main.ww b/selfhost/cmd/ww/main.ww index 90a0a66e..d13d6956 100644 --- a/selfhost/cmd/ww/main.ww +++ b/selfhost/cmd/ww/main.ww @@ -29,6 +29,10 @@ def SEP_LOCAL_IMPORT_PREFIX: str = "__wwlocal"; let selfpath: *u8; let sepfatalallocation: bool; +let sourcedocumentationpreloadpath: *u8; +let sourcedocumentationpreloadbuf: *u8; +let sourcedocumentationpreloadlen: u64; +let sourcedocumentationpreloadcap: u64; @symbol("rt_envp") fn rawenvp() **u8; @@ -495,32 +499,12 @@ fn locatemodule(dirs: *u8, name: *u8, namelen: u64, // Go's contract: only *_test.ww is a test source. Ask the compiler parser, // rather than a textual attribute scan, whether a production source contains // @test; otherwise valid whitespace/comments could silently drop a test. -fn dirfileattest(dirpath: *u8, name: *u8) i32 = { - let path: *u8 = joinpath(dirpath, name); - let fd: i32 = os.open(pathstr(path), os.flag.RDONLY, 0i32); - if (fd < 0) { return -1; }; - let sr: (i64 | os.oserror) = os.filesize(fd); - let n: i64 = -1i64; - match (sr) { - case let v: i64 => n = v; - case let e: os.oserror => { os.close(fd); return -1; }; - }; - if (n < 0i64) { os.close(fd); return -1; }; - let b: []u8 = alloc([], n: u64)!; - b.len = n: i32; - let rr: (i64 | os.oserror) = os.readall(fd, b.ptr, n: u64); - os.close(fd); - let got: i64 = -1i64; - match (rr) { - case let v: i64 => got = v; - case let e: os.oserror => return -1; - }; - if (got != n) { return -1; }; +fn dirfileattest(path: *u8, bufp: *u8, blen: u64) i32 = { // Preserve the directory loader's normalized package-clause diagnostic // before the full parser performs language-level recovery. This is also // the C/WW parity boundary for malformed clauses. let il: syntax.lex; - syntax.lexinit(&il, pathstr(path), b.ptr, n: u64); + syntax.lexinit(&il, pathstr(path), bufp, blen); let ips: syntax.parser; syntax.parserinit(&ips, &il); let imports: *syntax.node = syntax.parseimports(&ips); @@ -531,7 +515,7 @@ fn dirfileattest(dirpath: *u8, name: *u8) i32 = { return -1; }; let l: syntax.lex; - syntax.lexinit(&l, pathstr(path), b.ptr, n: u64); + syntax.lexinit(&l, pathstr(path), bufp, blen); let ps: syntax.parser; syntax.parserinit(&ps, &l); let f: *syntax.node = syntax.parsefile(&ps); @@ -666,26 +650,37 @@ fn sepsourcematchestarget(name: *u8, nlen: u64) bool = { return true; }; -// Classify a selected directory entry: 1 production, 2 test, 0 skipped, -// -1 @test outside *_test.ww, -2 non-regular source. -fn dirfileclass(dirpath: *u8, name: *u8, nlen: u64, - variant: i32) i32 = { - // nlen<=3 guard kept: a bare ".ww" (len 3) is rejected here but - // would pass strings.hassuffix(".ww"); preserves cstage parity. - if (nlen <= 3u64) { return 0; }; - if (name[0] == '.' || name[0] == '_') { return 0; }; - let s: str; - s.ptr = name; - s.len = nlen: i32; - if (!strings.hassuffix(s, ".ww")) { return 0; }; - if (!sepsourcematchestarget(name, nlen)) { return 0; }; - let istest: bool = strings.hassuffix(s, "_test.ww"); - if (istest && variant == SEP_VARIANT_PRODUCTION) { return 0; }; - if (!istest && variant == SEP_VARIANT_EXTERNAL) { return 0; }; - let source: *u8 = sepjoinpath(dirpath, name); - if (source == nil) { return -2; }; +// A graph owns one observation table per canonical directory. Membership is +// captured once, while source bytes remain lazy: a production-only request +// records *_test.ww names but never stats or opens them. Later variants reuse +// the same regular-file classification, parser attestation, package name, and +// exact byte snapshot. +type sepdirobservation = struct { + name: *u8, + nlen: u64, + path: *u8, + istest: bool, + state: i32, // 0 unseen, 1 source, 2 suppressed, -1 @test, -2 error + buf: *u8, + len: u64, + packagename: *u8, +}; + +type sepdircache = struct { + canon: *u8, + entry: *u8, + source: []sepdirobservation, + nsource: i32, + state: i32, // 0 not enumerated, 1 complete, -1 read, -2 diagnosed +}; + +fn dirsourceobserve(c: *sepdircache, i: i32) i32 = { + let o: *sepdirobservation = &c.source[i]; + if (o.state != 0) { return o.state; }; + o.path = sepjoinpath(c.entry, o.name); + if (o.path == nil) { o.state = -2; return o.state; }; let fi: os.filestat; - let sr: (void | os.oserror) = os.lstat(&fi, pathstr(source)); + let sr: (void | os.oserror) = os.lstat(&fi, pathstr(o.path)); let regular: bool = false; let directory: bool = false; match (sr) { @@ -694,7 +689,7 @@ fn dirfileclass(dirpath: *u8, name: *u8, nlen: u64, if (t == os.mode.REG: u32) { regular = true; } else if (t == os.mode.LINK: u32) { let target: os.filestat; - match (os.stat(&target, pathstr(source))) { + match (os.stat(&target, pathstr(o.path))) { case void => { let tt: u32 = (target.mode: u32) & 61440u32; if (tt == os.mode.REG: u32) { regular = true; }; @@ -707,32 +702,43 @@ fn dirfileclass(dirpath: *u8, name: *u8, nlen: u64, case let e: os.oserror => void; }; // go/build ignores a source-shaped symlink to a directory. - if (directory) { return 0; }; + if (directory) { o.state = 2; return o.state; }; if (!regular) { - cerr("ww: "); cerr(pathstr(source)); + cerr("ww: "); cerr(pathstr(o.path)); cerr(": package source is not a regular file\n"); - return -2; + o.state = -2; + return o.state; }; - if (!istest) { - let attest: i32 = dirfileattest(dirpath, name); - if (attest < 0) { return -2; }; - if (attest > 0) { return -1; }; + let body: *u8; + let bodylen: u64; + body, bodylen = slurp(o.path); + if (body == nil) { + cerrpath("ww: cannot read ", o.path, "\n"); + o.state = -2; + return o.state; }; - if (istest) { return 2; }; - return 1; + o.buf = body; + o.len = bodylen; + let documentation: i32 = + sourcedocumentationbuffer(o.path, o.buf, o.len); + if (documentation < 0) { o.state = -2; return o.state; }; + if (documentation > 0) { o.state = 2; return o.state; }; + if (o.istest) { + o.packagename = dirpackagename(o.path, o.buf, o.len); + if (o.packagename == nil) { o.state = -2; return o.state; }; + } else { + let attest: i32 = dirfileattest(o.path, o.buf, o.len); + if (attest < 0) { o.state = -2; return o.state; }; + if (attest > 0) { o.state = -1; return o.state; }; + }; + o.state = 1; + return o.state; }; -fn dirpackagename(path: *u8) *u8 = { +fn dirpackagename(path: *u8, bufp: *u8, blen: u64) *u8 = { let view: str; view.ptr = path; view.len = cstrlen(path): i32; - let bufp: *u8; - let blen: u64; - bufp, blen = slurp(path); - if (bufp == nil) { - cerr("ww: cannot read source\n"); - return nil; - }; let l: syntax.lex; syntax.lexinit(&l, view, bufp, blen); let ps: syntax.parser; @@ -766,29 +772,27 @@ fn bytecmp(a: *u8, alen: u64, b: *u8, blen: u64) i32 = { return 0; }; -// Enumerate a production, same-test, or external-test directory variant. -// Production files precede matching test files; each partition is byte-sorted. -fn enumeratedir(dirpath: *u8, variant: i32, - testpackage: *u8) (**u8, i32) = { - let fd: i32 = os.open(pathstr(dirpath), os.flag.RDONLY, 0i32); - if (fd < 0) { return nil: **u8, -1; }; +// Capture eligible membership exactly once. Eligibility precedes observation, +// so hidden, underscore-prefixed, wrong-suffix, and wrong-platform entries are +// never statted or opened by this cache. +fn sepdircachepopulate(c: *sepdircache) i32 = { + if (c.state != 0) { return c.state; }; + let fd: i32 = os.open(pathstr(c.entry), os.flag.RDONLY, 0i32); + if (fd < 0) { c.state = -1; return c.state; }; // #65: grow-dynamic (mirror cstage enumerate_dir_ww realloc-doubling, // cmd/ww/main.c:209). The old fixed 256-name cap silently dropped every // eligible file past it, diverging the package unit from cstage on a // module dir with >256 sources. - let names: []*u8 = []; - let nlens: []u64 = []; - let kinds: []i32 = []; - let n: i32 = 0; - if (!sepreservesources(&names, &nlens, &kinds, n, - SEP_INITIAL_CAP)) { + if (!sepreservedirobservations(c, SEP_INITIAL_CAP)) { os.close(fd); - return nil: **u8, -2; + c.state = -2; + return c.state; }; let buf: []u8; if (!sepmakebytes(8192u64, &buf)) { os.close(fd); - return nil: **u8, -2; + c.state = -2; + return c.state; }; let r: i64 = os.getdents64(fd, buf.ptr, 8192u64); for (r > 0i64) { @@ -804,26 +808,38 @@ fn enumeratedir(dirpath: *u8, variant: i32, let view: str; view.ptr = nm; view.len = nl: i32; - if (strings.hassuffix(view, ".ww")) { - if (n == SEP_COUNT_MAX) { + if (strings.hassuffix(view, ".ww") + && sepsourcematchestarget(nm, nl)) { + if (c.nsource == SEP_COUNT_MAX) { sepfailsize(); os.close(fd); - return nil: **u8, -2; + os.free(buf.ptr: *void, buf.cap: u64); + c.state = -2; + return c.state; }; - if (!sepreservesources(&names, &nlens, &kinds, n, - n + 1)) { + if (!sepreservedirobservations(c, c.nsource + 1)) { os.close(fd); - return nil: **u8, -2; + os.free(buf.ptr: *void, buf.cap: u64); + c.state = -2; + return c.state; }; let owned: *u8 = sepdupcstr(nm, nl); if (owned == nil) { os.close(fd); - return nil: **u8, -2; + os.free(buf.ptr: *void, buf.cap: u64); + c.state = -2; + return c.state; }; - names[n] = owned; - nlens[n] = nl; - kinds[n] = 0; - n += 1; + let o: *sepdirobservation = &c.source[c.nsource]; + o.name = owned; + o.nlen = nl; + o.path = nil; + o.istest = strings.hassuffix(view, "_test.ww"); + o.state = 0; + o.buf = nil; + o.len = 0u64; + o.packagename = nil; + c.nsource += 1; }; }; off += reclen; @@ -831,126 +847,182 @@ fn enumeratedir(dirpath: *u8, variant: i32, r = os.getdents64(fd, buf.ptr, 8192u64); }; os.close(fd); + os.free(buf.ptr: *void, buf.cap: u64); // A failed directory read is an ERROR, not EOF: mid-walk it // silently truncated the package source list, and on the first // read it was misdiagnosed as "directory contains no WW package // sources". -1 routes the caller's "cannot read directory" arm // (the cstage caller mapping). if (r < 0i64) { - return nil: **u8, -1; + c.state = -1; + return c.state; }; // Go's directory reader presents a byte-sorted name list to the loader. let i: i32 = 1; - for (i < n) { + for (i < c.nsource) { let j: i32 = i; for (j > 0) { - let c: i32 = bytecmp(names[j - 1], nlens[j - 1], - names[j], nlens[j]); - if (c <= 0) { j = 0; } + let cmp: i32 = bytecmp(c.source[j - 1].name, + c.source[j - 1].nlen, c.source[j].name, + c.source[j].nlen); + if (cmp <= 0) { j = 0; } else { - let t: *u8 = names[j]; - names[j] = names[j - 1]; - names[j - 1] = t; - let tl: u64 = nlens[j]; - nlens[j] = nlens[j - 1]; - nlens[j - 1] = tl; - let tk: i32 = kinds[j]; - kinds[j] = kinds[j - 1]; - kinds[j - 1] = tk; + let t: sepdirobservation = c.source[j]; + c.source[j] = c.source[j - 1]; + c.source[j - 1] = t; j -= 1; }; }; i += 1; }; - let raw: i32 = n; - let selected: i32 = 0; - let ri: i32 = 0; - for (ri < raw) { - let nm: *u8 = names[ri]; - let nl: u64 = nlens[ri]; - let cls: i32 = dirfileclass(dirpath, nm, nl, variant); - if (cls == -1) { - let badpath: *u8 = sepjoinpath(dirpath, nm); - if (badpath == nil) { return nil: **u8, -2; }; - cerr("ww: "); - cerr(pathstr(badpath)); - cerr(": @test declaration outside *_test.ww\n"); - return nil: **u8, -2; - }; - if (cls == -2) { return nil: **u8, -2; }; - if (cls > 0) { - let full: *u8 = sepjoinpath(dirpath, nm); - if (full == nil) { return nil: **u8, -2; }; - let keep: bool = true; - if (cls == 2) { - let pn: *u8 = dirpackagename(full); - if (pn == nil) { return nil: **u8, -2; }; - if (testpackage == nil || !cstreq(pn, testpackage)) { - keep = false; - }; - }; - if (keep) { - names[selected] = full; - nlens[selected] = cstrlen(full); - kinds[selected] = cls; - selected += 1; - }; - }; - ri += 1; - }; - n = selected; + c.state = 1; + return c.state; +}; - // Production files precede test files; each partition remains byte-sorted. - i = 1; - for (i < n) { - let j: i32 = i; - for (j > 0) { - let c: i32 = kinds[j - 1] - kinds[j]; - if (c == 0) { - c = bytecmp(names[j - 1], nlens[j - 1], - names[j], nlens[j]); - }; - if (c <= 0) { j = 0; } - else { - let t: *u8 = names[j]; - names[j] = names[j - 1]; - names[j - 1] = t; - let tl: u64 = nlens[j]; - nlens[j] = nlens[j - 1]; - nlens[j - 1] = tl; - let tk: i32 = kinds[j]; - kinds[j] = kinds[j - 1]; - kinds[j - 1] = tk; - j -= 1; - }; +fn dirsourcekept(o: *sepdirobservation, variant: i32, + testpackage: *u8) bool = { + if (o.istest && variant == SEP_VARIANT_PRODUCTION) { return false; }; + if (!o.istest && variant == SEP_VARIANT_EXTERNAL) { return false; }; + if (o.state != 1) { return false; }; + if (o.istest && (testpackage == nil + || !cstreq(o.packagename, testpackage))) { return false; }; + return true; +}; + +fn sepdirselectionfree(paths: []*u8, bodies: []*u8, lens: []u64, + owned: i32) void = { + let i: i32 = 0; + for (i < owned) { + if (paths.ptr != nil && paths[i] != nil) { + os.free(paths[i]: *void, os.PATH_MAX: u64); + }; + if (bodies.ptr != nil && bodies[i] != nil) { + os.free(bodies[i]: *void, lens[i] + 1u64); }; i += 1; }; - if (n == 0) { - os.free(names.ptr: *void, (names.cap: u64) * (size(*u8): u64)); - os.free(nlens.ptr: *void, (nlens.cap: u64) * (size(u64): u64)); - os.free(kinds.ptr: *void, (kinds.cap: u64) * (size(i32): u64)); - return nil: **u8, 0; + if (paths.ptr != nil) { + os.free(paths.ptr: *void, + (paths.cap: u64) * (size(*u8): u64)); }; + if (bodies.ptr != nil) { + os.free(bodies.ptr: *void, + (bodies.cap: u64) * (size(*u8): u64)); + }; + if (lens.ptr != nil) { + os.free(lens.ptr: *void, + (lens.cap: u64) * (size(u64): u64)); + }; +}; + +// Select a variant from a canonical cache and give the package node safe +// owned copies. Production files precede matching test files; each partition +// is byte-sorted. +fn enumeratedir(c: *sepdircache, variant: i32, + testpackage: *u8) (**u8, **u8, *u64, i32) = { + let populated: i32 = sepdircachepopulate(c); + if (populated < 0) { + return nil: **u8, nil: **u8, nil: *u64, populated; + }; + let n: i32 = 0; + let i: i32 = 0; + for (i < c.nsource) { + let o: *sepdirobservation = &c.source[i]; + if ((o.istest && variant == SEP_VARIANT_PRODUCTION) + || (!o.istest && variant == SEP_VARIANT_EXTERNAL)) { + i += 1; + continue; + }; + let observed: i32 = dirsourceobserve(c, i); + if (observed == -1) { + cerr("ww: "); cerr(pathstr(o.path)); + cerr(": @test declaration outside *_test.ww\n"); + return nil: **u8, nil: **u8, nil: *u64, -2; + }; + if (observed == -2) { + return nil: **u8, nil: **u8, nil: *u64, -2; + }; + if (dirsourcekept(o, variant, testpackage)) { n += 1; }; + i += 1; + }; + if (n == 0) { return nil: **u8, nil: **u8, nil: *u64, 0; }; let exactallocation: ([]*u8 | nomem) = sepallocptrs(n); let exact: []*u8; match (exactallocation) { case let value: []*u8 => exact = value; case nomem => { sepfailnomem(); - return nil: **u8, -2; + return nil: **u8, nil: **u8, nil: *u64, -2; }; }; exact.len = n; + let exactbufallocation: ([]*u8 | nomem) = sepallocptrs(n); + let exactbufs: []*u8; + match (exactbufallocation) { + case let value: []*u8 => exactbufs = value; + case nomem => { + sepfailnomem(); + let emptybodies: []*u8; + let emptylens: []u64; + sepdirselectionfree(exact, emptybodies, emptylens, 0); + return nil: **u8, nil: **u8, nil: *u64, -2; + }; + }; + exactbufs.len = n; + let exactlenallocation: ([]u64 | nomem) = sepallocu64s(n); + let exactlens: []u64; + match (exactlenallocation) { + case let value: []u64 => exactlens = value; + case nomem => { + sepfailnomem(); + let emptylens: []u64; + sepdirselectionfree(exact, exactbufs, emptylens, 0); + return nil: **u8, nil: **u8, nil: *u64, -2; + }; + }; + exactlens.len = n; let k: i32 = 0; - for (k < n) { exact[k] = names[k]; k += 1; }; - // rt_free is currently a no-op, but keep the concrete owner/release - // shape correct for the driver's allocations. - os.free(names.ptr: *void, (names.cap: u64) * (size(*u8): u64)); - os.free(nlens.ptr: *void, (nlens.cap: u64) * (size(u64): u64)); - os.free(kinds.ptr: *void, (kinds.cap: u64) * (size(i32): u64)); - return exact.ptr, n; + let zi: i32 = 0; + for (zi < n) { + exact[zi] = nil; + exactbufs[zi] = nil; + exactlens[zi] = 0u64; + zi += 1; + }; + let pass: i32 = 1; + for (pass <= 2) { + let i: i32 = 0; + for (i < c.nsource) { + let o: *sepdirobservation = &c.source[i]; + if ((pass == 1 && o.istest) || (pass == 2 && !o.istest) + || !dirsourcekept(o, variant, testpackage)) { + i += 1; + continue; + }; + let pathcopy: []u8; + if (!sepmakebytes(os.PATH_MAX: u64, &pathcopy)) { + sepdirselectionfree(exact, exactbufs, exactlens, k); + return nil: **u8, nil: **u8, nil: *u64, -2; + }; + let pn: u64 = cstrlen(o.path); + bytecpy(pathcopy.ptr, o.path, pn + 1u64); + let bodycopy: []u8; + if (!sepmakebytes(o.len + 1u64, &bodycopy)) { + os.free(pathcopy.ptr: *void, os.PATH_MAX: u64); + sepdirselectionfree(exact, exactbufs, exactlens, k); + return nil: **u8, nil: **u8, nil: *u64, -2; + }; + bytecpy(bodycopy.ptr, o.buf, o.len + 1u64); + exact[k] = pathcopy.ptr; + exactbufs[k] = bodycopy.ptr; + exactlens[k] = o.len; + k += 1; + i += 1; + }; + pass += 1; + }; + return exact.ptr, exactbufs.ptr, exactlens.ptr, n; }; fn slurp(pathcs: *u8) (*u8, u64) = { @@ -1087,6 +1159,157 @@ fn sourcebuildheader(path: *u8) bool = { return true; }; +// Quietly recognize only the exact package-clause spelling that can invoke +// Go 1.26.5's reserved documentation-package rule. Ordinary sources must not +// acquire diagnostics or an eager parse from this selection pass. +fn sourcedocumentationskiptrivia(src: *u8, len: u64, at: *u64) bool = { + let i: u64 = *at; + for (true) { + for (i < len && (src[i] == ' ': u8 || src[i] == '\t': u8 + || src[i] == '\r': u8 || src[i] == '\n': u8)) { i += 1u64; }; + if (i + 1u64 < len && src[i] == '/': u8 + && src[i + 1u64] == '/': u8) { + i += 2u64; + for (i < len && src[i] != '\n': u8) { i += 1u64; }; + continue; + }; + if (i + 1u64 < len && src[i] == '/': u8 + && src[i + 1u64] == '*': u8) { + i += 2u64; + for (i + 1u64 < len + && !(src[i] == '*': u8 && src[i + 1u64] == '/': u8)) { + i += 1u64; + }; + if (i + 1u64 >= len) { return false; }; + i += 2u64; + continue; + }; + break; + }; + *at = i; + return true; +}; + +fn sourcedocumentationident(c: u8) bool = { + return (c >= 'a': u8 && c <= 'z': u8) + || (c >= 'A': u8 && c <= 'Z': u8) + || (c >= '0': u8 && c <= '9': u8) || c == '_': u8; +}; + +fn sourcedocumentationword(src: *u8, len: u64, at: *u64, + word: str) bool = { + let i: u64 = *at; + if (i > len || len - i < word.len: u64) { return false; }; + let j: i32 = 0; + for (j < word.len) { + let ju: u64 = j: u64; + if (src[i + ju] != word[j]) { return false; }; + j += 1; + }; + let wordlen: u64 = word.len: u64; + if (len - i > word.len: u64 + && sourcedocumentationident(src[i + wordlen])) { + return false; + }; + *at = i + wordlen; + return true; +}; + +fn sourcedocumentationcandidate(src: *u8, len: u64) bool = { + let at: u64 = 0u64; + if (len >= 3u64 && src[0u64] == 0xefu8 && src[1u64] == 0xbbu8 + && src[2u64] == 0xbfu8) { at = 3u64; }; + if (!sourcedocumentationskiptrivia(src, len, &at) + || !sourcedocumentationword(src, len, &at, "package") + || !sourcedocumentationskiptrivia(src, len, &at) + || !sourcedocumentationword(src, len, &at, "documentation") + || !sourcedocumentationskiptrivia(src, len, &at)) { + return false; + }; + return at < len && src[at] == ';': u8; +}; + +// -1 is a diagnosed header error, 0 an ordinary source, and 1 an exact +// documentation source. The caller owns bufp. Exact import tokens are consumed +// (or diagnosed) by parsepackageheader; a remaining raw 'i' is readGoInfo's +// failed-keyword arm. +fn sourcedocumentationbuffer(path: *u8, bufp: *u8, blen: u64) i32 = { + let view: str = pathstr(path); + if (!sourcedocumentationcandidate(bufp, blen)) { return 0; }; + let l: syntax.lex; + syntax.lexinit(&l, view, bufp, blen); + let ps: syntax.parser; + syntax.parserinit(&ps, &l); + let header: *syntax.node = syntax.parsepackageheader(&ps); + if (l.errs > 0 || ps.errs > 0) { return -1; }; + if (header.nmod.len == 0) { + cerrpos(view, 1, 1); + cerr(": error: invalid or missing package clause\n"); + return -1; + }; + if (l.lpos < blen && bufp[l.lpos] == 'i': u8) { + cerrpos(view, l.line, l.col); + cerr(": error: expected top-level decl\n"); + return -1; + }; + let documentation: bool = strings.compare(header.nmod, + "documentation") == 0; + if (documentation) { return 1; }; + return 0; +}; + +fn sourcedocumentationpreloadclear() void = { + if (sourcedocumentationpreloadpath != nil) { + os.free(sourcedocumentationpreloadpath: *void, + cstrlen(sourcedocumentationpreloadpath) + 1u64); + }; + if (sourcedocumentationpreloadbuf != nil) { + os.free(sourcedocumentationpreloadbuf: *void, + sourcedocumentationpreloadcap); + }; + sourcedocumentationpreloadpath = nil; + sourcedocumentationpreloadbuf = nil; + sourcedocumentationpreloadlen = 0u64; + sourcedocumentationpreloadcap = 0u64; +}; + +// Read a regular named root exactly once and carry ordinary bytes into the +// graph. Nonregular sources never enter this preflight and retain their prior +// stream/error route. +fn sourcepreloaddocumentation(path: *u8) i32 = { + sourcedocumentationpreloadclear(); + let bufp: *u8; + let blen: u64; + bufp, blen = slurp(path); + if (bufp == nil) { + cerrpath("ww: cannot read ", path, "\n"); + return -1; + }; + let documentation: i32 = sourcedocumentationbuffer(path, bufp, blen); + if (documentation != 0) { + os.free(bufp: *void, blen + 1u64); + return documentation; + }; + let copy: *u8 = sepdupcstr(path, cstrlen(path)); + if (copy == nil) { + os.free(bufp: *void, blen + 1u64); + return -1; + }; + sourcedocumentationpreloadpath = copy; + sourcedocumentationpreloadbuf = bufp; + sourcedocumentationpreloadlen = blen; + sourcedocumentationpreloadcap = blen + 1u64; + return 0; +}; + +fn sourceregularfile(path: *u8) bool = { + let fi: os.filestat; + match (os.stat(&fi, pathstr(path))) { + case void => return ((fi.mode: u32) & 61440u32) == os.mode.REG: u32; + case let e: os.oserror => return false; + }; +}; + fn makestem(stem: *u8, src: *u8) void = { let n: u64 = cstrlen(src); let stop: u64 = n; @@ -1554,7 +1777,11 @@ type seppkg = struct { testpackage: *u8, fortest: *u8, sources: **u8, // owned, byte-sorted selected paths; dirs only + sourcebufs: **u8, // exact request snapshots parallel to sources + sourcelens: *u64, nsources: i32, + entrybuf: *u8, // request snapshot for a regular one-file root + entrylen: u64, isdir: i32, variant: i32, role: i32, @@ -1597,6 +1824,8 @@ type sepgraph = struct { npackagefolds: i32, filefolds: []sepfoldentry, nfilefolds: i32, + dircaches: []sepdircache, + ndircaches: i32, }; type sepproduct = struct { @@ -1690,6 +1919,16 @@ fn sepallocfoldentries(cap: i32) ([]sepfoldentry | nomem) = { return value; }; +fn sepallocdirobservations(cap: i32) ([]sepdirobservation | nomem) = { + let value: []sepdirobservation = alloc([], cap: u64)?; + return value; +}; + +fn sepallocdircaches(cap: i32) ([]sepdircache | nomem) = { + let value: []sepdircache = alloc([], cap: u64)?; + return value; +}; + fn sepdupstr(s: str) (str | nomem) = { let out: str; out.ptr = nil; @@ -1815,6 +2054,7 @@ fn sepfreetestenv(env: *septestenv) void = { fn sepallocgraph(pkg: []seppkg, context: []sepcontext) (*sepgraph | nomem) = { let emptyfolds: []sepfoldentry; + let emptydircaches: []sepdircache; let value: *sepgraph = alloc(sepgraph{ pkg = pkg, n = 0, @@ -1826,6 +2066,8 @@ fn sepallocgraph(pkg: []seppkg, context: []sepcontext) (*sepgraph | nomem) = { npackagefolds = 0, filefolds = emptyfolds, nfilefolds = 0, + dircaches = emptydircaches, + ndircaches = 0, })?; return value; }; @@ -1928,6 +2170,75 @@ fn sepreservecontexts(g: *sepgraph, need: i32) bool = { return true; }; +fn sepreservedirobservations(c: *sepdircache, need: i32) bool = { + if (need <= c.source.len) { return true; }; + let cap: i32 = sepgrowcap(c.source.len, need); + if (cap < 0) { return false; }; + let allocation: ([]sepdirobservation | nomem) = + sepallocdirobservations(cap); + let next: []sepdirobservation; + match (allocation) { + case let value: []sepdirobservation => next = value; + case nomem => { sepfailnomem(); return false; }; + }; + next.len = cap; + let i: i32 = 0; + for (i < c.nsource) { next[i] = c.source[i]; i += 1; }; + if (c.source.ptr != nil) { + os.free(c.source.ptr: *void, + (c.source.cap: u64) * (size(sepdirobservation): u64)); + }; + c.source = next; + return true; +}; + +fn sepreservedircaches(g: *sepgraph, need: i32) bool = { + if (need <= g.dircaches.len) { return true; }; + let cap: i32 = sepgrowcap(g.dircaches.len, need); + if (cap < 0) { return false; }; + let allocation: ([]sepdircache | nomem) = sepallocdircaches(cap); + let next: []sepdircache; + match (allocation) { + case let value: []sepdircache => next = value; + case nomem => { sepfailnomem(); return false; }; + }; + next.len = cap; + let i: i32 = 0; + for (i < g.ndircaches) { next[i] = g.dircaches[i]; i += 1; }; + if (g.dircaches.ptr != nil) { + os.free(g.dircaches.ptr: *void, + (g.dircaches.cap: u64) * (size(sepdircache): u64)); + }; + g.dircaches = next; + return true; +}; + +fn sepdircachefor(g: *sepgraph, canon: *u8, entry: *u8) *sepdircache = { + let i: i32 = 0; + for (i < g.ndircaches) { + if (cstreq(g.dircaches[i].canon, canon)) { return &g.dircaches[i]; }; + i += 1; + }; + if (g.ndircaches == SEP_COUNT_MAX + || !sepreservedircaches(g, g.ndircaches + 1)) { return nil; }; + let c: *sepdircache = &g.dircaches[g.ndircaches]; + let ownedcanon: *u8 = sepdupcstr(canon, cstrlen(canon)); + if (ownedcanon == nil) { return nil; }; + let ownedentry: *u8 = sepdupcstr(entry, cstrlen(entry)); + if (ownedentry == nil) { + os.free(ownedcanon: *void, cstrlen(ownedcanon) + 1u64); + return nil; + }; + c.canon = ownedcanon; + c.entry = ownedentry; + let emptysources: []sepdirobservation; + c.source = emptysources; + c.nsource = 0; + c.state = 0; + g.ndircaches += 1; + return c; +}; + fn sepreservedeps(p: *seppkg, need: i32) bool = { if (need <= p.deps.len) { return true; }; let cap: i32 = sepgrowcap(p.deps.len, need); @@ -3146,7 +3457,11 @@ fn sepfindoraddvariant(g: *sepgraph, path: *u8, entry: *u8, if (g.pkg[g.n].testpackage == nil) { return -1; }; }; g.pkg[g.n].sources = nil; + g.pkg[g.n].sourcebufs = nil; + g.pkg[g.n].sourcelens = nil; g.pkg[g.n].nsources = 0; + g.pkg[g.n].entrybuf = nil; + g.pkg[g.n].entrylen = 0u64; g.pkg[g.n].isdir = isdir; g.pkg[g.n].variant = variant; g.pkg[g.n].role = role; @@ -3216,13 +3531,29 @@ fn sepfindoraddrole(g: *sepgraph, path: *u8, entry: *u8, isdir: i32, fn seppkgfreeowned(p: *seppkg) void = { let j: i32 = 0; for (j < p.nsources) { - os.free(p.sources[j]: *void, os.PATH_MAX: u64); + if (p.sources != nil) { + os.free(p.sources[j]: *void, os.PATH_MAX: u64); + }; + if (p.sourcebufs != nil && p.sourcelens != nil) { + os.free(p.sourcebufs[j]: *void, p.sourcelens[j] + 1u64); + }; j += 1; }; if (p.sources != nil) { os.free(p.sources: *void, (p.nsources: u64) * (size(*u8): u64)); }; + if (p.sourcebufs != nil) { + os.free(p.sourcebufs: *void, + (p.nsources: u64) * (size(*u8): u64)); + }; + if (p.sourcelens != nil) { + os.free(p.sourcelens: *void, + (p.nsources: u64) * (size(u64): u64)); + }; + if (p.entrybuf != nil) { + os.free(p.entrybuf: *void, p.entrylen + 1u64); + }; if (p.name != nil) { os.free(p.name: *void, cstrlen(p.name) + 1u64); }; @@ -3308,6 +3639,43 @@ fn sepgraphfree(g: *sepgraph) void = { (g.filefolds.cap: u64) * (size(sepfoldentry): u64)); }; i = 0; + for (i < g.ndircaches) { + let c: *sepdircache = &g.dircaches[i]; + let j: i32 = 0; + for (j < c.nsource) { + let o: *sepdirobservation = &c.source[j]; + if (o.name != nil) { + os.free(o.name: *void, o.nlen + 1u64); + }; + if (o.path != nil) { + os.free(o.path: *void, cstrlen(o.path) + 1u64); + }; + if (o.buf != nil) { + os.free(o.buf: *void, o.len + 1u64); + }; + if (o.packagename != nil) { + os.free(o.packagename: *void, + cstrlen(o.packagename) + 1u64); + }; + j += 1; + }; + if (c.source.ptr != nil) { + os.free(c.source.ptr: *void, + (c.source.cap: u64) * (size(sepdirobservation): u64)); + }; + if (c.canon != nil) { + os.free(c.canon: *void, cstrlen(c.canon) + 1u64); + }; + if (c.entry != nil) { + os.free(c.entry: *void, cstrlen(c.entry) + 1u64); + }; + i += 1; + }; + if (g.dircaches.ptr != nil) { + os.free(g.dircaches.ptr: *void, + (g.dircaches.cap: u64) * (size(sepdircache): u64)); + }; + i = 0; for (i < g.ncontext) { if (g.context[i].root != nil) { os.free(g.context[i].root: *void, @@ -4091,7 +4459,7 @@ fn sepusecmp(a: *syntax.node, b: *syntax.node) i32 = { // sep_scan_file (collects PATHS, not bytes). fn sepscanfile(g: *sepgraph, pi: i32, file: *u8, context: i32, fv: *expctx, bindings: *[]sepbind, children: *[]sepchild, - ownedsource: i32) i32 = { + ownedsource: i32, selectedbuf: *u8, selectedlen: u64) i32 = { let fview: str; fview.ptr = file; fview.len = cstrlen(file): i32; @@ -4105,11 +4473,81 @@ fn sepscanfile(g: *sepgraph, pi: i32, file: *u8, context: i32, if (!visitadd(fv, fdup)) { return -1; }; let bufp: *u8; let blen: u64; - bufp, blen = slurp(file); + let documentationprechecked: bool = false; + if (selectedbuf != nil) { + let selectedcopy: []u8; + if (!sepmakebytes(selectedlen + 1u64, &selectedcopy)) { + return -1; + }; + let ci: u64 = 0u64; + for (ci <= selectedlen) { + selectedcopy[ci] = selectedbuf[ci]; + ci += 1u64; + }; + bufp = selectedcopy.ptr; + blen = selectedlen; + documentationprechecked = true; + } else { if (ownedsource == 0 && g.pkg[pi].entrybuf != nil + && cstreq(g.pkg[pi].entry, file)) { + let entrycopy: []u8; + if (!sepmakebytes(g.pkg[pi].entrylen + 1u64, &entrycopy)) { + return -1; + }; + let ci: u64 = 0u64; + for (ci <= g.pkg[pi].entrylen) { + entrycopy[ci] = g.pkg[pi].entrybuf[ci]; + ci += 1u64; + }; + bufp = entrycopy.ptr; + blen = g.pkg[pi].entrylen; + documentationprechecked = true; + } else { if (ownedsource == 0 && sourcedocumentationpreloadpath != nil + && cstreq(sourcedocumentationpreloadpath, file)) { + bufp = sourcedocumentationpreloadbuf; + blen = sourcedocumentationpreloadlen; + os.free(sourcedocumentationpreloadpath: *void, + cstrlen(sourcedocumentationpreloadpath) + 1u64); + sourcedocumentationpreloadpath = nil; + sourcedocumentationpreloadbuf = nil; + sourcedocumentationpreloadlen = 0u64; + sourcedocumentationpreloadcap = 0u64; + documentationprechecked = true; + } else { + bufp, blen = slurp(file); + }; }; }; if (bufp == nil) { cerr("ww: cannot read source\n"); return -1; }; + if (ownedsource == 0 && g.pkg[pi].entrybuf == nil + && cstreq(g.pkg[pi].entry, file) + && (documentationprechecked || sourceregularfile(file))) { + let entrysnapshot: []u8; + if (!sepmakebytes(blen + 1u64, &entrysnapshot)) { + os.free(bufp: *void, blen + 1u64); + return -1; + }; + let ci: u64 = 0u64; + for (ci <= blen) { + entrysnapshot[ci] = bufp[ci]; + ci += 1u64; + }; + g.pkg[pi].entrybuf = entrysnapshot.ptr; + g.pkg[pi].entrylen = blen; + }; + if (ownedsource == 0 && !documentationprechecked + && sourceregularfile(file)) { + let documentation: i32 = sourcedocumentationbuffer(file, bufp, blen); + if (documentation < 0) { + os.free(bufp: *void, blen + 1u64); + return -1; + }; + if (documentation > 0) { + sourceoperandnosources(file); + os.free(bufp: *void, blen + 1u64); + return -1; + }; + }; let l: syntax.lex; syntax.lexinit(&l, fdup, bufp, blen); let ps: syntax.parser; @@ -4522,7 +4960,11 @@ fn sepaddgeneratedmain(g: *sepgraph, product: *sepproduct, ordinal: i32, p.testpackage = nil; p.fortest = nil; p.sources = nil; + p.sourcebufs = nil; + p.sourcelens = nil; p.nsources = 0; + p.entrybuf = nil; + p.entrylen = 0u64; p.isdir = 0; p.variant = SEP_VARIANT_TEST_MAIN; p.role = SEP_ROLE_GENERATED_MAIN; @@ -4652,7 +5094,11 @@ fn sepclonefortest(g: *sepgraph, original: i32, owner: *u8, p.testpackage = src.testpackage; p.fortest = nil; p.sources = nil; + p.sourcebufs = nil; + p.sourcelens = nil; p.nsources = 0; + p.entrybuf = nil; + p.entrylen = 0u64; p.isdir = src.isdir; p.variant = SEP_VARIANT_TEST_COPY; p.role = src.role; @@ -4708,32 +5154,83 @@ fn sepclonefortest(g: *sepgraph, original: i32, owner: *u8, seppkgfreeowned(&p); return -1; }; + let sourcebufs: []*u8; + if (!sepmakeptrs(src.nsources, &sourcebufs)) { + os.free(sources.ptr: *void, + (src.nsources: u64) * (size(*u8): u64)); + seppkgfreeowned(&p); + return -1; + }; + let sourcelenallocation: ([]u64 | nomem) = + sepallocu64s(src.nsources); + let sourcelens: []u64; + match (sourcelenallocation) { + case let value: []u64 => sourcelens = value; + case nomem => { + sepfailnomem(); + os.free(sourcebufs.ptr: *void, + (src.nsources: u64) * (size(*u8): u64)); + os.free(sources.ptr: *void, + (src.nsources: u64) * (size(*u8): u64)); + seppkgfreeowned(&p); + return -1; + }; + }; + sourcelens.len = src.nsources; + let zi: i32 = 0; + for (zi < src.nsources) { + sources[zi] = nil; + sourcebufs[zi] = nil; + sourcelens[zi] = 0u64; + zi += 1; + }; + p.sources = sources.ptr; + p.sourcebufs = sourcebufs.ptr; + p.sourcelens = sourcelens.ptr; + p.nsources = src.nsources; let si: i32 = 0; for (si < src.nsources) { - let bytes: []u8; - if (!sepmakebytes(os.PATH_MAX: u64, &bytes)) { - let sj: i32 = 0; - for (sj < si) { - os.free(sources[sj]: *void, os.PATH_MAX: u64); - sj += 1; - }; - os.free(sources.ptr: *void, - (sources.cap: u64) * (size(*u8): u64)); + let pathbytes: []u8; + if (!sepmakebytes(os.PATH_MAX: u64, &pathbytes)) { seppkgfreeowned(&p); return -1; }; let n: u64 = cstrlen(src.sources[si]); let sj: u64 = 0u64; for (sj < n) { - bytes[sj] = src.sources[si][sj]; + pathbytes[sj] = src.sources[si][sj]; sj += 1u64; }; - bytes[n] = 0u8; - sources[si] = bytes.ptr; + pathbytes[n] = 0u8; + p.sources[si] = pathbytes.ptr; + let bodybytes: []u8; + if (!sepmakebytes(src.sourcelens[si] + 1u64, &bodybytes)) { + seppkgfreeowned(&p); + return -1; + }; + sj = 0u64; + for (sj <= src.sourcelens[si]) { + bodybytes[sj] = src.sourcebufs[si][sj]; + sj += 1u64; + }; + p.sourcebufs[si] = bodybytes.ptr; + p.sourcelens[si] = src.sourcelens[si]; si += 1; }; - p.sources = sources.ptr; - p.nsources = src.nsources; + }; + if (src.entrybuf != nil) { + let entrybytes: []u8; + if (!sepmakebytes(src.entrylen + 1u64, &entrybytes)) { + seppkgfreeowned(&p); + return -1; + }; + let ei: u64 = 0u64; + for (ei <= src.entrylen) { + entrybytes[ei] = src.entrybuf[ei]; + ei += 1u64; + }; + p.entrybuf = entrybytes.ptr; + p.entrylen = src.entrylen; }; if (src.contextstate.len > 0) { @@ -4853,10 +5350,24 @@ fn seppreparepkgcontext(g: *sepgraph, pi: i32, context: i32, g.pkg[pi].loaded = true; if (g.pkg[pi].isdir != 0) { let sources: **u8; + let sourcebufs: **u8; + let sourcelens: *u64; let nsources: i32; - sources, nsources = enumeratedir(g.pkg[pi].entry, - g.pkg[pi].variant, g.pkg[pi].testpackage); + let dircache: *sepdircache = sepdircachefor(g, + g.pkg[pi].canon, g.pkg[pi].entry); + if (dircache == nil) { + sources = nil; + sourcebufs = nil; + sourcelens = nil; + nsources = -2; + } else { + sources, sourcebufs, sourcelens, nsources = + enumeratedir(dircache, g.pkg[pi].variant, + g.pkg[pi].testpackage); + }; g.pkg[pi].sources = sources; + g.pkg[pi].sourcebufs = sourcebufs; + g.pkg[pi].sourcelens = sourcelens; g.pkg[pi].nsources = nsources; if (g.pkg[pi].nsources == -2) { // diagnosed in enumeratedir @@ -4885,13 +5396,14 @@ fn seppreparepkgcontext(g: *sepgraph, pi: i32, context: i32, for (i < g.pkg[pi].nsources) { if (rc == 0) { rc = sepscanfile(g, pi, g.pkg[pi].sources[i], - context, &fv, &bindings, children, 1); + context, &fv, &bindings, children, 1, + g.pkg[pi].sourcebufs[i], g.pkg[pi].sourcelens[i]); }; i += 1; }; } else { if (rc == 0) { rc = sepscanfile(g, pi, g.pkg[pi].entry, context, - &fv, &bindings, children, 0); + &fv, &bindings, children, 0, nil, 0u64); }; }; sepbindsort(&bindings); if (rc == 0 && !sepvalidatebindings(g, bindings)) { rc = -1; }; @@ -5664,13 +6176,16 @@ fn sepwriteall(fd: i32, buf: *u8, n: u64) bool = { }; }; -fn sepemitbody(fd: i32, path: *u8, modpath: *u8) i32 = { - let bufp: *u8; - let blen: u64; - bufp, blen = slurp(path); +fn sepemitbody(fd: i32, path: *u8, modpath: *u8, + selectedbuf: *u8, selectedlen: u64) i32 = { + let bufp: *u8 = selectedbuf; + let blen: u64 = selectedlen; if (bufp == nil) { - cerr("ww: cannot read source\n"); - return -1; + bufp, blen = slurp(path); + if (bufp == nil) { + cerr("ww: cannot read source\n"); + return -1; + }; }; // #57: tag the primary body by its full dotted import path so the // definer mangles == the importer reference; a root build (path "") @@ -5780,11 +6295,13 @@ fn sepcomposeunit(g: *sepgraph, pi: i32, scratch: *u8, unitf: *u8) i32 = { } else { if (g.pkg[pi].isdir != 0) { let i: i32 = 0; for (i < g.pkg[pi].nsources && bodyrc == 0) { - bodyrc = sepemitbody(u, g.pkg[pi].sources[i], g.pkg[pi].path); + bodyrc = sepemitbody(u, g.pkg[pi].sources[i], g.pkg[pi].path, + g.pkg[pi].sourcebufs[i], g.pkg[pi].sourcelens[i]); i += 1; }; } else { - bodyrc = sepemitbody(u, g.pkg[pi].entry, g.pkg[pi].path); + bodyrc = sepemitbody(u, g.pkg[pi].entry, g.pkg[pi].path, + g.pkg[pi].entrybuf, g.pkg[pi].entrylen); }; }; let ownsuffix: str = ""; let ownparents: u64 = 0u64; @@ -8618,6 +9135,7 @@ fn buildonesep(selfdir: *u8, src: *u8, entryisdir: i32, }; }; }; + sourcedocumentationpreloadclear(); return r; }; @@ -9086,6 +9604,14 @@ fn dobuild(selfdir: *u8, argv: **u8, argc: i32, start: i32) i32 = { cerrpath("ww build: cannot find module ", src, "\n"); return 1; }; + if (isdir == 0 && sourceregularfile(resolved)) { + let documentation: i32 = sourcepreloaddocumentation(resolved); + if (documentation < 0) { return 1; }; + if (documentation > 0) { + sourceoperandnosources(resolved); + return 1; + }; + }; let out: *u8 = nil; let objstem: *u8 = nil; let discardoutput: bool = outflag != nil @@ -9346,6 +9872,14 @@ fn dorun(selfdir: *u8, argv: **u8, argc: i32, start: i32) i32 = { cerrpath("ww run: cannot find module ", src, "\n"); return 1; }; + if (isdir == 0 && sourceregularfile(resolved)) { + let documentation: i32 = sourcepreloaddocumentation(resolved); + if (documentation < 0) { return 1; }; + if (documentation > 0) { + sourceoperandnosources(resolved); + return 1; + }; + }; let tmp: []u8 = alloc([], (os.PATH_MAX: u64))!; tmp.len = os.PATH_MAX; @@ -9420,6 +9954,17 @@ fn dorun(selfdir: *u8, argv: **u8, argc: i32, start: i32) i32 = { fn runsingletest(selfdir: *u8, src: *u8, incs: *u8, compileonly: i32, emitasm: i32, outstem: *u8, workdir: *u8, pattern: *u8) i32 = { + let documentation: i32 = 0; + if (sourceregularfile(src)) { + documentation = sourcepreloaddocumentation(src); + }; + if (documentation > 0) { sourceoperandnosources(src); }; + if (documentation != 0) { + if (compileonly == 0 && emitasm == 0) { + os.write(1, "FAIL\n".ptr, 5u64); + }; + return 1; + }; let tmp: []u8 = alloc([], (os.PATH_MAX: u64))!; tmp.len = os.PATH_MAX; // A retained -o redirects the binary and sepwork intermediates to its stem; diff --git a/test/package/package_test.ww b/test/package/package_test.ww index 769ab528..e7aefc51 100644 --- a/test/package/package_test.ww +++ b/test/package/package_test.ww @@ -14278,6 +14278,550 @@ fn runtimepath(relative: str) str = { return strings.concat(repo(), "/test/package/runtime/", relative); }; +// `documentation` is a source-loader reservation, not a package identity. +// Keep this matrix here rather than splitting it between command tests: both +// drivers and the shared package coordinator must make the same decision +// before a body, import edge, action, or public artifact exists. +fn documentationrecord(out: *commandout, code: i32, records: *[]str) void = { + expectexit(out, code); + append(*records, strings.dup(out.stdout)); + append(*records, strings.dup(out.stderr)); +}; + +fn documentationrecordssame(a: []str, b: []str) void = { + assert(a.len == b.len); + let i: i32 = 0; + for (i < a.len) { assert(same(a[i], b[i])); i += 1; }; +}; + +fn documentationclean(root: str) void = { + assert(!directoryhasnew(root) + && !directoryhasfragment(root, ".wwtxn.") + && !directoryhasfragment(root, ".install") + && !directoryhasfragment(root, ".sepwork") + && !wrongsuffixpathfragment(root, ".new")); +}; + +@test fn package_documentation_is_omitted_before_body_or_actions() void = { + let root: str = fresh(); + let docs: str = strings.concat(root, "/docs"); + let controls: str = strings.concat(root, "/controls"); + let bomdir: str = strings.concat(root, "/bom"); + let trivia: str = strings.concat(root, "/trivia"); + let rawidir: str = strings.concat(root, "/raw-i"); + let rawitree: str = strings.concat(root, "/raw-i-tree"); + let literal: str = strings.concat(root, "/literal"); + let include: str = strings.concat(root, "/include"); + let logicaldir: str = strings.concat(include, "/logic"); + let logicaltestdir: str = strings.concat(include, "/logicaltest"); + let tree: str = strings.concat(root, "/tree"); + let doconlytree: str = strings.concat(root, "/doconlytree"); + let doconlydocs: str = strings.concat(doconlytree, "/docs"); + let treedocs: str = strings.concat(tree, "/docs"); + let treemain: str = strings.concat(tree, "/main"); + let mixed: str = strings.concat(root, "/mixed"); + let control: str = strings.concat(root, "/control"); + let warm: str = strings.concat(root, "/warm"); + let exact: str = strings.concat(root, "/exact"); + let importer: str = strings.concat(root, "/importer"); + let importerone: str = strings.concat(root, "/importer-one"); + let docdep: str = strings.concat(include, "/docdep"); + mkdirall(docs); mkdirall(controls); mkdirall(bomdir); mkdirall(trivia); + mkdirall(rawidir); mkdirall(rawitree); mkdirall(literal); mkdirall(logicaldir); + mkdirall(logicaltestdir); mkdirall(treedocs); mkdirall(treemain); + mkdirall(doconlydocs); + mkdirall(mixed); mkdirall(control); mkdirall(warm); mkdirall(exact); mkdirall(importer); mkdirall(importerone); + mkdirall(docdep); + let doc: str = strings.concat(docs, "/doc.ww"); + let doclink: str = strings.concat(root, "/doclink.ww"); + let logical: str = strings.concat(logicaldir, "/doc.ww"); + let rawi: str = strings.concat(rawidir, "/rawi.ww"); + let rawitreefile: str = strings.concat(rawitree, "/rawi.ww"); + let badheader: str = strings.concat(controls, "/badheader.ww"); + let badpackage: str = strings.concat(controls, "/badpackage.ww"); + let bom: str = strings.concat(bomdir, "/bom.ww"); + let triviafile: str = strings.concat(trivia, "/trivia.ww"); + let literaltest: str = strings.concat(literal, "/literal_test.ww"); + let logicaltest: str = strings.concat(logicaltestdir, "/provider_test.ww"); + let mixdoc: str = strings.concat(mixed, "/doc.ww"); + let warmfile: str = strings.concat(warm, "/main.ww"); + let maintext: str = "package main;\nfn main() i32 = { return 31; };\n"; + let doctext: str = strings.concat("// leading trivia\npackage documentation;\n", + "this is deliberately malformed ordinary body;\n", + "import absent.after.boundary;\nfn main() i32 = { return 99; };\n"); + writefile(doc, doctext); writefile(logical, doctext); + assert(os.symlink(doc, doclink) == 0); + writefile(rawi, "package documentation;\ni this_is_not_import;\n"); + writefile(badheader, "package documentation;\nimport ;\n"); + writefile(badpackage, "package ;\n"); + putbomfile(bom, "// bom and comment\npackage documentation;\n", + "this malformed body is past the header;\n", false); + writefile(triviafile, strings.concat("//ww:module ignored.header\n", + "package /* comment between clause and name */ documentation;\n", + "this malformed body is past the header;\n")); + writefile(rawitreefile, + "package documentation;\ni this_is_not_import;\n"); + writefile(literaltest, "package documentation;\n"); + writefile(logicaltest, doctext); + writefile(mixdoc, doctext); writefile(strings.concat(mixed, "/main.ww"), maintext); + writefile(strings.concat(control, "/main.ww"), maintext); + writefile(warmfile, maintext); + writefile(strings.concat(treedocs, "/doc.ww"), doctext); + writefile(strings.concat(doconlydocs, "/doc.ww"), doctext); + writefile(strings.concat(treemain, "/main.ww"), maintext); + writefile(strings.concat(treemain, "/main_test.ww"), + "package main;\n@test fn retained() void = { assert(true); };\n"); + writefile(strings.concat(docdep, "/doc.ww"), doctext); + writefile(strings.concat(importer, "/user.ww"), strings.concat( + "package user;\nimport docdep;\n", + "export fn value() i32 = { return docdep.value(); };\n")); + writefile(strings.concat(include, "/docone.ww"), doctext); + writefile(strings.concat(importerone, "/user.ww"), strings.concat( + "package userone;\nimport docone;\n", + "export fn value() i32 = { return docone.value(); };\n")); + writefile(strings.concat(exact, "/x.ww"), + "package documentationx;\nimport absent.exact;\n"); + writefile(strings.concat(exact, "/t.ww"), + "package documentation_test;\nimport absent.exact;\n"); + + let stages: []str = ["ww", "ww_ww"]; + let tags: []str = ["c", "ww"]; + let compilers: []str = ["w6c", "w6c_ww"]; + let assemblers: []str = ["w6a", "w6a_ww"]; + let linkers: []str = ["w6l", "w6l_ww"]; + let baseenv: []str = os.getenvs(); + let records: []str = alloc([], 96u64)!; + let cross: []str = alloc([], 96u64)!; + let mixedcross: str = ""; + let mixedsemanticcross: str = ""; + let warmcross: str = ""; + let out: commandout; + let si: i32 = 0; + for (si < stages.len) { + records.len = 0; + let work: str = strings.concat(root, "/docs-work-", tags[si]); + let rawwork: str = strings.concat(root, "/raw-work-", tags[si]); + let mixwork: str = strings.concat(root, "/mixed-work-", tags[si]); + let ctlwork: str = strings.concat(root, "/control-work-", tags[si]); + let outputdir: str = strings.concat(root, "/outputs-", tags[si]); + let mixedout: str = strings.concat(root, "/mixed-", tags[si]); + let controlout: str = strings.concat(root, "/control-", tags[si]); + let trace: str = strings.concat(root, "/tools-", tags[si], ".trace"); + let cwrap: str = strings.concat(root, "/w6c-", tags[si], ".sh"); + let awrap: str = strings.concat(root, "/w6a-", tags[si], ".sh"); + let lwrap: str = strings.concat(root, "/w6l-", tags[si], ".sh"); + mkdirall(work); mkdirall(rawwork); mkdirall(mixwork); mkdirall(ctlwork); + mkdirall(outputdir); writefile(trace, ""); + writeexecutable(cwrap, "#!/bin/sh\nprintf C >> \"$WW_DOC_TRACE\"\nexec \"$WW_DOC_REAL_C\" \"$@\"\n"); + writeexecutable(awrap, "#!/bin/sh\nprintf A >> \"$WW_DOC_TRACE\"\nexec \"$WW_DOC_REAL_A\" \"$@\"\n"); + writeexecutable(lwrap, "#!/bin/sh\nprintf L >> \"$WW_DOC_TRACE\"\nexec \"$WW_DOC_REAL_L\" \"$@\"\n"); + let env: []str = alloc([], (baseenv.len + 6): u64)!; + let ei: i32 = 0; + for (ei < baseenv.len) { + if (!strings.hasprefix(baseenv[ei], "WW_W6C=") + && !strings.hasprefix(baseenv[ei], "WW_W6A=") + && !strings.hasprefix(baseenv[ei], "WW_W6L=") + && !strings.hasprefix(baseenv[ei], "WW_DOC_")) { append(env, baseenv[ei]); }; + ei += 1; + }; + append(env, strings.concat("WW_W6C=", cwrap)); + append(env, strings.concat("WW_W6A=", awrap)); + append(env, strings.concat("WW_W6L=", lwrap)); + append(env, strings.concat("WW_DOC_REAL_C=", driver(compilers[si]))); + append(env, strings.concat("WW_DOC_REAL_A=", driver(assemblers[si]))); + append(env, strings.concat("WW_DOC_REAL_L=", driver(linkers[si]))); + append(env, strings.concat("WW_DOC_TRACE=", trace)); + let directerr: str = strings.concat("ww: ", docs, + ": directory contains no WW package sources\n"); + let namederr: str = strings.concat("ww: ", docs, + ": directory contains no WW package sources\n"); + let coordinatorerr: str = strings.concat("wwtest package: ", docs, + ": directory contains no WW package sources\n"); + let before: str = treesnapshot(work); + let av: []str = [driver(stages[si]), "build", "-w", work, docs]; + runcommandenv(root, strings.concat("documentation-build-", tags[si]), av, + env, (30i64 * (time.second: i64)): time.duration, &out); + documentationrecord(&out, 1, &records); + assert(out.stdout.len == 0 && same(out.stderr, directerr) + && same(before, treesnapshot(work)) && readfile(trace).len == 0); + let namedav: []str = [driver(stages[si]), "build", "-w", work, doc]; + runcommandenv(root, strings.concat("documentation-named-", tags[si]), namedav, + env, (30i64 * (time.second: i64)): time.duration, &out); + documentationrecord(&out, 1, &records); + assert(out.stdout.len == 0 && same(out.stderr, namederr) + && same(before, treesnapshot(work)) && readfile(trace).len == 0); + let linkav: []str = [driver(stages[si]), "build", "-w", work, doclink]; + runcommandenv(root, strings.concat("documentation-named-link-", tags[si]), + linkav, env, (30i64 * (time.second: i64)): time.duration, &out); + documentationrecord(&out, 1, &records); + assert(out.stdout.len == 0 && same(out.stderr, strings.concat("ww: ", root, + ": directory contains no WW package sources\n")) + && same(before, treesnapshot(work)) && readfile(trace).len == 0); + let logicalav: []str = [driver(stages[si]), "build", "-w", work, + "-I", include, "logic.doc"]; + runcommandenv(root, strings.concat("documentation-logical-", tags[si]), logicalav, + env, (30i64 * (time.second: i64)): time.duration, &out); + documentationrecord(&out, 1, &records); + assert(out.stdout.len == 0 && same(out.stderr, strings.concat("ww: ", + logicaldir, ": directory contains no WW package sources\n"))); + let logicaltestav: []str = [driver(stages[si]), "build", "-w", work, + "-I", include, "logicaltest.provider_test"]; + runcommandenv(root, strings.concat("documentation-logical-test-", tags[si]), + logicaltestav, env, (30i64 * (time.second: i64)): time.duration, &out); + documentationrecord(&out, 1, &records); + assert(out.stdout.len == 0 && same(out.stderr, strings.concat("ww: ", + logicaltestdir, ": directory contains no WW package sources\n"))); + let coordav: []str = [driver(stages[si]), "build", "-o", outputdir, docs]; + runcommandenv(root, strings.concat("documentation-coordinator-", tags[si]), coordav, + env, (30i64 * (time.second: i64)): time.duration, &out); + documentationrecord(&out, 1, &records); + assert(out.stdout.len == 0 && same(out.stderr, coordinatorerr) + && !directoryhasnew(outputdir) && readfile(trace).len == 0); + + let runav: []str = [driver(stages[si]), "run", docs]; + runcommandenv(root, strings.concat("documentation-run-dir-", tags[si]), runav, + env, (30i64 * (time.second: i64)): time.duration, &out); + documentationrecord(&out, 1, &records); + assert(out.stdout.len == 0 && same(out.stderr, directerr) && readfile(trace).len == 0); + let runnamed: []str = [driver(stages[si]), "run", doc]; + runcommandenv(root, strings.concat("documentation-run-file-", tags[si]), runnamed, + env, (30i64 * (time.second: i64)): time.duration, &out); + documentationrecord(&out, 1, &records); + assert(out.stdout.len == 0 && same(out.stderr, namederr) && readfile(trace).len == 0); + let runlogical: []str = [driver(stages[si]), "run", "-I", include, "logic.doc"]; + runcommandenv(root, strings.concat("documentation-run-logical-", tags[si]), + runlogical, env, (30i64 * (time.second: i64)): time.duration, &out); + documentationrecord(&out, 1, &records); + assert(out.stdout.len == 0 && same(out.stderr, strings.concat("ww: ", + logicaldir, ": directory contains no WW package sources\n"))); + let literalplain: []str = [driver(stages[si]), "build", literaltest]; + runcommandenv(root, strings.concat("documentation-literal-plain-", tags[si]), + literalplain, env, (30i64 * (time.second: i64)): time.duration, &out); + documentationrecord(&out, 0, &records); + assert(out.stdout.len == 0 && out.stderr.len == 0); + let literalnull: []str = [driver(stages[si]), "build", "-o", "/dev/null", literaltest]; + runcommandenv(root, strings.concat("documentation-literal-null-", tags[si]), + literalnull, env, (30i64 * (time.second: i64)): time.duration, &out); + documentationrecord(&out, 0, &records); + assert(out.stdout.len == 0 && out.stderr.len == 0); + let literalfile: str = strings.concat(root, "/literal-file-", tags[si]); + let literalfileav: []str = [driver(stages[si]), "build", "-o", literalfile, literaltest]; + runcommandenv(root, strings.concat("documentation-literal-file-", tags[si]), + literalfileav, env, (30i64 * (time.second: i64)): time.duration, &out); + documentationrecord(&out, 1, &records); + assert(out.stdout.len == 0 && same(out.stderr, "ww: no packages to build\n") + && !os.exists(literalfile)); + let literaloutdir: str = strings.concat(root, "/literal-output-", tags[si]); + mkdirall(literaloutdir); + let literaldirav: []str = [driver(stages[si]), "build", "-o", literaloutdir, literaltest]; + runcommandenv(root, strings.concat("documentation-literal-dir-", tags[si]), + literaldirav, env, (30i64 * (time.second: i64)): time.duration, &out); + documentationrecord(&out, 1, &records); + assert(out.stdout.len == 0 && same(out.stderr, "ww: no main packages to build\n") + && !directoryhasnew(literaloutdir)); + + let rawbefore: str = treesnapshot(rawwork); + let rawtest: []str = [driver(stages[si]), "test", "-w", rawwork, doc]; + runcommandenv(root, strings.concat("documentation-test-raw-", tags[si]), rawtest, + env, (30i64 * (time.second: i64)): time.duration, &out); + documentationrecord(&out, 1, &records); + assert(same(out.stdout, "FAIL\n") && same(out.stderr, namederr) + && same(rawbefore, treesnapshot(rawwork)) && readfile(trace).len == 0); + let logicalraw: []str = [driver(stages[si]), "test", "-I", include, "logic.doc"]; + runcommandenv(root, strings.concat("documentation-test-logical-", tags[si]), + logicalraw, env, (30i64 * (time.second: i64)): time.duration, &out); + documentationrecord(&out, 1, &records); + assert(same(out.stdout, "FAIL\n") && same(out.stderr, strings.concat("ww: ", + logicaldir, ": directory contains no WW package sources\n"))); + let rawcompile: []str = [driver(stages[si]), "test", "-c", "-w", rawwork, doc]; + runcommandenv(root, strings.concat("documentation-test-c-", tags[si]), rawcompile, + env, (30i64 * (time.second: i64)): time.duration, &out); + documentationrecord(&out, 1, &records); + assert(out.stdout.len == 0 && same(out.stderr, namederr)); + let rawasmout: str = strings.concat(root, "/raw-asm-", tags[si]); + let rawasm: []str = [driver(stages[si]), "test", "-S", "-w", rawwork, + "-o", rawasmout, doc]; + runcommandenv(root, strings.concat("documentation-test-s-", tags[si]), rawasm, + env, (30i64 * (time.second: i64)): time.duration, &out); + documentationrecord(&out, 1, &records); + assert(out.stdout.len == 0 && same(out.stderr, namederr) && !os.exists(rawasmout)); + let dirtest: []str = [driver(stages[si]), "test", docs]; + runcommandenv(root, strings.concat("documentation-test-dir-", tags[si]), dirtest, + env, (30i64 * (time.second: i64)): time.duration, &out); + documentationrecord(&out, 1, &records); + assert(same(out.stdout, "FAIL\n") && same(out.stderr, coordinatorerr)); + let dircompile: []str = [driver(stages[si]), "test", "-c", docs]; + runcommandenv(root, strings.concat("documentation-test-dir-c-", tags[si]), dircompile, + env, (30i64 * (time.second: i64)): time.duration, &out); + documentationrecord(&out, 1, &records); + assert(out.stdout.len == 0 && same(out.stderr, coordinatorerr)); + let dirasm: []str = [driver(stages[si]), "test", "-S", docs]; + runcommandenv(root, strings.concat("documentation-test-dir-s-", tags[si]), dirasm, + env, (30i64 * (time.second: i64)): time.duration, &out); + documentationrecord(&out, 2, &records); + assert(out.stdout.len == 0 && same(out.stderr, + "ww test: -S needs -o\n")); + let dirasmout: str = strings.concat(root, "/dir-asm-", tags[si]); + let dirasmwithoutput: []str = [driver(stages[si]), "test", "-S", + "-o", dirasmout, docs]; + runcommandenv(root, strings.concat("documentation-test-dir-s-output-", tags[si]), + dirasmwithoutput, env, (30i64 * (time.second: i64)): time.duration, &out); + documentationrecord(&out, 2, &records); + assert(out.stdout.len == 0 && same(out.stderr, + "ww test: -S needs a single test file\n")); + + let spec: str = strings.concat(doconlytree, "/..."); + let recbuild: []str = [driver(stages[si]), "build", spec]; + runcommandenv(root, strings.concat("documentation-rec-build-", tags[si]), recbuild, + env, (30i64 * (time.second: i64)): time.duration, &out); + documentationrecord(&out, 0, &records); + assert(out.stdout.len == 0 && same(out.stderr, strings.concat("ww: warning: \"", + spec, "\" matched no packages\n"))); + let recbuildtwice: []str = [driver(stages[si]), "build", spec, spec]; + runcommandenv(root, strings.concat("documentation-rec-build-twice-", tags[si]), + recbuildtwice, env, (30i64 * (time.second: i64)): time.duration, &out); + documentationrecord(&out, 0, &records); + assert(out.stdout.len == 0 && same(out.stderr, strings.concat("ww: warning: \"", + spec, "\" matched no packages\nww: warning: \"", spec, + "\" matched no packages\n"))); + let rectest: []str = [driver(stages[si]), "test", spec]; + runcommandenv(root, strings.concat("documentation-rec-test-", tags[si]), rectest, + env, (30i64 * (time.second: i64)): time.duration, &out); + documentationrecord(&out, 1, &records); + assert(out.stdout.len == 0 && same(out.stderr, strings.concat("ww: warning: \"", + spec, "\" matched no packages\nww test: no packages to test\n"))); + let rectesttwice: []str = [driver(stages[si]), "test", spec, spec]; + runcommandenv(root, strings.concat("documentation-rec-test-twice-", tags[si]), + rectesttwice, env, (30i64 * (time.second: i64)): time.duration, &out); + documentationrecord(&out, 1, &records); + assert(out.stdout.len == 0 && same(out.stderr, strings.concat("ww: warning: \"", + spec, "\" matched no packages\nww: warning: \"", spec, + "\" matched no packages\nww test: no packages to test\n"))); + let mixedspec: str = strings.concat(tree, "/..."); + let mixedrecbuild: []str = [driver(stages[si]), "build", "-o", "/dev/null", + mixedspec]; + runcommandenv(root, strings.concat("documentation-rec-mixed-build-", tags[si]), + mixedrecbuild, env, (60i64 * (time.second: i64)): time.duration, &out); + documentationrecord(&out, 0, &records); + assert(out.stdout.len == 0 && out.stderr.len == 0); + let mixedrectest: []str = [driver(stages[si]), "test", mixedspec]; + runcommandenv(root, strings.concat("documentation-rec-mixed-test-", tags[si]), + mixedrectest, env, (60i64 * (time.second: i64)): time.duration, &out); + documentationrecord(&out, 0, &records); + assert(out.stderr.len == 0 && has(out.stdout, "retained ... ok\n") + && !has(out.stdout, "documentation")); + rewritefile(trace, ""); + + let rawiav: []str = [driver(stages[si]), "build", rawi]; + runcommandenv(root, strings.concat("documentation-raw-i-", tags[si]), rawiav, + env, (30i64 * (time.second: i64)): time.duration, &out); + documentationrecord(&out, 1, &records); + assert(out.stdout.len == 0 && same(out.stderr, strings.concat(rawi, + ":2:1: error: expected top-level decl\n")) && readfile(trace).len == 0); + let rawirun: []str = [driver(stages[si]), "run", rawi]; + runcommandenv(root, strings.concat("documentation-raw-i-run-", tags[si]), rawirun, + env, (30i64 * (time.second: i64)): time.duration, &out); + documentationrecord(&out, 1, &records); + assert(out.stdout.len == 0 && same(out.stderr, strings.concat(rawi, + ":2:1: error: expected top-level decl\n"))); + let rawitest: []str = [driver(stages[si]), "test", rawi]; + runcommandenv(root, strings.concat("documentation-raw-i-test-", tags[si]), rawitest, + env, (30i64 * (time.second: i64)): time.duration, &out); + documentationrecord(&out, 1, &records); + assert(same(out.stdout, "FAIL\n") && same(out.stderr, strings.concat(rawi, + ":2:1: error: expected top-level decl\n"))); + let rawicompile: []str = [driver(stages[si]), "test", "-c", rawi]; + runcommandenv(root, strings.concat("documentation-raw-i-c-", tags[si]), rawicompile, + env, (30i64 * (time.second: i64)): time.duration, &out); + documentationrecord(&out, 1, &records); + assert(out.stdout.len == 0 && same(out.stderr, strings.concat(rawi, + ":2:1: error: expected top-level decl\n"))); + let rawiasmout: str = strings.concat(root, "/raw-i-asm-", tags[si]); + let rawiasm: []str = [driver(stages[si]), "test", "-S", "-o", rawiasmout, rawi]; + runcommandenv(root, strings.concat("documentation-raw-i-s-", tags[si]), rawiasm, + env, (30i64 * (time.second: i64)): time.duration, &out); + documentationrecord(&out, 1, &records); + assert(out.stdout.len == 0 && same(out.stderr, strings.concat(rawi, + ":2:1: error: expected top-level decl\n")) && !os.exists(rawiasmout)); + let rawidirectorytest: []str = [driver(stages[si]), "test", rawidir]; + runcommandenv(root, strings.concat("documentation-raw-i-dir-test-", tags[si]), + rawidirectorytest, env, (30i64 * (time.second: i64)): time.duration, &out); + documentationrecord(&out, 1, &records); + assert(same(out.stdout, "FAIL\n") && same(out.stderr, strings.concat(rawi, + ":2:1: error: expected top-level decl\n"))); + let rawispec: str = strings.concat(rawitree, "/..."); + let rawirectest: []str = [driver(stages[si]), "test", rawispec]; + runcommandenv(root, strings.concat("documentation-raw-i-rec-test-", tags[si]), + rawirectest, env, (30i64 * (time.second: i64)): time.duration, &out); + documentationrecord(&out, 1, &records); + assert(same(out.stdout, "FAIL\n") && same(out.stderr, strings.concat(rawitreefile, + ":2:1: error: expected top-level decl\n"))); + let headerav: []str = [driver(stages[si]), "build", badheader]; + runcommandenv(root, strings.concat("documentation-header-", tags[si]), headerav, + env, (30i64 * (time.second: i64)): time.duration, &out); + documentationrecord(&out, 1, &records); + assert(out.stdout.len == 0 && same(out.stderr, strings.concat(badheader, + ":2:8: error: expected identifier, got ;\n"))); + let packageav: []str = [driver(stages[si]), "build", badpackage]; + runcommandenv(root, strings.concat("documentation-package-", tags[si]), packageav, + env, (30i64 * (time.second: i64)): time.duration, &out); + documentationrecord(&out, 1, &records); + assert(out.stdout.len == 0 && same(out.stderr, strings.concat(badpackage, + ":1:9: error: invalid or missing package clause\n"))); + let bomav: []str = [driver(stages[si]), "build", bom]; + runcommandenv(root, strings.concat("documentation-bom-", tags[si]), bomav, + env, (30i64 * (time.second: i64)): time.duration, &out); + documentationrecord(&out, 1, &records); + assert(out.stdout.len == 0 && same(out.stderr, strings.concat("ww: ", + bomdir, ": directory contains no WW package sources\n"))); + let triviaav: []str = [driver(stages[si]), "build", triviafile]; + runcommandenv(root, strings.concat("documentation-trivia-", tags[si]), triviaav, + env, (30i64 * (time.second: i64)): time.duration, &out); + documentationrecord(&out, 1, &records); + assert(out.stdout.len == 0 && same(out.stderr, strings.concat("ww: ", + trivia, ": directory contains no WW package sources\n"))); + let exactav: []str = [driver(stages[si]), "build", strings.concat(exact, "/x.ww")]; + runcommandenv(root, strings.concat("documentation-exact-x-", tags[si]), exactav, + env, (30i64 * (time.second: i64)): time.duration, &out); + expectexit(&out, 1); assert(!has(out.stderr, "directory contains no WW package sources")); + let exacttav: []str = [driver(stages[si]), "build", strings.concat(exact, "/t.ww")]; + runcommandenv(root, strings.concat("documentation-exact-t-", tags[si]), exacttav, + env, (30i64 * (time.second: i64)): time.duration, &out); + expectexit(&out, 1); assert(!has(out.stderr, "directory contains no WW package sources")); + let importav: []str = [driver(stages[si]), "build", "-I", include, importer]; + runcommandenv(root, strings.concat("documentation-imported-", tags[si]), importav, + env, (30i64 * (time.second: i64)): time.duration, &out); + expectexit(&out, 1); assert(!os.exists(strings.concat(root, "/imported-", tags[si])) + && readfile(trace).len == 0); + let importoneav: []str = [driver(stages[si]), "build", "-I", include, importerone]; + runcommandenv(root, strings.concat("documentation-imported-one-", tags[si]), + importoneav, env, (30i64 * (time.second: i64)): time.duration, &out); + expectexit(&out, 1); assert(has(out.stderr, + ":2:1: error: cannot find package docone\n") + && !has(out.stderr, "directory contains no WW package sources") + && readfile(trace).len == 0); + + let mixav: []str = [driver(stages[si]), "build", "-w", mixwork, + "-o", mixedout, mixed]; + let ctlav: []str = [driver(stages[si]), "build", "-w", ctlwork, + "-o", controlout, control]; + runcommandenv(root, strings.concat("documentation-mixed-cold-", tags[si]), mixav, + env, (60i64 * (time.second: i64)): time.duration, &out); + expectexit(&out, 0); assert(out.stdout.len == 0 && out.stderr.len == 0 && os.exists(mixedout)); + runcommandenv(root, strings.concat("documentation-control-cold-", tags[si]), ctlav, + env, (60i64 * (time.second: i64)): time.duration, &out); + expectexit(&out, 0); assert(out.stdout.len == 0 && out.stderr.len == 0 + && same(readfile(mixedout), readfile(controlout))); + let mixedbytes: str = strings.dup(readfile(mixedout)); + let mixedtree: str = strings.dup(artifacttreesnapshot(mixwork)); + let mixedsemantic: str = strings.dup(wrongsuffixsemantictreesnapshot(mixwork)); + if (si == 0) { + mixedcross = strings.dup(mixedbytes); + mixedsemanticcross = strings.dup(mixedsemantic); + } else { + assert(same(mixedcross, mixedbytes) + && same(mixedsemanticcross, mixedsemantic)); + }; + let mixabsent: str = strings.concat(mixdoc, ".absent"); + assert(os.rename(mixdoc, mixabsent) == 0); + rewritefile(trace, ""); + runcommandenv(root, strings.concat("documentation-mixed-absent-", tags[si]), mixav, + env, (60i64 * (time.second: i64)): time.duration, &out); + documentationrecord(&out, 0, &records); + assert(out.stdout.len == 0 && out.stderr.len == 0 + && same(mixedbytes, readfile(mixedout)) + && same(mixedtree, artifacttreesnapshot(mixwork)) + && same(mixedsemantic, wrongsuffixsemantictreesnapshot(mixwork)) + && same(readfile(trace), "L")); + assert(os.rename(mixabsent, mixdoc) == 0); + rewritefile(trace, ""); rewritefile(mixdoc, strings.concat(doctext, "still ignored\n")); + runcommandenv(root, strings.concat("documentation-mixed-warm-", tags[si]), mixav, + env, (60i64 * (time.second: i64)): time.duration, &out); + expectexit(&out, 0); assert(out.stdout.len == 0 && out.stderr.len == 0 + && same(mixedbytes, readfile(mixedout)) && same(mixedtree, artifacttreesnapshot(mixwork)) + && same(readfile(trace), "L")); + let warmwork: str = strings.concat(root, "/warm-work-", tags[si]); + let warmout: str = strings.concat(root, "/warm-output-", tags[si]); + mkdirall(warmwork); + let warmav: []str = [driver(stages[si]), "build", "-w", warmwork, + "-o", warmout, warm]; + runcommandenv(root, strings.concat("documentation-warm-cold-", tags[si]), warmav, + env, (60i64 * (time.second: i64)): time.duration, &out); + expectexit(&out, 0); assert(out.stdout.len == 0 && out.stderr.len == 0); + let warmbytes: str = strings.dup(readfile(warmout)); + let warmtree: str = strings.dup(artifacttreesnapshot(warmwork)); + if (si == 0) { warmcross = strings.dup(warmbytes); } + else { assert(same(warmcross, warmbytes)); }; + rewritefile(trace, ""); rewritefile(warmfile, doctext); + runcommandenv(root, strings.concat("documentation-warm-docs-", tags[si]), warmav, + env, (30i64 * (time.second: i64)): time.duration, &out); + expectexit(&out, 1); + assert(out.stdout.len == 0 && same(out.stderr, strings.concat("ww: ", warm, + ": directory contains no WW package sources\n")) && same(warmbytes, readfile(warmout)) + && same(warmtree, artifacttreesnapshot(warmwork)) && readfile(trace).len == 0); + rewritefile(warmfile, "package documentation;\nimport ;\n"); + runcommandenv(root, strings.concat("documentation-warm-header-", tags[si]), warmav, + env, (30i64 * (time.second: i64)): time.duration, &out); + expectexit(&out, 1); + assert(out.stdout.len == 0 && same(out.stderr, strings.concat(warmfile, + ":2:8: error: expected identifier, got ;\n")) && same(warmbytes, readfile(warmout)) + && same(warmtree, artifacttreesnapshot(warmwork)) && readfile(trace).len == 0); + rewritefile(warmfile, maintext); + documentationclean(root); + if (si == 0) { + let ri: i32 = 0; + for (ri < records.len) { append(cross, strings.dup(records[ri])); ri += 1; }; + } else { documentationrecordssame(cross, records); }; + si += 1; + }; + // Header classification has no shared action identity or lock: simultaneous + // Cstage/WWstage requests may read the same source but must leave separate + // caller-owned work roots untouched. + let cwork: str = strings.concat(root, "/parallel-c-work"); + let wwork: str = strings.concat(root, "/parallel-ww-work"); + mkdirall(cwork); mkdirall(wwork); + let cbefore: str = treesnapshot(cwork); + let wbefore: str = treesnapshot(wwork); + let cav: []str = [driver("ww"), "build", "-w", cwork, docs]; + let wav: []str = [driver("ww_ww"), "build", "-w", wwork, docs]; + let cc: exec.command; + cc.path = cav[0]; cc.argv = cav; cc.env = os.getenvs(); cc.dir = repo(); + cc.stdoutpath = strings.concat(root, "/documentation-parallel-c.stdout"); + cc.stderrpath = strings.concat(root, "/documentation-parallel-c.stderr"); + cc.deadline = time.add(time.now(time.clock.monotonic), + (30i64 * (time.second: i64)): time.duration); + cc.grace = (100i64 * (time.millisecond: i64)): time.duration; + let wc: exec.command; + wc.path = wav[0]; wc.argv = wav; wc.env = os.getenvs(); wc.dir = repo(); + wc.stdoutpath = strings.concat(root, "/documentation-parallel-ww.stdout"); + wc.stderrpath = strings.concat(root, "/documentation-parallel-ww.stderr"); + wc.deadline = time.add(time.now(time.clock.monotonic), + (30i64 * (time.second: i64)): time.duration); + wc.grace = (100i64 * (time.millisecond: i64)): time.duration; + let cp: exec.process; + let wp: exec.process; + exec.start(&cp, &cc); exec.start(&wp, &wc); + let cdone: bool = false; + let wdone: bool = false; + for (!cdone || !wdone) { + if (!cdone) { cdone = exec.poll(&cp); }; + if (!wdone) { wdone = exec.poll(&wp); }; + if (!cdone || !wdone) { time.sleep(time.millisecond, time.clock.monotonic); }; + }; + let parallelerr: str = strings.concat("ww: ", docs, + ": directory contains no WW package sources\n"); + assert(cp.result.errno == 0 && cp.result.cleanuperrno == 0 + && cp.result.termination == exec.termination.EXIT && cp.result.code == 1 + && wp.result.errno == 0 && wp.result.cleanuperrno == 0 + && wp.result.termination == exec.termination.EXIT && wp.result.code == 1 + && readfile(cc.stdoutpath).len == 0 && readfile(wc.stdoutpath).len == 0 + && same(readfile(cc.stderrpath), parallelerr) + && same(readfile(wc.stderrpath), parallelerr) + && same(cbefore, treesnapshot(cwork)) && same(wbefore, treesnapshot(wwork))); + documentationclean(root); + clean(root); +}; + // The single-FILE `ww test ` route wraps the same in-binary test // runtime as the directory-package route (cases/*), but through the // driver's single-file leg. One row per outcome class keeps that leg's