diff --git a/Makefile b/Makefile index 2dba1027..542c21b5 100644 --- a/Makefile +++ b/Makefile @@ -606,7 +606,7 @@ $(BIN)/test_test_filter: test/wcc/989_test_filter.c $(BIN)/ww $(BIN)/ww_ww \ $(BIN)/w6c $(BIN)/w6a $(BIN)/w6l \ $(BIN)/w6c_ww $(BIN)/w6a_ww $(BIN)/w6l_ww \ $(LIB)/libwwrt.a | $(BIN) - $(CC) $(CFLAGS) -o $@ $< + $(CC) $(CFLAGS) -Werror -o $@ $< # 989_callarg_typecheck drives `ww build` + `ww test -c` then ` -T` # on BOTH driver/compiler twins (#24 general call-arg typecheck + #34 @@ -2740,10 +2740,13 @@ sizelint: peellint: @sh tools/peellint -test: all sizelint peellint $(TESTS) test-lang test-lang-byteid test-run +test-harness: test/run test/run_test.sh + @sh test/run_test.sh + +test: all sizelint peellint test-harness $(TESTS) test-lang test-lang-byteid test-run @WW=$(BIN)/ww BIN=$(BIN) sh test/run -test-unit: all sizelint peellint $(TESTS) +test-unit: all sizelint peellint test-harness $(TESTS) @WW=$(BIN)/ww BIN=$(BIN) UNIT=1 sh test/run # ---- test-run: ww-native behavior harness (fold 2) --------------------- @@ -2840,12 +2843,13 @@ test-lang-byteid: all # ---- test-commit: per-commit default (task #10 T1) --------------------- # Full suite, but content-key-SKIPS the expensive wwstage byte-id / -# self-compile gates (950/989_lib_byteid/990-997) when their input set is +# self-compile gates (950/989_lib_byteid/990-995/997) when their input set is # byte-identical to the last green run — reported as `cached`, never a # silent skip. NOT a substitute for the push bar: `make test` stays the # no-skip bootstrap gate. Cache lives in out/.testcache (gitignored, wiped -# by `make clean`); a green `make test` primes it. See test/run. -test-commit: all sizelint peellint $(TESTS) test-lang +# by `make clean`); a green `make test` primes it. The host-/usr/lib dynamic +# gate 996 always runs and is never cached. See test/run. +test-commit: all sizelint peellint test-harness $(TESTS) test-lang @WW=$(BIN)/ww BIN=$(BIN) COMMIT=1 sh test/run # ---- smoke: inner-loop cross-stage byte-id check on a SMALL input ------ @@ -3028,4 +3032,4 @@ nocc: @echo @echo "NOCC OK: $(STAGE0)/* reproduces itself from source. cc not invoked." -.PHONY: all cstage wwstage test test-unit test-commit test-run test-lang test-lang-byteid smoke sizelint peellint install clean bootstrap nocc bootstrap-snapshot +.PHONY: all cstage wwstage test test-unit test-commit test-harness test-run test-lang test-lang-byteid smoke sizelint peellint install clean bootstrap nocc bootstrap-snapshot diff --git a/PROJECT_PLAN.md b/PROJECT_PLAN.md new file mode 100644 index 00000000..c90e315c --- /dev/null +++ b/PROJECT_PLAN.md @@ -0,0 +1,827 @@ +# WW Project Continuation Plan + +Status date: 2026-08-05 + +Canonical baseline: `master` at `3d7973596422f1c9e2275016a5f264a2ae504ceb` + +This is the durable handoff for the next Codex session. Read this file before +starting implementation. It records the repository audit, the test-suite audit, +the branch cleanup, the first T0 implementation slice, and the recommended order +of work. + +## 0. T0 first-slice update + +The focused topic branch is `t0-harness-integrity`, based directly on the +canonical `master` commit above. This slice did not touch compiler features, +CSP, bootstrap design, the cast-diagnostic worktree, recovery artifacts, or the +existing unrelated generated/untracked paths. + +Completed in this slice: + +- `test/run` now materializes the complete 337-source corpus before fan-out, + records atomic started and terminal records, captures launcher failures, and + checks discovered, started, completed, classified, cached, excluded, and + not-run counts by identity. +- Worker results distinguish pass, fail, explicit exit-77 skip, timeout, and + harness error. Every C wrapper has a 900-second outer GNU `timeout` with a + kill-after grace period. A private child-status record plus a separate GNU + `timeout --verbose` diagnostic distinguishes actual expiration from tests + that themselves return reserved codes 124-127 or 137. +- `make test-unit` prints all eight intentional phase-2 exclusions. Unexpected + skips, timeouts, missing binaries, malformed/incomplete records, launcher + failures, empty corpora, and corpus shrinkage make required targets fail. +- Successful wrapper output is visible. The whole-wrapper prerequisite skips in + `530_w6a_parsenum`, `810_dyn`, `989_dynentry_run`, and + `989_structlocal_frame` now return the explicit skip code instead of success. +- The cache key includes tracked and relevant nonignored untracked inputs, + unstaged-deletion markers, built tools, the runtime archive, cached gate + executables, and harness configuration. Ambient `WW_W6*`/library/cache + overrides are removed from required-suite workers. Inputs are fingerprinted + again after both cache hits and misses, publication is atomic, custom workers + disable caching, and the host-dependent `996_dyn_ww` gate is never cached. +- `test/run_test.sh`, wired as `make test-harness`, exercises pass, fail, skip, + TERM-resistant timeout, genuine reserved exit codes, timeout-wrapper failure, + missing binary/capture/result, empty/shrunken corpora, failed launchers, + zero-duration rejection, cache publication/hits, unstaged tracked deletion, + uncached host-dependent gates, and visible `test-unit` exclusions entirely in + temporary directories. +- The regex gate now runs the external black-box unit and the in-package module + root as two separate `-T` binaries. It requires all 33 black-box tests and all + 15 named white-box tests exactly once. The obsolete zero-test + `lib/regex/wb/regex_test.ww` scaffold and its duplicate byte-ID row are gone. +- `989_test_filter.c` uses one named four-entry bound for both expectation arrays + and loops; the redundant fifth initializer is gone, and its dedicated Make + rule compiles with `-Werror`. +- `989_lib_byteid.c` now pins its enrolled corpus at 42 entries, so deleting a + byte-ID row cannot silently shrink that cross-stage gate. + +Verification on 2026-08-05: + +| Command | Result | Reported scope and time | +| --- | --- | --- | +| `sh test/run_test.sh` | Pass | All 16 synthetic harness scenarios | +| `make test-harness out/bin/test_regex_run out/bin/test_test_filter` | Pass | Harness self-test plus focused binaries | +| `BIN="$PWD/out/bin" timeout 300 ./out/bin/test_regex_run` | Pass | Regex black-box 33/33; white-box 15/15 | +| `BIN="$PWD/out/bin" timeout 60 ./out/bin/test_test_filter` | Pass | Five filter rows; `ww == ww_ww` output | +| `BIN=out/bin out/bin/test_lib_byteid` | Pass | 36 byte-identical, four pinned divergences, two pinned WW rejects across 42 units | +| `make test-unit` | Pass | Discovered 337; started/completed/pass 329; fail/skip/timeout/harness-error 0; eight explicit exclusions; 76 seconds | +| `make test` | Pass | C harness 337/337 with zero fail/skip/timeout/harness-error; 159 language files containing 1,224 tests; 153 language byte-ID files; 106/106 `runww` cases; 308 seconds | + +The pre/post generated-artifact path scan contained the same 533 paths; neither +required run introduced a new source-tree artifact path. Final logs are in +`/tmp/ww-t0-test-unit-final.log` and `/tmp/ww-t0-test-full-final.log`, which are +not durable. + +Remaining T0 and adjacent risks: + +- `runww` still uses shared `/tmp/runww.err` and `/tmp/runww.bin`, has no per-case + timeout, and has no pinned 106-case minimum. Close these paths in the first + replacement-runner vertical slice rather than extending the legacy wrapper. +- The WW-native `lib/test` runner still needs T2's child completion token, + timeout/capture protocol, skip reason API, and failure-count clamping; an + explicit stale-record/256-failure regression therefore remains outstanding. +- Several legacy wrappers have partial stage-missing branches that print a skip + and continue after other assertions. Required Make prerequisites made none of + those branches fire in either verified run, and their output is now visible, + but converting them to structured whole-test/subtest outcomes remains work. +- Warning-fatal compilation is local to `989_test_filter.c`; broader harness + `-Werror` requires resolving existing unrelated missing-initializer and path + truncation warnings first. +- The regex module-root route is deliberately temporary and C-driver-only. + T1 must implement package-aware white-box/external test planning symmetrically + in `ww` and `ww_ww`, after which `regex_whitebox.ww` can become a canonical + `*_test.ww` file. +- Manifest and planning appends still rely on the fresh temporary filesystem + remaining writable, and cached gates treat ordinary host `PATH` utilities as + part of the machine baseline rather than fingerprinting each utility. These + are reasons not to expand the legacy cache/harness design. + +Strategic direction recorded on 2026-08-05: the 337-wrapper/manual-Make test +architecture is a temporary migration oracle, not a design to keep extending. +After landing this branch, design the declarative replacement and migrate +`runww` as its first vertical slice; the new runner must provide unique temp +roots, per-case timeouts, a pinned corpus, and structured outcomes from the +start. Retire old wrappers and manual rules only after coverage-equivalence +checks pass. Do not integrate the cast-diagnostic WIP into this branch. + +## 1. Executive decision + +The project is alive and its current test commands pass. The first T0 slice has +closed the host C harness's missing-worker and vacuous-regex false greens, but the +remaining `runww`, package-driver, and WW-native runner contracts still prevent +the test system from being the final trustworthy development gate. The largest +immediate risk remains test architecture, not a single compiler feature. + +Proceed in this order: + +1. Preserve the current state and make the test result trustworthy. +2. Split compiler unit tests, compiler conformance tests, package unit tests, + integrations, differential tests, and bootstrap tests into explicit suites. +3. Finish the small compiler-stability tail, including the active cast-diagnostic + work. +4. Produce a usable developer release: SDK installation, documentation, CI, and + import/export and error-handling cleanup. +5. Start the CSP/runtime vertical slice only after renewed explicit approval. +6. Treat bootstrap independence as the final milestone, not the next task. + +Do not start by rewriting the compiler, changing the backend, or doing a large +test-directory rename. First eliminate false-green paths and define stable suite +contracts. + +## 2. Preservation and branch cleanup + +### 2.1 Repository state + +- There is no configured Git remote. Local history has no off-machine safety net. +- The main worktree is `/home/kimchi/src/ww` on `t0-harness-integrity`, based + directly on `master` at the canonical commit above. +- `docs/spec.md` exists but is untracked. It appears to be intended as a + normative language specification and must not be discarded. +- `ref/hare` and `ref/plan9front` are gitlinks, but there is no `.gitmodules` + file. `ref/inferno-os` and `ref/qbe` are untracked trees. Make their provenance + reproducible before calling the repository portable. +- Ignored bootstrap binaries exist under `bootstrap/amd64`. +- Generated output currently exists under `build/`, several + `test/wcc/data/*/out/` directories, `scratch/`, and other untracked paths. + Classify these paths before deleting them; they were not part of the branch + cleanup. +- `.ai/` is ignored and contains older handoffs. Use this document as the current + source of truth unless an older note is explicitly cited. + +### 2.2 Recovery artifacts created before cleanup + +The following files are in the repository root and are currently untracked: + +| File | Purpose | SHA-256 | +| --- | --- | --- | +| `branch-backup-20260805.bundle` | Complete pre-cleanup Git history and refs | `a96f48588aa406ab74c52ba8e94fdd635bfbb4e80c2602811d33c5b28a7793aa` | +| `wip-29-castdiag-20260805.patch` | Tracked dirty changes from the active cast-diagnostic worktree | `c7a5de8606ee8d9cbba32dbc5fd420c694b0c2929b673f35155db4f7066ba2d7` | +| `wip-29-cast-fixture-20260805.tar.gz` | Untracked `cast_unknown_type` fixture | `27a16b43ee83dde28c207f22050738591c1414ab917509529fb2d9d5519b5192` | + +`git bundle verify branch-backup-20260805.bundle` reports a complete history. +The bundle contains the original 107 local branch refs plus the stash, tag, and +worktree refs. Copy these three artifacts to a different disk or remote storage; +an untracked backup on the same disk is not a permanent backup. + +To recover one deleted branch without disturbing current refs: + +```sh +git bundle verify branch-backup-20260805.bundle +git fetch ./branch-backup-20260805.bundle \ + refs/heads/OLD_NAME:refs/heads/restore/OLD_NAME +``` + +To reconstruct the dirty cast work on a fresh worktree: + +```sh +git worktree add ../ww-cast-restore -b restore/castdiag master +git -C ../ww-cast-restore apply \ + /home/kimchi/src/ww/wip-29-castdiag-20260805.patch +tar -xzf /home/kimchi/src/ww/wip-29-cast-fixture-20260805.tar.gz \ + -C ../ww-cast-restore +``` + +### 2.3 Branch cleanup result + +The repository started with 107 local branches. Cleanup deleted 105: + +- 90 branch tips were direct ancestors of `master`. +- 14 branches looked unmerged by graph ancestry but were audited and found to be + patch-equivalent to, or superseded by, later work on `master`. +- `wip-buga` was already merged; its clean worktree was removed and the branch + was deleted. + +Before T0 implementation, only these branches remained. The focused T0 branch +was then added: + +| Branch | State | Decision | +| --- | --- | --- | +| `master` | Clean tracked baseline at `3d79735` | Keep | +| `wip-29-castdiag` | Checked out at `/home/kimchi/ww-29`; contains uncommitted source and fixture changes | Keep until reviewed and integrated | +| `t0-harness-integrity` | Checked out at `/home/kimchi/src/ww`; first T0 gate-integrity slice | Review and merge separately | + +The active WIP adds a clean diagnostic for a cast to an unknown type in +`selfhost/cmd/wcc/check.ww`, plus a fixture under +`test/wcc/data/cast_unknown_type/`. Do not delete this worktree merely because +its branch tip is already reachable from `master`; the valuable part is the +dirty working tree. The patch and fixture archive above are secondary recovery +copies. + +A detached worktree remains at `/home/kimchi/src/ww-f0` on `e8977a4`. It is not +a branch and was therefore not removed. Audit it separately before deciding +whether to remove it. + +The 14 non-ancestor branches were matched to later `master` work as follows: + +- `e3-c1-flip`, `e3-oracle`: separate-compilation flip later landed around + `33edc38`. +- `f1-stack-v1`, `f1-stack-v2`, `f1-stack-v3`: superseded by `9bd0d8b`, + `b9dd297`, `c138605`, `ac63951`, and later type-chase work. +- `impl-26`: superseded by `9fcb3be`. +- `impl-ascii-bytes-cut`, `impl-ascii2-complete`: superseded by `07fed80` and + `2b893b9`. +- `impl-strconv`: finalized through `6a5cdbd`, `a117852`, and `db5c5b6`. +- `impl-strings-cutrcut`: patch-equivalent to `38a906c`. +- `t28`, `t28-parked`: superseded by `c40b2df` and `330792b`. +- `wip/io5-handle-seeker`: superseded by `06c00e0` and `532b0a8`. +- `worktree-impl-e8`: superseded by `b842f93`, `10cb835`, and `7d39f6d`. + +The bundle remains the authoritative record if any of those classifications +must be revisited. + +## 3. Verified build and test baseline + +The following commands were run on `master` at the baseline commit before this +document and branch-only cleanup: + +| Command | Result | Observed scope and time | +| --- | --- | --- | +| `make test-unit` | Pass, 329/329 C harness entries | About 95 seconds | +| `make test` | Pass | C harness 337 entries; `test/lang` 159 files; language byte-ID pass over 153 files; `runww` 106 cases; about 313 seconds total | + +The full audit log was written to `/tmp/ww-tests-health-full.log`; `/tmp` is not +durable, so rerun the commands when a fresh log is needed. + +These historical green results prove that the canonical baseline is buildable. +They did not prove that every named test executed useful assertions; the first +T0 slice above closes the missing-worker and empty regex-table paths without +changing this preserved baseline record. + +No source code changed during the branch cleanup and English-document rewrite, +so the full suite was not rerun afterward. + +## 4. Current test architecture + +The project already contains substantial test material. The problem is that +several different kinds of tests are mixed behind numeric C wrappers and broad +Make targets. + +### 4.1 C harness under `test/wcc` + +- 337 C test sources are registered through roughly 337 manual Make rules. +- The sources contain about 89,519 lines. +- Static inspection found subprocess orchestration in approximately 333 tests + (`fork`, `exec`, `system`, or `popen`). +- Only a handful, approximately the `000`, `100`, `200`, `300`, and `738` + families, resemble direct compiler-component unit tests. +- Most files are integration wrappers around compiler commands, generated + programs, output matching, byte-ID checks, or bootstrap behavior. + +Therefore `make test-unit` is currently misnamed. It omits a few heavy phase-2 +tests, but still runs most integrations and includes `989_lib_byteid`, which +alone took roughly 42 seconds in the audit. It does not run the 1,224-test +`test/lang` corpus or the `runww` fixtures, but C wrappers do indirectly launch +the library `@test` files. The first T0 slice makes all 458 library tests +effective, including the 15 regex white-box tests. + +### 4.2 Language tests under `test/lang` + +- 159 source files contain exactly 1,224 source-level `@test` functions. +- Six files are `_runonly_test.ww` variants. +- The current Make targets compile/run this corpus and also perform byte-ID + comparison for most of it. + +This is valuable language and compiler coverage, but file layout and target +naming do not clearly separate package unit behavior from compiler conformance +behavior. + +### 4.3 Library tests + +- 35 test-named library source files contain 443 `@test` functions. Eleven use + `_test.ww`; 24 use other names ending in `test.ww`. +- `lib/regex/regex_whitebox.ww`, a production-named source file, contains 15 + additional `@test` functions. +- There are therefore 36 files that actually contain tests and 458 source-level + library tests in total. + +Together, `test/lang` and `lib` contain 1,682 real source `@test` functions, +excluding generated combined files. + +### 4.4 Directive fixtures through `runww` + +There are 106 directive-driven cases: + +- 100 `//ww:error` cases. +- 5 `//ww:run-exit` cases. +- 1 `//ww:run` case. +- 0 `//ww:compile` cases. + +The rejection coverage is useful, but this is not yet a balanced compiler +conformance suite. Rejections do not identify whether failure was expected in +lexing, parsing, checking, code generation, assembly, or linking. Accept/run +cases use only the C stage, while rejects exercise two stages. + +### 4.5 Current `@test` implementation + +The compiler's `-T` path synthesizes a `__wwtests` value table and test main. +`@test` functions are expected to have the shape `fn() void`. Normal builds +type-check tests and then strip them. The runtime in `lib/test` forks once per +test, so an abort, signal, or nonzero exit does not stop subsequent tests. The +runner also supports glob filtering and treats zero matches as success. + +The per-test fork model and compiler-generated value table are good foundations +and should be preserved. The package driver and child/parent result protocol need +to be completed. + +### 4.6 Target coupling and differential caveats + +- Despite its name, `make test-run` is not an isolated `runww` target. Its + order-only prerequisites build the C harness and run `test-lang` plus + `test-lang-byteid` first. The redesigned target graph must remove this + surprising coupling. +- `test-lang-byteid` reads only `test/lang`. The six positive `runww` fixtures + are not checked for stage-by-stage assembly identity. +- `989_lib_byteid` has 42 rows: 36 require identity, four are declared + divergences, and two are expected WW-stage rejects. A green result does not + mean complete standard-library stage parity. +- `runww` itself is compiled by the C-stage compiler and then invokes that same + stage. It is useful coverage but not an independent sole oracle. Retain an + independent host-side runner or cross-validate the harness during migration. + +## 5. Correctness risks in the current test result + +Treat the following as gate-integrity bugs, not cosmetic cleanup. + +### 5.1 Vacuous regex test: closed in the first T0 slice + +The former `lib/regex/wb/regex_test.ww` root contained no tests while importing +regex as a dependency, so `-T` stripped all 15 intended white-box tests. The T0 +gate now resolves `regex` itself as the root module, keeps it separate from the +external `regex_test` binary, and asserts every expected name plus the exact +15/15 summary. The empty scaffold was removed. T1 still needs to replace this +temporary module-root route with ordinary package-aware test assembly. + +### 5.2 Harness false-green paths + +Closed in the first T0 slice: + +- `test/run` no longer masks either `xargs` launcher. It collects from a complete + manifest and fails on missing starts, missing/malformed terminal records, + launcher errors, or any count/identity mismatch. +- Pass, fail, explicit skip, timeout, and harness error are distinct outcomes; + required targets fail on every unexpected non-pass outcome. +- Successful wrapper output and every expected `test-unit` exclusion are visible. +- The cache hashes nonignored untracked inputs plus effective built inputs, + represents tracked deletions, revalidates hits and misses at run end, and + publishes atomically only after complete green accounting. The host-dependent + dynamic gate is deliberately uncached. +- The dead nested `test/lang` scan is gone. +- `989_test_filter.c` has one named expectation count, no excess initializer, + and a rule-local `-Werror` gate. + +Remaining: + +- Existing generated artifacts outside the new harness result/temp directories + have not been broadly relocated or classified. +- Partial prerequisite skips inside legacy wrappers are not yet structured + subtest outcomes. None fired in the recorded runs, and whole-wrapper skips now + use exit 77. +- Broad warning-fatal harness compilation remains blocked by pre-existing + warnings outside `989_test_filter.c`. + +### 5.3 `runww` limitations + +- It uses shared fixed paths such as `/tmp/runww.err` and a fixed binary name, + which prevents safe parallel execution and risks stale-file interference. +- It supplies a nil environment to subprocesses. +- Negative cases check a diagnostic substring but do not identify the failure + phase. +- There is no per-case timeout. +- Missing arguments fail, but there is no expected/minimum corpus count. A suite + accidentally reduced from 106 fixtures to one can still pass without a + corpus-shrinkage alarm. + +### 5.4 Package-driver limitations + +- `ww test DIR` discovers only immediate `_test.ww` files and does not recurse. +- Files are not sorted deterministically. +- Each discovered file is built separately instead of assembling one package + test unit. Same-package helper sharing therefore does not work as expected. +- Production enumeration excludes all `*test.ww`, while test discovery includes + only `_test.ww`; the naming rules disagree. +- Only the first glob argument reaches the runner even though the runner can + support more than one. +- A single file with no tests can silently return success, while a directory + with no recognized tests can error. Define one consistent contract. +- Imported dependency tests are stripped. This is normally correct, but root + package files containing tests must be assembled into the root test build. +- `-c` without `-o` can leave a temporary test executable behind. + +### 5.5 Runner-protocol limitations + +- A test can call `os.exit(0)` before completing and the parent reports it as a + pass. The parent needs an explicit completion token from the child. +- The runner returns the raw number of failures. At 256 failures, the Unix exit + status can wrap to zero. +- There is no timeout, structured capture, duration reporting, skip reason, + expected-abort protocol, cleanup hook, or per-test temporary directory. +- Invalid `@test` declarations are checked too narrowly and mainly under `-T`. + Exported tests, prototypes, arguments, variadics, non-void returns, duplicate + annotations, and other invalid forms need deterministic diagnostics in normal + and test builds. + +## 6. Target model: Go/Hare-style separation + +The desired result is not a copy of either project, but it should adopt their +separation of concerns. + +### 6.1 Lessons from Go + +The local Go 1.25.4 source tree demonstrates two distinct systems: + +- Compiler/runtime conformance fixtures live outside ordinary package unit + tests and use explicit execution/error directives. +- Ordinary `*_test.go` files are assembled per package into a generated test + main, with support for same-package tests, external-package tests, filtering, + caching, timeouts, parallelism, and structured output. +- Compiler internals also have direct component unit tests rather than wrapping + every assertion in a compiler subprocess. + +WW needs the same conceptual split: compiler correctness fixtures are not the +same thing as user-facing package unit tests. + +### 6.2 Lessons from Hare and harec + +The vendored `ref/hare` test command demonstrates useful package-test behavior: + +- Recursive module discovery when no explicit module is given. +- Deterministic sorting, multiple globs, and a test-only build tag. +- First-class skip, require, expected-abort, current-test, capture, timing, and + backtrace behavior. + +The upstream Hare compiler suite uses feature-grouped positive and negative +fixtures and distinguishes outcomes such as success, user error, lex failure, +parse failure, check failure, and abnormal failure. The vendored `ref/harec` +copy is incomplete and lacks the full compiler test corpus; pin an upstream +version or import a documented reference snapshot before using it as a durable +oracle. + +Keep WW's fork isolation and generated source value table. Do not copy Hare's +same-process recovery mechanism or unrelated linker-specific structure. + +## 7. Proposed suite contracts + +Use semantic directories and explicit metadata. Numeric prefixes may remain as +stable case IDs during migration, but they must no longer determine what a test +means or when it runs. + +```text +test/ + unit/ Direct host-side compiler/library API tests; no subprocesses + compiler/ Source conformance fixtures grouped by language feature + package/ Fixtures for `ww test` discovery and package assembly + integration/ CLI, import graph, archive, ELF, FFI, and toolchain behavior + differential/ C-stage versus WW-stage and byte-ID comparisons + bootstrap/ Fixed-point, self-host, and no-C-compiler scenarios + harness/ Shared runners, manifests, and assertion helpers +``` + +Actual package unit tests should stay beside their code as canonical +`*_test.ww` files. `test/package` is for testing the package-test driver itself, +including multi-file layouts, helper sharing, external test packages, empty +packages, recursion, and failures. + +Define these Make targets: + +| Target | Contract | +| --- | --- | +| `test-unit` | Direct in-process compiler and library API tests only; zero child compiler processes | +| `test-compiler` | Positive, negative, run, compile, assembly, and multi-file language conformance fixtures | +| `test-package` | Real `ww test` package tests and package-driver fixtures | +| `test-integration` | CLI, separate compilation, archives, ELF, FFI, and graph behavior | +| `test-differential` | C-stage/WW-stage output and byte-identity checks | +| `test-bootstrap` | Fixed point, self-host, and `nocc`; explicitly slow | +| `test-lint` | Harness integrity, fixture metadata, duplicate IDs, empty corpora, and warning checks | +| `test-fast` | Unit tests plus a documented compiler/package smoke set | +| `test` | Complete required pre-merge set, with no hidden skips | +| `test-all` | `test` plus bootstrap and any explicitly labeled slow suites | + +Every target must print discovered, passed, failed, skipped, timed out, and not +run counts. A target may intentionally exclude another named suite, but it may +not silently skip entries within its declared corpus. + +Compiler fixtures need explicit metadata for at least: + +- Mode: check, compile, run, run-with-exit, reject, assembly match, or multi-file. +- Expected stage: lex, parse, check, codegen, assemble, link, run, signal, or + success. +- Expected diagnostic, stdout, stderr, exit status, or signal. +- Timeout and optional platform/tool prerequisites. +- Stable feature group and case ID. + +Use one declarative manifest or one clearly documented directive grammar. Do not +continue adding one manual Make rule and one custom C subprocess wrapper for +every source fixture. + +## 8. Required `ww test` semantics + +The language specification requires both same-package white-box tests and an +external `_test` black-box package. A package-under-test invocation must +produce one top-level result covering both, but the implementation cannot simply +link two current `-T` units: every `-T` unit owns global `main` and `__wwtests` +symbols, dependencies are compiled without `-T`, and `.wwi` does not carry test +descriptors. + +Use this implementation sequence: + +1. Treat `*_test.ww` only as the candidate-file discovery rule. Classify a file + as white-box or black-box from its declared package (`package foo` versus + `package foo_test`), not from its suffix alone. +2. Compile production files plus same-package test files into one white-box root + unit. +3. Compile the external package against only the public API. Initially, build a + second test binary and aggregate both binary results in the driver. A later + design may instead emit linkable test descriptors and generate one aggregator + main, but it must first solve the current symbol and interface limitations. +4. Exclude test bodies from imported dependencies unless that dependency is the + package under test. +5. Give each test a canonical package-qualified ID such as + `.`. Bare leaf names are ambiguous across the white-box + and black-box units and are not sufficient for output or filtering. The + generated descriptor must also refer to the callable through its owning + package rather than assuming a bare function reference. +6. Produce one top-level package result even if it is internally implemented by + two test binaries. +7. Support multiple glob filters and a recursive package pattern equivalent in + purpose to `./...`. +8. Use one canonical file naming rule: `*_test.ww`. + +Normal production enumeration must exclude the entire `*_test.ww` file, +including test-only helpers and imports. An `@test` declaration placed in a +production file must still be fully checked in a normal build before its function +is stripped. These are separate driver and checker responsibilities. + +Make ordering an explicit compatibility decision. The recommended first +contract is to sort package and source discovery paths, then preserve declaration +order within each source. Do not silently change the existing documented +source/collection behavior to a global name sort. + +Define all empty cases separately: + +- An explicitly requested package with no tests reports `[no tests]` and + succeeds. +- An explicitly requested test file containing no `@test` declaration is an + error because it is likely a fixture mistake. +- A filter matching zero tests reports `[no matches]` and succeeds, preserving + the current useful filtering behavior. +- Recursive discovery reports packages with no tests visibly, but a pattern that + resolves to zero packages is an error. +- A required suite with an unexpectedly empty or shrunken corpus is a harness + error regardless of the user-facing command rules. + +Validate every `@test` declaration during normal checking, even when test bodies +are later stripped. At minimum reject parameters, variadics, non-void returns, +prototypes without bodies, invalid export combinations, `@test(...)`, unknown +attributes, placement on non-functions, and duplicate attributes with stable +diagnostic text. Put both the C-stage and self-hosted parser/checker through the +same acceptance fixtures. + +Define compile-only behavior rather than deleting its output: `ww test -c` +should preserve a deterministic `.test` binary by default, and `-o` +should select its path. Specify whether `-o` without `-c` is invalid; the +recommended contract is to require `-c` so output preservation is unambiguous. + +Keep one process per test, but add a control pipe from child to parent. A normal +return must write a completion record. `os.exit(0)` without that record is a +failure. The same protocol can carry skip, expected-abort, failure reason, and +captured-output metadata. Clamp the process exit code to `0` for no failures and +`1` for any failure instead of returning the raw failure count. + +## 9. Test-system implementation plan + +### T0: Make the existing green result trustworthy + +Do this before moving directories. Status after the first focused slice: + +- [x] Make `test/run` compare discovered, started, and completed test counts. +- [x] Remove `|| true` masking around worker execution; collect worker failure + as a harness failure. +- [x] Represent pass, fail, skip, timeout, and harness error separately. +- [x] Print skip reasons/output and fail required targets on unexpected skips. +- [~] Harness records and new regression artifacts are private temporary files, + and relevant source/configuration inputs are in the cache key. Broad legacy + source-tree artifact relocation remains. +- [x] Give every host C-harness subprocess an outer timeout and kill its process + group on expiry. WW-native enforcement remains T2 work. +- [ ] Replace shared `runww` paths with a unique per-case temporary directory. +- [x] Fail the C-harness target on zero or shrunken corpora and incomplete worker + results. Add the equivalent pinned minimum to `runww` next. +- [x] Fix the `mustnot[4]` initializer and related loops through one named count. +- [x] Prove all 15 regex white-box tests execute in a separate root-module + binary without combining conflicting `-T` units. +- [~] Sixteen harness scenarios cover fail, skip, TERM-resistant timeout, + reserved child exits, timeout-wrapper failure, lost/incomplete workers, + failed launchers, missing binary/result/captures, empty/shrunken corpora, + invalid bounds, cache safety, and expected exclusion. A fresh per-run result + directory prevents stale reuse; the WW-native 256-failure case remains. + +The host C-harness portion of the completion criterion is met: a killed or +skipped worker cannot leave a required target green, and the regex test reports +15 discovered and 15 passed white-box tests. Finish the unchecked `runww` items +before declaring all of T0 complete. + +### T1: Implement package-aware `ww test` + +- Introduce a package test-plan data structure in the driver. +- Unify production-file exclusion and candidate discovery on `*_test.ww`, then + classify candidates by their package declarations. +- Assemble production files and same-package tests into one white-box root. +- Build the required external `_test` unit as a second test binary first, + and aggregate its result with the white-box binary at the driver level. Do not + attempt to link two current `-T` mains together. +- Keep imported dependency tests stripped. +- Add package-qualified test IDs, deterministic recursive discovery, and + multiple filter arguments. +- Implement the documented `-c` output contract and all four no-test/no-match + cases above. +- Add C-stage and self-hosted compiler diagnostics for all invalid `@test` and + attribute shapes. +- Convert the regex white-box case into an ordinary, non-vacuous package test. + +Completion criterion: two test files in one package can share a private helper, +dependency tests do not leak in, filters work deterministically, and empty or +invalid packages produce the documented result. The C-stage `ww` and self-hosted +`ww_ww` must discover the same fixtures and produce matching output classes and +exit statuses. + +### T2: Finish runner semantics + +- Add the required OS/runtime wrappers for pipes, process-group creation, and + process-group signals before depending on them in the WW-native runner. +- Add the child completion/control protocol. +- Add skip and require helpers, expected abort, current test identity, cleanup, + per-test temporary directories, timeouts, captured output, and duration. +- Drain stdout, stderr, and control pipes while the child is running; waiting + before draining can deadlock when a pipe buffer fills. Add polling/nonblocking + support as needed. +- Ensure signals and premature `os.exit(0)` are classified correctly. +- Offer a stable human-readable output and a machine-readable event stream. + +Completion criterion: the runner's own conformance suite covers every outcome +and never wraps failures into exit status zero. + +### T3: Build the compiler conformance engine + +- Create the explicit fixture schema and stage-aware outcome model. +- Migrate `runww` first because it is already directive-driven. +- Keep an independent host-side fixture runner as the gate, or cross-check the + WW-native harness so the compiler under test is not its own sole oracle. +- Add balanced positive/check/compile/run cases beside rejection cases. +- Add multi-file, diagnostic location, exit, signal, stdout/stderr, and assembly + matching support. +- Pin the external Hare/harec reference version used for design comparison. + +This is a replacement program, not a permanent adapter around `test/run`. +Freeze the current C wrapper layer as a comparison oracle, migrate by declared +suite and outcome, and remove each old path when the new engine proves the same +coverage. Do not add new numeric wrappers or hand-written Make registrations for +cases that fit the new fixture schema. + +Completion criterion: adding a language fixture normally requires source plus +metadata, not a new C wrapper or Make rule. + +### T4: Migrate and shrink `test/wcc` + +Classify every existing C wrapper as unit, compiler fixture, package, integration, +differential, bootstrap, or obsolete duplicate. Move direct API tests first. +Mechanically migrate simple subprocess wrappers into fixtures. Keep C only where +the assertion genuinely needs host APIs, ABI inspection, or custom orchestration. + +Completion criterion: `test-unit` launches no compiler subprocess, suite +membership is discoverable without numeric-prefix knowledge, and the giant +manual Make registry is replaced by generated discovery plus a small manifest. + +### T5: Add CI cadence + +- Pre-commit/local: `test-fast`. +- Required pre-merge: `test` plus harness lint. +- Scheduled or release: `test-all`, bootstrap, sanitizers, and expensive + differential/byte-ID jobs. +- Archive structured results and fail on unexpected skips or corpus shrinkage. + +Completion criterion: a fresh checkout can reproduce the declared test matrix +without ignored local binaries or undocumented reference trees. + +## 10. Broader project roadmap + +### P0: Preserve and normalize the repository + +- Copy the recovery artifacts off-machine. +- Add a remote or another durable backup. +- Decide which of `docs/spec.md`, reference trees, probes, and scratch artifacts + are source, vendored dependencies, or disposable output. +- Track reference provenance through submodules, pinned archives, or documented + fetch scripts. +- Commit this plan and the chosen specification files. Do not commit generated + output or the bundle merely to make `git status` quiet. + +### P1: Establish the test foundation + +Execute T0 through at least T3 before broad compiler feature work. T4 can proceed +incrementally after the contracts stabilize. + +### P2: Finish the compiler-stability tail + +- Review and integrate `wip-29-castdiag`, including a regression fixture. +- Finish or explicitly close the large aggregate return (`sret`, especially + values over 24 bytes) edge cases. +- Re-audit stale issue-number tests and convert unresolved cases into explicit + fixtures. +- Finish type-information (`tinfo`) consistency work and remove duplicated or + stage-specific behavior where possible. +- Preserve separate-compilation and byte-ID guarantees while changing these + paths. + +Each item needs a focused branch, a regression test in the correct new suite, +and a clean full required test run before merge. + +### P3: Produce a developer-usable v0.1 release + +- Finish import/export membership rules and diagnostics. +- Normalize `errno` and error propagation across the standard library, especially + the existing error/I/O tail. +- Turn `make install` into an actual SDK installation. It currently copies only + the `ww` executable and `libwcc.a`; define and install the compiler, standard + library sources/artifacts, headers or metadata, and version information. +- Add a README with bootstrap prerequisites, build, test, install, and first + program instructions. +- Track the language specification and label implemented, reserved, and planned + behavior. +- Add CI from a fresh checkout and publish one reproducible release artifact. +- Document the current allocation lifetime honestly: both the compiler's + `free(x)` lowering and `os.free(p, n)` ultimately reach the current no-op + `rt_free`, so neither reclaims memory today. Decide and test a real reclamation + policy separately before promising `free` semantics. + +Release criterion: a developer can clone, build, run the required tests, install +to a staging prefix, and compile a small program using the installed SDK without +knowledge of this worktree. + +### P4: CSP/runtime vertical slice, only after explicit approval + +The existing design direction is a cooperative M:1 runtime with `swapctx`, a +run queue, and an unbuffered channel implemented library-first, followed by +syntax. `epoll`, `alt`/`select`, broader scheduling, and optimization come later. + +However, the recorded project status says CSP work was explicitly put on hold on +2026-06-18. Do not infer that a general request to resume the project cancels +that hold. Ask for renewed explicit approval before implementing CSP/runtime +features. The specification may reserve the design in the meantime. + +When approved, implement one narrow end-to-end slice: spawn two cooperative +tasks, rendezvous over one unbuffered channel, shut down deterministically, and +cover it in package, integration, and differential tests. + +### P5: Bootstrap independence last + +`make bootstrap` already checks a fixed point, `make nocc` exists, and ignored +stage-0 binaries are present. The process is not yet a clean independent bootstrap +because the C driver and local artifacts still orchestrate important steps. + +Freeze bootstrap only after the language, package graph, tests, SDK layout, and +runtime surface are stable. Then pin stage 0, document trust inputs, build stage +1 and stage 2 in a clean environment, compare the intended artifacts, and prove +the process without a C compiler where promised. + +## 11. Next-session checklist + +The next Codex session should do the following, in order: + +1. Read `CLAUDE.md`, this file, and `docs/spec.md`; do not rely on `.ai/` notes + without checking their date and status. +2. Inspect `t0-harness-integrity` and its final commit/diff against `master` at + `3d79735`; do not repeat the repository-wide audit unless that state changed. +3. Verify the three recovery artifact hashes and arrange an off-machine copy. +4. Inspect `/home/kimchi/ww-29` and compare its working tree with + `wip-29-castdiag-20260805.patch` and the fixture archive. Do not delete it. +5. Review and merge the first T0 branch separately, preserving all unrelated + untracked/generated paths. Re-run required targets if the diff changes. +6. Specify the replacement test manifest, structured result protocol, suite + boundaries, and coverage-equivalence report; keep fork-per-test isolation. +7. Migrate `runww` as the first vertical slice, implementing unique temp roots, + per-case timeout, and the pinned 106-case corpus in the new engine rather + than further entrenching the old wrapper architecture. +8. Begin package-aware `ww test` assembly after that first replacement slice; + keep white-box and external-package binaries separate while generated + `main`/`__wwtests` symbols still conflict. +9. Integrate the cast-diagnostic WIP as its own reviewed branch once the new test + placement is available, then remove `wip-29-castdiag` and its worktree. +10. Do not start CSP or bootstrap redesign unless the prerequisites and approval + above have been satisfied. + +## 12. Working rules for future branches + +- One branch should represent one reviewable outcome, not an indefinite work + queue. +- Record the intended suite and completion criterion before implementing. +- Rebase or merge promptly, then delete the branch after verifying reachability + and a durable backup. +- Never classify a branch only with `git branch --merged`; inspect dirty + worktrees, patch equivalence, fixture files, and later superseding commits. +- Keep generated output out of topic branches. +- A green target is acceptable only when it reports a complete corpus and + explicit skips. + +The immediate success condition is not more test count. It is the ability to +state exactly which compiler unit tests, compiler fixtures, package tests, and +integrations ran—and to know that none of them passed vacuously. diff --git a/lib/regex/regex_test.ww b/lib/regex/regex_test.ww index 6413aaa8..b16509a0 100644 --- a/lib/regex/regex_test.ww +++ b/lib/regex/regex_test.ww @@ -7,8 +7,8 @@ // unexported internals (thread/newmatch, run_thread / add_thread / // delete_thread / search, find_last_groupstart / shift / // parse_repetition, is_consuming_inst) — lives in -// lib/regex/regex_whitebox.ww (`package regex`, unified with regex.ww), -// run via the lib/regex/wb/ driver. +// lib/regex/regex_whitebox.ww (`package regex`, unified with regex.ww) and +// runs as a separate root-package binary. // // Fold 2a ports compile()'s lit/any/match arms only; exec lives in // later folds, so the compile_* cases pin the emitted inst PROGRAM diff --git a/lib/regex/regex_whitebox.ww b/lib/regex/regex_whitebox.ww index 417ba239..cc97208f 100644 --- a/lib/regex/regex_whitebox.ww +++ b/lib/regex/regex_whitebox.ww @@ -10,14 +10,12 @@ // the non-T @test drop, #6). Load-bearing ww-compiler coverage (drove // #34/#38/#44/#45/#48), not black-box-reachable. // -// `_whitebox.ww` (NOT *_test.ww) is the ww convention for an in-package -// white-box test file: the enumerator skips *test.ww, so the non-suffixed -// name is what keeps this a bundled regex module source. It is exercised -// by the sibling external driver lib/regex/wb/regex_test.ww, whose -// different-dir `import regex` dir-resolves and bundles this file (a -// same-dir import file-resolves regex.ww only — probe E; task #33 would -// delete the driver). The public-API @tests stay black-box in -// lib/regex/regex_test.ww (Go's foo / foo_test split; CLAUDE.md rule-9). +// `_whitebox.ww` (NOT *_test.ww) keeps this file in the production +// enumerator's root-package assembly until package-aware testing can classify +// same-package `*_test.ww` files (T1). The T0 gate resolves `regex` as the root +// directory so -T sees these tests directly; importing regex as a dependency +// would strip them. Public-API tests remain a separate `package regex_test` +// binary in lib/regex/regex_test.ww (CLAUDE.md rule 9). package regex; import io; diff --git a/lib/regex/wb/regex_test.ww b/lib/regex/wb/regex_test.ww deleted file mode 100644 index 23c7a482..00000000 --- a/lib/regex/wb/regex_test.ww +++ /dev/null @@ -1,20 +0,0 @@ -// wb/regex_test — DOCUMENTED-TEMPORARY scaffolding (rule-7), NOT part of -// the blessed white-box convention: the permanent shape is the colocated -// lib/regex/regex_whitebox.ww (`package regex`) file alone. This driver -// only exists to RUN it, and task #33 ("resolver: dir-enumerate same-dir -// same-package siblings on import") DELETES this file when it lands — the -// true E-endpoint. -// -// Why it is load-bearing until then, not incidental: a `import regex` from -// a file sitting IN lib/regex/ (e.g. regex_test.ww) FILE-resolves to the -// sibling regex.ww and pulls ONLY that file, so the colocated -// regex_whitebox.ww is never bundled and its @tests are never collected -// (probe E, negative). Placed in a DIFFERENT dir with no regex.ww to -// shadow, this driver's `import regex` instead DIRECTORY-resolves the -// lib/regex package and bundles every non-*test.ww member — regex.ww + -// regex_whitebox.ww — unifying them so -T collects the white-box @tests. -// The dir is `wb/`, NOT `test/`: a lib//test/ subdir would shadow the -// lib/test runner module in the -T auto-bundle search path (task #32). -package regex_test; - -import regex; diff --git a/test/run b/test/run index 67f8af4e..9cfe4e7b 100755 --- a/test/run +++ b/test/run @@ -1,126 +1,331 @@ #!/bin/sh -# test/run — driver for `make test`. Plan 9 rc-flavoured but plain sh. -# -# Walks test/wcc/_.c → out/bin/test_ binary, fans out -# across $JOBS (default $(nproc)) workers via xargs -P. Each worker -# writes its status to a per-job file; the collector enumerates them in -# lexicographic order so output stays deterministic across runs even -# though execution interleaves. Exit non-zero if any worker failed. +# A manifest-backed collector prevents a lost parallel worker from shrinking a +# required corpus into a false green. BIN=${BIN:-out/bin} WW=${WW:-$BIN/ww} +TEST_DIR=${TEST_DIR:-test/wcc} +TEST_EXPECTED_MIN=${TEST_EXPECTED_MIN:-337} +TEST_TIMEOUT=${TEST_TIMEOUT:-900s} +TEST_KILL_AFTER=${TEST_KILL_AFTER:-10s} +TEST_WORKER=${TEST_WORKER:-$0} -# Worker mode: invoked once per test by xargs. Argv: --one . -# Writes .status; touches .fail on a non-zero binary exit. -if [ "$1" = "--one" ]; then +# Required-suite results describe the toolchain under BIN, not ambient driver +# overrides from a developer shell. Individual tests still set overrides +# explicitly when that behavior is what they exercise. +unset WW_W6C WW_W6A WW_W6L WW_LIB WW_SRCLIB WW_PKGCACHE + +atomic_record() { + record_path=$1 + record_text=$2 + record_tmp=$record_path.tmp.$$ + if ! printf '%s\n' "$record_text" > "$record_tmp"; then + rm -f "$record_tmp" + return 1 + fi + if ! mv "$record_tmp" "$record_path"; then + rm -f "$record_tmp" + return 1 + fi + return 0 +} + +# Worker mode owns one result prefix. A result becomes visible only after every +# capture and timing file is complete, so the collector cannot accept a torn +# terminal record as a finished test. +if [ "${1:-}" = "--one" ]; then + [ "$#" -eq 3 ] || exit 64 prefix=$2 t=$3 name=${t##*/} name=${name%.c} short=${name#[0-9][0-9][0-9]_} bin=$BIN/test_$short - if [ ! -x "$bin" ]; then - # A missing binary means an unwired test/wcc file: `make test` - # builds every $(TESTS) target before this runs, so the only way - # to get here is a .c with no Makefile rule. That used to SKIP - # and still count toward "all N tests passed" — 953_arrlit_slice - # sat dark for weeks under a green gate. Fail loud instead. - { - printf 'FAIL %s (no binary %s)\n' "$name" "$bin" - printf 'unwired test: add $(BIN)/test_%s to TESTS + a build rule in Makefile\n' "$short" - } > "$prefix.status" - : > "$prefix.fail" - exit 0 + + if ! atomic_record "$prefix.started" "WWTEST_STARTED 1"; then + exit 70 fi - # Per-test wall-clock (task #10 baseline). Stamp around the binary run - # only; build cost lives in `make all`, not here. Duration lands in - # .dur as "\t" for the collector's timing section — - # kept out of .status so the ok/FAIL log format is untouched. + if ! : > "$prefix.out" || ! : > "$prefix.err"; then + exit 70 + fi + t0=$(date +%s.%N) - if "$bin" > "$prefix.out" 2> "$prefix.err"; then - t1=$(date +%s.%N) - printf 'ok %s\n' "$name" > "$prefix.status" + outcome=pass + rc=0 + if [ ! -x "$bin" ]; then + printf 'unwired test: no executable %s\n' "$bin" > "$prefix.err" + outcome=harness_error + rc=127 else - rc=$? - t1=$(date +%s.%N) - { - printf 'FAIL %s (rc=%d)\n' "$name" "$rc" - echo '--- stdout ---' - cat "$prefix.out" - echo '--- stderr ---' - cat "$prefix.err" - } > "$prefix.status" - : > "$prefix.fail" + # The child records its own status before the timeout wrapper exits. + # That distinguishes a real child exit 124/137 from expiration, and + # child exits 125-127 from timeout's reserved infrastructure codes. + command_status=$prefix.command-status + timeout_diag=$prefix.timeout-diag + if LC_ALL=C timeout --verbose --kill-after="$TEST_KILL_AFTER" \ + "$TEST_TIMEOUT" sh -c ' + "$1" > "$2" 2> "$3" + command_rc=$? + status_tmp=$4.tmp.$$ + printf "%s\n" "$command_rc" > "$status_tmp" || exit 70 + mv "$status_tmp" "$4" || exit 70 + ' sh "$bin" "$prefix.out" "$prefix.err" "$command_status" \ + 2> "$timeout_diag"; then + timeout_rc=0 + else + timeout_rc=$? + fi + deadline_expired=0 + if grep -Eq '^timeout: sending signal (TERM|KILL) to command ' \ + "$timeout_diag"; then + deadline_expired=1 + fi + if [ -s "$timeout_diag" ]; then + cat "$timeout_diag" >> "$prefix.err" + fi + if [ "$deadline_expired" -eq 1 ]; then + outcome=timeout + rc=$timeout_rc + elif [ -f "$command_status" ]; then + rc="" extra="" + IFS=' ' read -r rc extra < "$command_status" + case $rc in + ''|*[!0-9]*) rc_valid=0 ;; + *) rc_valid=1 ;; + esac + if [ "$rc_valid" -ne 1 ] || [ -n "$extra" ] \ + || [ "$timeout_rc" -ne 0 ]; then + printf 'timeout wrapper returned %s with child status %s\n' \ + "$timeout_rc" "${rc:-missing}" >> "$prefix.err" + outcome=harness_error + rc=125 + elif [ "$rc" -eq 0 ]; then + outcome=pass + elif [ "$rc" -eq 77 ]; then + outcome=skip + else + outcome=fail + fi + else + printf 'timeout wrapper produced no child status (rc=%s)\n' \ + "$timeout_rc" >> "$prefix.err" + outcome=harness_error + case $timeout_rc in + 125|126|127) rc=$timeout_rc ;; + *) rc=125 ;; + esac + fi + fi + t1=$(date +%s.%N) + dur_tmp=$prefix.dur.tmp.$$ + if ! awk -v a="$t0" -v b="$t1" -v n="$name" \ + 'BEGIN{printf "%.3f\t%s\n", b-a, n}' > "$dur_tmp"; then + rm -f "$dur_tmp" + exit 70 + fi + if ! mv "$dur_tmp" "$prefix.dur"; then + rm -f "$dur_tmp" + exit 70 + fi + if ! atomic_record "$prefix.result" "WWTEST_RESULT 1 $outcome $rc"; then + exit 70 fi - awk -v a="$t0" -v b="$t1" -v n="$name" 'BEGIN{printf "%.3f\t%s\n", b-a, n}' \ - > "$prefix.dur" exit 0 fi -set -e - -RESULTS=$(mktemp -d "${TMPDIR:-/tmp}/wwtest.XXXXXX") +RESULTS=$(mktemp -d "${TMPDIR:-/tmp}/wwtest.XXXXXX") || exit 2 trap 'rm -rf "$RESULTS"' EXIT +trap 'exit 1' HUP INT TERM -JOBS=${JOBS:-$(nproc 2>/dev/null || echo 4)} +ALL_MANIFEST=$RESULTS/all.manifest +PHASE1_MANIFEST=$RESULTS/phase1.manifest +PHASE2_MANIFEST=$RESULTS/phase2.manifest +RUN1_MANIFEST=$RESULTS/run1.manifest +RUN2_MANIFEST=$RESULTS/run2.manifest +SELECTED_NAMES=$RESULTS/selected.names +CACHED_NAMES=$RESULTS/cached.names +ALL_NAMES=$RESULTS/all.names +for manifest in "$ALL_MANIFEST" "$PHASE1_MANIFEST" "$PHASE2_MANIFEST" \ + "$RUN1_MANIFEST" "$RUN2_MANIFEST" "$SELECTED_NAMES" \ + "$CACHED_NAMES" "$ALL_NAMES"; do + if ! : > "$manifest"; then + printf 'HARNESS suite (cannot initialize manifest %s)\n' "$manifest" + exit 2 + fi +done -# ---- content-keyed gate cache (task #10 T1; .ai/rob-testperf-go.md L1) ------- -# `make test-commit` (COMMIT=1) reports the expensive wwstage byte-id / -# self-compile gates as `cached` when their full input set is byte-identical to -# the last green run. Those gate outcomes are a PURE function of (stage binaries -# + selfhost/lib/cmd/rt sources + test fixtures + this harness + the Makefile); -# if that content hash equals the recorded last-green key the result is provably -# unchanged, so `cached` is sound — not a guess. `make test` NEVER reads the -# cache (it stays the no-skip pre-push bar) but a green run of the gates DOES -# record the key, as the natural primer. A red run never records: the write is -# guarded on fail=0 (and the fail>0 path exits before it). Cache lives under -# out/.testcache (out/ is gitignored); `make clean` wipes out/ and thus the -# cache, which is correct — the key is keyed on the just-built binaries, so a -# from-scratch rebuild must start from a clean cache. -OUT_DIR=$(dirname "$BIN") -CACHE_DIR=$OUT_DIR/.testcache -CACHE_FILE=$CACHE_DIR/last-green - -# The exact cached set, confirmed from the Makefile/test/run: the UNIT-guarded -# phase-2 gates (950 + 990-997) plus the single phase-1 lib byte-id gate -# (989_lib_byteid). The other 989_*/998/999 files are cheap functional _run -# tests, NOT byte-id gates, and stay always-on. Every member is a deterministic -# pure function of the hashed inputs (no /tmp/env/clock/network reads); a future -# gate that reads outside that set must be kept OUT of this list (Go marks such -# tests uncacheable for the same reason). -is_cached_gate() { +is_phase2() { case $1 in - 950_selfcheck.c|989_lib_byteid.c|\ - 990_*|991_*|992_*|993_*|994_*|995_*|996_*|997_*) return 0 ;; + 950_*|990_*|991_*|992_*|993_*|994_*|995_*|996_*|997_*) return 0 ;; esac return 1 } -emit_cached() { - printf 'cached %s\n' "$1" > "$RESULTS/$1.status" - : > "$RESULTS/$1.cached" +is_cached_gate() { + case $1 in + 950_selfcheck.c|989_lib_byteid.c|\ + 990_*|991_*|992_*|993_*|994_*|995_*|997_*) return 0 ;; + esac + return 1 } -# Content key over EVERYTHING that determines the cached gates' outcomes: the -# stage binary md5s first (fixed order), then tracked source/fixture/harness -# content (git ls-files is sorted → deterministic; md5sum hashes working-tree -# bytes, so uncommitted edits flip the key too). Any changed byte flips the key -# → the gates RUN. A missed input would be a false green (rob's veto), so the -# set is a deliberate conservative superset (wwstage binary md5s overlap the -# selfhost/lib source hashes, rt/ is hashed though it only reaches the gates via -# libwwrt); an extra input only ever costs a needless honest re-run. examples/ -# is hashed because 996_dyn_ww links examples/mandelbrot/mandelbrot.{ww,o} — a -# real gate input outside the source tree. +for t in "$TEST_DIR"/*.c; do + [ -f "$t" ] || continue + printf '%s\n' "$t" >> "$ALL_MANIFEST" + name=${t##*/} + name=${name%.c} + printf '%s\n' "$name" >> "$ALL_NAMES" + if is_phase2 "${t##*/}"; then + printf '%s\n' "$t" >> "$PHASE2_MANIFEST" + else + printf '%s\n' "$t" >> "$PHASE1_MANIFEST" + fi +done + +discovered=$(awk 'END{print NR + 0}' "$ALL_MANIFEST") +excluded=0 +preflight_error=0 +harness_error=0 + +case $TEST_EXPECTED_MIN in +''|*[!0-9]*) + printf 'HARNESS suite (invalid TEST_EXPECTED_MIN: %s)\n' "$TEST_EXPECTED_MIN" + preflight_error=1 + harness_error=$((harness_error + 1)) + ;; +*) + if [ "$discovered" -eq 0 ]; then + echo 'HARNESS suite (empty corpus)' + preflight_error=1 + harness_error=$((harness_error + 1)) + elif [ "$discovered" -lt "$TEST_EXPECTED_MIN" ]; then + printf 'HARNESS suite (corpus shrank: %d < %d)\n' \ + "$discovered" "$TEST_EXPECTED_MIN" + preflight_error=1 + harness_error=$((harness_error + 1)) + fi + ;; +esac + +duplicate_names=$(sort "$ALL_NAMES" | uniq -d) +if [ -n "$duplicate_names" ]; then + echo 'HARNESS suite (duplicate test identifiers)' + printf '%s\n' "$duplicate_names" + preflight_error=1 + harness_error=$((harness_error + 1)) +fi + +JOBS=${JOBS:-$(nproc 2>/dev/null || echo 4)} +case $JOBS in +''|*[!0-9]*|0) + printf 'HARNESS suite (invalid JOBS: %s)\n' "$JOBS" + preflight_error=1 + harness_error=$((harness_error + 1)) + ;; +esac +if ! command -v timeout >/dev/null 2>&1; then + echo 'HARNESS suite (GNU timeout is unavailable)' + preflight_error=1 + harness_error=$((harness_error + 1)) +fi +valid_duration() { + awk -v duration="$1" 'BEGIN { + if (duration !~ /^[0-9]+([.][0-9]+)?[smhd]?$/) exit 1 + sub(/[smhd]$/, "", duration) + exit !(duration + 0 > 0) + }' +} +if ! valid_duration "$TEST_TIMEOUT"; then + printf 'HARNESS suite (invalid TEST_TIMEOUT: %s)\n' "$TEST_TIMEOUT" + preflight_error=1 + harness_error=$((harness_error + 1)) +fi +if ! valid_duration "$TEST_KILL_AFTER"; then + printf 'HARNESS suite (invalid TEST_KILL_AFTER: %s)\n' "$TEST_KILL_AFTER" + preflight_error=1 + harness_error=$((harness_error + 1)) +fi + +OUT_DIR=$(dirname "$BIN") +CACHE_DIR=$OUT_DIR/.testcache +CACHE_FILE=$CACHE_DIR/last-green + +# The cache includes nonignored untracked inputs because a local fixture or +# source file can change a gate even before it is added to the index. testcache_key() { + cache_inputs=$RESULTS/cache-inputs + if ! : > "$cache_inputs"; then + return 1 + fi + # Hash the effective built tools, runtime archive, and every executable + # that the last-green policy may replace with a cached result. for b in ww w6c w6a w6l wwdump ww_ww w6c_ww w6a_ww w6l_ww wwdump_ww; do - [ -f "$BIN/$b" ] && md5sum "$BIN/$b" + printf '%s\0' "$BIN/$b" >> "$cache_inputs" || return 1 done - git ls-files selfhost lib cmd rt examples test/wcc test/run Makefile | xargs md5sum + printf '%s\0' "$OUT_DIR/lib/libwwrt.a" >> "$cache_inputs" || return 1 + while IFS= read -r t; do + [ -n "$t" ] || continue + if is_cached_gate "${t##*/}"; then + name=${t##*/} + name=${name%.c} + short=${name#[0-9][0-9][0-9]_} + printf '%s\0' "$BIN/test_$short" >> "$cache_inputs" || return 1 + fi + printf '%s\0' "$t" >> "$cache_inputs" || return 1 + done < "$ALL_MANIFEST" + if ! git ls-files --cached -z -- \ + selfhost lib cmd rt examples test/wcc test/run test/run_test.sh Makefile \ + >> "$cache_inputs"; then + return 1 + fi + # Generated package-cache state is not fixture input; hashing it would make + # a gate mutate its own key. Other untracked files under data/ remain input. + if ! git ls-files --others --exclude-standard -z -- \ + selfhost lib cmd rt examples test/wcc test/run test/run_test.sh Makefile \ + ':(exclude,glob)test/wcc/data/**/.pkgcache/**' >> "$cache_inputs"; then + return 1 + fi + if ! sort -z -u "$cache_inputs" -o "$cache_inputs"; then + return 1 + fi + for setting in \ + "TEST_DIR=$TEST_DIR" \ + "TEST_EXPECTED_MIN=$TEST_EXPECTED_MIN" \ + "TEST_TIMEOUT=$TEST_TIMEOUT" \ + "TEST_KILL_AFTER=$TEST_KILL_AFTER" \ + "TEST_WORKER=$TEST_WORKER" \ + "JOBS=$JOBS"; do + setting_key=$(printf '%s' "$setting" | md5sum) || return 1 + printf 'setting %s\n' "${setting_key%% *}" + done + # The index can name an unstaged deletion. Preserve that state in the + # stream instead of letting md5sum abort and silently disable caching. + xargs -0 -r sh -c ' + for cache_file do + path_key=$(printf "%s" "$cache_file" | md5sum) || exit + path_key=${path_key%% *} + if [ -f "$cache_file" ]; then + content_key=$(md5sum < "$cache_file") || exit + content_key=${content_key%% *} + mode=- + [ -x "$cache_file" ] && mode=x + printf "file %s %s %s\n" \ + "$path_key" "$mode" "$content_key" + elif [ ! -e "$cache_file" ] && [ ! -L "$cache_file" ]; then + printf "missing %s\n" "$path_key" + else + printf "unsupported cache input %s\n" "$path_key" >&2 + exit 1 + fi + done + ' sh < "$cache_inputs" } KEY="" -if [ "${UNIT:-0}" != "1" ]; then - KEY=$(testcache_key 2>/dev/null | md5sum 2>/dev/null | cut -d' ' -f1) || KEY="" +if [ "${UNIT:-0}" != "1" ] && [ "$TEST_WORKER" = "$0" ]; then + if key_stream=$(testcache_key 2>/dev/null); then + KEY=$(printf '%s\n' "$key_stream" | md5sum | cut -d' ' -f1) + fi fi CACHE_HIT=0 if [ "${COMMIT:-0}" = "1" ] && [ -n "$KEY" ] && [ -f "$CACHE_FILE" ] \ @@ -128,122 +333,311 @@ if [ "${COMMIT:-0}" = "1" ] && [ -n "$KEY" ] && [ -f "$CACHE_FILE" ] \ CACHE_HIT=1 fi -# Phase wall-clock stamps (task #10 baseline). Each phase records start/end -# so the collector can report wall shares onto ken's three poles. -run_start=$(date +%s.%N) - -# Phase 2 runs the wwstage byte-id / self-compile gates (990-997) + -# 950_selfcheck. They're kept out of phase 1 only so the heavy 43k -# self-compiles don't contend with the phase-1 fixture swarm; within -# phase 2 they all run in parallel (post-T3 each build's intermediates -# follow its output via `-o`/temp, so no member writes a path another -# reads — see the phase-2 block below). -for t in test/wcc/*.c; do - [ -f "$t" ] || continue - case ${t##*/} in - 950_*|990_*|991_*|992_*|993_*|994_*|995_*|996_*|997_*) continue ;; - esac - name=${t##*/}; name=${name%.c} - # The only cached gate that lives in phase 1 is 989_lib_byteid; on a - # content-key hit report it cached instead of re-running it. - if [ "$CACHE_HIT" = 1 ] && is_cached_gate "${t##*/}"; then - emit_cached "$name" - continue +emit_cached() { + cached_name=$1 + cached_prefix=$RESULTS/$cached_name + if ! atomic_record "$cached_prefix.result" "WWTEST_RESULT 1 cached 0"; then + return 1 fi - printf '%s/%s\0%s\0' "$RESULTS" "$name" "$t" -done | xargs -0 -n2 -P "$JOBS" "$0" --one || true -p1_end=$(date +%s.%N) + printf '%s\n' "$cached_name" >> "$CACHED_NAMES" + return 0 +} -# UNIT=1 (make test-unit) is the inner-loop short path: skip the wwstage -# byte-id gates (990-997) and 950_selfcheck entirely. Pre-push uses -# `make test` for the full suite. -if [ "${UNIT:-0}" != "1" ]; then - if [ "$CACHE_HIT" = 1 ]; then - # Content key matched last-green → the wwstage byte-id / self-compile - # gates are provably unchanged. Report them cached; do NOT re-run. - for t in test/wcc/990_*.c test/wcc/991_*.c test/wcc/992_*.c \ - test/wcc/993_*.c test/wcc/994_*.c test/wcc/995_*.c \ - test/wcc/996_*.c test/wcc/997_*.c test/wcc/950_*.c; do - [ -f "$t" ] || continue - name=${t##*/}; name=${name%.c} - emit_cached "$name" - done +plan_manifest() { + plan_input=$1 + plan_run=$2 + while IFS= read -r t; do + [ -n "$t" ] || continue + name=${t##*/} + name=${name%.c} + printf '%s\n' "$name" >> "$SELECTED_NAMES" + if [ "$CACHE_HIT" = 1 ] && is_cached_gate "${t##*/}"; then + if ! emit_cached "$name"; then + printf 'HARNESS %s (cannot record cached result)\n' "$name" + harness_error=$((harness_error + 1)) + fi + else + printf '%s\n' "$t" >> "$plan_run" + fi + done < "$plan_input" +} + +if [ "$preflight_error" -eq 0 ]; then + plan_manifest "$PHASE1_MANIFEST" "$RUN1_MANIFEST" + if [ "${UNIT:-0}" = "1" ]; then + while IFS= read -r t; do + [ -n "$t" ] || continue + name=${t##*/} + name=${name%.c} + printf 'excluded %s (test-unit phase-2)\n' "$name" + excluded=$((excluded + 1)) + done < "$PHASE2_MANIFEST" else - p2par_start=$(date +%s.%N) - # Phase 2 — the wwstage byte-id / self-compile gates, all parallel. - # Pre-T3 the source-tree writers (993/995/950) ran in a serial tail - # because ww_ww wrote intermediates next to every traversed source, so - # they raced each other AND the 990/991/992/994 readers. T3 made every - # build's .combined.ww/.s/.o follow its OUTPUT (`-o`/per-pid temp): 993 - # → its dc/dw workdirs, 995 → /tmp/wwsr__, 950 → /tmp only. - # No member now writes a stem another reads (the readers see only the - # stable make-all regen), so the whole group fans out via xargs -P. - for t in test/wcc/990_*.c test/wcc/991_*.c test/wcc/992_*.c \ - test/wcc/993_*.c test/wcc/994_*.c test/wcc/995_*.c \ - test/wcc/996_*.c test/wcc/997_*.c test/wcc/950_*.c; do - [ -f "$t" ] || continue - name=${t##*/}; name=${name%.c} + plan_manifest "$PHASE2_MANIFEST" "$RUN2_MANIFEST" + fi +fi + +launch_manifest() { + launch_file=$1 + [ -s "$launch_file" ] || return 0 + while IFS= read -r t; do + name=${t##*/} + name=${name%.c} printf '%s/%s\0%s\0' "$RESULTS" "$name" "$t" - done | xargs -0 -n2 -P "$JOBS" "$0" --one || true - p2par_end=$(date +%s.%N) + done < "$launch_file" | \ + xargs -0 -r -n2 -P "$JOBS" "$TEST_WORKER" --one +} + +run_start=$(date +%s.%N) +launcher_error=0 +p1_end=$run_start +p2par_start="" +p2par_end="" +if [ "$preflight_error" -eq 0 ]; then + if launch_manifest "$RUN1_MANIFEST"; then + : + else + rc=$? + printf 'HARNESS launcher (phase 1 xargs rc=%d)\n' "$rc" + launcher_error=1 + harness_error=$((harness_error + 1)) + fi + p1_end=$(date +%s.%N) + if [ "${UNIT:-0}" != "1" ] && [ -s "$RUN2_MANIFEST" ]; then + p2par_start=$(date +%s.%N) + if launch_manifest "$RUN2_MANIFEST"; then + : + else + rc=$? + printf 'HARNESS launcher (phase 2 xargs rc=%d)\n' "$rc" + launcher_error=1 + harness_error=$((harness_error + 1)) + fi + p2par_end=$(date +%s.%N) fi fi -fail=0 -ran=0 -cached=0 -for s in "$RESULTS"/*.status; do - [ -f "$s" ] || continue - cat "$s" - prefix=${s%.status} - if [ -f "$prefix.cached" ]; then - cached=$((cached + 1)) - else - ran=$((ran + 1)) +emit_output() { + output_prefix=$1 + output_name=$2 + if [ -s "$output_prefix.out" ]; then + printf '%s\n' "--- stdout: $output_name ---" + cat "$output_prefix.out" fi - [ -f "$prefix.fail" ] && fail=$((fail + 1)) -done + if [ -s "$output_prefix.err" ]; then + printf '%s\n' "--- stderr: $output_name ---" + cat "$output_prefix.err" + fi +} -# ---- ww source-level tests ---------------------------------------------- -if [ -d test/lang ]; then - for f in test/lang/*/*.ww; do - [ -f "$f" ] || continue - echo "skip $f (ww run not online yet)" +started=0 +completed=0 +pass=0 +fail=0 +skip=0 +timed_out=0 +cached=0 +worker_harness_error=0 + +if [ "$preflight_error" -eq 0 ]; then + while IFS= read -r name; do + [ -n "$name" ] || continue + prefix=$RESULTS/$name + expect_cached=0 + if grep -Fqx "$name" "$CACHED_NAMES"; then + expect_cached=1 + fi + has_started=0 + if [ -f "$prefix.started" ]; then + if grep -qx 'WWTEST_STARTED 1' "$prefix.started"; then + has_started=1 + started=$((started + 1)) + else + printf 'HARNESS %s (malformed started record)\n' "$name" + harness_error=$((harness_error + 1)) + fi + fi + + if [ ! -f "$prefix.result" ]; then + if [ "$has_started" -eq 1 ]; then + printf 'HARNESS %s (started but no terminal result)\n' "$name" + else + printf 'HARNESS %s (worker never started)\n' "$name" + fi + harness_error=$((harness_error + 1)) + continue + fi + + magic="" version="" outcome="" result_rc="" extra="" + IFS=' ' read -r magic version outcome result_rc extra < "$prefix.result" + if [ "$magic" != WWTEST_RESULT ] || [ "$version" != 1 ] \ + || [ -z "$outcome" ] || [ -z "$result_rc" ] || [ -n "$extra" ]; then + printf 'HARNESS %s (malformed terminal result)\n' "$name" + harness_error=$((harness_error + 1)) + continue + fi + case $result_rc in + ''|*[!0-9]*) + printf 'HARNESS %s (nonnumeric terminal rc)\n' "$name" + harness_error=$((harness_error + 1)) + continue + ;; + esac + + if [ "$outcome" = cached ]; then + if [ "$expect_cached" -ne 1 ] || [ "$has_started" -ne 0 ] \ + || [ "$result_rc" != 0 ]; then + printf 'HARNESS %s (invalid cached result)\n' "$name" + harness_error=$((harness_error + 1)) + continue + fi + printf 'cached %s\n' "$name" + cached=$((cached + 1)) + continue + fi + + capture_error=0 + for suffix in out err; do + if [ ! -f "$prefix.$suffix" ]; then + printf 'HARNESS %s (missing %s capture)\n' "$name" "$suffix" + capture_error=1 + fi + done + if [ ! -s "$prefix.dur" ]; then + printf 'HARNESS %s (missing duration record)\n' "$name" + capture_error=1 + fi + if [ "$capture_error" -ne 0 ]; then + harness_error=$((harness_error + 1)) + continue + fi + + completed=$((completed + 1)) + if [ "$expect_cached" -eq 1 ] || [ "$has_started" -ne 1 ]; then + printf 'HARNESS %s (terminal result without matching start)\n' "$name" + harness_error=$((harness_error + 1)) + fi + case $outcome:$result_rc in + pass:0) + printf 'ok %s\n' "$name" + pass=$((pass + 1)) + ;; + skip:77) + printf 'skip %s (rc=77)\n' "$name" + skip=$((skip + 1)) + if [ ! -s "$prefix.out" ] && [ ! -s "$prefix.err" ]; then + echo 'skip reason: exit 77 (no output)' + fi + ;; + timeout:124|timeout:137) + printf 'TIMEOUT %s\n' "$name" + timed_out=$((timed_out + 1)) + ;; + fail:*) + printf 'FAIL %s (rc=%s)\n' "$name" "$result_rc" + fail=$((fail + 1)) + ;; + harness_error:125|harness_error:126|harness_error:127) + printf 'HARNESS %s (worker rc=%s)\n' "$name" "$result_rc" + worker_harness_error=$((worker_harness_error + 1)) + harness_error=$((harness_error + 1)) + ;; + *) + printf 'HARNESS %s (invalid outcome/rc %s/%s)\n' \ + "$name" "$outcome" "$result_rc" + worker_harness_error=$((worker_harness_error + 1)) + harness_error=$((harness_error + 1)) + ;; + esac + emit_output "$prefix" "$name" + done < "$SELECTED_NAMES" + + for record in "$RESULTS"/*.result "$RESULTS"/*.started; do + [ -f "$record" ] || continue + record_name=${record##*/} + record_name=${record_name%.result} + record_name=${record_name%.started} + if ! grep -Fqx "$record_name" "$SELECTED_NAMES"; then + printf 'HARNESS %s (unexpected worker record)\n' "$record_name" + harness_error=$((harness_error + 1)) + fi done fi -# ---- timing (task #10 baseline) ----------------------------------------- -# Per-test durations + phase walls. Emitted after the per-test ok/FAIL dump -# but before the summary line's exit checks, so timing prints even on the -# failure path; the "all N tests passed" summary stays the last line. No log -# consumer is disturbed: the "---" delimiters and "\t" rows match -# neither the "all N tests passed" nor the "FAIL" grep keys. +selected=$((discovered - excluded)) +planned=$((selected - cached)) +terminal=$((pass + fail + skip + timed_out + worker_harness_error)) +if [ "$preflight_error" -eq 0 ]; then + if [ "$started" -ne "$planned" ]; then + printf 'HARNESS accounting (started %d != planned %d)\n' \ + "$started" "$planned" + harness_error=$((harness_error + 1)) + fi + if [ "$completed" -ne "$planned" ]; then + printf 'HARNESS accounting (completed %d != planned %d)\n' \ + "$completed" "$planned" + harness_error=$((harness_error + 1)) + fi + if [ "$terminal" -ne "$completed" ]; then + printf 'HARNESS accounting (terminal outcomes %d != completed %d)\n' \ + "$terminal" "$completed" + harness_error=$((harness_error + 1)) + fi + if [ $((completed + cached + excluded)) -ne "$discovered" ]; then + printf 'HARNESS accounting (discovered corpus is incomplete)\n' + harness_error=$((harness_error + 1)) + fi +fi + +# A cache decision is valid only if its complete input fingerprint is still +# current after all workers have finished. This applies to hits as well as to +# newly published keys. +if [ "${UNIT:-0}" != "1" ] && [ -n "$KEY" ]; then + end_key="" + if end_stream=$(testcache_key 2>/dev/null); then + end_key=$(printf '%s\n' "$end_stream" | md5sum | cut -d' ' -f1) + fi + if [ -z "$end_key" ] || [ "$end_key" != "$KEY" ]; then + echo 'HARNESS cache (inputs changed during test run)' + harness_error=$((harness_error + 1)) + fi +fi + +not_run=$((discovered - completed - cached)) run_end=$(date +%s.%N) echo '--- timing: per-test (sec, desc) ---' cat "$RESULTS"/*.dur 2>/dev/null | sort -rn echo '--- timing: phase walls (sec) ---' awk -v rs="$run_start" -v p1e="$p1_end" \ -v ppar_s="$p2par_start" -v ppar_e="$p2par_end" \ - -v pser_s="$p2ser_start" -v pser_e="$p2ser_end" \ -v re="$run_end" 'BEGIN{ printf "phase1_parallel\t%.3f\n", p1e - rs if (ppar_e != "") printf "phase2_parallel\t%.3f\n", ppar_e - ppar_s - if (pser_e != "") printf "phase2_serial_tail\t%.3f\n", pser_e - pser_s printf "total\t%.3f\n", re - rs }' -if [ $ran -eq 0 ] && [ $cached -eq 0 ]; then - echo "no tests were run" - exit 1 +printf 'summary: discovered=%d started=%d completed=%d pass=%d fail=%d skip=%d timeout=%d harness_error=%d cached=%d not_run=%d\n' \ + "$discovered" "$started" "$completed" "$pass" "$fail" "$skip" \ + "$timed_out" "$harness_error" "$cached" "$not_run" + +green=1 +if [ "$preflight_error" -ne 0 ] || [ "$launcher_error" -ne 0 ] \ + || [ "$fail" -ne 0 ] || [ "$skip" -ne 0 ] || [ "$timed_out" -ne 0 ] \ + || [ "$harness_error" -ne 0 ]; then + green=0 fi -if [ $fail -gt 0 ]; then - echo "$fail test(s) failed" - exit 1 +if [ "$preflight_error" -eq 0 ] && [ "$not_run" -ne "$excluded" ]; then + green=0 fi -# Green and the gates actually RAN (not a cache hit, not test-unit): record the -# content key as the last-green primer. A red run exits above, never here. -if [ "${UNIT:-0}" != "1" ] && [ "$CACHE_HIT" != 1 ] && [ -n "$KEY" ]; then - mkdir -p "$CACHE_DIR" && printf '%s\n' "$KEY" > "$CACHE_FILE" + +if [ "$green" -eq 1 ] && [ "${UNIT:-0}" != "1" ] \ + && [ "$CACHE_HIT" != 1 ] && [ -n "$KEY" ]; then + if ! mkdir -p "$CACHE_DIR" \ + || ! atomic_record "$CACHE_FILE" "$KEY"; then + echo 'HARNESS cache (cannot publish last-green key)' + exit 1 + fi fi -[ $cached -gt 0 ] && echo "$cached gate(s) cached (content key matched last green)" -echo "all $ran tests passed" + +[ "$green" -eq 1 ] || exit 1 +exit 0 diff --git a/test/run_test.sh b/test/run_test.sh new file mode 100644 index 00000000..ee30de1b --- /dev/null +++ b/test/run_test.sh @@ -0,0 +1,316 @@ +#!/bin/sh +# Synthetic corpora keep harness failure-path coverage independent of compiler builds. + +set -eu + +ROOT=$(cd "$(dirname "$0")/.." && pwd) +RUN=$ROOT/test/run +WORK=$(mktemp -d "${TMPDIR:-/tmp}/ww-run-test.XXXXXX") +trap 'rm -rf "$WORK"' EXIT +trap 'exit 1' HUP INT TERM + +fail() +{ + printf 'run_test: FAIL: %s\n' "$*" >&2 + exit 1 +} + +new_case() +{ + CASE=$WORK/$1 + mkdir -p "$CASE/corpus" "$CASE/bin" +} + +add_source() +{ + : > "$CASE/corpus/$1.c" +} + +add_binary() +{ + name=$1 + body=$2 + { + printf '%s\n' '#!/bin/sh' + printf '%s\n' "$body" + } > "$CASE/bin/test_$name" + chmod +x "$CASE/bin/test_$name" +} + +run_case() +{ + want=$1 + shift + set +e + env BIN="$CASE/bin" JOBS=2 "$@" sh "$RUN" > "$CASE/output" 2>&1 + GOT=$? + set -e + [ "$GOT" -eq "$want" ] || { + cat "$CASE/output" >&2 + fail "$(basename "$CASE"): exit $GOT, want $want" + } +} + +run_case_from() +{ + want=$1 + run_dir=$2 + shift 2 + set +e + ( + cd "$run_dir" + env BIN="$CASE/bin" JOBS=2 "$@" sh "$RUN" + ) > "$CASE/output" 2>&1 + GOT=$? + set -e + [ "$GOT" -eq "$want" ] || { + cat "$CASE/output" >&2 + fail "$(basename "$CASE"): exit $GOT, want $want" + } +} + +assert_line() +{ + grep -Fqx "$1" "$CASE/output" || { + cat "$CASE/output" >&2 + fail "$(basename "$CASE"): missing line: $1" + } +} + +assert_text() +{ + grep -Fq "$1" "$CASE/output" || { + cat "$CASE/output" >&2 + fail "$(basename "$CASE"): missing text: $1" + } +} + +assert_summary_field() +{ + key=$1 + want=$2 + summary=$(grep '^summary:' "$CASE/output" || true) + [ "$(printf '%s\n' "$summary" | grep -c '^summary:')" -eq 1 ] || { + cat "$CASE/output" >&2 + fail "$(basename "$CASE"): expected one summary line" + } + case " $summary " in + *" $key=$want "*) ;; + *) + cat "$CASE/output" >&2 + fail "$(basename "$CASE"): summary $key is not $want" + ;; + esac +} + +assert_summary_nonzero() +{ + key=$1 + summary=$(grep '^summary:' "$CASE/output" || true) + [ "$(printf '%s\n' "$summary" | grep -c '^summary:')" -eq 1 ] || { + cat "$CASE/output" >&2 + fail "$(basename "$CASE"): expected one summary line" + } + case " $summary " in + *" $key=0 "*) + cat "$CASE/output" >&2 + fail "$(basename "$CASE"): summary $key is zero" + ;; + *" $key="*) ;; + *) + cat "$CASE/output" >&2 + fail "$(basename "$CASE"): summary has no $key field" + ;; + esac +} + +new_case all_pass +add_source 001_alpha +add_source 002_beta +add_binary alpha 'exit 0' +add_binary beta 'exit 0' +run_case 0 TEST_DIR="$CASE/corpus" TEST_EXPECTED_MIN=2 UNIT=1 +assert_line 'ok 001_alpha' +assert_line 'ok 002_beta' +assert_line 'summary: discovered=2 started=2 completed=2 pass=2 fail=0 skip=0 timeout=0 harness_error=0 cached=0 not_run=0' + +new_case cache_publish +add_source 990_cache +add_source 996_dynamic +add_binary cache '[ -z "${WW_W6C+x}${WW_W6A+x}${WW_W6L+x}${WW_LIB+x}${WW_SRCLIB+x}${WW_PKGCACHE+x}" ] || exit 9; exit 0' +add_binary dynamic 'exit 0' +run_case 0 TEST_DIR="$CASE/corpus" TEST_EXPECTED_MIN=2 COMMIT=1 \ + WW_W6C=/first/w6c WW_W6A=/first/w6a WW_W6L=/first/w6l \ + WW_LIB=/first/lib WW_SRCLIB=/first/src WW_PKGCACHE=/first/cache +assert_summary_field pass 2 +assert_summary_field harness_error 0 +assert_summary_field cached 0 +[ -s "$CASE/.testcache/last-green" ] || fail "cache_publish: no cache key" +run_case 0 TEST_DIR="$CASE/corpus" TEST_EXPECTED_MIN=2 COMMIT=1 \ + WW_W6C=/second/w6c WW_W6A=/second/w6a WW_W6L=/second/w6l \ + WW_LIB=/second/lib WW_SRCLIB=/second/src WW_PKGCACHE=/second/cache +assert_line 'cached 990_cache' +assert_line 'ok 996_dynamic' +assert_line 'summary: discovered=2 started=1 completed=1 pass=1 fail=0 skip=0 timeout=0 harness_error=0 cached=1 not_run=0' + +new_case cache_deleted_tracked +add_source 990_cache +add_binary cache 'exit 0' +mkdir -p "$CASE/repo" +: > "$CASE/repo/Makefile" +git -C "$CASE/repo" init -q +git -C "$CASE/repo" add Makefile +rm -f "$CASE/repo/Makefile" +run_case_from 0 "$CASE/repo" TEST_DIR="$CASE/corpus" \ + TEST_EXPECTED_MIN=1 COMMIT=1 +assert_summary_field pass 1 +[ -s "$CASE/.testcache/last-green" ] || \ + fail "cache_deleted_tracked: no cache key" +run_case_from 0 "$CASE/repo" TEST_DIR="$CASE/corpus" \ + TEST_EXPECTED_MIN=1 COMMIT=1 +assert_summary_field cached 1 +assert_summary_field started 0 + +new_case fail +add_source 001_fail +add_binary fail 'printf "%s\n" "synthetic failure" >&2; exit 3' +run_case 1 TEST_DIR="$CASE/corpus" TEST_EXPECTED_MIN=1 UNIT=1 +assert_line 'FAIL 001_fail (rc=3)' +assert_text 'synthetic failure' +assert_summary_field fail 1 +assert_summary_field harness_error 0 + +new_case skip +add_source 001_skip +add_binary skip 'printf "%s\n" "synthetic prerequisite unavailable"; exit 77' +run_case 1 TEST_DIR="$CASE/corpus" TEST_EXPECTED_MIN=1 UNIT=1 +assert_line 'skip 001_skip (rc=77)' +assert_text 'synthetic prerequisite unavailable' +assert_summary_field skip 1 +assert_summary_field pass 0 + +new_case timeout +add_source 001_slow +add_binary slow 'trap "" TERM; sleep 5; exit 0' +run_case 1 TEST_DIR="$CASE/corpus" TEST_EXPECTED_MIN=1 \ + TEST_TIMEOUT=1 TEST_KILL_AFTER=1 UNIT=1 +assert_line 'TIMEOUT 001_slow' +assert_summary_field timeout 1 +assert_summary_field pass 0 + +new_case reserved_exit_codes +for item in 001:124 002:125 003:126 004:127 005:137; do + prefix=${item%%:*} + rc=${item#*:} + add_source "${prefix}_exit${rc}" + add_binary "exit${rc}" "exit $rc" +done +run_case 1 TEST_DIR="$CASE/corpus" TEST_EXPECTED_MIN=5 UNIT=1 +assert_summary_field fail 5 +assert_summary_field timeout 0 +assert_summary_field harness_error 0 + +new_case timeout_wrapper_failure +add_source 001_probe +add_binary probe 'exit 0' +mkdir -p "$CASE/path" +{ + printf '%s\n' '#!/bin/sh' + printf '%s\n' 'exit 124' +} > "$CASE/path/timeout" +chmod +x "$CASE/path/timeout" +run_case 1 PATH="$CASE/path:$PATH" TEST_DIR="$CASE/corpus" \ + TEST_EXPECTED_MIN=1 UNIT=1 +assert_summary_field timeout 0 +assert_summary_nonzero harness_error + +new_case missing_binary +add_source 001_missing +run_case 1 TEST_DIR="$CASE/corpus" TEST_EXPECTED_MIN=1 UNIT=1 +assert_text 'HARNESS 001_missing' +assert_summary_field harness_error 1 +assert_summary_field completed 1 + +new_case empty +run_case 1 TEST_DIR="$CASE/corpus" TEST_EXPECTED_MIN=0 UNIT=1 +assert_summary_field discovered 0 +assert_summary_field started 0 +assert_summary_field completed 0 +assert_summary_field harness_error 1 + +new_case below_minimum +add_source 001_only +add_binary only 'exit 0' +run_case 1 TEST_DIR="$CASE/corpus" TEST_EXPECTED_MIN=2 UNIT=1 +assert_summary_field discovered 1 +assert_summary_field harness_error 1 + +new_case zero_timeout +add_source 001_bounded +add_binary bounded 'exit 0' +run_case 1 TEST_DIR="$CASE/corpus" TEST_EXPECTED_MIN=1 \ + TEST_TIMEOUT=0 UNIT=1 +assert_summary_field discovered 1 +assert_summary_field started 0 +assert_summary_field harness_error 1 + +new_case launcher_failure +add_source 001_lost +add_binary lost 'exit 0' +mkdir -p "$CASE/path" +{ + printf '%s\n' '#!/bin/sh' + printf '%s\n' 'exit 23' +} > "$CASE/path/xargs" +chmod +x "$CASE/path/xargs" +run_case 1 PATH="$CASE/path:$PATH" TEST_DIR="$CASE/corpus" \ + TEST_EXPECTED_MIN=1 UNIT=1 +assert_summary_field discovered 1 +assert_summary_field started 0 +assert_summary_field completed 0 +assert_summary_nonzero harness_error + +new_case lost_result +add_source 001_lost +add_binary lost 'exit 0' +{ + printf '%s\n' '#!/bin/sh' + printf '%s\n' 'prefix=$2' + printf '%s\n' "printf '%s\\n' 'WWTEST_STARTED 1' > \"\$prefix.started\"" + printf '%s\n' 'exit 0' +} > "$CASE/worker" +chmod +x "$CASE/worker" +run_case 1 TEST_DIR="$CASE/corpus" TEST_EXPECTED_MIN=1 \ + TEST_WORKER="$CASE/worker" UNIT=1 +assert_summary_field discovered 1 +assert_summary_field started 1 +assert_summary_field completed 0 +assert_summary_nonzero harness_error + +new_case incomplete_result +add_source 001_incomplete +add_binary incomplete 'exit 0' +{ + printf '%s\n' '#!/bin/sh' + printf '%s\n' 'prefix=$2' + printf '%s\n' "printf '%s\\n' 'WWTEST_STARTED 1' > \"\$prefix.started\"" + printf '%s\n' "printf '%s\\n' 'WWTEST_RESULT 1 pass 0' > \"\$prefix.result\"" + printf '%s\n' 'exit 0' +} > "$CASE/worker" +chmod +x "$CASE/worker" +run_case 1 TEST_DIR="$CASE/corpus" TEST_EXPECTED_MIN=1 \ + TEST_WORKER="$CASE/worker" UNIT=1 +assert_summary_field discovered 1 +assert_summary_field completed 0 +assert_summary_field pass 0 +assert_summary_nonzero harness_error + +new_case unit_exclusion +add_source 001_fast +add_source 990_phase +add_binary fast 'exit 0' +run_case 0 TEST_DIR="$CASE/corpus" TEST_EXPECTED_MIN=2 UNIT=1 +assert_line 'excluded 990_phase (test-unit phase-2)' +assert_line 'summary: discovered=2 started=1 completed=1 pass=1 fail=0 skip=0 timeout=0 harness_error=0 cached=0 not_run=1' + +printf 'run_test: all harness regressions passed\n' diff --git a/test/wcc/530_w6a_parsenum.c b/test/wcc/530_w6a_parsenum.c index 5f402b04..76c1bc75 100644 --- a/test/wcc/530_w6a_parsenum.c +++ b/test/wcc/530_w6a_parsenum.c @@ -106,7 +106,7 @@ main(void) if (access(wtool, X_OK) != 0) { fprintf(stderr, "w6a_parsenum: skip (no %s)\n", wtool); printf("w6a_parsenum: skipped\n"); - return 0; + return 77; } int nid = (int)(sizeof idrows / sizeof idrows[0]); diff --git a/test/wcc/810_dyn.c b/test/wcc/810_dyn.c index 0e752d8f..904cd653 100644 --- a/test/wcc/810_dyn.c +++ b/test/wcc/810_dyn.c @@ -13,7 +13,7 @@ * the loader requires the right .gnu.version_r entry to bind * the default vDSO-aware impl. * - * Skipped (passing trivially) on systems without /usr/lib/libc.so.6. + * Reports the harness skip status on systems without a supported libc.so.6. */ #include #include @@ -108,7 +108,7 @@ main(void) && access("/lib/x86_64-linux-gnu/libc.so.6", 0) != 0 && access("/lib64/libc.so.6", 0) != 0) { puts("dyn: no libc.so.6 on this system — skipping"); - return 0; + return 77; } const char *bin = getenv("BIN"); diff --git a/test/wcc/989_dynentry_run.c b/test/wcc/989_dynentry_run.c index d71a538d..9068c6b1 100644 --- a/test/wcc/989_dynentry_run.c +++ b/test/wcc/989_dynentry_run.c @@ -23,8 +23,9 @@ * small pins the headers-fit-page path stays correct and byte-identical. * * Generates the dyn-sym set from `nm -D ` (default-versioned text - * exports) so it adapts to whatever glibc is present; skips (passing) when - * libc / nm are unavailable or too few symbols to overflow the first page. + * exports) so it adapts to whatever glibc is present; reports the harness + * skip status when libc / nm are unavailable or too few symbols are available + * to overflow the first page. */ #include #include @@ -87,14 +88,14 @@ main(void) if (access("/usr/lib/libc.so.6", 0) == 0) { libc = "/usr/lib/libc.so.6"; libdir = "/usr/lib"; } else if (access("/lib/x86_64-linux-gnu/libc.so.6", 0) == 0) { libc = "/lib/x86_64-linux-gnu/libc.so.6"; libdir = "/lib/x86_64-linux-gnu"; } else if (access("/lib64/libc.so.6", 0) == 0) { libc = "/lib64/libc.so.6"; libdir = "/lib64"; } - if (!libc) { puts("dynentry: no libc.so.6 — skipping"); return 0; } + if (!libc) { puts("dynentry: no libc.so.6 — skipping"); return 77; } static char syms[200][32]; int ns = harvest_syms(libc, syms, 200); if (ns < 130) { printf("dynentry: only %d libc syms harvested (<130, can't overflow " "first page) — skipping\n", ns); - return 0; + return 77; } const char *bin = getenv("BIN"); diff --git a/test/wcc/989_lib_byteid.c b/test/wcc/989_lib_byteid.c index c4b2fb9b..5533dccc 100644 --- a/test/wcc/989_lib_byteid.c +++ b/test/wcc/989_lib_byteid.c @@ -53,7 +53,7 @@ #include #include -enum { M_ID, M_DIVERGE, M_WWREJECT }; +enum { M_ID, M_DIVERGE, M_WWREJECT, NENT_EXPECT = 42 }; struct ent { const char *fixture; /* repo-relative .ww; NULL → probe entry */ @@ -95,10 +95,6 @@ static const struct ent ents[] = { { .fixture = "lib/memio/memiotest.ww", .mode = M_ID }, { .fixture = "lib/os/ostest.ww", .mode = M_ID }, { .fixture = "lib/regex/regex_test.ww", .mode = M_ID }, - /* in-package white-box half (#9): the wb/ driver dir-resolves - * `import regex` and bundles lib/regex/regex_whitebox.ww (package - * regex), so this fixture byte-ids the moved engine probes. */ - { .fixture = "lib/regex/wb/regex_test.ww", .mode = M_ID }, { .fixture = "lib/strconv/test/ftostest.ww", .mode = M_ID }, /* graduated from #59.10 DIVERGE by the #62 float-literal fold fix * (wwstage lexer now folds through strconv.stof64, matching @@ -457,6 +453,12 @@ main(void) if (!bin) return 1; char cwd[1024]; if (getcwd(cwd, sizeof cwd) == NULL) return 1; + int nent = (int)(sizeof ents / sizeof ents[0]) - 1; + if (nent != NENT_EXPECT) { + fprintf(stderr, "lib_byteid FAIL: corpus has %d entries, want %d\n", + nent, NENT_EXPECT); + return 1; + } int fail = 0, n = 0, nid = 0, ndiv = 0, nrej = 0; if (corpus_complete(cwd) != 0) fail++; diff --git a/test/wcc/989_regex_run.c b/test/wcc/989_regex_run.c index 420cc51a..eff875bf 100644 --- a/test/wcc/989_regex_run.c +++ b/test/wcc/989_regex_run.c @@ -1,67 +1,201 @@ /* - * 989_regex_run — execute BOTH lib/regex @test halves under the C-side - * `ww test` driver and assert exit 0: - * 1. lib/regex/regex_test.ww — BLACK-BOX, package regex_test. - * 2. lib/regex/wb/regex_test.ww — the WHITE-BOX driver (#9): its - * `import regex` dir-resolves and bundles lib/regex/regex_whitebox.ww - * (package regex), so -T collects the in-package engine probes. - * Both are main-less @test files; -T synthesizes the entry, runs the - * fork-per-test record-and-continue harness, and exits nonzero iff any - * @test fails. - * - * Sibling to 984_base64_run / 989_sha256_run (9xx is full so this - * shares the 989 prefix — the `short` name keys the binary, cf the - * 949_* / 989_sha256 precedent). + * The white-box half must be the -T root: dependency packages lose @test + * declarations before code generation. Keep it separate from the external + * regex_test binary because each -T unit owns main and __wwtests. */ #include #include +#include #include #include +static const char *whitebox[] = { + "thread_shape", + "newmatch_discriminates", + "is_consuming_kinds", + "delete_thread_middle", + "add_thread_dedup_inherit", + "add_thread_dup_independence", + "run_thread_literal_program", + "run_thread_anchored_route", + "search_matches", + "search_early_exit", + "search_no_match", + "find_last_groupstart_cases", + "shift_direct", + "run_thread_group_arms", + "parse_repetition_cases", +}; + static int runwait(const char *cmd) { - int rc = system(cmd); - if (rc == -1) return -1; - if (WIFEXITED(rc)) return WEXITSTATUS(rc); + int rc; + + rc = system(cmd); + if (rc == -1) + return -1; + if (WIFEXITED(rc)) + return WEXITSTATUS(rc); return 1; } +static int +runcapture(const char *cmd, char *out, size_t outsz) +{ + FILE *p; + size_t n; + int ch, overflow, status; + + p = popen(cmd, "r"); + if (p == NULL) + return -1; + n = 0; + overflow = 0; + while ((ch = fgetc(p)) != EOF) { + if (n + 1 < outsz) + out[n++] = ch; + else + overflow = 1; + } + out[n] = '\0'; + status = pclose(p); + if (overflow || status == -1) + return -1; + if (WIFEXITED(status)) + return WEXITSTATUS(status); + return 1; +} + +static int +linecount(const char *out, const char *want) +{ + const char *p, *end; + size_t n; + int count; + + n = strlen(want); + count = 0; + for (p = out; *p != '\0'; p = *end == '\0' ? end : end + 1) { + end = strchr(p, '\n'); + if (end == NULL) + end = p + strlen(p); + if ((size_t)(end - p) == n && memcmp(p, want, n) == 0) + count++; + } + return count; +} + +static int +okcount(const char *out) +{ + const char *p, *end; + static const char suffix[] = " ... ok"; + size_t n, slen; + int count; + + slen = sizeof suffix - 1; + count = 0; + for (p = out; *p != '\0'; p = *end == '\0' ? end : end + 1) { + end = strchr(p, '\n'); + if (end == NULL) + end = p + strlen(p); + n = (size_t)(end - p); + if (n >= slen && memcmp(end - slen, suffix, slen) == 0) + count++; + } + return count; +} + +static int +checkresult(const char *label, const char *out, int expected, + const char *const *names, size_t nnames) +{ + char summary[64], row[256]; + int bad; + + bad = 0; + snprintf(summary, sizeof summary, "%d passed, 0 failed", expected); + if (linecount(out, summary) != 1 || okcount(out) != expected + || strstr(out, "FAIL") != NULL || strstr(out, "No tests run") != NULL) { + fprintf(stderr, "regex_run FAIL: %s result was incomplete\n%s", + label, out); + bad = 1; + } + for (size_t i = 0; i < nnames; i++) { + snprintf(row, sizeof row, "%s ... ok", names[i]); + if (linecount(out, row) != 1) { + fprintf(stderr, "regex_run FAIL: %s did not execute exactly once\n", + names[i]); + bad = 1; + } + } + return bad ? -1 : 0; +} + int main(void) { - const char *bin = getenv("BIN"); - if (!bin) bin = "out/bin"; - char absbin[1024]; + const char *bin; + char absbin[1024], cwd[1024], tmp[] = "/tmp/wwregex.XXXXXX"; + char cmd[4096], output[32768], cleanup[1200]; + int rc, failed; + + bin = getenv("BIN"); + if (bin == NULL) + bin = "out/bin"; if (bin[0] != '/') { - char cwd[1024]; - if (getcwd(cwd, sizeof cwd) == NULL) return 1; - snprintf(absbin, sizeof absbin, "%s/%s", cwd, bin); + if (getcwd(cwd, sizeof cwd) == NULL) + return 1; + if (snprintf(absbin, sizeof absbin, "%s/%s", cwd, bin) + >= (int)sizeof absbin) + return 1; bin = absbin; } - char cwd[1024]; - if (getcwd(cwd, sizeof cwd) == NULL) return 1; + if (getcwd(cwd, sizeof cwd) == NULL) + return 1; + if (mkdtemp(tmp) == NULL) + return 1; - const char *srcs[] = { - "lib/regex/regex_test.ww", - "lib/regex/wb/regex_test.ww", - }; - for (size_t i = 0; i < sizeof srcs / sizeof srcs[0]; i++) { - const char *src = srcs[i]; - char path[1024], cmd[2048]; - snprintf(path, sizeof path, "%s/%s", cwd, src); - /* timeout 180 per repo convention (732/775; test/run does not - * bound runtime): the fold-2c zero-length findall rows turn a - * regression of the ha:946-952 rune-advancement guard into an - * infinite loop (frees are no-ops, so OOM is the only other - * exit) — timeout converts the hang into a loud 124. */ - snprintf(cmd, sizeof cmd, "timeout 180 %s/ww test %s", bin, path); - int rc = runwait(cmd); - if (rc != 0) { - fprintf(stderr, "regex_run FAIL: %s exited %d\n", src, rc); - return 1; + failed = 0; + /* A unique cwd prevents an unrelated local `regex` path from winning + * the module lookup; the private cache keeps the source tree untouched. */ + if (snprintf(cmd, sizeof cmd, + "cd '%s' && WW_PKGCACHE='%s/cache-black' timeout 180 '%s/ww' " + "test -I '%s/lib' '%s/lib/regex/regex_test.ww'", + tmp, tmp, bin, cwd, cwd) >= (int)sizeof cmd) { + failed = 1; + } else { + rc = runcapture(cmd, output, sizeof output); + if (rc != 0 || checkresult("black-box", output, 33, NULL, 0) != 0) { + fprintf(stderr, "regex_run FAIL: black-box exited %d\n", rc); + failed = 1; + } else { + printf("regex_run: black-box 33 discovered, 33 passed, 0 failed\n"); } - printf("regex_run: %s ok\n", src); } - return 0; + + /* Resolving the package directory as the root keeps -T off imported + * dependencies while making every in-package probe visible to the synth. */ + if (snprintf(cmd, sizeof cmd, + "cd '%s' && WW_PKGCACHE='%s/cache-white' timeout 180 '%s/ww' " + "test -I '%s/lib' regex", tmp, tmp, bin, cwd) >= (int)sizeof cmd) { + failed = 1; + } else { + rc = runcapture(cmd, output, sizeof output); + if (rc != 0 || checkresult("white-box", output, 15, whitebox, + sizeof whitebox / sizeof whitebox[0]) != 0) { + fprintf(stderr, "regex_run FAIL: white-box exited %d\n", rc); + failed = 1; + } else { + printf("regex_run: white-box 15 discovered, 15 passed, 0 failed\n"); + } + } + + snprintf(cleanup, sizeof cleanup, "rm -rf -- '%s'", tmp); + if (runwait(cleanup) != 0) { + fprintf(stderr, "regex_run FAIL: cannot clean temporary directory\n"); + failed = 1; + } + return failed ? 1 : 0; } diff --git a/test/wcc/989_structlocal_frame.c b/test/wcc/989_structlocal_frame.c index 5f46db69..1d7a9c8a 100644 --- a/test/wcc/989_structlocal_frame.c +++ b/test/wcc/989_structlocal_frame.c @@ -169,7 +169,7 @@ main(void) if (access(wdrv, X_OK) != 0) { fprintf(stderr, "structlocal_frame: skip (no %s)\n", wdrv); printf("structlocal_frame: skipped (no wwstage)\n"); - return 0; + return 77; } int n = (int)(sizeof rows / sizeof rows[0]); diff --git a/test/wcc/989_test_filter.c b/test/wcc/989_test_filter.c index 76b9908f..1253a5a0 100644 --- a/test/wcc/989_test_filter.c +++ b/test/wcc/989_test_filter.c @@ -27,11 +27,13 @@ #include #include +enum { Nexpect = 4 }; + struct row { const char *pat; /* pattern arg, or NULL for none */ int rc; /* expected exit code */ - const char *must[4]; /* substrings required in stdout */ - const char *mustnot[4]; /* substrings forbidden in stdout */ + const char *must[Nexpect]; /* substrings required in stdout */ + const char *mustnot[Nexpect]; /* substrings forbidden in stdout */ }; static const struct row rows[] = { @@ -46,7 +48,7 @@ static const struct row rows[] = { { "alpha", "beta", NULL } }, /* no match → "No tests run", success */ { "zzz", 0, { "No tests run", NULL }, - { "alpha", "beta", "gamma", "passed", NULL } }, + { "alpha", "beta", "gamma", "passed" } }, /* star matches everything */ { "*", 0, { "alpha ... ok", "beta ... ok", "gamma ... ok", "3 passed, 0 failed" }, { NULL } }, @@ -93,7 +95,12 @@ main(void) if (bin[0] != '/') { char cwd[1024]; if (getcwd(cwd, sizeof cwd) == NULL) return 1; - snprintf(absbin, sizeof absbin, "%s/%s", cwd, bin); + size_t ncwd = strlen(cwd); + size_t nbin = strlen(bin); + if (nbin >= sizeof absbin - ncwd - 1) return 1; + memcpy(absbin, cwd, ncwd); + absbin[ncwd] = '/'; + memcpy(absbin + ncwd + 1, bin, nbin + 1); bin = absbin; } @@ -110,13 +117,13 @@ main(void) pat, crc, rows[i].rc); fail = 1; } - for (int k = 0; k < 4 && rows[i].must[k]; k++) + for (int k = 0; k < Nexpect && rows[i].must[k]; k++) if (!strstr(cout, rows[i].must[k])) { fprintf(stderr, "989 FAIL: ww pat=%s stdout missing " "'%s' (got '%s')\n", pat, rows[i].must[k], cout); fail = 1; } - for (int k = 0; k < 4 && rows[i].mustnot[k]; k++) + for (int k = 0; k < Nexpect && rows[i].mustnot[k]; k++) if (strstr(cout, rows[i].mustnot[k])) { fprintf(stderr, "989 FAIL: ww pat=%s stdout has " "forbidden '%s' (got '%s')\n",