Hojun-Cho/ww
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

test: make T0 harness results complete

Browse Source
This commit is contained in:
Hojun-Cho
2026-08-05 18:19:40 +09:00
parent 3d79735964
commit 0bb1f86c14
14 changed files with 1944 additions and 281 deletions
Download Patch File Download Diff File Expand all files Collapse all files

756
test/run
View File

@@ -1,126 +1,331 @@
#!/bin/sh
# test/run — driver for `make test`. Plan 9 rc-flavoured but plain sh.
#
# Walks test/wcc/<NNN>_<name>.c → out/bin/test_<name> binary, fans out
# across $JOBS (default $(nproc)) workers via xargs -P. Each worker
# writes its status to a per-job file; the collector enumerates them in
# lexicographic order so output stays deterministic across runs even
# though execution interleaves. Exit non-zero if any worker failed.
# A manifest-backed collector prevents a lost parallel worker from shrinking a
# required corpus into a false green.
BIN=${BIN:-out/bin}
WW=${WW:-$BIN/ww}
TEST_DIR=${TEST_DIR:-test/wcc}
TEST_EXPECTED_MIN=${TEST_EXPECTED_MIN:-337}
TEST_TIMEOUT=${TEST_TIMEOUT:-900s}
TEST_KILL_AFTER=${TEST_KILL_AFTER:-10s}
TEST_WORKER=${TEST_WORKER:-$0}
# Worker mode: invoked once per test by xargs. Argv: --one <prefix> <test.c>.
# Writes <prefix>.status; touches <prefix>.fail on a non-zero binary exit.
if [ "$1" = "--one" ]; then
# Required-suite results describe the toolchain under BIN, not ambient driver
# overrides from a developer shell. Individual tests still set overrides
# explicitly when that behavior is what they exercise.
unset WW_W6C WW_W6A WW_W6L WW_LIB WW_SRCLIB WW_PKGCACHE
atomic_record() {
record_path=$1
record_text=$2
record_tmp=$record_path.tmp.$$
if ! printf '%s\n' "$record_text" > "$record_tmp"; then
rm -f "$record_tmp"
return 1
fi
if ! mv "$record_tmp" "$record_path"; then
rm -f "$record_tmp"
return 1
fi
return 0
}
# Worker mode owns one result prefix. A result becomes visible only after every
# capture and timing file is complete, so the collector cannot accept a torn
# terminal record as a finished test.
if [ "${1:-}" = "--one" ]; then
[ "$#" -eq 3 ] || exit 64
prefix=$2
t=$3
name=${t##*/}
name=${name%.c}
short=${name#[0-9][0-9][0-9]_}
bin=$BIN/test_$short
if [ ! -x "$bin" ]; then
# A missing binary means an unwired test/wcc file: `make test`
# builds every $(TESTS) target before this runs, so the only way
# to get here is a .c with no Makefile rule. That used to SKIP
# and still count toward "all N tests passed" — 953_arrlit_slice
# sat dark for weeks under a green gate. Fail loud instead.
{
printf 'FAIL %s (no binary %s)\n' "$name" "$bin"
printf 'unwired test: add $(BIN)/test_%s to TESTS + a build rule in Makefile\n' "$short"
} > "$prefix.status"
: > "$prefix.fail"
exit 0
if ! atomic_record "$prefix.started" "WWTEST_STARTED 1"; then
exit 70
fi
# Per-test wall-clock (task #10 baseline). Stamp around the binary run
# only; build cost lives in `make all`, not here. Duration lands in
# <prefix>.dur as "<sec>\t<name>" for the collector's timing section —
# kept out of <prefix>.status so the ok/FAIL log format is untouched.
if ! : > "$prefix.out" || ! : > "$prefix.err"; then
exit 70
fi
t0=$(date +%s.%N)
if "$bin" > "$prefix.out" 2> "$prefix.err"; then
t1=$(date +%s.%N)
printf 'ok %s\n' "$name" > "$prefix.status"
outcome=pass
rc=0
if [ ! -x "$bin" ]; then
printf 'unwired test: no executable %s\n' "$bin" > "$prefix.err"
outcome=harness_error
rc=127
else
rc=$?
t1=$(date +%s.%N)
{
printf 'FAIL %s (rc=%d)\n' "$name" "$rc"
echo '--- stdout ---'
cat "$prefix.out"
echo '--- stderr ---'
cat "$prefix.err"
} > "$prefix.status"
: > "$prefix.fail"
# The child records its own status before the timeout wrapper exits.
# That distinguishes a real child exit 124/137 from expiration, and
# child exits 125-127 from timeout's reserved infrastructure codes.
command_status=$prefix.command-status
timeout_diag=$prefix.timeout-diag
if LC_ALL=C timeout --verbose --kill-after="$TEST_KILL_AFTER" \
"$TEST_TIMEOUT" sh -c '
"$1" > "$2" 2> "$3"
command_rc=$?
status_tmp=$4.tmp.$$
printf "%s\n" "$command_rc" > "$status_tmp" || exit 70
mv "$status_tmp" "$4" || exit 70
' sh "$bin" "$prefix.out" "$prefix.err" "$command_status" \
2> "$timeout_diag"; then
timeout_rc=0
else
timeout_rc=$?
fi
deadline_expired=0
if grep -Eq '^timeout: sending signal (TERM|KILL) to command ' \
"$timeout_diag"; then
deadline_expired=1
fi
if [ -s "$timeout_diag" ]; then
cat "$timeout_diag" >> "$prefix.err"
fi
if [ "$deadline_expired" -eq 1 ]; then
outcome=timeout
rc=$timeout_rc
elif [ -f "$command_status" ]; then
rc="" extra=""
IFS=' ' read -r rc extra < "$command_status"
case $rc in
''|*[!0-9]*) rc_valid=0 ;;
*) rc_valid=1 ;;
esac
if [ "$rc_valid" -ne 1 ] || [ -n "$extra" ] \
|| [ "$timeout_rc" -ne 0 ]; then
printf 'timeout wrapper returned %s with child status %s\n' \
"$timeout_rc" "${rc:-missing}" >> "$prefix.err"
outcome=harness_error
rc=125
elif [ "$rc" -eq 0 ]; then
outcome=pass
elif [ "$rc" -eq 77 ]; then
outcome=skip
else
outcome=fail
fi
else
printf 'timeout wrapper produced no child status (rc=%s)\n' \
"$timeout_rc" >> "$prefix.err"
outcome=harness_error
case $timeout_rc in
125|126|127) rc=$timeout_rc ;;
*) rc=125 ;;
esac
fi
fi
t1=$(date +%s.%N)
dur_tmp=$prefix.dur.tmp.$$
if ! awk -v a="$t0" -v b="$t1" -v n="$name" \
'BEGIN{printf "%.3f\t%s\n", b-a, n}' > "$dur_tmp"; then
rm -f "$dur_tmp"
exit 70
fi
if ! mv "$dur_tmp" "$prefix.dur"; then
rm -f "$dur_tmp"
exit 70
fi
if ! atomic_record "$prefix.result" "WWTEST_RESULT 1 $outcome $rc"; then
exit 70
fi
awk -v a="$t0" -v b="$t1" -v n="$name" 'BEGIN{printf "%.3f\t%s\n", b-a, n}' \
> "$prefix.dur"
exit 0
fi
set -e
RESULTS=$(mktemp -d "${TMPDIR:-/tmp}/wwtest.XXXXXX")
RESULTS=$(mktemp -d "${TMPDIR:-/tmp}/wwtest.XXXXXX") || exit 2
trap 'rm -rf "$RESULTS"' EXIT
trap 'exit 1' HUP INT TERM
JOBS=${JOBS:-$(nproc 2>/dev/null || echo 4)}
ALL_MANIFEST=$RESULTS/all.manifest
PHASE1_MANIFEST=$RESULTS/phase1.manifest
PHASE2_MANIFEST=$RESULTS/phase2.manifest
RUN1_MANIFEST=$RESULTS/run1.manifest
RUN2_MANIFEST=$RESULTS/run2.manifest
SELECTED_NAMES=$RESULTS/selected.names
CACHED_NAMES=$RESULTS/cached.names
ALL_NAMES=$RESULTS/all.names
for manifest in "$ALL_MANIFEST" "$PHASE1_MANIFEST" "$PHASE2_MANIFEST" \
"$RUN1_MANIFEST" "$RUN2_MANIFEST" "$SELECTED_NAMES" \
"$CACHED_NAMES" "$ALL_NAMES"; do
if ! : > "$manifest"; then
printf 'HARNESS suite (cannot initialize manifest %s)\n' "$manifest"
exit 2
fi
done
# ---- content-keyed gate cache (task #10 T1; .ai/rob-testperf-go.md L1) -------
# `make test-commit` (COMMIT=1) reports the expensive wwstage byte-id /
# self-compile gates as `cached` when their full input set is byte-identical to
# the last green run. Those gate outcomes are a PURE function of (stage binaries
# + selfhost/lib/cmd/rt sources + test fixtures + this harness + the Makefile);
# if that content hash equals the recorded last-green key the result is provably
# unchanged, so `cached` is sound — not a guess. `make test` NEVER reads the
# cache (it stays the no-skip pre-push bar) but a green run of the gates DOES
# record the key, as the natural primer. A red run never records: the write is
# guarded on fail=0 (and the fail>0 path exits before it). Cache lives under
# out/.testcache (out/ is gitignored); `make clean` wipes out/ and thus the
# cache, which is correct — the key is keyed on the just-built binaries, so a
# from-scratch rebuild must start from a clean cache.
OUT_DIR=$(dirname "$BIN")
CACHE_DIR=$OUT_DIR/.testcache
CACHE_FILE=$CACHE_DIR/last-green
# The exact cached set, confirmed from the Makefile/test/run: the UNIT-guarded
# phase-2 gates (950 + 990-997) plus the single phase-1 lib byte-id gate
# (989_lib_byteid). The other 989_*/998/999 files are cheap functional _run
# tests, NOT byte-id gates, and stay always-on. Every member is a deterministic
# pure function of the hashed inputs (no /tmp/env/clock/network reads); a future
# gate that reads outside that set must be kept OUT of this list (Go marks such
# tests uncacheable for the same reason).
is_cached_gate() {
is_phase2() {
case $1 in
950_selfcheck.c|989_lib_byteid.c|\
990_*|991_*|992_*|993_*|994_*|995_*|996_*|997_*) return 0 ;;
950_*|990_*|991_*|992_*|993_*|994_*|995_*|996_*|997_*) return 0 ;;
esac
return 1
}
emit_cached() {
printf 'cached %s\n' "$1" > "$RESULTS/$1.status"
: > "$RESULTS/$1.cached"
is_cached_gate() {
case $1 in
950_selfcheck.c|989_lib_byteid.c|\
990_*|991_*|992_*|993_*|994_*|995_*|997_*) return 0 ;;
esac
return 1
}
# Content key over EVERYTHING that determines the cached gates' outcomes: the
# stage binary md5s first (fixed order), then tracked source/fixture/harness
# content (git ls-files is sorted → deterministic; md5sum hashes working-tree
# bytes, so uncommitted edits flip the key too). Any changed byte flips the key
# → the gates RUN. A missed input would be a false green (rob's veto), so the
# set is a deliberate conservative superset (wwstage binary md5s overlap the
# selfhost/lib source hashes, rt/ is hashed though it only reaches the gates via
# libwwrt); an extra input only ever costs a needless honest re-run. examples/
# is hashed because 996_dyn_ww links examples/mandelbrot/mandelbrot.{ww,o} — a
# real gate input outside the source tree.
for t in "$TEST_DIR"/*.c; do
[ -f "$t" ] || continue
printf '%s\n' "$t" >> "$ALL_MANIFEST"
name=${t##*/}
name=${name%.c}
printf '%s\n' "$name" >> "$ALL_NAMES"
if is_phase2 "${t##*/}"; then
printf '%s\n' "$t" >> "$PHASE2_MANIFEST"
else
printf '%s\n' "$t" >> "$PHASE1_MANIFEST"
fi
done
discovered=$(awk 'END{print NR + 0}' "$ALL_MANIFEST")
excluded=0
preflight_error=0
harness_error=0
case $TEST_EXPECTED_MIN in
''|*[!0-9]*)
printf 'HARNESS suite (invalid TEST_EXPECTED_MIN: %s)\n' "$TEST_EXPECTED_MIN"
preflight_error=1
harness_error=$((harness_error + 1))
;;
*)
if [ "$discovered" -eq 0 ]; then
echo 'HARNESS suite (empty corpus)'
preflight_error=1
harness_error=$((harness_error + 1))
elif [ "$discovered" -lt "$TEST_EXPECTED_MIN" ]; then
printf 'HARNESS suite (corpus shrank: %d < %d)\n' \
"$discovered" "$TEST_EXPECTED_MIN"
preflight_error=1
harness_error=$((harness_error + 1))
fi
;;
esac
duplicate_names=$(sort "$ALL_NAMES" | uniq -d)
if [ -n "$duplicate_names" ]; then
echo 'HARNESS suite (duplicate test identifiers)'
printf '%s\n' "$duplicate_names"
preflight_error=1
harness_error=$((harness_error + 1))
fi
JOBS=${JOBS:-$(nproc 2>/dev/null || echo 4)}
case $JOBS in
''|*[!0-9]*|0)
printf 'HARNESS suite (invalid JOBS: %s)\n' "$JOBS"
preflight_error=1
harness_error=$((harness_error + 1))
;;
esac
if ! command -v timeout >/dev/null 2>&1; then
echo 'HARNESS suite (GNU timeout is unavailable)'
preflight_error=1
harness_error=$((harness_error + 1))
fi
valid_duration() {
awk -v duration="$1" 'BEGIN {
if (duration !~ /^[0-9]+([.][0-9]+)?[smhd]?$/) exit 1
sub(/[smhd]$/, "", duration)
exit !(duration + 0 > 0)
}'
}
if ! valid_duration "$TEST_TIMEOUT"; then
printf 'HARNESS suite (invalid TEST_TIMEOUT: %s)\n' "$TEST_TIMEOUT"
preflight_error=1
harness_error=$((harness_error + 1))
fi
if ! valid_duration "$TEST_KILL_AFTER"; then
printf 'HARNESS suite (invalid TEST_KILL_AFTER: %s)\n' "$TEST_KILL_AFTER"
preflight_error=1
harness_error=$((harness_error + 1))
fi
OUT_DIR=$(dirname "$BIN")
CACHE_DIR=$OUT_DIR/.testcache
CACHE_FILE=$CACHE_DIR/last-green
# The cache includes nonignored untracked inputs because a local fixture or
# source file can change a gate even before it is added to the index.
testcache_key() {
cache_inputs=$RESULTS/cache-inputs
if ! : > "$cache_inputs"; then
return 1
fi
# Hash the effective built tools, runtime archive, and every executable
# that the last-green policy may replace with a cached result.
for b in ww w6c w6a w6l wwdump ww_ww w6c_ww w6a_ww w6l_ww wwdump_ww; do
[ -f "$BIN/$b" ] && md5sum "$BIN/$b"
printf '%s\0' "$BIN/$b" >> "$cache_inputs" || return 1
done
git ls-files selfhost lib cmd rt examples test/wcc test/run Makefile | xargs md5sum
printf '%s\0' "$OUT_DIR/lib/libwwrt.a" >> "$cache_inputs" || return 1
while IFS= read -r t; do
[ -n "$t" ] || continue
if is_cached_gate "${t##*/}"; then
name=${t##*/}
name=${name%.c}
short=${name#[0-9][0-9][0-9]_}
printf '%s\0' "$BIN/test_$short" >> "$cache_inputs" || return 1
fi
printf '%s\0' "$t" >> "$cache_inputs" || return 1
done < "$ALL_MANIFEST"
if ! git ls-files --cached -z -- \
selfhost lib cmd rt examples test/wcc test/run test/run_test.sh Makefile \
>> "$cache_inputs"; then
return 1
fi
# Generated package-cache state is not fixture input; hashing it would make
# a gate mutate its own key. Other untracked files under data/ remain input.
if ! git ls-files --others --exclude-standard -z -- \
selfhost lib cmd rt examples test/wcc test/run test/run_test.sh Makefile \
':(exclude,glob)test/wcc/data/**/.pkgcache/**' >> "$cache_inputs"; then
return 1
fi
if ! sort -z -u "$cache_inputs" -o "$cache_inputs"; then
return 1
fi
for setting in \
"TEST_DIR=$TEST_DIR" \
"TEST_EXPECTED_MIN=$TEST_EXPECTED_MIN" \
"TEST_TIMEOUT=$TEST_TIMEOUT" \
"TEST_KILL_AFTER=$TEST_KILL_AFTER" \
"TEST_WORKER=$TEST_WORKER" \
"JOBS=$JOBS"; do
setting_key=$(printf '%s' "$setting" | md5sum) || return 1
printf 'setting %s\n' "${setting_key%% *}"
done
# The index can name an unstaged deletion. Preserve that state in the
# stream instead of letting md5sum abort and silently disable caching.
xargs -0 -r sh -c '
for cache_file do
path_key=$(printf "%s" "$cache_file" | md5sum) || exit
path_key=${path_key%% *}
if [ -f "$cache_file" ]; then
content_key=$(md5sum < "$cache_file") || exit
content_key=${content_key%% *}
mode=-
[ -x "$cache_file" ] && mode=x
printf "file %s %s %s\n" \
"$path_key" "$mode" "$content_key"
elif [ ! -e "$cache_file" ] && [ ! -L "$cache_file" ]; then
printf "missing %s\n" "$path_key"
else
printf "unsupported cache input %s\n" "$path_key" >&2
exit 1
fi
done
' sh < "$cache_inputs"
}
KEY=""
if [ "${UNIT:-0}" != "1" ]; then
KEY=$(testcache_key 2>/dev/null | md5sum 2>/dev/null | cut -d' ' -f1) || KEY=""
if [ "${UNIT:-0}" != "1" ] && [ "$TEST_WORKER" = "$0" ]; then
if key_stream=$(testcache_key 2>/dev/null); then
KEY=$(printf '%s\n' "$key_stream" | md5sum | cut -d' ' -f1)
fi
fi
CACHE_HIT=0
if [ "${COMMIT:-0}" = "1" ] && [ -n "$KEY" ] && [ -f "$CACHE_FILE" ] \
@@ -128,122 +333,311 @@ if [ "${COMMIT:-0}" = "1" ] && [ -n "$KEY" ] && [ -f "$CACHE_FILE" ] \
CACHE_HIT=1
fi
# Phase wall-clock stamps (task #10 baseline). Each phase records start/end
# so the collector can report wall shares onto ken's three poles.
run_start=$(date +%s.%N)
# Phase 2 runs the wwstage byte-id / self-compile gates (990-997) +
# 950_selfcheck. They're kept out of phase 1 only so the heavy 43k
# self-compiles don't contend with the phase-1 fixture swarm; within
# phase 2 they all run in parallel (post-T3 each build's intermediates
# follow its output via `-o`/temp, so no member writes a path another
# reads — see the phase-2 block below).
for t in test/wcc/*.c; do
[ -f "$t" ] || continue
case ${t##*/} in
950_*|990_*|991_*|992_*|993_*|994_*|995_*|996_*|997_*) continue ;;
esac
name=${t##*/}; name=${name%.c}
# The only cached gate that lives in phase 1 is 989_lib_byteid; on a
# content-key hit report it cached instead of re-running it.
if [ "$CACHE_HIT" = 1 ] && is_cached_gate "${t##*/}"; then
emit_cached "$name"
continue
emit_cached() {
cached_name=$1
cached_prefix=$RESULTS/$cached_name
if ! atomic_record "$cached_prefix.result" "WWTEST_RESULT 1 cached 0"; then
return 1
fi
printf '%s/%s\0%s\0' "$RESULTS" "$name" "$t"
done | xargs -0 -n2 -P "$JOBS" "$0" --one || true
p1_end=$(date +%s.%N)
printf '%s\n' "$cached_name" >> "$CACHED_NAMES"
return 0
}
# UNIT=1 (make test-unit) is the inner-loop short path: skip the wwstage
# byte-id gates (990-997) and 950_selfcheck entirely. Pre-push uses
# `make test` for the full suite.
if [ "${UNIT:-0}" != "1" ]; then
if [ "$CACHE_HIT" = 1 ]; then
# Content key matched last-green → the wwstage byte-id / self-compile
# gates are provably unchanged. Report them cached; do NOT re-run.
for t in test/wcc/990_*.c test/wcc/991_*.c test/wcc/992_*.c \
test/wcc/993_*.c test/wcc/994_*.c test/wcc/995_*.c \
test/wcc/996_*.c test/wcc/997_*.c test/wcc/950_*.c; do
[ -f "$t" ] || continue
name=${t##*/}; name=${name%.c}
emit_cached "$name"
done
plan_manifest() {
plan_input=$1
plan_run=$2
while IFS= read -r t; do
[ -n "$t" ] || continue
name=${t##*/}
name=${name%.c}
printf '%s\n' "$name" >> "$SELECTED_NAMES"
if [ "$CACHE_HIT" = 1 ] && is_cached_gate "${t##*/}"; then
if ! emit_cached "$name"; then
printf 'HARNESS %s (cannot record cached result)\n' "$name"
harness_error=$((harness_error + 1))
fi
else
printf '%s\n' "$t" >> "$plan_run"
fi
done < "$plan_input"
}
if [ "$preflight_error" -eq 0 ]; then
plan_manifest "$PHASE1_MANIFEST" "$RUN1_MANIFEST"
if [ "${UNIT:-0}" = "1" ]; then
while IFS= read -r t; do
[ -n "$t" ] || continue
name=${t##*/}
name=${name%.c}
printf 'excluded %s (test-unit phase-2)\n' "$name"
excluded=$((excluded + 1))
done < "$PHASE2_MANIFEST"
else
p2par_start=$(date +%s.%N)
# Phase 2 — the wwstage byte-id / self-compile gates, all parallel.
# Pre-T3 the source-tree writers (993/995/950) ran in a serial tail
# because ww_ww wrote intermediates next to every traversed source, so
# they raced each other AND the 990/991/992/994 readers. T3 made every
# build's .combined.ww/.s/.o follow its OUTPUT (`-o`/per-pid temp): 993
# → its dc/dw workdirs, 995 → /tmp/wwsr_<pid>_<tool>, 950 → /tmp only.
# No member now writes a stem another reads (the readers see only the
# stable make-all regen), so the whole group fans out via xargs -P.
for t in test/wcc/990_*.c test/wcc/991_*.c test/wcc/992_*.c \
test/wcc/993_*.c test/wcc/994_*.c test/wcc/995_*.c \
test/wcc/996_*.c test/wcc/997_*.c test/wcc/950_*.c; do
[ -f "$t" ] || continue
name=${t##*/}; name=${name%.c}
plan_manifest "$PHASE2_MANIFEST" "$RUN2_MANIFEST"
fi
fi
launch_manifest() {
launch_file=$1
[ -s "$launch_file" ] || return 0
while IFS= read -r t; do
name=${t##*/}
name=${name%.c}
printf '%s/%s\0%s\0' "$RESULTS" "$name" "$t"
done | xargs -0 -n2 -P "$JOBS" "$0" --one || true
p2par_end=$(date +%s.%N)
done < "$launch_file" | \
xargs -0 -r -n2 -P "$JOBS" "$TEST_WORKER" --one
}
run_start=$(date +%s.%N)
launcher_error=0
p1_end=$run_start
p2par_start=""
p2par_end=""
if [ "$preflight_error" -eq 0 ]; then
if launch_manifest "$RUN1_MANIFEST"; then
:
else
rc=$?
printf 'HARNESS launcher (phase 1 xargs rc=%d)\n' "$rc"
launcher_error=1
harness_error=$((harness_error + 1))
fi
p1_end=$(date +%s.%N)
if [ "${UNIT:-0}" != "1" ] && [ -s "$RUN2_MANIFEST" ]; then
p2par_start=$(date +%s.%N)
if launch_manifest "$RUN2_MANIFEST"; then
:
else
rc=$?
printf 'HARNESS launcher (phase 2 xargs rc=%d)\n' "$rc"
launcher_error=1
harness_error=$((harness_error + 1))
fi
p2par_end=$(date +%s.%N)
fi
fi
fail=0
ran=0
cached=0
for s in "$RESULTS"/*.status; do
[ -f "$s" ] || continue
cat "$s"
prefix=${s%.status}
if [ -f "$prefix.cached" ]; then
cached=$((cached + 1))
else
ran=$((ran + 1))
emit_output() {
output_prefix=$1
output_name=$2
if [ -s "$output_prefix.out" ]; then
printf '%s\n' "--- stdout: $output_name ---"
cat "$output_prefix.out"
fi
[ -f "$prefix.fail" ] && fail=$((fail + 1))
done
if [ -s "$output_prefix.err" ]; then
printf '%s\n' "--- stderr: $output_name ---"
cat "$output_prefix.err"
fi
}
# ---- ww source-level tests ----------------------------------------------
if [ -d test/lang ]; then
for f in test/lang/*/*.ww; do
[ -f "$f" ] || continue
echo "skip $f (ww run not online yet)"
started=0
completed=0
pass=0
fail=0
skip=0
timed_out=0
cached=0
worker_harness_error=0
if [ "$preflight_error" -eq 0 ]; then
while IFS= read -r name; do
[ -n "$name" ] || continue
prefix=$RESULTS/$name
expect_cached=0
if grep -Fqx "$name" "$CACHED_NAMES"; then
expect_cached=1
fi
has_started=0
if [ -f "$prefix.started" ]; then
if grep -qx 'WWTEST_STARTED 1' "$prefix.started"; then
has_started=1
started=$((started + 1))
else
printf 'HARNESS %s (malformed started record)\n' "$name"
harness_error=$((harness_error + 1))
fi
fi
if [ ! -f "$prefix.result" ]; then
if [ "$has_started" -eq 1 ]; then
printf 'HARNESS %s (started but no terminal result)\n' "$name"
else
printf 'HARNESS %s (worker never started)\n' "$name"
fi
harness_error=$((harness_error + 1))
continue
fi
magic="" version="" outcome="" result_rc="" extra=""
IFS=' ' read -r magic version outcome result_rc extra < "$prefix.result"
if [ "$magic" != WWTEST_RESULT ] || [ "$version" != 1 ] \
|| [ -z "$outcome" ] || [ -z "$result_rc" ] || [ -n "$extra" ]; then
printf 'HARNESS %s (malformed terminal result)\n' "$name"
harness_error=$((harness_error + 1))
continue
fi
case $result_rc in
''|*[!0-9]*)
printf 'HARNESS %s (nonnumeric terminal rc)\n' "$name"
harness_error=$((harness_error + 1))
continue
;;
esac
if [ "$outcome" = cached ]; then
if [ "$expect_cached" -ne 1 ] || [ "$has_started" -ne 0 ] \
|| [ "$result_rc" != 0 ]; then
printf 'HARNESS %s (invalid cached result)\n' "$name"
harness_error=$((harness_error + 1))
continue
fi
printf 'cached %s\n' "$name"
cached=$((cached + 1))
continue
fi
capture_error=0
for suffix in out err; do
if [ ! -f "$prefix.$suffix" ]; then
printf 'HARNESS %s (missing %s capture)\n' "$name" "$suffix"
capture_error=1
fi
done
if [ ! -s "$prefix.dur" ]; then
printf 'HARNESS %s (missing duration record)\n' "$name"
capture_error=1
fi
if [ "$capture_error" -ne 0 ]; then
harness_error=$((harness_error + 1))
continue
fi
completed=$((completed + 1))
if [ "$expect_cached" -eq 1 ] || [ "$has_started" -ne 1 ]; then
printf 'HARNESS %s (terminal result without matching start)\n' "$name"
harness_error=$((harness_error + 1))
fi
case $outcome:$result_rc in
pass:0)
printf 'ok %s\n' "$name"
pass=$((pass + 1))
;;
skip:77)
printf 'skip %s (rc=77)\n' "$name"
skip=$((skip + 1))
if [ ! -s "$prefix.out" ] && [ ! -s "$prefix.err" ]; then
echo 'skip reason: exit 77 (no output)'
fi
;;
timeout:124|timeout:137)
printf 'TIMEOUT %s\n' "$name"
timed_out=$((timed_out + 1))
;;
fail:*)
printf 'FAIL %s (rc=%s)\n' "$name" "$result_rc"
fail=$((fail + 1))
;;
harness_error:125|harness_error:126|harness_error:127)
printf 'HARNESS %s (worker rc=%s)\n' "$name" "$result_rc"
worker_harness_error=$((worker_harness_error + 1))
harness_error=$((harness_error + 1))
;;
*)
printf 'HARNESS %s (invalid outcome/rc %s/%s)\n' \
"$name" "$outcome" "$result_rc"
worker_harness_error=$((worker_harness_error + 1))
harness_error=$((harness_error + 1))
;;
esac
emit_output "$prefix" "$name"
done < "$SELECTED_NAMES"
for record in "$RESULTS"/*.result "$RESULTS"/*.started; do
[ -f "$record" ] || continue
record_name=${record##*/}
record_name=${record_name%.result}
record_name=${record_name%.started}
if ! grep -Fqx "$record_name" "$SELECTED_NAMES"; then
printf 'HARNESS %s (unexpected worker record)\n' "$record_name"
harness_error=$((harness_error + 1))
fi
done
fi
# ---- timing (task #10 baseline) -----------------------------------------
# Per-test durations + phase walls. Emitted after the per-test ok/FAIL dump
# but before the summary line's exit checks, so timing prints even on the
# failure path; the "all N tests passed" summary stays the last line. No log
# consumer is disturbed: the "---" delimiters and "<sec>\t<name>" rows match
# neither the "all N tests passed" nor the "FAIL" grep keys.
selected=$((discovered - excluded))
planned=$((selected - cached))
terminal=$((pass + fail + skip + timed_out + worker_harness_error))
if [ "$preflight_error" -eq 0 ]; then
if [ "$started" -ne "$planned" ]; then
printf 'HARNESS accounting (started %d != planned %d)\n' \
"$started" "$planned"
harness_error=$((harness_error + 1))
fi
if [ "$completed" -ne "$planned" ]; then
printf 'HARNESS accounting (completed %d != planned %d)\n' \
"$completed" "$planned"
harness_error=$((harness_error + 1))
fi
if [ "$terminal" -ne "$completed" ]; then
printf 'HARNESS accounting (terminal outcomes %d != completed %d)\n' \
"$terminal" "$completed"
harness_error=$((harness_error + 1))
fi
if [ $((completed + cached + excluded)) -ne "$discovered" ]; then
printf 'HARNESS accounting (discovered corpus is incomplete)\n'
harness_error=$((harness_error + 1))
fi
fi
# A cache decision is valid only if its complete input fingerprint is still
# current after all workers have finished. This applies to hits as well as to
# newly published keys.
if [ "${UNIT:-0}" != "1" ] && [ -n "$KEY" ]; then
end_key=""
if end_stream=$(testcache_key 2>/dev/null); then
end_key=$(printf '%s\n' "$end_stream" | md5sum | cut -d' ' -f1)
fi
if [ -z "$end_key" ] || [ "$end_key" != "$KEY" ]; then
echo 'HARNESS cache (inputs changed during test run)'
harness_error=$((harness_error + 1))
fi
fi
not_run=$((discovered - completed - cached))
run_end=$(date +%s.%N)
echo '--- timing: per-test (sec, desc) ---'
cat "$RESULTS"/*.dur 2>/dev/null | sort -rn
echo '--- timing: phase walls (sec) ---'
awk -v rs="$run_start" -v p1e="$p1_end" \
-v ppar_s="$p2par_start" -v ppar_e="$p2par_end" \
-v pser_s="$p2ser_start" -v pser_e="$p2ser_end" \
-v re="$run_end" 'BEGIN{
printf "phase1_parallel\t%.3f\n", p1e - rs
if (ppar_e != "") printf "phase2_parallel\t%.3f\n", ppar_e - ppar_s
if (pser_e != "") printf "phase2_serial_tail\t%.3f\n", pser_e - pser_s
printf "total\t%.3f\n", re - rs
}'
if [ $ran -eq 0 ] && [ $cached -eq 0 ]; then
echo "no tests were run"
exit 1
printf 'summary: discovered=%d started=%d completed=%d pass=%d fail=%d skip=%d timeout=%d harness_error=%d cached=%d not_run=%d\n' \
"$discovered" "$started" "$completed" "$pass" "$fail" "$skip" \
"$timed_out" "$harness_error" "$cached" "$not_run"
green=1
if [ "$preflight_error" -ne 0 ] || [ "$launcher_error" -ne 0 ] \
|| [ "$fail" -ne 0 ] || [ "$skip" -ne 0 ] || [ "$timed_out" -ne 0 ] \
|| [ "$harness_error" -ne 0 ]; then
green=0
fi
if [ $fail -gt 0 ]; then
echo "$fail test(s) failed"
exit 1
if [ "$preflight_error" -eq 0 ] && [ "$not_run" -ne "$excluded" ]; then
green=0
fi
# Green and the gates actually RAN (not a cache hit, not test-unit): record the
# content key as the last-green primer. A red run exits above, never here.
if [ "${UNIT:-0}" != "1" ] && [ "$CACHE_HIT" != 1 ] && [ -n "$KEY" ]; then
mkdir -p "$CACHE_DIR" && printf '%s\n' "$KEY" > "$CACHE_FILE"
if [ "$green" -eq 1 ] && [ "${UNIT:-0}" != "1" ] \
&& [ "$CACHE_HIT" != 1 ] && [ -n "$KEY" ]; then
if ! mkdir -p "$CACHE_DIR" \
|| ! atomic_record "$CACHE_FILE" "$KEY"; then
echo 'HARNESS cache (cannot publish last-green key)'
exit 1
fi
fi
[ $cached -gt 0 ] && echo "$cached gate(s) cached (content key matched last green)"
echo "all $ran tests passed"
[ "$green" -eq 1 ] || exit 1
exit 0