test: make T0 harness results complete
This commit is contained in:
316
test/run_test.sh
Normal file
316
test/run_test.sh
Normal file
@@ -0,0 +1,316 @@
|
||||
#!/bin/sh
|
||||
# Synthetic corpora keep harness failure-path coverage independent of compiler builds.
|
||||
|
||||
set -eu
|
||||
|
||||
ROOT=$(cd "$(dirname "$0")/.." && pwd)
|
||||
RUN=$ROOT/test/run
|
||||
WORK=$(mktemp -d "${TMPDIR:-/tmp}/ww-run-test.XXXXXX")
|
||||
trap 'rm -rf "$WORK"' EXIT
|
||||
trap 'exit 1' HUP INT TERM
|
||||
|
||||
fail()
|
||||
{
|
||||
printf 'run_test: FAIL: %s\n' "$*" >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
new_case()
|
||||
{
|
||||
CASE=$WORK/$1
|
||||
mkdir -p "$CASE/corpus" "$CASE/bin"
|
||||
}
|
||||
|
||||
add_source()
|
||||
{
|
||||
: > "$CASE/corpus/$1.c"
|
||||
}
|
||||
|
||||
add_binary()
|
||||
{
|
||||
name=$1
|
||||
body=$2
|
||||
{
|
||||
printf '%s\n' '#!/bin/sh'
|
||||
printf '%s\n' "$body"
|
||||
} > "$CASE/bin/test_$name"
|
||||
chmod +x "$CASE/bin/test_$name"
|
||||
}
|
||||
|
||||
run_case()
|
||||
{
|
||||
want=$1
|
||||
shift
|
||||
set +e
|
||||
env BIN="$CASE/bin" JOBS=2 "$@" sh "$RUN" > "$CASE/output" 2>&1
|
||||
GOT=$?
|
||||
set -e
|
||||
[ "$GOT" -eq "$want" ] || {
|
||||
cat "$CASE/output" >&2
|
||||
fail "$(basename "$CASE"): exit $GOT, want $want"
|
||||
}
|
||||
}
|
||||
|
||||
run_case_from()
|
||||
{
|
||||
want=$1
|
||||
run_dir=$2
|
||||
shift 2
|
||||
set +e
|
||||
(
|
||||
cd "$run_dir"
|
||||
env BIN="$CASE/bin" JOBS=2 "$@" sh "$RUN"
|
||||
) > "$CASE/output" 2>&1
|
||||
GOT=$?
|
||||
set -e
|
||||
[ "$GOT" -eq "$want" ] || {
|
||||
cat "$CASE/output" >&2
|
||||
fail "$(basename "$CASE"): exit $GOT, want $want"
|
||||
}
|
||||
}
|
||||
|
||||
assert_line()
|
||||
{
|
||||
grep -Fqx "$1" "$CASE/output" || {
|
||||
cat "$CASE/output" >&2
|
||||
fail "$(basename "$CASE"): missing line: $1"
|
||||
}
|
||||
}
|
||||
|
||||
assert_text()
|
||||
{
|
||||
grep -Fq "$1" "$CASE/output" || {
|
||||
cat "$CASE/output" >&2
|
||||
fail "$(basename "$CASE"): missing text: $1"
|
||||
}
|
||||
}
|
||||
|
||||
assert_summary_field()
|
||||
{
|
||||
key=$1
|
||||
want=$2
|
||||
summary=$(grep '^summary:' "$CASE/output" || true)
|
||||
[ "$(printf '%s\n' "$summary" | grep -c '^summary:')" -eq 1 ] || {
|
||||
cat "$CASE/output" >&2
|
||||
fail "$(basename "$CASE"): expected one summary line"
|
||||
}
|
||||
case " $summary " in
|
||||
*" $key=$want "*) ;;
|
||||
*)
|
||||
cat "$CASE/output" >&2
|
||||
fail "$(basename "$CASE"): summary $key is not $want"
|
||||
;;
|
||||
esac
|
||||
}
|
||||
|
||||
assert_summary_nonzero()
|
||||
{
|
||||
key=$1
|
||||
summary=$(grep '^summary:' "$CASE/output" || true)
|
||||
[ "$(printf '%s\n' "$summary" | grep -c '^summary:')" -eq 1 ] || {
|
||||
cat "$CASE/output" >&2
|
||||
fail "$(basename "$CASE"): expected one summary line"
|
||||
}
|
||||
case " $summary " in
|
||||
*" $key=0 "*)
|
||||
cat "$CASE/output" >&2
|
||||
fail "$(basename "$CASE"): summary $key is zero"
|
||||
;;
|
||||
*" $key="*) ;;
|
||||
*)
|
||||
cat "$CASE/output" >&2
|
||||
fail "$(basename "$CASE"): summary has no $key field"
|
||||
;;
|
||||
esac
|
||||
}
|
||||
|
||||
new_case all_pass
|
||||
add_source 001_alpha
|
||||
add_source 002_beta
|
||||
add_binary alpha 'exit 0'
|
||||
add_binary beta 'exit 0'
|
||||
run_case 0 TEST_DIR="$CASE/corpus" TEST_EXPECTED_MIN=2 UNIT=1
|
||||
assert_line 'ok 001_alpha'
|
||||
assert_line 'ok 002_beta'
|
||||
assert_line 'summary: discovered=2 started=2 completed=2 pass=2 fail=0 skip=0 timeout=0 harness_error=0 cached=0 not_run=0'
|
||||
|
||||
new_case cache_publish
|
||||
add_source 990_cache
|
||||
add_source 996_dynamic
|
||||
add_binary cache '[ -z "${WW_W6C+x}${WW_W6A+x}${WW_W6L+x}${WW_LIB+x}${WW_SRCLIB+x}${WW_PKGCACHE+x}" ] || exit 9; exit 0'
|
||||
add_binary dynamic 'exit 0'
|
||||
run_case 0 TEST_DIR="$CASE/corpus" TEST_EXPECTED_MIN=2 COMMIT=1 \
|
||||
WW_W6C=/first/w6c WW_W6A=/first/w6a WW_W6L=/first/w6l \
|
||||
WW_LIB=/first/lib WW_SRCLIB=/first/src WW_PKGCACHE=/first/cache
|
||||
assert_summary_field pass 2
|
||||
assert_summary_field harness_error 0
|
||||
assert_summary_field cached 0
|
||||
[ -s "$CASE/.testcache/last-green" ] || fail "cache_publish: no cache key"
|
||||
run_case 0 TEST_DIR="$CASE/corpus" TEST_EXPECTED_MIN=2 COMMIT=1 \
|
||||
WW_W6C=/second/w6c WW_W6A=/second/w6a WW_W6L=/second/w6l \
|
||||
WW_LIB=/second/lib WW_SRCLIB=/second/src WW_PKGCACHE=/second/cache
|
||||
assert_line 'cached 990_cache'
|
||||
assert_line 'ok 996_dynamic'
|
||||
assert_line 'summary: discovered=2 started=1 completed=1 pass=1 fail=0 skip=0 timeout=0 harness_error=0 cached=1 not_run=0'
|
||||
|
||||
new_case cache_deleted_tracked
|
||||
add_source 990_cache
|
||||
add_binary cache 'exit 0'
|
||||
mkdir -p "$CASE/repo"
|
||||
: > "$CASE/repo/Makefile"
|
||||
git -C "$CASE/repo" init -q
|
||||
git -C "$CASE/repo" add Makefile
|
||||
rm -f "$CASE/repo/Makefile"
|
||||
run_case_from 0 "$CASE/repo" TEST_DIR="$CASE/corpus" \
|
||||
TEST_EXPECTED_MIN=1 COMMIT=1
|
||||
assert_summary_field pass 1
|
||||
[ -s "$CASE/.testcache/last-green" ] || \
|
||||
fail "cache_deleted_tracked: no cache key"
|
||||
run_case_from 0 "$CASE/repo" TEST_DIR="$CASE/corpus" \
|
||||
TEST_EXPECTED_MIN=1 COMMIT=1
|
||||
assert_summary_field cached 1
|
||||
assert_summary_field started 0
|
||||
|
||||
new_case fail
|
||||
add_source 001_fail
|
||||
add_binary fail 'printf "%s\n" "synthetic failure" >&2; exit 3'
|
||||
run_case 1 TEST_DIR="$CASE/corpus" TEST_EXPECTED_MIN=1 UNIT=1
|
||||
assert_line 'FAIL 001_fail (rc=3)'
|
||||
assert_text 'synthetic failure'
|
||||
assert_summary_field fail 1
|
||||
assert_summary_field harness_error 0
|
||||
|
||||
new_case skip
|
||||
add_source 001_skip
|
||||
add_binary skip 'printf "%s\n" "synthetic prerequisite unavailable"; exit 77'
|
||||
run_case 1 TEST_DIR="$CASE/corpus" TEST_EXPECTED_MIN=1 UNIT=1
|
||||
assert_line 'skip 001_skip (rc=77)'
|
||||
assert_text 'synthetic prerequisite unavailable'
|
||||
assert_summary_field skip 1
|
||||
assert_summary_field pass 0
|
||||
|
||||
new_case timeout
|
||||
add_source 001_slow
|
||||
add_binary slow 'trap "" TERM; sleep 5; exit 0'
|
||||
run_case 1 TEST_DIR="$CASE/corpus" TEST_EXPECTED_MIN=1 \
|
||||
TEST_TIMEOUT=1 TEST_KILL_AFTER=1 UNIT=1
|
||||
assert_line 'TIMEOUT 001_slow'
|
||||
assert_summary_field timeout 1
|
||||
assert_summary_field pass 0
|
||||
|
||||
new_case reserved_exit_codes
|
||||
for item in 001:124 002:125 003:126 004:127 005:137; do
|
||||
prefix=${item%%:*}
|
||||
rc=${item#*:}
|
||||
add_source "${prefix}_exit${rc}"
|
||||
add_binary "exit${rc}" "exit $rc"
|
||||
done
|
||||
run_case 1 TEST_DIR="$CASE/corpus" TEST_EXPECTED_MIN=5 UNIT=1
|
||||
assert_summary_field fail 5
|
||||
assert_summary_field timeout 0
|
||||
assert_summary_field harness_error 0
|
||||
|
||||
new_case timeout_wrapper_failure
|
||||
add_source 001_probe
|
||||
add_binary probe 'exit 0'
|
||||
mkdir -p "$CASE/path"
|
||||
{
|
||||
printf '%s\n' '#!/bin/sh'
|
||||
printf '%s\n' 'exit 124'
|
||||
} > "$CASE/path/timeout"
|
||||
chmod +x "$CASE/path/timeout"
|
||||
run_case 1 PATH="$CASE/path:$PATH" TEST_DIR="$CASE/corpus" \
|
||||
TEST_EXPECTED_MIN=1 UNIT=1
|
||||
assert_summary_field timeout 0
|
||||
assert_summary_nonzero harness_error
|
||||
|
||||
new_case missing_binary
|
||||
add_source 001_missing
|
||||
run_case 1 TEST_DIR="$CASE/corpus" TEST_EXPECTED_MIN=1 UNIT=1
|
||||
assert_text 'HARNESS 001_missing'
|
||||
assert_summary_field harness_error 1
|
||||
assert_summary_field completed 1
|
||||
|
||||
new_case empty
|
||||
run_case 1 TEST_DIR="$CASE/corpus" TEST_EXPECTED_MIN=0 UNIT=1
|
||||
assert_summary_field discovered 0
|
||||
assert_summary_field started 0
|
||||
assert_summary_field completed 0
|
||||
assert_summary_field harness_error 1
|
||||
|
||||
new_case below_minimum
|
||||
add_source 001_only
|
||||
add_binary only 'exit 0'
|
||||
run_case 1 TEST_DIR="$CASE/corpus" TEST_EXPECTED_MIN=2 UNIT=1
|
||||
assert_summary_field discovered 1
|
||||
assert_summary_field harness_error 1
|
||||
|
||||
new_case zero_timeout
|
||||
add_source 001_bounded
|
||||
add_binary bounded 'exit 0'
|
||||
run_case 1 TEST_DIR="$CASE/corpus" TEST_EXPECTED_MIN=1 \
|
||||
TEST_TIMEOUT=0 UNIT=1
|
||||
assert_summary_field discovered 1
|
||||
assert_summary_field started 0
|
||||
assert_summary_field harness_error 1
|
||||
|
||||
new_case launcher_failure
|
||||
add_source 001_lost
|
||||
add_binary lost 'exit 0'
|
||||
mkdir -p "$CASE/path"
|
||||
{
|
||||
printf '%s\n' '#!/bin/sh'
|
||||
printf '%s\n' 'exit 23'
|
||||
} > "$CASE/path/xargs"
|
||||
chmod +x "$CASE/path/xargs"
|
||||
run_case 1 PATH="$CASE/path:$PATH" TEST_DIR="$CASE/corpus" \
|
||||
TEST_EXPECTED_MIN=1 UNIT=1
|
||||
assert_summary_field discovered 1
|
||||
assert_summary_field started 0
|
||||
assert_summary_field completed 0
|
||||
assert_summary_nonzero harness_error
|
||||
|
||||
new_case lost_result
|
||||
add_source 001_lost
|
||||
add_binary lost 'exit 0'
|
||||
{
|
||||
printf '%s\n' '#!/bin/sh'
|
||||
printf '%s\n' 'prefix=$2'
|
||||
printf '%s\n' "printf '%s\\n' 'WWTEST_STARTED 1' > \"\$prefix.started\""
|
||||
printf '%s\n' 'exit 0'
|
||||
} > "$CASE/worker"
|
||||
chmod +x "$CASE/worker"
|
||||
run_case 1 TEST_DIR="$CASE/corpus" TEST_EXPECTED_MIN=1 \
|
||||
TEST_WORKER="$CASE/worker" UNIT=1
|
||||
assert_summary_field discovered 1
|
||||
assert_summary_field started 1
|
||||
assert_summary_field completed 0
|
||||
assert_summary_nonzero harness_error
|
||||
|
||||
new_case incomplete_result
|
||||
add_source 001_incomplete
|
||||
add_binary incomplete 'exit 0'
|
||||
{
|
||||
printf '%s\n' '#!/bin/sh'
|
||||
printf '%s\n' 'prefix=$2'
|
||||
printf '%s\n' "printf '%s\\n' 'WWTEST_STARTED 1' > \"\$prefix.started\""
|
||||
printf '%s\n' "printf '%s\\n' 'WWTEST_RESULT 1 pass 0' > \"\$prefix.result\""
|
||||
printf '%s\n' 'exit 0'
|
||||
} > "$CASE/worker"
|
||||
chmod +x "$CASE/worker"
|
||||
run_case 1 TEST_DIR="$CASE/corpus" TEST_EXPECTED_MIN=1 \
|
||||
TEST_WORKER="$CASE/worker" UNIT=1
|
||||
assert_summary_field discovered 1
|
||||
assert_summary_field completed 0
|
||||
assert_summary_field pass 0
|
||||
assert_summary_nonzero harness_error
|
||||
|
||||
new_case unit_exclusion
|
||||
add_source 001_fast
|
||||
add_source 990_phase
|
||||
add_binary fast 'exit 0'
|
||||
run_case 0 TEST_DIR="$CASE/corpus" TEST_EXPECTED_MIN=2 UNIT=1
|
||||
assert_line 'excluded 990_phase (test-unit phase-2)'
|
||||
assert_line 'summary: discovered=2 started=1 completed=1 pass=1 fail=0 skip=0 timeout=0 harness_error=0 cached=0 not_run=1'
|
||||
|
||||
printf 'run_test: all harness regressions passed\n'
|
||||
Reference in New Issue
Block a user