test: stop the runners reporting green on suites that never ran
run_bats() was an if with no else, so a missing or non-executable run-bats.sh made all 166 bats tests disappear with no diagnostic while the run printed a green summary and exited 0. --bats-only became a total no-op. This is the same defect the PR fixed one level down, left open in the dispatcher that pre-push actually invokes -- and nothing tested run-tests.sh at all. run-bats.sh's aggregation was asserted by nothing. Three separate mutations to its failure-detection line all survived the existing suite, because real bats emits both a nonzero exit and "not ok" lines, so each signal masked the other. The new cases produce each signal without the other; all three mutants now die. Also in this pass: - zero discovered .bats files exited 0, so a widened path exclusion retired the suite silently. Replaced with a file-count floor: a collapse to one or two files is the same failure as a collapse to zero - an existing-but-empty status file counted as a pass, because [[ "" -eq 0 ]] is arithmetic-true and the || echo 1 fallback only covered a missing file. The repro is deterministic: the stub truncates its own status file, then kill -9s its parent so the real exit-code write never happens Adds tests/test-run-tests.sh, 9 cases pinning the exit 0 / 1 / 77 three-way split against a single summary line. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01X7GvKuJfy2WrdBmUttV4DT
This commit is contained in:
@@ -12,6 +12,14 @@
|
||||
#
|
||||
# Both branches were code-only and asserted by nothing, which is the same
|
||||
# "green either way" hole the guard itself closes. This file covers them.
|
||||
#
|
||||
# It also covers the aggregation those counts feed, which the parallelization
|
||||
# rewrite left unasserted: three separate mutations of the `if [[ "$file_not_ok"
|
||||
# -gt 0 || "$status" != "0" ]]` line -- dropping either half, or deleting the
|
||||
# whole branch -- all survived this file. They survived because every stub here
|
||||
# exited 0 and emitted no `not ok`, so neither signal was ever exercised, and
|
||||
# because real bats emits both at once each half masks the other. The cases
|
||||
# below produce each signal *without* the other, so each mutation dies alone.
|
||||
set -euo pipefail
|
||||
|
||||
REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||
@@ -61,11 +69,21 @@ seed_bats_files() {
|
||||
}
|
||||
|
||||
# Runs the fixture's run-bats.sh, capturing output and exit code separately.
|
||||
#
|
||||
# BATS_FILE_FLOOR is lowered to 1 for every case that is not specifically about
|
||||
# the floor: these fixtures hold one or two .bats files by design, which is well
|
||||
# under the real repo's floor. FAKE_FLOOR lets the floor cases opt back into the
|
||||
# script's own default. TMPDIR is a private per-run directory so a stub can find
|
||||
# the scratch dir run-bats.sh mktemp'd for itself -- see case 7.
|
||||
FAKE_OUT=""
|
||||
FAKE_RC=0
|
||||
FAKE_FLOOR=""
|
||||
run_fake() {
|
||||
local priv
|
||||
priv="$(mktemp -d)"
|
||||
FIXTURES+=("$priv")
|
||||
FAKE_RC=0
|
||||
FAKE_OUT="$(bash "$1/tests/run-bats.sh" 2>&1)" || FAKE_RC=$?
|
||||
FAKE_OUT="$(TMPDIR="$priv" BATS_FILE_FLOOR="${FAKE_FLOOR:-1}" bash "$1/tests/run-bats.sh" 2>&1)" || FAKE_RC=$?
|
||||
}
|
||||
|
||||
# --- 1. A stub emitting nothing at all is a broken harness, not a clean run ---
|
||||
@@ -165,6 +183,146 @@ else
|
||||
fi
|
||||
fi
|
||||
|
||||
# --- 5. `not ok` lines with a zero exit still fail the run. This is the half of
|
||||
# the aggregation that a bats wrapper swallowing the exit status would leave as
|
||||
# the only surviving evidence of a failure, and it is the case that kills the
|
||||
# `drop "$file_not_ok" -gt 0 ||` mutation: without that half the run reports
|
||||
# "2 tests, 1 failures" and exits 0, calling a failing test suite green.
|
||||
echo ""
|
||||
echo "--- a failing test whose process still exits 0 fails the run ---"
|
||||
DIR5="$(make_fake_repo)"
|
||||
FIXTURES+=("$DIR5")
|
||||
seed_bats_files "$DIR5"
|
||||
install_stub_bats "$DIR5" <<'EOF'
|
||||
#!/usr/bin/env bash
|
||||
echo "1..1"
|
||||
echo "not ok 1 a failing test"
|
||||
exit 0
|
||||
EOF
|
||||
run_fake "$DIR5"
|
||||
if [[ $FAKE_RC -eq 0 ]]; then
|
||||
fail "a 'not ok' TAP result exited 0 — a failing test reported as a pass because only the process status was consulted"
|
||||
elif echo "$FAKE_OUT" | grep -q "^2 tests, 2 failures$"; then
|
||||
pass "'not ok' lines fail the run even when every bats process exits 0"
|
||||
else
|
||||
fail "the run failed but with the wrong count: $FAKE_OUT"
|
||||
fi
|
||||
|
||||
# --- 6. A non-zero exit with no `not ok` line still fails the run. This is the
|
||||
# other half: a crash, a timeout, or an unbound variable in setup_file kills bats
|
||||
# before it can emit a result line, so the exit status is the only evidence. It
|
||||
# kills the `drop || "$status" -ne 0` mutation. The stub emits a passing result
|
||||
# first so the zero-count guard cannot be what fails the run -- without the
|
||||
# status half this stub reports "2 tests, 0 failures" and exits 0.
|
||||
echo ""
|
||||
echo "--- a bats exiting non-zero with no 'not ok' line fails the run ---"
|
||||
DIR6="$(make_fake_repo)"
|
||||
FIXTURES+=("$DIR6")
|
||||
seed_bats_files "$DIR6"
|
||||
install_stub_bats "$DIR6" <<'EOF'
|
||||
#!/usr/bin/env bash
|
||||
echo "1..2"
|
||||
echo "ok 1 first"
|
||||
echo "bats: setup_file failed" >&2
|
||||
exit 1
|
||||
EOF
|
||||
run_fake "$DIR6"
|
||||
if [[ $FAKE_RC -eq 0 ]]; then
|
||||
fail "a bats process exiting 1 was reported as a pass because only the TAP text was consulted"
|
||||
elif echo "$FAKE_OUT" | grep -q "^2 tests, 0 failures$"; then
|
||||
pass "a non-zero bats exit fails the run even with no 'not ok' line to find"
|
||||
else
|
||||
fail "the run failed but with the wrong count: $FAKE_OUT"
|
||||
fi
|
||||
|
||||
# --- 7. An empty status file is a failure, not a pass. The status is read back
|
||||
# with `cat ... || echo 1`, which covers a *missing* file; a file that exists but
|
||||
# is empty is what a job killed between the `>` truncating it and the `echo`
|
||||
# completing leaves behind, and what ENOSPC leaves behind. Under the arithmetic
|
||||
# `-ne` that used to compare it, `[[ "" -ne 0 ]]` is false and the job read as a
|
||||
# clean exit.
|
||||
#
|
||||
# The stub reproduces that state exactly: it emits a healthy TAP stream, creates
|
||||
# the empty status file itself, then SIGKILLs the subshell that would have
|
||||
# written the real status. It finds the scratch directory through the private
|
||||
# TMPDIR run_fake sets -- run-bats.sh mktemp -d's under it, and the log file for
|
||||
# job 1 is already open by the time the stub runs.
|
||||
echo ""
|
||||
echo "--- an empty status file fails the run rather than counting as exit 0 ---"
|
||||
DIR7="$(make_fake_repo)"
|
||||
FIXTURES+=("$DIR7")
|
||||
printf '@test "a" { false; }\n' > "$DIR7/tests/a.bats"
|
||||
install_stub_bats "$DIR7" <<'EOF'
|
||||
#!/usr/bin/env bash
|
||||
echo "1..1"
|
||||
echo "ok 1 looked fine"
|
||||
for d in "$TMPDIR"/*/; do
|
||||
if [[ -e "$d/1.log" ]]; then
|
||||
: > "$d/1.status"
|
||||
fi
|
||||
done
|
||||
kill -9 $PPID
|
||||
sleep 5
|
||||
EOF
|
||||
run_fake "$DIR7"
|
||||
# The count line is asserted alongside the exit code so this can only pass for
|
||||
# the stated reason: the TAP stream the stub emitted is healthy, so "1 tests, 0
|
||||
# failures" proves the zero-count guard did not fire and the empty status is the
|
||||
# only thing left that can have failed the run.
|
||||
if ! echo "$FAKE_OUT" | grep -q "^1 tests, 0 failures$"; then
|
||||
fail "the killed job did not leave the healthy TAP stream the case needs: $FAKE_OUT"
|
||||
elif [[ $FAKE_RC -eq 0 ]]; then
|
||||
fail "an empty status file was counted as a clean exit — a killed job reported as a pass"
|
||||
else
|
||||
pass "an empty status file fails the run despite a healthy TAP stream"
|
||||
fi
|
||||
|
||||
# --- 8. Too few discovered .bats files is a hard error, not a green run. Zero
|
||||
# files used to print a note to stderr and `exit 0`; the floor makes both zero
|
||||
# and a collapsed-but-nonzero count fail, because a moved tests/ tree or a
|
||||
# widened `-not -path` exclusion produces exactly that and nothing else notices.
|
||||
echo ""
|
||||
echo "--- a discovered-file count under the floor fails the run ---"
|
||||
DIR8="$(make_fake_repo)"
|
||||
FIXTURES+=("$DIR8")
|
||||
seed_bats_files "$DIR8"
|
||||
install_stub_bats "$DIR8" <<'EOF'
|
||||
#!/usr/bin/env bash
|
||||
echo "1..1"
|
||||
echo "ok 1 first"
|
||||
exit 0
|
||||
EOF
|
||||
FAKE_FLOOR=8
|
||||
run_fake "$DIR8"
|
||||
FAKE_FLOOR=""
|
||||
if [[ $FAKE_RC -eq 0 ]]; then
|
||||
fail "2 .bats files under a floor of 8 passed — a collapsed search path reads as green"
|
||||
elif echo "$FAKE_OUT" | grep -q "below the floor of 8"; then
|
||||
pass "a file count under the floor fails and names the floor it missed"
|
||||
else
|
||||
fail "the run failed but not with the floor message: $FAKE_OUT"
|
||||
fi
|
||||
|
||||
# --- 9. Zero discovered .bats files is the same hard error. Kept separate from
|
||||
# case 8 because it is the state the old `exit 0` branch handled by name, and it
|
||||
# is the one a path change actually produces.
|
||||
echo ""
|
||||
echo "--- zero discovered .bats files fails the run ---"
|
||||
DIR9="$(make_fake_repo)"
|
||||
FIXTURES+=("$DIR9")
|
||||
install_stub_bats "$DIR9" <<'EOF'
|
||||
#!/usr/bin/env bash
|
||||
exit 0
|
||||
EOF
|
||||
run_fake "$DIR9"
|
||||
if [[ $FAKE_RC -eq 0 ]]; then
|
||||
fail "finding no .bats files at all exited 0 — the whole suite can vanish and the run stays green"
|
||||
elif echo "$FAKE_OUT" | grep -q "found 0 .bats file"; then
|
||||
pass "finding no .bats files fails the run and says so"
|
||||
else
|
||||
fail "the run failed but not with the zero-files message: $FAKE_OUT"
|
||||
fi
|
||||
|
||||
echo ""
|
||||
echo "Results: $PASS passed, $FAIL failed"
|
||||
[[ $FAIL -eq 0 ]]
|
||||
|
||||
Reference in New Issue
Block a user