diff --git a/tests/run-bats.sh b/tests/run-bats.sh index 391fa24..d26cb05 100755 --- a/tests/run-bats.sh +++ b/tests/run-bats.sh @@ -32,9 +32,24 @@ done < <( | sort ) -if [[ ${#TEST_FILES[@]} -eq 0 ]]; then - echo "No .bats test files found." >&2 - exit 0 +# A floor on the discovered file count, not merely a zero check, and a hard error +# rather than the `exit 0` this used to be. Zero discovered files was the likelier +# of the two silent-green failures -- a moved tests/ tree, a renamed skill +# directory, or the `-not -path` exclusions above widening -- and it exited 0 with +# a note on stderr nobody reads, while the zero-*result* guard further down was +# already a hard error. A floor rather than `-gt 0` because the count collapsing +# to 1 or 2 is the same failure as collapsing to 0 and only a floor names it. +# Same reasoning, and the same "set it a little under the current count" rule, as +# the per-glob floors in tests/test-vale-wrap.sh -- ordinary file churn does not +# trip it, a broken or renamed path does. +# +# BATS_FILE_FLOOR overrides it. That override exists for the fixture repos in +# tests/test-run-bats.sh and tests/test-run-tests.sh, which hold one or two .bats +# files by design; it is not an escape hatch for a real run. +BATS_FILE_FLOOR="${BATS_FILE_FLOOR:-8}" +if [[ ${#TEST_FILES[@]} -lt $BATS_FILE_FLOOR ]]; then + echo "Error: found ${#TEST_FILES[@]} .bats file(s) under $REPO_ROOT, below the floor of $BATS_FILE_FLOOR — the search path is wrong or the suite has been gutted" >&2 + exit 1 fi # Each file gets its own `bats` process, run concurrently (bounded by core @@ -83,17 +98,29 @@ for f in ${TEST_FILES[@]+"${TEST_FILES[@]}"}; do # file bats really did run, so it has to be distinguishable from a file that # produced nothing whatsoever. file_plan="$(grep -c '^1\.\.[0-9]' "$SCRATCH_ROOT/$i.log" || true)" + # String-compared below, not `-ne`. `-ne` is arithmetic and bash evaluates an + # empty string as 0 there -- `[[ "" -ne 0 ]]` is false -- so an *empty* status + # file read as a clean exit. The `|| echo 1` fallback only covers a *missing* + # file; an existing-but-empty one is what a job killed between the `>` and the + # `echo` leaves behind, or what ENOSPC leaves behind. status="$(cat "$SCRATCH_ROOT/$i.status" 2>/dev/null || echo 1)" TOTAL_OK=$((TOTAL_OK + file_ok)) TOTAL_NOT_OK=$((TOTAL_NOT_OK + file_not_ok)) TOTAL_PLANS=$((TOTAL_PLANS + file_plan)) - if [[ "$file_not_ok" -gt 0 || "$status" -ne 0 ]]; then + # Two independent failure signals, deliberately OR-ed: a file can report `not + # ok` lines while its process still exits 0 (a bats formatter or wrapper that + # swallows the status), and a file can exit non-zero having emitted no `not + # ok` at all (a crash, a timeout, an unbound variable in setup_file). Real + # bats normally emits both at once, so each signal masks the other and + # dropping either half is invisible without tests that produce one without + # the other -- tests/test-run-bats.sh has those. + if [[ "$file_not_ok" -gt 0 || "$status" != "0" ]]; then FAIL=1 fi done -# Zero counted tests is never a clean run: files were found (the empty-TEST_FILES -# case exits above), so nothing was executed. Without this, a `bats` that emits +# Zero counted tests is never a clean run: enough files were found (a count under +# the floor, zero included, exits non-zero above), so nothing was executed. Without this, a `bats` that emits # nothing and exits 0 -- a broken binary, a formatter change, or a wholesale # `@test` removal -- reports "0 tests, 0 failures" and exits green, silently # turning a total harness failure into a pass. diff --git a/tests/run-tests.sh b/tests/run-tests.sh index bf9569b..1211ff2 100755 --- a/tests/run-tests.sh +++ b/tests/run-tests.sh @@ -20,12 +20,22 @@ SKIPPED=() PASSED=0 SKIP_EXIT=77 +# A missing or non-executable run-bats.sh is a hard error, never a silent skip. +# This was `if [[ -x "$BATS" ]]; then ... fi` with no else and no assertion that +# bats ran at all, so renaming, moving, or dropping the executable bit off +# run-bats.sh made the entire bats suite vanish with zero diagnostic and the run +# still printed "Summary: N passed, 0 failed" and exited 0 -- and --bats-only +# degraded to a no-op that printed nothing and exited 0. That is the same +# green-either-way hole run-bats.sh's own zero-count guard closes one level down; +# this closes it in the dispatcher that pre-push actually invokes. run_bats() { - if [[ -x "$BATS" ]]; then - echo "=== bats ===" - bash "$BATS" - echo "" + if [[ ! -x "$BATS" ]]; then + echo "Error: bats runner not found or not executable at $BATS — the bats suite cannot be skipped silently" >&2 + exit 1 fi + echo "=== bats ===" + bash "$BATS" + echo "" } if $BATS_ONLY; then @@ -89,9 +99,16 @@ for script in ${SCRIPTS[@]+"${SCRIPTS[@]}"}; do echo "=== $rel ===" cat "$SCRATCH_ROOT/$idx.log" rc="$(cat "$SCRATCH_ROOT/$idx.status" 2>/dev/null || echo 1)" - if [[ $rc -eq 0 ]]; then + # String comparison, not `-eq`. `-eq` is arithmetic, and bash evaluates an + # empty string as 0 there -- `[[ "" -eq 0 ]]` is true -- so an *empty* status + # file counted as a pass. The `|| echo 1` fallback above only covers a + # *missing* file; a file that exists but is empty is what you get when the job + # is killed between the `>` truncating it and the `echo` completing, or on + # ENOSPC. Under `==` an empty status falls through to FAILED, which is the only + # safe reading of "the job did not report a result". + if [[ "$rc" == "0" ]]; then PASSED=$((PASSED + 1)) - elif [[ $rc -eq $SKIP_EXIT ]]; then + elif [[ "$rc" == "$SKIP_EXIT" ]]; then SKIPPED+=("$rel") else FAILED+=("$rel") diff --git a/tests/test-run-bats.sh b/tests/test-run-bats.sh index f18a86b..c9bd38b 100644 --- a/tests/test-run-bats.sh +++ b/tests/test-run-bats.sh @@ -12,6 +12,14 @@ # # Both branches were code-only and asserted by nothing, which is the same # "green either way" hole the guard itself closes. This file covers them. +# +# It also covers the aggregation those counts feed, which the parallelization +# rewrite left unasserted: three separate mutations of the `if [[ "$file_not_ok" +# -gt 0 || "$status" != "0" ]]` line -- dropping either half, or deleting the +# whole branch -- all survived this file. They survived because every stub here +# exited 0 and emitted no `not ok`, so neither signal was ever exercised, and +# because real bats emits both at once each half masks the other. The cases +# below produce each signal *without* the other, so each mutation dies alone. set -euo pipefail REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" @@ -61,11 +69,21 @@ seed_bats_files() { } # Runs the fixture's run-bats.sh, capturing output and exit code separately. +# +# BATS_FILE_FLOOR is lowered to 1 for every case that is not specifically about +# the floor: these fixtures hold one or two .bats files by design, which is well +# under the real repo's floor. FAKE_FLOOR lets the floor cases opt back into the +# script's own default. TMPDIR is a private per-run directory so a stub can find +# the scratch dir run-bats.sh mktemp'd for itself -- see case 7. FAKE_OUT="" FAKE_RC=0 +FAKE_FLOOR="" run_fake() { + local priv + priv="$(mktemp -d)" + FIXTURES+=("$priv") FAKE_RC=0 - FAKE_OUT="$(bash "$1/tests/run-bats.sh" 2>&1)" || FAKE_RC=$? + FAKE_OUT="$(TMPDIR="$priv" BATS_FILE_FLOOR="${FAKE_FLOOR:-1}" bash "$1/tests/run-bats.sh" 2>&1)" || FAKE_RC=$? } # --- 1. A stub emitting nothing at all is a broken harness, not a clean run --- @@ -165,6 +183,146 @@ else fi fi +# --- 5. `not ok` lines with a zero exit still fail the run. This is the half of +# the aggregation that a bats wrapper swallowing the exit status would leave as +# the only surviving evidence of a failure, and it is the case that kills the +# `drop "$file_not_ok" -gt 0 ||` mutation: without that half the run reports +# "2 tests, 1 failures" and exits 0, calling a failing test suite green. +echo "" +echo "--- a failing test whose process still exits 0 fails the run ---" +DIR5="$(make_fake_repo)" +FIXTURES+=("$DIR5") +seed_bats_files "$DIR5" +install_stub_bats "$DIR5" <<'EOF' +#!/usr/bin/env bash +echo "1..1" +echo "not ok 1 a failing test" +exit 0 +EOF +run_fake "$DIR5" +if [[ $FAKE_RC -eq 0 ]]; then + fail "a 'not ok' TAP result exited 0 — a failing test reported as a pass because only the process status was consulted" +elif echo "$FAKE_OUT" | grep -q "^2 tests, 2 failures$"; then + pass "'not ok' lines fail the run even when every bats process exits 0" +else + fail "the run failed but with the wrong count: $FAKE_OUT" +fi + +# --- 6. A non-zero exit with no `not ok` line still fails the run. This is the +# other half: a crash, a timeout, or an unbound variable in setup_file kills bats +# before it can emit a result line, so the exit status is the only evidence. It +# kills the `drop || "$status" -ne 0` mutation. The stub emits a passing result +# first so the zero-count guard cannot be what fails the run -- without the +# status half this stub reports "2 tests, 0 failures" and exits 0. +echo "" +echo "--- a bats exiting non-zero with no 'not ok' line fails the run ---" +DIR6="$(make_fake_repo)" +FIXTURES+=("$DIR6") +seed_bats_files "$DIR6" +install_stub_bats "$DIR6" <<'EOF' +#!/usr/bin/env bash +echo "1..2" +echo "ok 1 first" +echo "bats: setup_file failed" >&2 +exit 1 +EOF +run_fake "$DIR6" +if [[ $FAKE_RC -eq 0 ]]; then + fail "a bats process exiting 1 was reported as a pass because only the TAP text was consulted" +elif echo "$FAKE_OUT" | grep -q "^2 tests, 0 failures$"; then + pass "a non-zero bats exit fails the run even with no 'not ok' line to find" +else + fail "the run failed but with the wrong count: $FAKE_OUT" +fi + +# --- 7. An empty status file is a failure, not a pass. The status is read back +# with `cat ... || echo 1`, which covers a *missing* file; a file that exists but +# is empty is what a job killed between the `>` truncating it and the `echo` +# completing leaves behind, and what ENOSPC leaves behind. Under the arithmetic +# `-ne` that used to compare it, `[[ "" -ne 0 ]]` is false and the job read as a +# clean exit. +# +# The stub reproduces that state exactly: it emits a healthy TAP stream, creates +# the empty status file itself, then SIGKILLs the subshell that would have +# written the real status. It finds the scratch directory through the private +# TMPDIR run_fake sets -- run-bats.sh mktemp -d's under it, and the log file for +# job 1 is already open by the time the stub runs. +echo "" +echo "--- an empty status file fails the run rather than counting as exit 0 ---" +DIR7="$(make_fake_repo)" +FIXTURES+=("$DIR7") +printf '@test "a" { false; }\n' > "$DIR7/tests/a.bats" +install_stub_bats "$DIR7" <<'EOF' +#!/usr/bin/env bash +echo "1..1" +echo "ok 1 looked fine" +for d in "$TMPDIR"/*/; do + if [[ -e "$d/1.log" ]]; then + : > "$d/1.status" + fi +done +kill -9 $PPID +sleep 5 +EOF +run_fake "$DIR7" +# The count line is asserted alongside the exit code so this can only pass for +# the stated reason: the TAP stream the stub emitted is healthy, so "1 tests, 0 +# failures" proves the zero-count guard did not fire and the empty status is the +# only thing left that can have failed the run. +if ! echo "$FAKE_OUT" | grep -q "^1 tests, 0 failures$"; then + fail "the killed job did not leave the healthy TAP stream the case needs: $FAKE_OUT" +elif [[ $FAKE_RC -eq 0 ]]; then + fail "an empty status file was counted as a clean exit — a killed job reported as a pass" +else + pass "an empty status file fails the run despite a healthy TAP stream" +fi + +# --- 8. Too few discovered .bats files is a hard error, not a green run. Zero +# files used to print a note to stderr and `exit 0`; the floor makes both zero +# and a collapsed-but-nonzero count fail, because a moved tests/ tree or a +# widened `-not -path` exclusion produces exactly that and nothing else notices. +echo "" +echo "--- a discovered-file count under the floor fails the run ---" +DIR8="$(make_fake_repo)" +FIXTURES+=("$DIR8") +seed_bats_files "$DIR8" +install_stub_bats "$DIR8" <<'EOF' +#!/usr/bin/env bash +echo "1..1" +echo "ok 1 first" +exit 0 +EOF +FAKE_FLOOR=8 +run_fake "$DIR8" +FAKE_FLOOR="" +if [[ $FAKE_RC -eq 0 ]]; then + fail "2 .bats files under a floor of 8 passed — a collapsed search path reads as green" +elif echo "$FAKE_OUT" | grep -q "below the floor of 8"; then + pass "a file count under the floor fails and names the floor it missed" +else + fail "the run failed but not with the floor message: $FAKE_OUT" +fi + +# --- 9. Zero discovered .bats files is the same hard error. Kept separate from +# case 8 because it is the state the old `exit 0` branch handled by name, and it +# is the one a path change actually produces. +echo "" +echo "--- zero discovered .bats files fails the run ---" +DIR9="$(make_fake_repo)" +FIXTURES+=("$DIR9") +install_stub_bats "$DIR9" <<'EOF' +#!/usr/bin/env bash +exit 0 +EOF +run_fake "$DIR9" +if [[ $FAKE_RC -eq 0 ]]; then + fail "finding no .bats files at all exited 0 — the whole suite can vanish and the run stays green" +elif echo "$FAKE_OUT" | grep -q "found 0 .bats file"; then + pass "finding no .bats files fails the run and says so" +else + fail "the run failed but not with the zero-files message: $FAKE_OUT" +fi + echo "" echo "Results: $PASS passed, $FAIL failed" [[ $FAIL -eq 0 ]] diff --git a/tests/test-run-tests.sh b/tests/test-run-tests.sh new file mode 100644 index 0000000..3b9f87c --- /dev/null +++ b/tests/test-run-tests.sh @@ -0,0 +1,316 @@ +#!/usr/bin/env bash +# Regression test for tests/run-tests.sh, the dispatcher pre-push actually +# invokes. Nothing tested it at all before this file, and two of the holes that +# left open are the same "green either way" defect class the runner one level +# down (tests/run-bats.sh) had already been fixed for: +# +# * run_bats() was `if [[ -x "$BATS" ]]; then ... fi` with no else. Renaming, +# moving, or dropping the executable bit off run-bats.sh made the entire bats +# suite disappear with no diagnostic while the run printed a green summary and +# exited 0, and turned `--bats-only` into a no-op that printed nothing. +# * The per-script status was compared with `-eq`, which is arithmetic, and bash +# evaluates an empty string as 0 there -- so a status file that existed but was +# empty counted as a pass. +# +# The rest of the cases pin behaviour that already worked, so the two fixes above +# cannot be "fixed" into a blanket failure: a healthy run is still green, a +# non-zero exit is still FAILED, and exit 77 is still SKIPPED rather than either. +set -euo pipefail + +REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +RUN_TESTS="$REPO_ROOT/tests/run-tests.sh" +PASS=0 +FAIL=0 + +pass() { echo " PASS: $1"; PASS=$((PASS + 1)); } +fail() { echo " FAIL: $1"; FAIL=$((FAIL + 1)); } + +FIXTURES=() +cleanup() { [[ ${#FIXTURES[@]} -eq 0 ]] || rm -rf "${FIXTURES[@]}"; } +trap cleanup EXIT + +# Builds a throwaway tree that a copy of run-tests.sh resolves as its own +# REPO_ROOT (it derives that from its own location), so these cases drive the real +# script with a stub bats runner and a stub set of test-*.sh scripts. The fixtures +# live under TMPDIR, never inside the repo, so the real suite cannot pick the case +# scripts up as tests of its own. +# +# Prints the directory and does NOT register it for cleanup -- every caller uses +# `$(make_fake_repo)`, so an append made in here would land in the command +# substitution's subshell and be lost. Registration is the caller's job. Same +# convention as tests/test-run-bats.sh. +make_fake_repo() { + local dir + dir="$(mktemp -d)" + mkdir -p "$dir/tests" "$dir/scripts/lib" "$dir/cases" + cp "$REPO_ROOT/scripts/lib/batch-run.sh" "$dir/scripts/lib/batch-run.sh" + cp "$RUN_TESTS" "$dir/tests/run-tests.sh" + echo "$dir" +} + +# Writes a stub tests/run-bats.sh from stdin, executable. Every case that is not +# about the bats runner installs the healthy one so the bats leg is a constant. +install_stub_bats_runner() { + cat > "$1/tests/run-bats.sh" + chmod +x "$1/tests/run-bats.sh" +} +install_healthy_bats_runner() { + install_stub_bats_runner "$1" <<'EOF' +#!/usr/bin/env bash +echo "12 tests, 0 failures" +exit 0 +EOF +} + +# Writes one case script into the fixture's TEST_DIR. Name must match test-*.sh +# or run-tests.sh will not discover it. +add_case() { + cat > "$1/cases/$2" +} + +# Runs the fixture's run-tests.sh over its cases/ directory, capturing output and +# exit code separately. TMPDIR is private per run so a case script can locate the +# scratch directory run-tests.sh mktemp -d's for itself -- see case 6. +FAKE_OUT="" +FAKE_RC=0 +run_fake() { + local dir="$1" priv + shift + priv="$(mktemp -d)" + FIXTURES+=("$priv") + FAKE_RC=0 + FAKE_OUT="$(TMPDIR="$priv" TEST_DIR="$dir/cases" bash "$dir/tests/run-tests.sh" "$@" 2>&1)" || FAKE_RC=$? +} + +# --- 1. A healthy run is green, runs the bats leg, and says so --- +# The control for cases 2 and 3: it proves those fail because the bats runner is +# unusable, not because run_bats() now fails unconditionally. +echo "" +echo "--- a healthy run passes and reports the bats leg ---" +DIR1="$(make_fake_repo)" +FIXTURES+=("$DIR1") +install_healthy_bats_runner "$DIR1" +add_case "$DIR1" test-ok.sh <<'EOF' +#!/usr/bin/env bash +echo "fine" +EOF +run_fake "$DIR1" +if [[ $FAKE_RC -ne 0 ]]; then + fail "a healthy fixture failed: $FAKE_OUT" +elif ! echo "$FAKE_OUT" | grep -q "^=== bats ===$"; then + fail "a healthy run never announced the bats leg: $FAKE_OUT" +elif echo "$FAKE_OUT" | grep -q "^=== Summary: 1 passed, 0 skipped, 0 failed ===$"; then + pass "a passing case script and a healthy bats runner report 1 passed, 0 failed" +else + fail "a healthy run produced the wrong summary: $FAKE_OUT" +fi + +# --- 2. A non-executable run-bats.sh is a hard error --- +# Reproduced on the real repo before the fix: `chmod -x tests/run-bats.sh && +# bash tests/run-tests.sh` printed "Summary: 1 passed, 0 skipped, 0 failed", +# exited 0, and never mentioned bats -- 166 tests gone with no diagnostic. +echo "" +echo "--- a non-executable run-bats.sh fails the run instead of vanishing ---" +DIR2="$(make_fake_repo)" +FIXTURES+=("$DIR2") +install_healthy_bats_runner "$DIR2" +chmod -x "$DIR2/tests/run-bats.sh" +add_case "$DIR2" test-ok.sh <<'EOF' +#!/usr/bin/env bash +echo "fine" +EOF +run_fake "$DIR2" +if [[ $FAKE_RC -eq 0 ]]; then + fail "a non-executable run-bats.sh exited 0 — the whole bats suite can vanish silently" +elif echo "$FAKE_OUT" | grep -q "bats runner not found or not executable"; then + pass "a non-executable run-bats.sh fails the run and names what is missing" +else + fail "the run failed but not with the missing-runner message: $FAKE_OUT" +fi + +# --- 3. An absent run-bats.sh is the same hard error --- +# The likelier spelling of case 2 in practice: the file is renamed or moved +# rather than losing its mode bit. +echo "" +echo "--- an absent run-bats.sh fails the run ---" +DIR3="$(make_fake_repo)" +FIXTURES+=("$DIR3") +add_case "$DIR3" test-ok.sh <<'EOF' +#!/usr/bin/env bash +echo "fine" +EOF +run_fake "$DIR3" +if [[ $FAKE_RC -eq 0 ]]; then + fail "a missing run-bats.sh exited 0 — a rename deletes the bats suite from the run with no diagnostic" +elif echo "$FAKE_OUT" | grep -q "bats runner not found or not executable"; then + pass "a missing run-bats.sh fails the run and names what is missing" +else + fail "the run failed but not with the missing-runner message: $FAKE_OUT" +fi + +# --- 4. --bats-only with an unusable runner is a hard error, not a silent no-op --- +# This mode has nothing else to run, so the old code path printed nothing at all +# and exited 0 -- the single most misleading form of the same bug. +echo "" +echo "--- --bats-only fails loudly when the runner is missing ---" +DIR4="$(make_fake_repo)" +FIXTURES+=("$DIR4") +add_case "$DIR4" test-ok.sh <<'EOF' +#!/usr/bin/env bash +echo "fine" +EOF +run_fake "$DIR4" --bats-only +if [[ $FAKE_RC -eq 0 ]]; then + fail "--bats-only with no runner exited 0 having printed nothing — a total no-op reported as a pass" +elif echo "$FAKE_OUT" | grep -q "bats runner not found or not executable"; then + pass "--bats-only fails when the runner is missing rather than doing nothing quietly" +else + fail "--bats-only failed but not with the missing-runner message: $FAKE_OUT" +fi + +# --- 5. A failing bats run propagates --- +# run_bats calls the runner under `set -e`, so a red bats suite aborts the whole +# dispatcher. Asserted here so that stays deliberate rather than incidental. +echo "" +echo "--- a failing bats runner fails the whole run ---" +DIR5="$(make_fake_repo)" +FIXTURES+=("$DIR5") +install_stub_bats_runner "$DIR5" <<'EOF' +#!/usr/bin/env bash +echo "3 tests, 1 failures" +exit 1 +EOF +add_case "$DIR5" test-ok.sh <<'EOF' +#!/usr/bin/env bash +echo "fine" +EOF +run_fake "$DIR5" +if [[ $FAKE_RC -eq 0 ]]; then + fail "a bats runner exiting 1 did not fail the dispatcher" +else + pass "a failing bats runner propagates out of run-tests.sh" +fi + +# --- 6. An empty status file is FAILED, not a pass --- +# The status is read back with `cat ... || echo 1`, which covers a *missing* +# file. A file that exists but is empty is what a job killed between the `>` +# truncating it and the `echo` completing leaves behind, and what ENOSPC leaves +# behind. Under the arithmetic `-eq` this used to be compared with, +# `[[ "" -eq 0 ]]` is true and the job counted as a pass. +# +# The case script reproduces that state exactly: it truncates its own status file +# and then SIGKILLs the subshell that would have written the real one. It finds +# the scratch directory through the private TMPDIR run_fake sets -- run-tests.sh +# mktemp -d's under it, and with a single case script the index is always 1. +echo "" +echo "--- an empty status file is reported as FAILED ---" +DIR6="$(make_fake_repo)" +FIXTURES+=("$DIR6") +install_healthy_bats_runner "$DIR6" +add_case "$DIR6" test-empty-status.sh <<'EOF' +#!/usr/bin/env bash +echo "about to be killed mid-write" +for d in "$TMPDIR"/*/; do + if [[ -e "$d/1.log" ]]; then + : > "$d/1.status" + fi +done +kill -9 $PPID +sleep 5 +EOF +run_fake "$DIR6" +if echo "$FAKE_OUT" | grep -q "^=== Summary: 1 passed, 0 skipped, 0 failed ===$"; then + fail "an empty status file counted as a pass — a killed job reads as green" +elif [[ $FAKE_RC -eq 0 ]]; then + fail "an empty status file did not fail the run: $FAKE_OUT" +elif echo "$FAKE_OUT" | grep -q "^=== Summary: 0 passed, 0 skipped, 1 failed ===$"; then + pass "an empty status file is counted as FAILED" +else + fail "an empty status file failed the run with the wrong summary: $FAKE_OUT" +fi + +# --- 7. A job killed outright, leaving no status file at all, is FAILED --- +# The sibling of case 6 and the path the `|| echo 1` fallback exists for. Both +# are asserted because the fix to case 6 must not be a change that only happens +# to work when the file is absent. +echo "" +echo "--- a SIGKILLed job with no status file is reported as FAILED ---" +DIR7="$(make_fake_repo)" +FIXTURES+=("$DIR7") +install_healthy_bats_runner "$DIR7" +add_case "$DIR7" test-killed.sh <<'EOF' +#!/usr/bin/env bash +echo "about to be killed" +kill -9 $PPID +sleep 5 +EOF +run_fake "$DIR7" +if [[ $FAKE_RC -eq 0 ]]; then + fail "a SIGKILLed job did not fail the run: $FAKE_OUT" +elif echo "$FAKE_OUT" | grep -q "^=== Summary: 0 passed, 0 skipped, 1 failed ===$"; then + pass "a job killed with no status file written is counted as FAILED" +else + fail "a SIGKILLed job failed the run with the wrong summary: $FAKE_OUT" +fi + +# --- 8. A case script with a syntax error is FAILED --- +# Bash exits 2 on a parse error, which is neither 0 nor the skip code -- the +# case that proves the classification is a three-way split and not "0 or not 0". +echo "" +echo "--- a case script that does not parse is reported as FAILED ---" +DIR8="$(make_fake_repo)" +FIXTURES+=("$DIR8") +install_healthy_bats_runner "$DIR8" +add_case "$DIR8" test-syntax.sh <<'EOF' +#!/usr/bin/env bash +if [ 1 -eq 1 ]; then + echo "never closed" +EOF +run_fake "$DIR8" +if [[ $FAKE_RC -eq 0 ]]; then + fail "a case script with a syntax error did not fail the run: $FAKE_OUT" +elif echo "$FAKE_OUT" | grep -q "^=== Summary: 0 passed, 0 skipped, 1 failed ===$"; then + pass "a case script that fails to parse is counted as FAILED" +else + fail "a syntax error failed the run with the wrong summary: $FAKE_OUT" +fi + +# --- 9. exit 1 is FAILED and exit 77 is SKIPPED, in the same run --- +# One fixture holding both so the split is asserted against a single summary +# line: a skip must not be counted as a pass and must not be counted as a +# failure. +echo "" +echo "--- exit 1 is FAILED and exit 77 is SKIPPED ---" +DIR9="$(make_fake_repo)" +FIXTURES+=("$DIR9") +install_healthy_bats_runner "$DIR9" +add_case "$DIR9" test-a-fails.sh <<'EOF' +#!/usr/bin/env bash +echo "nope" +exit 1 +EOF +add_case "$DIR9" test-b-skips.sh <<'EOF' +#!/usr/bin/env bash +echo "SKIP: a required binary is missing" +exit 77 +EOF +add_case "$DIR9" test-c-passes.sh <<'EOF' +#!/usr/bin/env bash +echo "fine" +EOF +run_fake "$DIR9" +if [[ $FAKE_RC -eq 0 ]]; then + fail "a case script exiting 1 did not fail the run: $FAKE_OUT" +elif ! echo "$FAKE_OUT" | grep -q "^=== Summary: 1 passed, 1 skipped, 1 failed ===$"; then + fail "the pass/skip/fail split was miscounted: $FAKE_OUT" +elif ! echo "$FAKE_OUT" | grep -q "^ test-b-skips.sh$"; then + fail "the skipped script was not named in the skip list: $FAKE_OUT" +elif echo "$FAKE_OUT" | grep -q "^ test-a-fails.sh$"; then + pass "exit 1 is FAILED, exit 77 is SKIPPED, and both are named in their lists" +else + fail "the failed script was not named in the failure list: $FAKE_OUT" +fi + +echo "" +echo "Results: $PASS passed, $FAIL failed" +[[ $FAIL -eq 0 ]]