#!/usr/bin/env bash # Regression test for tests/run-bats.sh's TAP-result accounting. # # run-bats.sh runs each .bats file in its own process and aggregates the TAP # streams. It used to derive its test count from that text without ever asserting # the count was non-zero, so a `bats` that produced no output and exited 0 was # reported as "0 tests, 0 failures" with exit 0 -- a total harness failure # rendered as a clean pass. The guard added for that distinguishes two causes, # because `1..0` is itself valid TAP: no plan lines at all means bats emitted # nothing to parse, while plans present with zero results means bats ran fine and # the files genuinely declare no tests. # # Both branches were code-only and asserted by nothing, which is the same # "green either way" hole the guard itself closes. This file covers them. set -euo pipefail REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" RUN_BATS="$REPO_ROOT/tests/run-bats.sh" PASS=0 FAIL=0 pass() { echo " PASS: $1"; PASS=$((PASS + 1)); } fail() { echo " FAIL: $1"; FAIL=$((FAIL + 1)); } FIXTURES=() cleanup() { [[ ${#FIXTURES[@]} -eq 0 ]] || rm -rf "${FIXTURES[@]}"; } trap cleanup EXIT # Builds a throwaway tree that a copy of run-bats.sh will resolve as its own # REPO_ROOT (it derives that from its own location), so these cases drive the # real script without the repo's actual .bats files being involved. The fixtures # live under TMPDIR, never inside the repo, so the real suite cannot pick them up. # # This prints the directory and does NOT register it for cleanup: every caller # invokes it as `$(make_fake_repo)`, and an append made in here would land in the # command substitution's subshell and be lost. Registration is the caller's job. make_fake_repo() { local dir dir="$(mktemp -d)" mkdir -p "$dir/tests" "$dir/scripts/lib" cp "$REPO_ROOT/scripts/lib/batch-run.sh" "$dir/scripts/lib/batch-run.sh" cp "$RUN_BATS" "$dir/tests/run-bats.sh" echo "$dir" } # Writes a stub `bats` from stdin. Executable, so run-bats.sh never falls through # to its submodule-init branch. install_stub_bats() { mkdir -p "$1/tests/bats/bin" cat > "$1/tests/bats/bin/bats" chmod +x "$1/tests/bats/bin/bats" } # Two .bats files whose tests would fail if anything actually ran them. Their # content is irrelevant to the stub cases -- what matters is that files exist, so # run-bats.sh gets past its "no .bats test files found" early exit and the zero # count it then sees can only have come from the TAP stream. seed_bats_files() { printf '@test "a" { false; }\n' > "$1/tests/a.bats" printf '@test "b" { false; }\n' > "$1/tests/b.bats" } # Runs the fixture's run-bats.sh, capturing output and exit code separately. FAKE_OUT="" FAKE_RC=0 run_fake() { FAKE_RC=0 FAKE_OUT="$(bash "$1/tests/run-bats.sh" 2>&1)" || FAKE_RC=$? } # --- 1. A stub emitting nothing at all is a broken harness, not a clean run --- echo "" echo "--- a bats that emits no TAP output at all fails the run ---" DIR1="$(make_fake_repo)" FIXTURES+=("$DIR1") seed_bats_files "$DIR1" install_stub_bats "$DIR1" <<'EOF' #!/usr/bin/env bash exit 0 EOF run_fake "$DIR1" if [[ $FAKE_RC -eq 0 ]]; then fail "a bats emitting nothing exited 0 — a total harness failure reported as a pass" elif echo "$FAKE_OUT" | grep -q "no TAP output at all"; then pass "an empty TAP stream fails the run and names the broken harness" else fail "the run failed but not with the broken-harness message: $FAKE_OUT" fi # --- 2. A stub emitting only a plan ran fine but declares no tests --- # This is what real bats produces for a .bats file with every @test removed, so # it must fail for a different, accurately-worded reason than case 1. echo "" echo "--- a bats emitting only a zero plan fails with the no-tests-declared message ---" DIR2="$(make_fake_repo)" FIXTURES+=("$DIR2") seed_bats_files "$DIR2" install_stub_bats "$DIR2" <<'EOF' #!/usr/bin/env bash echo "1..0" exit 0 EOF run_fake "$DIR2" if [[ $FAKE_RC -eq 0 ]]; then fail "every test declaring zero tests exited 0 — wholesale @test removal reported as a pass" elif echo "$FAKE_OUT" | grep -q "declared 0 tests"; then pass "a plan-only TAP stream fails the run and names the removed tests" elif echo "$FAKE_OUT" | grep -q "no TAP output at all"; then fail "a valid '1..0' plan was misreported as a broken harness — the two branches are not distinguished" else fail "the run failed but not with the no-tests-declared message: $FAKE_OUT" fi # --- 3. A healthy TAP stream still passes and still counts correctly. The guard # must not turn into a blanket failure: this is the case that proves the two # above fail for their stated reason rather than because the guard fails always. echo "" echo "--- a healthy TAP stream passes with its full count ---" DIR3="$(make_fake_repo)" FIXTURES+=("$DIR3") seed_bats_files "$DIR3" install_stub_bats "$DIR3" <<'EOF' #!/usr/bin/env bash echo "1..2" echo "ok 1 first" echo "ok 2 second" exit 0 EOF run_fake "$DIR3" if [[ $FAKE_RC -ne 0 ]]; then fail "a healthy TAP stream was failed by the zero-count guard: $FAKE_OUT" elif echo "$FAKE_OUT" | grep -q "^4 tests, 0 failures$"; then pass "two files reporting two passing tests each aggregate to 4 tests, 0 failures" else fail "a healthy TAP stream produced the wrong count: $FAKE_OUT" fi # --- 4. The same shapes out of the real bats binary. The stubs above encode an # assumption about what real bats emits; this pins that assumption. A genuinely # empty .bats file yields `1..0` and exit 0, so a suite holding one alongside a # real test file must still pass -- the zero-count guard fires on the aggregate, # not per file, and one declared test is enough to clear it. echo "" echo "--- an empty .bats file beside a real one still passes under the real bats ---" if [[ ! -x "$REPO_ROOT/tests/bats/bin/bats" ]]; then echo " SKIP: real bats is not initialized — run tests/run-bats.sh once to fetch the submodule" else DIR4="$(make_fake_repo)" FIXTURES+=("$DIR4") # Symlinked rather than copied: bats resolves its libexec relative to its own # path, so the tree has to stay intact. run-bats.sh excludes */tests/bats/* from # its own file search, so bats's bundled .bats suites are not collected here. ln -s "$REPO_ROOT/tests/bats" "$DIR4/tests/bats" : > "$DIR4/tests/empty.bats" printf '@test "a real passing test" { true; }\n' > "$DIR4/tests/real.bats" run_fake "$DIR4" if [[ $FAKE_RC -ne 0 ]]; then fail "an empty .bats file beside a real one failed the run: $FAKE_OUT" elif ! echo "$FAKE_OUT" | grep -q "^1\.\.0$"; then fail "real bats did not emit '1..0' for an empty file, so the case-2 stub no longer matches it: $FAKE_OUT" elif echo "$FAKE_OUT" | grep -q "^1 tests, 0 failures$"; then pass "an empty .bats file contributes a '1..0' plan and the suite still passes" else fail "the real-bats run passed with an unexpected count: $FAKE_OUT" fi fi echo "" echo "Results: $PASS passed, $FAIL failed" [[ $FAIL -eq 0 ]]