holocron/tests/test-run-bats.sh

#!/usr/bin/env bash
# Regression test for tests/run-bats.sh's TAP-result accounting.
#
# run-bats.sh runs each .bats file in its own process and aggregates the TAP
# streams. It used to derive its test count from that text without ever asserting
# the count was non-zero, so a `bats` that produced no output and exited 0 was
# reported as "0 tests, 0 failures" with exit 0 -- a total harness failure
# rendered as a clean pass. The guard added for that distinguishes two causes,
# because `1..0` is itself valid TAP: no plan lines at all means bats emitted
# nothing to parse, while plans present with zero results means bats ran fine and
# the files genuinely declare no tests.
#
# Both branches were code-only and asserted by nothing, which is the same
# "green either way" hole the guard itself closes. This file covers them.
#
# It also covers the aggregation those counts feed, which the parallelization
# rewrite left unasserted: three separate mutations of the `if [[ "$file_not_ok"
# -gt 0 || "$status" != "0" ]]` line -- dropping either half, or deleting the
# whole branch -- all survived this file. They survived because every stub here
# exited 0 and emitted no `not ok`, so neither signal was ever exercised, and
# because real bats emits both at once each half masks the other. The cases
# below produce each signal *without* the other, so each mutation dies alone.
set -euo pipefail

REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
RUN_BATS="$REPO_ROOT/tests/run-bats.sh"
PASS=0
FAIL=0

pass() { echo "  PASS: $1"; PASS=$((PASS + 1)); }
fail() { echo "  FAIL: $1"; FAIL=$((FAIL + 1)); }

FIXTURES=()
cleanup() { [[ ${#FIXTURES[@]} -eq 0 ]] || rm -rf "${FIXTURES[@]}"; }
trap cleanup EXIT

# Builds a throwaway tree that a copy of run-bats.sh will resolve as its own
# REPO_ROOT (it derives that from its own location), so these cases drive the
# real script without the repo's actual .bats files being involved. The fixtures
# live under TMPDIR, never inside the repo, so the real suite cannot pick them up.
#
# This prints the directory and does NOT register it for cleanup: every caller
# invokes it as `$(make_fake_repo)`, and an append made in here would land in the
# command substitution's subshell and be lost. Registration is the caller's job.
make_fake_repo() {
  local dir
  dir="$(mktemp -d)"
  mkdir -p "$dir/tests" "$dir/scripts/lib"
  cp "$REPO_ROOT/scripts/lib/batch-run.sh" "$dir/scripts/lib/batch-run.sh"
  cp "$RUN_BATS" "$dir/tests/run-bats.sh"
  echo "$dir"
}

# Writes a stub `bats` from stdin. Executable, so run-bats.sh never falls through
# to its submodule-init branch.
install_stub_bats() {
  mkdir -p "$1/tests/bats/bin"
  cat > "$1/tests/bats/bin/bats"
  chmod +x "$1/tests/bats/bin/bats"
}

# Two .bats files whose tests would fail if anything actually ran them. Their
# content is irrelevant to the stub cases -- what matters is that files exist, so
# run-bats.sh gets past its "no .bats test files found" early exit and the zero
# count it then sees can only have come from the TAP stream.
seed_bats_files() {
  printf '@test "a" { false; }\n' > "$1/tests/a.bats"
  printf '@test "b" { false; }\n' > "$1/tests/b.bats"
}

# Runs the fixture's run-bats.sh, capturing output and exit code separately.
#
# BATS_FILE_FLOOR is lowered to 1 for every case that is not specifically about
# the floor: these fixtures hold one or two .bats files by design, which is well
# under the real repo's floor. FAKE_FLOOR lets the floor cases opt back into the
# script's own default. TMPDIR is a private per-run directory so a stub can find
# the scratch dir run-bats.sh mktemp'd for itself -- see case 7.
FAKE_OUT=""
FAKE_RC=0
FAKE_FLOOR=""
run_fake() {
  local priv
  priv="$(mktemp -d)"
  FIXTURES+=("$priv")
  FAKE_RC=0
  FAKE_OUT="$(TMPDIR="$priv" BATS_FILE_FLOOR="${FAKE_FLOOR:-1}" bash "$1/tests/run-bats.sh" 2>&1)" || FAKE_RC=$?
}

# --- 1. A stub emitting nothing at all is a broken harness, not a clean run ---
echo ""
echo "--- a bats that emits no TAP output at all fails the run ---"
DIR1="$(make_fake_repo)"
FIXTURES+=("$DIR1")
seed_bats_files "$DIR1"
install_stub_bats "$DIR1" <<'EOF'
#!/usr/bin/env bash
exit 0
EOF
run_fake "$DIR1"
if [[ $FAKE_RC -eq 0 ]]; then
  fail "a bats emitting nothing exited 0 — a total harness failure reported as a pass"
elif echo "$FAKE_OUT" | grep -q "no TAP output at all"; then
  pass "an empty TAP stream fails the run and names the broken harness"
else
  fail "the run failed but not with the broken-harness message: $FAKE_OUT"
fi

# --- 2. A stub emitting only a plan ran fine but declares no tests ---
# This is what real bats produces for a .bats file with every @test removed, so
# it must fail for a different, accurately-worded reason than case 1.
echo ""
echo "--- a bats emitting only a zero plan fails with the no-tests-declared message ---"
DIR2="$(make_fake_repo)"
FIXTURES+=("$DIR2")
seed_bats_files "$DIR2"
install_stub_bats "$DIR2" <<'EOF'
#!/usr/bin/env bash
echo "1..0"
exit 0
EOF
run_fake "$DIR2"
if [[ $FAKE_RC -eq 0 ]]; then
  fail "every test declaring zero tests exited 0 — wholesale @test removal reported as a pass"
elif echo "$FAKE_OUT" | grep -q "declared 0 tests"; then
  pass "a plan-only TAP stream fails the run and names the removed tests"
elif echo "$FAKE_OUT" | grep -q "no TAP output at all"; then
  fail "a valid '1..0' plan was misreported as a broken harness — the two branches are not distinguished"
else
  fail "the run failed but not with the no-tests-declared message: $FAKE_OUT"
fi

# --- 3. A healthy TAP stream still passes and still counts correctly. The guard
# must not turn into a blanket failure: this is the case that proves the two
# above fail for their stated reason rather than because the guard fails always.
echo ""
echo "--- a healthy TAP stream passes with its full count ---"
DIR3="$(make_fake_repo)"
FIXTURES+=("$DIR3")
seed_bats_files "$DIR3"
install_stub_bats "$DIR3" <<'EOF'
#!/usr/bin/env bash
echo "1..2"
echo "ok 1 first"
echo "ok 2 second"
exit 0
EOF
run_fake "$DIR3"
if [[ $FAKE_RC -ne 0 ]]; then
  fail "a healthy TAP stream was failed by the zero-count guard: $FAKE_OUT"
elif echo "$FAKE_OUT" | grep -q "^4 tests, 0 failures$"; then
  pass "two files reporting two passing tests each aggregate to 4 tests, 0 failures"
else
  fail "a healthy TAP stream produced the wrong count: $FAKE_OUT"
fi

# --- 4. The same shapes out of the real bats binary. The stubs above encode an
# assumption about what real bats emits; this pins that assumption. A genuinely
# empty .bats file yields `1..0` and exit 0, so a suite holding one alongside a
# real test file must still pass -- the zero-count guard fires on the aggregate,
# not per file, and one declared test is enough to clear it.
echo ""
echo "--- an empty .bats file beside a real one still passes under the real bats ---"
if [[ ! -x "$REPO_ROOT/tests/bats/bin/bats" ]]; then
  echo "  SKIP: real bats is not initialized — run tests/run-bats.sh once to fetch the submodule"
else
  DIR4="$(make_fake_repo)"
  FIXTURES+=("$DIR4")
  # Symlinked rather than copied: bats resolves its libexec relative to its own
  # path, so the tree has to stay intact. run-bats.sh excludes */tests/bats/* from
  # its own file search, so bats's bundled .bats suites are not collected here.
  ln -s "$REPO_ROOT/tests/bats" "$DIR4/tests/bats"
  : > "$DIR4/tests/empty.bats"
  printf '@test "a real passing test" { true; }\n' > "$DIR4/tests/real.bats"
  run_fake "$DIR4"
  if [[ $FAKE_RC -ne 0 ]]; then
    fail "an empty .bats file beside a real one failed the run: $FAKE_OUT"
  elif ! echo "$FAKE_OUT" | grep -q "^1\.\.0$"; then
    fail "real bats did not emit '1..0' for an empty file, so the case-2 stub no longer matches it: $FAKE_OUT"
  elif echo "$FAKE_OUT" | grep -q "^1 tests, 0 failures$"; then
    pass "an empty .bats file contributes a '1..0' plan and the suite still passes"
  else
    fail "the real-bats run passed with an unexpected count: $FAKE_OUT"
  fi
fi

# --- 5. `not ok` lines with a zero exit still fail the run. This is the half of
# the aggregation that a bats wrapper swallowing the exit status would leave as
# the only surviving evidence of a failure, and it is the case that kills the
# `drop "$file_not_ok" -gt 0 ||` mutation: without that half the run reports
# "2 tests, 1 failures" and exits 0, calling a failing test suite green.
echo ""
echo "--- a failing test whose process still exits 0 fails the run ---"
DIR5="$(make_fake_repo)"
FIXTURES+=("$DIR5")
seed_bats_files "$DIR5"
install_stub_bats "$DIR5" <<'EOF'
#!/usr/bin/env bash
echo "1..1"
echo "not ok 1 a failing test"
exit 0
EOF
run_fake "$DIR5"
if [[ $FAKE_RC -eq 0 ]]; then
  fail "a 'not ok' TAP result exited 0 — a failing test reported as a pass because only the process status was consulted"
elif echo "$FAKE_OUT" | grep -q "^2 tests, 2 failures$"; then
  pass "'not ok' lines fail the run even when every bats process exits 0"
else
  fail "the run failed but with the wrong count: $FAKE_OUT"
fi

# --- 6. A non-zero exit with no `not ok` line still fails the run. This is the
# other half: a crash, a timeout, or an unbound variable in setup_file kills bats
# before it can emit a result line, so the exit status is the only evidence. It
# kills the `drop || "$status" -ne 0` mutation. The stub emits a passing result
# first so the zero-count guard cannot be what fails the run -- without the
# status half this stub reports "2 tests, 0 failures" and exits 0.
echo ""
echo "--- a bats exiting non-zero with no 'not ok' line fails the run ---"
DIR6="$(make_fake_repo)"
FIXTURES+=("$DIR6")
seed_bats_files "$DIR6"
install_stub_bats "$DIR6" <<'EOF'
#!/usr/bin/env bash
echo "1..2"
echo "ok 1 first"
echo "bats: setup_file failed" >&2
exit 1
EOF
run_fake "$DIR6"
if [[ $FAKE_RC -eq 0 ]]; then
  fail "a bats process exiting 1 was reported as a pass because only the TAP text was consulted"
elif echo "$FAKE_OUT" | grep -q "^2 tests, 0 failures$"; then
  pass "a non-zero bats exit fails the run even with no 'not ok' line to find"
else
  fail "the run failed but with the wrong count: $FAKE_OUT"
fi

# --- 7. An empty status file is a failure, not a pass. The status is read back
# with `cat ... || echo 1`, which covers a *missing* file; a file that exists but
# is empty is what a job killed between the `>` truncating it and the `echo`
# completing leaves behind, and what ENOSPC leaves behind. Under the arithmetic
# `-ne` that used to compare it, `[[ "" -ne 0 ]]` is false and the job read as a
# clean exit.
#
# The stub reproduces that state exactly: it emits a healthy TAP stream, creates
# the empty status file itself, then SIGKILLs the subshell that would have
# written the real status. It finds the scratch directory through the private
# TMPDIR run_fake sets -- run-bats.sh mktemp -d's under it, and the log file for
# job 1 is already open by the time the stub runs.
echo ""
echo "--- an empty status file fails the run rather than counting as exit 0 ---"
DIR7="$(make_fake_repo)"
FIXTURES+=("$DIR7")
printf '@test "a" { false; }\n' > "$DIR7/tests/a.bats"
install_stub_bats "$DIR7" <<'EOF'
#!/usr/bin/env bash
echo "1..1"
echo "ok 1 looked fine"
for d in "$TMPDIR"/*/; do
    if [[ -e "$d/1.log" ]]; then
        : > "$d/1.status"
    fi
done
kill -9 $PPID
sleep 5
EOF
run_fake "$DIR7"
# The count line is asserted alongside the exit code so this can only pass for
# the stated reason: the TAP stream the stub emitted is healthy, so "1 tests, 0
# failures" proves the zero-count guard did not fire and the empty status is the
# only thing left that can have failed the run.
if ! echo "$FAKE_OUT" | grep -q "^1 tests, 0 failures$"; then
  fail "the killed job did not leave the healthy TAP stream the case needs: $FAKE_OUT"
elif [[ $FAKE_RC -eq 0 ]]; then
  fail "an empty status file was counted as a clean exit — a killed job reported as a pass"
else
  pass "an empty status file fails the run despite a healthy TAP stream"
fi

# --- 8. Too few discovered .bats files is a hard error, not a green run. Zero
# files used to print a note to stderr and `exit 0`; the floor makes both zero
# and a collapsed-but-nonzero count fail, because a moved tests/ tree or a
# widened `-not -path` exclusion produces exactly that and nothing else notices.
echo ""
echo "--- a discovered-file count under the floor fails the run ---"
DIR8="$(make_fake_repo)"
FIXTURES+=("$DIR8")
seed_bats_files "$DIR8"
install_stub_bats "$DIR8" <<'EOF'
#!/usr/bin/env bash
echo "1..1"
echo "ok 1 first"
exit 0
EOF
FAKE_FLOOR=8
run_fake "$DIR8"
FAKE_FLOOR=""
if [[ $FAKE_RC -eq 0 ]]; then
  fail "2 .bats files under a floor of 8 passed — a collapsed search path reads as green"
elif echo "$FAKE_OUT" | grep -q "below the floor of 8"; then
  pass "a file count under the floor fails and names the floor it missed"
else
  fail "the run failed but not with the floor message: $FAKE_OUT"
fi

# --- 9. Zero discovered .bats files is the same hard error. Kept separate from
# case 8 because it is the state the old `exit 0` branch handled by name, and it
# is the one a path change actually produces.
echo ""
echo "--- zero discovered .bats files fails the run ---"
DIR9="$(make_fake_repo)"
FIXTURES+=("$DIR9")
install_stub_bats "$DIR9" <<'EOF'
#!/usr/bin/env bash
exit 0
EOF
run_fake "$DIR9"
if [[ $FAKE_RC -eq 0 ]]; then
  fail "finding no .bats files at all exited 0 — the whole suite can vanish and the run stays green"
elif echo "$FAKE_OUT" | grep -q "found 0 .bats file"; then
  pass "finding no .bats files fails the run and says so"
else
  fail "the run failed but not with the zero-files message: $FAKE_OUT"
fi

echo ""
echo "Results: $PASS passed, $FAIL failed"
[[ $FAIL -eq 0 ]]