holocron/tests/test-run-bats.sh

#!/usr/bin/env bash
# Regression test for tests/run-bats.sh's TAP-result accounting.
#
# run-bats.sh runs each .bats file in its own process and aggregates the TAP
# streams. It used to derive its test count from that text without ever asserting
# the count was non-zero, so a `bats` that produced no output and exited 0 was
# reported as "0 tests, 0 failures" with exit 0 -- a total harness failure
# rendered as a clean pass. The guard added for that distinguishes two causes,
# because `1..0` is itself valid TAP: no plan lines at all means bats emitted
# nothing to parse, while plans present with zero results means bats ran fine and
# the files genuinely declare no tests.
#
# Both branches were code-only and asserted by nothing, which is the same
# "green either way" hole the guard itself closes. This file covers them.
#
# It also covers the aggregation those counts feed, which the parallelization
# rewrite left unasserted: three separate mutations of the `if [[ "$file_not_ok"
# -gt 0 || "$status" != "0" ]]` line -- dropping either half, or deleting the
# whole branch -- all survived this file. They survived because every stub here
# exited 0 and emitted no `not ok`, so neither signal was ever exercised, and
# because real bats emits both at once each half masks the other. The cases
# below produce each signal *without* the other, so each mutation dies alone.
set -euo pipefail

REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
RUN_BATS="$REPO_ROOT/tests/run-bats.sh"
PASS=0
FAIL=0

pass() { echo "  PASS: $1"; PASS=$((PASS + 1)); }
fail() { echo "  FAIL: $1"; FAIL=$((FAIL + 1)); }

FIXTURES=()
cleanup() { [[ ${#FIXTURES[@]} -eq 0 ]] || rm -rf "${FIXTURES[@]}"; }
trap cleanup EXIT

# Builds a throwaway tree that a copy of run-bats.sh will resolve as its own
# REPO_ROOT (it derives that from its own location), so these cases drive the
# real script without the repo's actual .bats files being involved. The fixtures
# live under TMPDIR, never inside the repo, so the real suite cannot pick them up.
#
# This prints the directory and does NOT register it for cleanup: every caller
# invokes it as `$(make_fake_repo)`, and an append made in here would land in the
# command substitution's subshell and be lost. Registration is the caller's job.
make_fake_repo() {
  local dir
  dir="$(mktemp -d)"
  mkdir -p "$dir/tests" "$dir/scripts/lib"
  cp "$REPO_ROOT/scripts/lib/batch-run.sh" "$dir/scripts/lib/batch-run.sh"
  cp "$RUN_BATS" "$dir/tests/run-bats.sh"
  echo "$dir"
}

# Writes a stub `bats` from stdin. Executable, so run-bats.sh never falls through
# to its submodule-init branch.
install_stub_bats() {
  mkdir -p "$1/tests/bats/bin"
  cat > "$1/tests/bats/bin/bats"
  chmod +x "$1/tests/bats/bin/bats"
}

# Two .bats files whose tests would fail if anything actually ran them. Their
# content is irrelevant to the stub cases -- what matters is that files exist, so
# run-bats.sh gets past its "no .bats test files found" early exit and the zero
# count it then sees can only have come from the TAP stream.
seed_bats_files() {
  printf '@test "a" { false; }\n' > "$1/tests/a.bats"
  printf '@test "b" { false; }\n' > "$1/tests/b.bats"
}

# Runs the fixture's run-bats.sh, capturing output and exit code separately.
#
# No file-count knob is passed any more, and none is needed: the expected file
# set is derived from `git ls-files`, and a mktemp fixture is not a git worktree
# root, so run-bats.sh announces that it could not derive an expectation and
# falls back to the unconditional zero-file check. Cases 8-8c drive the derived
# path deliberately by `git init`-ing their fixtures.
#
# TMPDIR is a private per-run directory so a stub can find the scratch dir
# run-bats.sh mktemp'd for itself -- see case 7.
FAKE_OUT=""
FAKE_RC=0
run_fake() {
  local priv
  priv="$(mktemp -d)"
  FIXTURES+=("$priv")
  FAKE_RC=0
  FAKE_OUT="$(TMPDIR="$priv" bash "$1/tests/run-bats.sh" 2>&1)" || FAKE_RC=$?
}

# A fixture whose root IS a git worktree root, so run-bats.sh derives its
# expected set from the index instead of degrading. Only `git add` is used --
# `git ls-files` reads the index, so nothing needs committing and no user
# identity is required.
make_git_fake_repo() {
  local dir
  dir="$(make_fake_repo)"
  git -C "$dir" init -q
  echo "$dir"
}

# --- 1. A stub emitting nothing at all is a broken harness, not a clean run ---
echo ""
echo "--- a bats that emits no TAP output at all fails the run ---"
DIR1="$(make_fake_repo)"
FIXTURES+=("$DIR1")
seed_bats_files "$DIR1"
install_stub_bats "$DIR1" <<'EOF'
#!/usr/bin/env bash
exit 0
EOF
run_fake "$DIR1"
if [[ $FAKE_RC -eq 0 ]]; then
  fail "a bats emitting nothing exited 0 — a total harness failure reported as a pass"
elif grep -q "no TAP output at all" <<< "$FAKE_OUT"; then
  pass "an empty TAP stream fails the run and names the broken harness"
else
  fail "the run failed but not with the broken-harness message: $FAKE_OUT"
fi

# --- 2. A stub emitting only a plan ran fine but declares no tests ---
# This is what real bats produces for a .bats file with every @test removed, so
# it must fail for a different, accurately-worded reason than case 1.
echo ""
echo "--- a bats emitting only a zero plan fails with the no-tests-declared message ---"
DIR2="$(make_fake_repo)"
FIXTURES+=("$DIR2")
seed_bats_files "$DIR2"
install_stub_bats "$DIR2" <<'EOF'
#!/usr/bin/env bash
echo "1..0"
exit 0
EOF
run_fake "$DIR2"
if [[ $FAKE_RC -eq 0 ]]; then
  fail "every test declaring zero tests exited 0 — wholesale @test removal reported as a pass"
elif grep -q "declared 0 tests" <<< "$FAKE_OUT"; then
  pass "a plan-only TAP stream fails the run and names the removed tests"
elif grep -q "no TAP output at all" <<< "$FAKE_OUT"; then
  fail "a valid '1..0' plan was misreported as a broken harness — the two branches are not distinguished"
else
  fail "the run failed but not with the no-tests-declared message: $FAKE_OUT"
fi

# --- 3. A healthy TAP stream still passes and still counts correctly. The guard
# must not turn into a blanket failure: this is the case that proves the two
# above fail for their stated reason rather than because the guard fails always.
echo ""
echo "--- a healthy TAP stream passes with its full count ---"
DIR3="$(make_fake_repo)"
FIXTURES+=("$DIR3")
seed_bats_files "$DIR3"
install_stub_bats "$DIR3" <<'EOF'
#!/usr/bin/env bash
echo "1..2"
echo "ok 1 first"
echo "ok 2 second"
exit 0
EOF
run_fake "$DIR3"
if [[ $FAKE_RC -ne 0 ]]; then
  fail "a healthy TAP stream was failed by the zero-count guard: $FAKE_OUT"
elif grep -q "^4 tests, 0 failures$" <<< "$FAKE_OUT"; then
  pass "two files reporting two passing tests each aggregate to 4 tests, 0 failures"
else
  fail "a healthy TAP stream produced the wrong count: $FAKE_OUT"
fi

# --- 4. The same shapes out of the real bats binary. The stubs above encode an
# assumption about what real bats emits; this pins that assumption. A genuinely
# empty .bats file yields `1..0` and exit 0, so a suite holding one alongside a
# real test file must still pass -- the zero-count guard fires on the aggregate,
# not per file, and one declared test is enough to clear it.
echo ""
echo "--- an empty .bats file beside a real one still passes under the real bats ---"
if [[ ! -x "$REPO_ROOT/tests/bats/bin/bats" ]]; then
  echo "  SKIP: real bats is not initialized — run tests/run-bats.sh once to fetch the submodule"
else
  DIR4="$(make_fake_repo)"
  FIXTURES+=("$DIR4")
  # Symlinked rather than copied: bats resolves its libexec relative to its own
  # path, so the tree has to stay intact. run-bats.sh excludes */tests/bats/* from
  # its own file search, so bats's bundled .bats suites are not collected here.
  ln -s "$REPO_ROOT/tests/bats" "$DIR4/tests/bats"
  : > "$DIR4/tests/empty.bats"
  printf '@test "a real passing test" { true; }\n' > "$DIR4/tests/real.bats"
  run_fake "$DIR4"
  if [[ $FAKE_RC -ne 0 ]]; then
    fail "an empty .bats file beside a real one failed the run: $FAKE_OUT"
  elif ! grep -q "^1\.\.0$" <<< "$FAKE_OUT"; then
    fail "real bats did not emit '1..0' for an empty file, so the case-2 stub no longer matches it: $FAKE_OUT"
  elif grep -q "^1 tests, 0 failures$" <<< "$FAKE_OUT"; then
    pass "an empty .bats file contributes a '1..0' plan and the suite still passes"
  else
    fail "the real-bats run passed with an unexpected count: $FAKE_OUT"
  fi
fi

# --- 5. `not ok` lines with a zero exit still fail the run. This is the half of
# the aggregation that a bats wrapper swallowing the exit status would leave as
# the only surviving evidence of a failure, and it is the case that kills the
# `drop "$file_not_ok" -gt 0 ||` mutation: without that half the run reports
# "2 tests, 1 failures" and exits 0, calling a failing test suite green.
echo ""
echo "--- a failing test whose process still exits 0 fails the run ---"
DIR5="$(make_fake_repo)"
FIXTURES+=("$DIR5")
seed_bats_files "$DIR5"
install_stub_bats "$DIR5" <<'EOF'
#!/usr/bin/env bash
echo "1..1"
echo "not ok 1 a failing test"
exit 0
EOF
run_fake "$DIR5"
if [[ $FAKE_RC -eq 0 ]]; then
  fail "a 'not ok' TAP result exited 0 — a failing test reported as a pass because only the process status was consulted"
elif grep -q "^2 tests, 2 failures$" <<< "$FAKE_OUT"; then
  pass "'not ok' lines fail the run even when every bats process exits 0"
else
  fail "the run failed but with the wrong count: $FAKE_OUT"
fi

# --- 6. A non-zero exit with no `not ok` line still fails the run. This is the
# other half: a crash, a timeout, or an unbound variable in setup_file kills bats
# before it can emit a result line, so the exit status is the only evidence. It
# kills the `drop || "$status" -ne 0` mutation. The stub emits a passing result
# first so the zero-count guard cannot be what fails the run -- without the
# status half this stub reports "2 tests, 0 failures" and exits 0.
echo ""
echo "--- a bats exiting non-zero with no 'not ok' line fails the run ---"
DIR6="$(make_fake_repo)"
FIXTURES+=("$DIR6")
seed_bats_files "$DIR6"
install_stub_bats "$DIR6" <<'EOF'
#!/usr/bin/env bash
echo "1..2"
echo "ok 1 first"
echo "bats: setup_file failed" >&2
exit 1
EOF
run_fake "$DIR6"
if [[ $FAKE_RC -eq 0 ]]; then
  fail "a bats process exiting 1 was reported as a pass because only the TAP text was consulted"
elif grep -q "^2 tests, 0 failures$" <<< "$FAKE_OUT"; then
  pass "a non-zero bats exit fails the run even with no 'not ok' line to find"
else
  fail "the run failed but with the wrong count: $FAKE_OUT"
fi

# --- 7. An empty status file is a failure, not a pass. The status is read back
# with `cat ... || echo 1`, which covers a *missing* file; a file that exists but
# is empty is what a job killed between the `>` truncating it and the `echo`
# completing leaves behind, and what ENOSPC leaves behind. Under the arithmetic
# `-ne` that used to compare it, `[[ "" -ne 0 ]]` is false and the job read as a
# clean exit.
#
# The stub reproduces that state exactly: it emits a healthy TAP stream, creates
# the empty status file itself, then SIGKILLs the subshell that would have
# written the real status. It finds the scratch directory through the private
# TMPDIR run_fake sets -- run-bats.sh mktemp -d's under it, and the log file for
# job 1 is already open by the time the stub runs.
echo ""
echo "--- an empty status file fails the run rather than counting as exit 0 ---"
DIR7="$(make_fake_repo)"
FIXTURES+=("$DIR7")
printf '@test "a" { false; }\n' > "$DIR7/tests/a.bats"
install_stub_bats "$DIR7" <<'EOF'
#!/usr/bin/env bash
echo "1..1"
echo "ok 1 looked fine"
for d in "$TMPDIR"/*/; do
    if [[ -e "$d/1.log" ]]; then
        : > "$d/1.status"
    fi
done
kill -9 $PPID
sleep 5
EOF
run_fake "$DIR7"
# The count line is asserted alongside the exit code so this can only pass for
# the stated reason: the TAP stream the stub emitted is healthy, so "1 tests, 0
# failures" proves the zero-count guard did not fire and the empty status is the
# only thing left that can have failed the run.
if ! grep -q "^1 tests, 0 failures$" <<< "$FAKE_OUT"; then
  fail "the killed job did not leave the healthy TAP stream the case needs: $FAKE_OUT"
elif [[ $FAKE_RC -eq 0 ]]; then
  fail "an empty status file was counted as a clean exit — a killed job reported as a pass"
else
  pass "an empty status file fails the run despite a healthy TAP stream"
fi

# --- 8. A tracked .bats file the walk did not discover is a hard error. This
# replaces a `BATS_FILE_FLOOR=8` guess against a real count of 10 -- two files of
# slack, which is not hypothetical: deleting two real .bats files left the suite
# reporting "155 tests, 0 failures" and exiting 0 with 11 tests silently gone.
# The expectation is now derived from `git ls-files`, so it is exact and needs no
# magic number.
echo ""
echo "--- a tracked .bats file missing from the walk fails the run and names it ---"
DIR8="$(make_git_fake_repo)"
FIXTURES+=("$DIR8")
seed_bats_files "$DIR8"
install_stub_bats "$DIR8" <<'EOF'
#!/usr/bin/env bash
echo "1..1"
echo "ok 1 first"
exit 0
EOF
git -C "$DIR8" add tests/a.bats tests/b.bats
rm "$DIR8/tests/b.bats"
run_fake "$DIR8"
if [[ $FAKE_RC -eq 0 ]]; then
  fail "a tracked .bats file gone from the worktree passed — a deleted suite reads as green"
elif ! grep -q "tracked .bats file(s) were not discovered" <<< "$FAKE_OUT"; then
  fail "the run failed but not with the undiscovered-tracked-file message: $FAKE_OUT"
elif grep -q "^  tests/b.bats$" <<< "$FAKE_OUT"; then
  pass "a tracked .bats file missing from the walk fails the run and names the file"
else
  fail "the run failed without naming the missing file: $FAKE_OUT"
fi

# --- 8b. An UNTRACKED .bats file is not a finding. The derived expectation runs
# one way only: every tracked file must have been discovered, but a discovered
# file need not be tracked. Without this the check would fail on ordinary
# not-yet-committed work, which is how a correct guard gets disabled.
echo ""
echo "--- an untracked new .bats file does not fail the run ---"
DIR8B="$(make_git_fake_repo)"
FIXTURES+=("$DIR8B")
seed_bats_files "$DIR8B"
install_stub_bats "$DIR8B" <<'EOF'
#!/usr/bin/env bash
echo "1..1"
echo "ok 1 first"
exit 0
EOF
git -C "$DIR8B" add tests/a.bats
run_fake "$DIR8B"
if [[ $FAKE_RC -ne 0 ]]; then
  fail "an untracked .bats file was reported as a finding: $FAKE_OUT"
elif grep -q "^2 tests, 0 failures$" <<< "$FAKE_OUT"; then
  pass "an untracked .bats file is run without being demanded of the index"
else
  fail "the untracked-file run passed with the wrong count: $FAKE_OUT"
fi

# --- 8c. A newly added .bats file joins the expectation immediately. This is the
# half a floor can never have: adding files only ever widens a floor's slack,
# while `git add` alone makes the new file required from the next run on, with no
# edit to this script and no number to bump.
echo ""
echo "--- a newly git-added .bats file is required from the next run on ---"
git -C "$DIR8B" add tests/b.bats
rm "$DIR8B/tests/b.bats"
run_fake "$DIR8B"
if [[ $FAKE_RC -eq 0 ]]; then
  fail "the .bats file added to the index a moment ago was not demanded back: $FAKE_OUT"
elif grep -q "^  tests/b.bats$" <<< "$FAKE_OUT"; then
  pass "a file added to the index joins the expected set with no floor to bump"
else
  fail "the run failed but did not name the newly tracked file: $FAKE_OUT"
fi

# --- 8d. Outside a git worktree the run still works, and says the expectation
# could not be derived. That degradation is what every other fixture here relies
# on, and it must be announced rather than silent -- an unannounced fallback is
# how a derived check quietly becomes no check at all on a tarball export.
echo ""
echo "--- a non-git tree runs, and announces that no expectation could be derived ---"
DIR8D="$(make_fake_repo)"
FIXTURES+=("$DIR8D")
seed_bats_files "$DIR8D"
install_stub_bats "$DIR8D" <<'EOF'
#!/usr/bin/env bash
echo "1..1"
echo "ok 1 first"
exit 0
EOF
run_fake "$DIR8D"
if [[ $FAKE_RC -ne 0 ]]; then
  fail "a non-git tree failed the run: $FAKE_OUT"
elif ! grep -q "not a git worktree root" <<< "$FAKE_OUT"; then
  fail "a non-git tree silently skipped the derived expectation with no note: $FAKE_OUT"
elif grep -q "^2 tests, 0 failures$" <<< "$FAKE_OUT"; then
  pass "a non-git tree runs the suite and says the expected set could not be derived"
else
  fail "the non-git run passed with the wrong count: $FAKE_OUT"
fi

# --- 9. Zero discovered .bats files is a hard error regardless, and is checked
# unconditionally rather than through the derived set: a tree with nothing
# tracked at all (a tarball export, a fresh scaffold) must still not run on an
# empty set and call it green. It is the state the old `exit 0` branch handled by
# name, and the one a path change actually produces.
echo ""
echo "--- zero discovered .bats files fails the run ---"
DIR9="$(make_fake_repo)"
FIXTURES+=("$DIR9")
install_stub_bats "$DIR9" <<'EOF'
#!/usr/bin/env bash
exit 0
EOF
run_fake "$DIR9"
if [[ $FAKE_RC -eq 0 ]]; then
  fail "finding no .bats files at all exited 0 — the whole suite can vanish and the run stays green"
elif grep -q "found 0 .bats file" <<< "$FAKE_OUT"; then
  pass "finding no .bats files fails the run and says so"
else
  fail "the run failed but not with the zero-files message: $FAKE_OUT"
fi

echo ""
echo "Results: $PASS passed, $FAIL failed"
[[ $FAIL -eq 0 ]]