holocron/tests/test-run-bats.sh

#!/usr/bin/env bash
# Regression test for tests/run-bats.sh's TAP-result accounting.
#
# run-bats.sh runs each .bats file in its own process and aggregates the TAP
# streams. It used to derive its test count from that text without ever asserting
# the count was non-zero, so a `bats` that produced no output and exited 0 was
# reported as "0 tests, 0 failures" with exit 0 -- a total harness failure
# rendered as a clean pass. The guard added for that distinguishes two causes,
# because `1..0` is itself valid TAP: no plan lines at all means bats emitted
# nothing to parse, while plans present with zero results means bats ran fine and
# the files genuinely declare no tests.
#
# Both branches were code-only and asserted by nothing, which is the same
# "green either way" hole the guard itself closes. This file covers them.
set -euo pipefail

REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
RUN_BATS="$REPO_ROOT/tests/run-bats.sh"
PASS=0
FAIL=0

pass() { echo "  PASS: $1"; PASS=$((PASS + 1)); }
fail() { echo "  FAIL: $1"; FAIL=$((FAIL + 1)); }

FIXTURES=()
cleanup() { [[ ${#FIXTURES[@]} -eq 0 ]] || rm -rf "${FIXTURES[@]}"; }
trap cleanup EXIT

# Builds a throwaway tree that a copy of run-bats.sh will resolve as its own
# REPO_ROOT (it derives that from its own location), so these cases drive the
# real script without the repo's actual .bats files being involved. The fixtures
# live under TMPDIR, never inside the repo, so the real suite cannot pick them up.
#
# This prints the directory and does NOT register it for cleanup: every caller
# invokes it as `$(make_fake_repo)`, and an append made in here would land in the
# command substitution's subshell and be lost. Registration is the caller's job.
make_fake_repo() {
  local dir
  dir="$(mktemp -d)"
  mkdir -p "$dir/tests" "$dir/scripts/lib"
  cp "$REPO_ROOT/scripts/lib/batch-run.sh" "$dir/scripts/lib/batch-run.sh"
  cp "$RUN_BATS" "$dir/tests/run-bats.sh"
  echo "$dir"
}

# Writes a stub `bats` from stdin. Executable, so run-bats.sh never falls through
# to its submodule-init branch.
install_stub_bats() {
  mkdir -p "$1/tests/bats/bin"
  cat > "$1/tests/bats/bin/bats"
  chmod +x "$1/tests/bats/bin/bats"
}

# Two .bats files whose tests would fail if anything actually ran them. Their
# content is irrelevant to the stub cases -- what matters is that files exist, so
# run-bats.sh gets past its "no .bats test files found" early exit and the zero
# count it then sees can only have come from the TAP stream.
seed_bats_files() {
  printf '@test "a" { false; }\n' > "$1/tests/a.bats"
  printf '@test "b" { false; }\n' > "$1/tests/b.bats"
}

# Runs the fixture's run-bats.sh, capturing output and exit code separately.
FAKE_OUT=""
FAKE_RC=0
run_fake() {
  FAKE_RC=0
  FAKE_OUT="$(bash "$1/tests/run-bats.sh" 2>&1)" || FAKE_RC=$?
}

# --- 1. A stub emitting nothing at all is a broken harness, not a clean run ---
echo ""
echo "--- a bats that emits no TAP output at all fails the run ---"
DIR1="$(make_fake_repo)"
FIXTURES+=("$DIR1")
seed_bats_files "$DIR1"
install_stub_bats "$DIR1" <<'EOF'
#!/usr/bin/env bash
exit 0
EOF
run_fake "$DIR1"
if [[ $FAKE_RC -eq 0 ]]; then
  fail "a bats emitting nothing exited 0 — a total harness failure reported as a pass"
elif echo "$FAKE_OUT" | grep -q "no TAP output at all"; then
  pass "an empty TAP stream fails the run and names the broken harness"
else
  fail "the run failed but not with the broken-harness message: $FAKE_OUT"
fi

# --- 2. A stub emitting only a plan ran fine but declares no tests ---
# This is what real bats produces for a .bats file with every @test removed, so
# it must fail for a different, accurately-worded reason than case 1.
echo ""
echo "--- a bats emitting only a zero plan fails with the no-tests-declared message ---"
DIR2="$(make_fake_repo)"
FIXTURES+=("$DIR2")
seed_bats_files "$DIR2"
install_stub_bats "$DIR2" <<'EOF'
#!/usr/bin/env bash
echo "1..0"
exit 0
EOF
run_fake "$DIR2"
if [[ $FAKE_RC -eq 0 ]]; then
  fail "every test declaring zero tests exited 0 — wholesale @test removal reported as a pass"
elif echo "$FAKE_OUT" | grep -q "declared 0 tests"; then
  pass "a plan-only TAP stream fails the run and names the removed tests"
elif echo "$FAKE_OUT" | grep -q "no TAP output at all"; then
  fail "a valid '1..0' plan was misreported as a broken harness — the two branches are not distinguished"
else
  fail "the run failed but not with the no-tests-declared message: $FAKE_OUT"
fi

# --- 3. A healthy TAP stream still passes and still counts correctly. The guard
# must not turn into a blanket failure: this is the case that proves the two
# above fail for their stated reason rather than because the guard fails always.
echo ""
echo "--- a healthy TAP stream passes with its full count ---"
DIR3="$(make_fake_repo)"
FIXTURES+=("$DIR3")
seed_bats_files "$DIR3"
install_stub_bats "$DIR3" <<'EOF'
#!/usr/bin/env bash
echo "1..2"
echo "ok 1 first"
echo "ok 2 second"
exit 0
EOF
run_fake "$DIR3"
if [[ $FAKE_RC -ne 0 ]]; then
  fail "a healthy TAP stream was failed by the zero-count guard: $FAKE_OUT"
elif echo "$FAKE_OUT" | grep -q "^4 tests, 0 failures$"; then
  pass "two files reporting two passing tests each aggregate to 4 tests, 0 failures"
else
  fail "a healthy TAP stream produced the wrong count: $FAKE_OUT"
fi

# --- 4. The same shapes out of the real bats binary. The stubs above encode an
# assumption about what real bats emits; this pins that assumption. A genuinely
# empty .bats file yields `1..0` and exit 0, so a suite holding one alongside a
# real test file must still pass -- the zero-count guard fires on the aggregate,
# not per file, and one declared test is enough to clear it.
echo ""
echo "--- an empty .bats file beside a real one still passes under the real bats ---"
if [[ ! -x "$REPO_ROOT/tests/bats/bin/bats" ]]; then
  echo "  SKIP: real bats is not initialized — run tests/run-bats.sh once to fetch the submodule"
else
  DIR4="$(make_fake_repo)"
  FIXTURES+=("$DIR4")
  # Symlinked rather than copied: bats resolves its libexec relative to its own
  # path, so the tree has to stay intact. run-bats.sh excludes */tests/bats/* from
  # its own file search, so bats's bundled .bats suites are not collected here.
  ln -s "$REPO_ROOT/tests/bats" "$DIR4/tests/bats"
  : > "$DIR4/tests/empty.bats"
  printf '@test "a real passing test" { true; }\n' > "$DIR4/tests/real.bats"
  run_fake "$DIR4"
  if [[ $FAKE_RC -ne 0 ]]; then
    fail "an empty .bats file beside a real one failed the run: $FAKE_OUT"
  elif ! echo "$FAKE_OUT" | grep -q "^1\.\.0$"; then
    fail "real bats did not emit '1..0' for an empty file, so the case-2 stub no longer matches it: $FAKE_OUT"
  elif echo "$FAKE_OUT" | grep -q "^1 tests, 0 failures$"; then
    pass "an empty .bats file contributes a '1..0' plan and the suite still passes"
  else
    fail "the real-bats run passed with an unexpected count: $FAKE_OUT"
  fi
fi

echo ""
echo "Results: $PASS passed, $FAIL failed"
[[ $FAIL -eq 0 ]]