holocron/tests/test-run-tests.sh

#!/usr/bin/env bash
# Regression test for tests/run-tests.sh, the dispatcher pre-push actually
# invokes. Nothing tested it at all before this file, and two of the holes that
# left open are the same "green either way" defect class the runner one level
# down (tests/run-bats.sh) had already been fixed for:
#
#   * run_bats() was `if [[ -x "$BATS" ]]; then ... fi` with no else. Renaming,
#     moving, or dropping the executable bit off run-bats.sh made the entire bats
#     suite disappear with no diagnostic while the run printed a green summary and
#     exited 0, and turned `--bats-only` into a no-op that printed nothing.
#   * The per-script status was compared with `-eq`, which is arithmetic, and bash
#     evaluates an empty string as 0 there -- so a status file that existed but was
#     empty counted as a pass.
#
# The rest of the cases pin behaviour that already worked, so the two fixes above
# cannot be "fixed" into a blanket failure: a healthy run is still green, a
# non-zero exit is still FAILED, and exit 77 is still SKIPPED rather than either.
set -euo pipefail

REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
RUN_TESTS="$REPO_ROOT/tests/run-tests.sh"
PASS=0
FAIL=0

pass() { echo "  PASS: $1"; PASS=$((PASS + 1)); }
fail() { echo "  FAIL: $1"; FAIL=$((FAIL + 1)); }

FIXTURES=()
cleanup() { [[ ${#FIXTURES[@]} -eq 0 ]] || rm -rf "${FIXTURES[@]}"; }
trap cleanup EXIT

# Builds a throwaway tree that a copy of run-tests.sh resolves as its own
# REPO_ROOT (it derives that from its own location), so these cases drive the real
# script with a stub bats runner and a stub set of test-*.sh scripts. The fixtures
# live under TMPDIR, never inside the repo, so the real suite cannot pick the case
# scripts up as tests of its own.
#
# Prints the directory and does NOT register it for cleanup -- every caller uses
# `$(make_fake_repo)`, so an append made in here would land in the command
# substitution's subshell and be lost. Registration is the caller's job. Same
# convention as tests/test-run-bats.sh.
make_fake_repo() {
  local dir
  dir="$(mktemp -d)"
  mkdir -p "$dir/tests" "$dir/scripts/lib" "$dir/cases"
  cp "$REPO_ROOT/scripts/lib/batch-run.sh" "$dir/scripts/lib/batch-run.sh"
  cp "$RUN_TESTS" "$dir/tests/run-tests.sh"
  echo "$dir"
}

# Writes a stub tests/run-bats.sh from stdin, executable. Every case that is not
# about the bats runner installs the healthy one so the bats leg is a constant.
install_stub_bats_runner() {
  cat > "$1/tests/run-bats.sh"
  chmod +x "$1/tests/run-bats.sh"
}
install_healthy_bats_runner() {
  install_stub_bats_runner "$1" <<'EOF'
#!/usr/bin/env bash
echo "12 tests, 0 failures"
exit 0
EOF
}

# Writes one case script into the fixture's TEST_DIR. Name must match test-*.sh
# or run-tests.sh will not discover it.
add_case() {
  cat > "$1/cases/$2"
}

# Runs the fixture's run-tests.sh over its cases/ directory, capturing output and
# exit code separately. TMPDIR is private per run so a case script can locate the
# scratch directory run-tests.sh mktemp -d's for itself -- see case 6.
FAKE_OUT=""
FAKE_RC=0
run_fake() {
  local dir="$1" priv
  shift
  priv="$(mktemp -d)"
  FIXTURES+=("$priv")
  FAKE_RC=0
  FAKE_OUT="$(TMPDIR="$priv" TEST_DIR="$dir/cases" bash "$dir/tests/run-tests.sh" "$@" 2>&1)" || FAKE_RC=$?
}

# --- 1. A healthy run is green, runs the bats leg, and says so ---
# The control for cases 2 and 3: it proves those fail because the bats runner is
# unusable, not because run_bats() now fails unconditionally.
echo ""
echo "--- a healthy run passes and reports the bats leg ---"
DIR1="$(make_fake_repo)"
FIXTURES+=("$DIR1")
install_healthy_bats_runner "$DIR1"
add_case "$DIR1" test-ok.sh <<'EOF'
#!/usr/bin/env bash
echo "fine"
EOF
run_fake "$DIR1"
if [[ $FAKE_RC -ne 0 ]]; then
  fail "a healthy fixture failed: $FAKE_OUT"
elif ! echo "$FAKE_OUT" | grep -q "^=== bats ===$"; then
  fail "a healthy run never announced the bats leg: $FAKE_OUT"
elif echo "$FAKE_OUT" | grep -q "^=== Summary: 1 passed, 0 skipped, 0 failed ===$"; then
  pass "a passing case script and a healthy bats runner report 1 passed, 0 failed"
else
  fail "a healthy run produced the wrong summary: $FAKE_OUT"
fi

# --- 2. A non-executable run-bats.sh is a hard error ---
# Reproduced on the real repo before the fix: `chmod -x tests/run-bats.sh &&
# bash tests/run-tests.sh` printed "Summary: 1 passed, 0 skipped, 0 failed",
# exited 0, and never mentioned bats -- 166 tests gone with no diagnostic.
echo ""
echo "--- a non-executable run-bats.sh fails the run instead of vanishing ---"
DIR2="$(make_fake_repo)"
FIXTURES+=("$DIR2")
install_healthy_bats_runner "$DIR2"
chmod -x "$DIR2/tests/run-bats.sh"
add_case "$DIR2" test-ok.sh <<'EOF'
#!/usr/bin/env bash
echo "fine"
EOF
run_fake "$DIR2"
if [[ $FAKE_RC -eq 0 ]]; then
  fail "a non-executable run-bats.sh exited 0 — the whole bats suite can vanish silently"
elif echo "$FAKE_OUT" | grep -q "bats runner not found or not executable"; then
  pass "a non-executable run-bats.sh fails the run and names what is missing"
else
  fail "the run failed but not with the missing-runner message: $FAKE_OUT"
fi

# --- 3. An absent run-bats.sh is the same hard error ---
# The likelier spelling of case 2 in practice: the file is renamed or moved
# rather than losing its mode bit.
echo ""
echo "--- an absent run-bats.sh fails the run ---"
DIR3="$(make_fake_repo)"
FIXTURES+=("$DIR3")
add_case "$DIR3" test-ok.sh <<'EOF'
#!/usr/bin/env bash
echo "fine"
EOF
run_fake "$DIR3"
if [[ $FAKE_RC -eq 0 ]]; then
  fail "a missing run-bats.sh exited 0 — a rename deletes the bats suite from the run with no diagnostic"
elif echo "$FAKE_OUT" | grep -q "bats runner not found or not executable"; then
  pass "a missing run-bats.sh fails the run and names what is missing"
else
  fail "the run failed but not with the missing-runner message: $FAKE_OUT"
fi

# --- 4. --bats-only with an unusable runner is a hard error, not a silent no-op ---
# This mode has nothing else to run, so the old code path printed nothing at all
# and exited 0 -- the single most misleading form of the same bug.
echo ""
echo "--- --bats-only fails loudly when the runner is missing ---"
DIR4="$(make_fake_repo)"
FIXTURES+=("$DIR4")
add_case "$DIR4" test-ok.sh <<'EOF'
#!/usr/bin/env bash
echo "fine"
EOF
run_fake "$DIR4" --bats-only
if [[ $FAKE_RC -eq 0 ]]; then
  fail "--bats-only with no runner exited 0 having printed nothing — a total no-op reported as a pass"
elif echo "$FAKE_OUT" | grep -q "bats runner not found or not executable"; then
  pass "--bats-only fails when the runner is missing rather than doing nothing quietly"
else
  fail "--bats-only failed but not with the missing-runner message: $FAKE_OUT"
fi

# --- 5. A failing bats run propagates ---
# run_bats calls the runner under `set -e`, so a red bats suite aborts the whole
# dispatcher. Asserted here so that stays deliberate rather than incidental.
echo ""
echo "--- a failing bats runner fails the whole run ---"
DIR5="$(make_fake_repo)"
FIXTURES+=("$DIR5")
install_stub_bats_runner "$DIR5" <<'EOF'
#!/usr/bin/env bash
echo "3 tests, 1 failures"
exit 1
EOF
add_case "$DIR5" test-ok.sh <<'EOF'
#!/usr/bin/env bash
echo "fine"
EOF
run_fake "$DIR5"
if [[ $FAKE_RC -eq 0 ]]; then
  fail "a bats runner exiting 1 did not fail the dispatcher"
else
  pass "a failing bats runner propagates out of run-tests.sh"
fi

# --- 6. An empty status file is FAILED, not a pass ---
# The status is read back with `cat ... || echo 1`, which covers a *missing*
# file. A file that exists but is empty is what a job killed between the `>`
# truncating it and the `echo` completing leaves behind, and what ENOSPC leaves
# behind. Under the arithmetic `-eq` this used to be compared with,
# `[[ "" -eq 0 ]]` is true and the job counted as a pass.
#
# The case script reproduces that state exactly: it truncates its own status file
# and then SIGKILLs the subshell that would have written the real one. It finds
# the scratch directory through the private TMPDIR run_fake sets -- run-tests.sh
# mktemp -d's under it, and with a single case script the index is always 1.
echo ""
echo "--- an empty status file is reported as FAILED ---"
DIR6="$(make_fake_repo)"
FIXTURES+=("$DIR6")
install_healthy_bats_runner "$DIR6"
add_case "$DIR6" test-empty-status.sh <<'EOF'
#!/usr/bin/env bash
echo "about to be killed mid-write"
for d in "$TMPDIR"/*/; do
    if [[ -e "$d/1.log" ]]; then
        : > "$d/1.status"
    fi
done
kill -9 $PPID
sleep 5
EOF
run_fake "$DIR6"
if echo "$FAKE_OUT" | grep -q "^=== Summary: 1 passed, 0 skipped, 0 failed ===$"; then
  fail "an empty status file counted as a pass — a killed job reads as green"
elif [[ $FAKE_RC -eq 0 ]]; then
  fail "an empty status file did not fail the run: $FAKE_OUT"
elif echo "$FAKE_OUT" | grep -q "^=== Summary: 0 passed, 0 skipped, 1 failed ===$"; then
  pass "an empty status file is counted as FAILED"
else
  fail "an empty status file failed the run with the wrong summary: $FAKE_OUT"
fi

# --- 7. A job killed outright, leaving no status file at all, is FAILED ---
# The sibling of case 6 and the path the `|| echo 1` fallback exists for. Both
# are asserted because the fix to case 6 must not be a change that only happens
# to work when the file is absent.
echo ""
echo "--- a SIGKILLed job with no status file is reported as FAILED ---"
DIR7="$(make_fake_repo)"
FIXTURES+=("$DIR7")
install_healthy_bats_runner "$DIR7"
add_case "$DIR7" test-killed.sh <<'EOF'
#!/usr/bin/env bash
echo "about to be killed"
kill -9 $PPID
sleep 5
EOF
run_fake "$DIR7"
if [[ $FAKE_RC -eq 0 ]]; then
  fail "a SIGKILLed job did not fail the run: $FAKE_OUT"
elif echo "$FAKE_OUT" | grep -q "^=== Summary: 0 passed, 0 skipped, 1 failed ===$"; then
  pass "a job killed with no status file written is counted as FAILED"
else
  fail "a SIGKILLed job failed the run with the wrong summary: $FAKE_OUT"
fi

# --- 8. A case script with a syntax error is FAILED ---
# Bash exits 2 on a parse error, which is neither 0 nor the skip code -- the
# case that proves the classification is a three-way split and not "0 or not 0".
echo ""
echo "--- a case script that does not parse is reported as FAILED ---"
DIR8="$(make_fake_repo)"
FIXTURES+=("$DIR8")
install_healthy_bats_runner "$DIR8"
add_case "$DIR8" test-syntax.sh <<'EOF'
#!/usr/bin/env bash
if [ 1 -eq 1 ]; then
  echo "never closed"
EOF
run_fake "$DIR8"
if [[ $FAKE_RC -eq 0 ]]; then
  fail "a case script with a syntax error did not fail the run: $FAKE_OUT"
elif echo "$FAKE_OUT" | grep -q "^=== Summary: 0 passed, 0 skipped, 1 failed ===$"; then
  pass "a case script that fails to parse is counted as FAILED"
else
  fail "a syntax error failed the run with the wrong summary: $FAKE_OUT"
fi

# --- 9. exit 1 is FAILED and exit 77 is SKIPPED, in the same run ---
# One fixture holding both so the split is asserted against a single summary
# line: a skip must not be counted as a pass and must not be counted as a
# failure.
echo ""
echo "--- exit 1 is FAILED and exit 77 is SKIPPED ---"
DIR9="$(make_fake_repo)"
FIXTURES+=("$DIR9")
install_healthy_bats_runner "$DIR9"
add_case "$DIR9" test-a-fails.sh <<'EOF'
#!/usr/bin/env bash
echo "nope"
exit 1
EOF
add_case "$DIR9" test-b-skips.sh <<'EOF'
#!/usr/bin/env bash
echo "SKIP: a required binary is missing"
exit 77
EOF
add_case "$DIR9" test-c-passes.sh <<'EOF'
#!/usr/bin/env bash
echo "fine"
EOF
run_fake "$DIR9"
if [[ $FAKE_RC -eq 0 ]]; then
  fail "a case script exiting 1 did not fail the run: $FAKE_OUT"
elif ! echo "$FAKE_OUT" | grep -q "^=== Summary: 1 passed, 1 skipped, 1 failed ===$"; then
  fail "the pass/skip/fail split was miscounted: $FAKE_OUT"
elif ! echo "$FAKE_OUT" | grep -q "^  test-b-skips.sh$"; then
  fail "the skipped script was not named in the skip list: $FAKE_OUT"
elif echo "$FAKE_OUT" | grep -q "^  test-a-fails.sh$"; then
  pass "exit 1 is FAILED, exit 77 is SKIPPED, and both are named in their lists"
else
  fail "the failed script was not named in the failure list: $FAKE_OUT"
fi

echo ""
echo "Results: $PASS passed, $FAIL failed"
[[ $FAIL -eq 0 ]]