holocron/tests/test-run-tests.sh

#!/usr/bin/env bash
# Regression test for tests/run-tests.sh, the dispatcher pre-push actually
# invokes. Nothing tested it at all before this file, and two of the holes that
# left open are the same "green either way" defect class the runner one level
# down (tests/run-bats.sh) had already been fixed for:
#
#   * run_bats() was `if [[ -x "$BATS" ]]; then ... fi` with no else. Renaming,
#     moving, or dropping the executable bit off run-bats.sh made the entire bats
#     suite disappear with no diagnostic while the run printed a green summary and
#     exited 0, and turned `--bats-only` into a no-op that printed nothing.
#   * run_bats() then checked only that run-bats.sh was present and executable,
#     never that it PRODUCED anything. `bash` on an empty run-bats.sh exits 0
#     having printed nothing, so the dispatcher printed `=== bats ===` and a green
#     summary. The runner's `N tests, M failures` line is now required, with a
#     non-zero count.
#   * The per-script status was compared with `-eq`, which is arithmetic, and bash
#     evaluates an empty string as 0 there -- so a status file that existed but was
#     empty counted as a pass.
#
# The rest of the cases pin behaviour that already worked, so the two fixes above
# cannot be "fixed" into a blanket failure: a healthy run is still green, a
# non-zero exit is still FAILED, and exit 77 is still SKIPPED rather than either.
set -euo pipefail

REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
RUN_TESTS="$REPO_ROOT/tests/run-tests.sh"
PASS=0
FAIL=0

pass() { echo "  PASS: $1"; PASS=$((PASS + 1)); }
fail() { echo "  FAIL: $1"; FAIL=$((FAIL + 1)); }

FIXTURES=()
cleanup() { [[ ${#FIXTURES[@]} -eq 0 ]] || rm -rf "${FIXTURES[@]}"; }
trap cleanup EXIT

# Builds a throwaway tree that a copy of run-tests.sh resolves as its own
# REPO_ROOT (it derives that from its own location), so these cases drive the real
# script with a stub bats runner and a stub set of test-*.sh scripts. The fixtures
# live under TMPDIR, never inside the repo, so the real suite cannot pick the case
# scripts up as tests of its own.
#
# Prints the directory and does NOT register it for cleanup -- every caller uses
# `$(make_fake_repo)`, so an append made in here would land in the command
# substitution's subshell and be lost. Registration is the caller's job. Same
# convention as tests/test-run-bats.sh.
make_fake_repo() {
  local dir
  dir="$(mktemp -d)"
  mkdir -p "$dir/tests" "$dir/scripts/lib" "$dir/cases"
  cp "$REPO_ROOT/scripts/lib/batch-run.sh" "$dir/scripts/lib/batch-run.sh"
  cp "$RUN_TESTS" "$dir/tests/run-tests.sh"
  echo "$dir"
}

# Writes a stub tests/run-bats.sh from stdin, executable. Every case that is not
# about the bats runner installs the healthy one so the bats leg is a constant.
install_stub_bats_runner() {
  cat > "$1/tests/run-bats.sh"
  chmod +x "$1/tests/run-bats.sh"
}
install_healthy_bats_runner() {
  install_stub_bats_runner "$1" <<'EOF'
#!/usr/bin/env bash
echo "12 tests, 0 failures"
exit 0
EOF
}

# Writes one case script into the fixture's TEST_DIR. Name must match test-*.sh
# or run-tests.sh will not discover it.
add_case() {
  cat > "$1/cases/$2"
}

# Runs the fixture's run-tests.sh over its cases/ directory, capturing output and
# exit code separately. TMPDIR is private per run so a case script can locate the
# scratch directory run-tests.sh mktemp -d's for itself -- see case 6.
#
# RUN_TESTS_STRICT is REMOVED from the child environment, not merely left alone.
# This suite is itself discovered and run by run-tests.sh, so when the outer run
# is the gate (`bash tests/run-tests.sh --strict`, or RUN_TESTS_STRICT=1 in CI)
# the variable is exported down the whole process tree and every fixture below
# silently became a strict run. Case 10c -- the control asserting a skip is
# tolerated WITHOUT strict -- then failed, and the suite passed ad hoc while
# failing under the exact invocation the run-tests pre-push hook uses. The
# fixture's strictness must be a property of the case, never of how this file
# happened to be launched, so strict is opted into per case: `--strict` on the
# argv (cases 10, 10d, 10f, 10h) or an explicit RUN_TESTS_STRICT=1 on the one
# invocation testing the env var (case 10b). Case 10g pins the scrub itself.
FAKE_OUT=""
FAKE_RC=0
run_fake() {
  local dir="$1" priv
  shift
  priv="$(mktemp -d)"
  FIXTURES+=("$priv")
  FAKE_RC=0
  FAKE_OUT="$(env -u RUN_TESTS_STRICT TMPDIR="$priv" TEST_DIR="$dir/cases" \
    bash "$dir/tests/run-tests.sh" "$@" 2>&1)" || FAKE_RC=$?
}

# --- 1. A healthy run is green, runs the bats leg, and says so ---
# The control for cases 2 and 3: it proves those fail because the bats runner is
# unusable, not because run_bats() now fails unconditionally.
echo ""
echo "--- a healthy run passes and reports the bats leg ---"
DIR1="$(make_fake_repo)"
FIXTURES+=("$DIR1")
install_healthy_bats_runner "$DIR1"
add_case "$DIR1" test-ok.sh <<'EOF'
#!/usr/bin/env bash
echo "fine"
EOF
run_fake "$DIR1"
if [[ $FAKE_RC -ne 0 ]]; then
  fail "a healthy fixture failed: $FAKE_OUT"
elif ! echo "$FAKE_OUT" | grep -q "^=== bats ===$"; then
  fail "a healthy run never announced the bats leg: $FAKE_OUT"
elif echo "$FAKE_OUT" | grep -q "^=== Summary: 1 passed, 0 skipped, 0 failed ===$"; then
  pass "a passing case script and a healthy bats runner report 1 passed, 0 failed"
else
  fail "a healthy run produced the wrong summary: $FAKE_OUT"
fi

# --- 2. A non-executable run-bats.sh is a hard error ---
# Reproduced on the real repo before the fix: `chmod -x tests/run-bats.sh &&
# bash tests/run-tests.sh` printed "Summary: 1 passed, 0 skipped, 0 failed",
# exited 0, and never mentioned bats -- 166 tests gone with no diagnostic.
echo ""
echo "--- a non-executable run-bats.sh fails the run instead of vanishing ---"
DIR2="$(make_fake_repo)"
FIXTURES+=("$DIR2")
install_healthy_bats_runner "$DIR2"
chmod -x "$DIR2/tests/run-bats.sh"
add_case "$DIR2" test-ok.sh <<'EOF'
#!/usr/bin/env bash
echo "fine"
EOF
run_fake "$DIR2"
if [[ $FAKE_RC -eq 0 ]]; then
  fail "a non-executable run-bats.sh exited 0 — the whole bats suite can vanish silently"
elif echo "$FAKE_OUT" | grep -q "bats runner not found or not executable"; then
  pass "a non-executable run-bats.sh fails the run and names what is missing"
else
  fail "the run failed but not with the missing-runner message: $FAKE_OUT"
fi

# --- 3. An absent run-bats.sh is the same hard error ---
# The likelier spelling of case 2 in practice: the file is renamed or moved
# rather than losing its mode bit.
echo ""
echo "--- an absent run-bats.sh fails the run ---"
DIR3="$(make_fake_repo)"
FIXTURES+=("$DIR3")
add_case "$DIR3" test-ok.sh <<'EOF'
#!/usr/bin/env bash
echo "fine"
EOF
run_fake "$DIR3"
if [[ $FAKE_RC -eq 0 ]]; then
  fail "a missing run-bats.sh exited 0 — a rename deletes the bats suite from the run with no diagnostic"
elif echo "$FAKE_OUT" | grep -q "bats runner not found or not executable"; then
  pass "a missing run-bats.sh fails the run and names what is missing"
else
  fail "the run failed but not with the missing-runner message: $FAKE_OUT"
fi

# --- 4. --bats-only with an unusable runner is a hard error, not a silent no-op ---
# This mode has nothing else to run, so the old code path printed nothing at all
# and exited 0 -- the single most misleading form of the same bug.
echo ""
echo "--- --bats-only fails loudly when the runner is missing ---"
DIR4="$(make_fake_repo)"
FIXTURES+=("$DIR4")
add_case "$DIR4" test-ok.sh <<'EOF'
#!/usr/bin/env bash
echo "fine"
EOF
run_fake "$DIR4" --bats-only
if [[ $FAKE_RC -eq 0 ]]; then
  fail "--bats-only with no runner exited 0 having printed nothing — a total no-op reported as a pass"
elif echo "$FAKE_OUT" | grep -q "bats runner not found or not executable"; then
  pass "--bats-only fails when the runner is missing rather than doing nothing quietly"
else
  fail "--bats-only failed but not with the missing-runner message: $FAKE_OUT"
fi

# --- 5. A failing bats run propagates ---
# run_bats calls the runner under `set -e`, so a red bats suite aborts the whole
# dispatcher. Asserted here so that stays deliberate rather than incidental.
echo ""
echo "--- a failing bats runner fails the whole run ---"
DIR5="$(make_fake_repo)"
FIXTURES+=("$DIR5")
install_stub_bats_runner "$DIR5" <<'EOF'
#!/usr/bin/env bash
echo "3 tests, 1 failures"
exit 1
EOF
add_case "$DIR5" test-ok.sh <<'EOF'
#!/usr/bin/env bash
echo "fine"
EOF
run_fake "$DIR5"
if [[ $FAKE_RC -eq 0 ]]; then
  fail "a bats runner exiting 1 did not fail the dispatcher"
else
  pass "a failing bats runner propagates out of run-tests.sh"
fi

# --- 5b. An EMPTY run-bats.sh is a hard error, not a green no-op ---
# Present and executable was still not "it ran". `bash` on a zero-byte script
# exits 0 having printed nothing, so the dispatcher printed `=== bats ===`, a
# blank line, and `Summary: 1 passed, 0 skipped, 0 failed` with rc=0 -- the whole
# bats suite gone, exactly the defect cases 2-4 close for the other spellings.
# Truncation, a partial write, an editor saving an empty buffer, and a `set -e`
# abort in a run-bats.sh preamble all produce this file.
#
# It only failed on the real repo incidentally, because tests/test-run-bats.sh
# copies run-bats.sh into its own fixtures and blows up there; rename or retire
# that file and the hole is live in the gate pre-push invokes. This asserts it
# directly.
echo ""
echo "--- an empty run-bats.sh fails the run instead of passing silently ---"
DIR5B="$(make_fake_repo)"
FIXTURES+=("$DIR5B")
: > "$DIR5B/tests/run-bats.sh"
chmod +x "$DIR5B/tests/run-bats.sh"
add_case "$DIR5B" test-ok.sh <<'EOF'
#!/usr/bin/env bash
echo "fine"
EOF
run_fake "$DIR5B"
if echo "$FAKE_OUT" | grep -q "^=== Summary: 1 passed, 0 skipped, 0 failed ===$"; then
  fail "an empty run-bats.sh produced a green summary — the bats suite vanished with no diagnostic"
elif [[ $FAKE_RC -eq 0 ]]; then
  fail "an empty run-bats.sh exited 0: $FAKE_OUT"
elif echo "$FAKE_OUT" | grep -q "without reporting an 'N tests, M failures' summary"; then
  pass "an empty run-bats.sh fails the run and says the bats suite was never verified"
else
  fail "the run failed but not with the no-summary message: $FAKE_OUT"
fi

# --- 5c. A runner reporting zero tests is a hard error too ---
# The other half of "ran but produced nothing": the summary line is there and the
# process exits 0, but it accounts for no tests. run-bats.sh has its own guard for
# this one file down; asserting it here means the dispatcher does not depend on
# that guard surviving, and it pins the count as the thing being read rather than
# the mere presence of a line matching the pattern.
echo ""
echo "--- a bats runner reporting 0 tests fails the run ---"
DIR5C="$(make_fake_repo)"
FIXTURES+=("$DIR5C")
install_stub_bats_runner "$DIR5C" <<'EOF'
#!/usr/bin/env bash
echo "0 tests, 0 failures"
exit 0
EOF
add_case "$DIR5C" test-ok.sh <<'EOF'
#!/usr/bin/env bash
echo "fine"
EOF
run_fake "$DIR5C"
if [[ $FAKE_RC -eq 0 ]]; then
  fail "a bats runner reporting 0 tests exited 0 — a suite that executed nothing read as green: $FAKE_OUT"
elif echo "$FAKE_OUT" | grep -q "reported 0 tests"; then
  pass "a bats runner reporting 0 tests fails the run and says the suite executed nothing"
else
  fail "the run failed but not with the zero-tests message: $FAKE_OUT"
fi

# --- 6. An empty status file is FAILED, not a pass ---
# The status is read back with `cat ... || echo 1`, which covers a *missing*
# file. A file that exists but is empty is what a job killed between the `>`
# truncating it and the `echo` completing leaves behind, and what ENOSPC leaves
# behind. Under the arithmetic `-eq` this used to be compared with,
# `[[ "" -eq 0 ]]` is true and the job counted as a pass.
#
# The case script reproduces that state exactly: it truncates its own status file
# and then SIGKILLs the subshell that would have written the real one. It finds
# the scratch directory through the private TMPDIR run_fake sets -- run-tests.sh
# mktemp -d's under it, and with a single case script the index is always 1.
echo ""
echo "--- an empty status file is reported as FAILED ---"
DIR6="$(make_fake_repo)"
FIXTURES+=("$DIR6")
install_healthy_bats_runner "$DIR6"
add_case "$DIR6" test-empty-status.sh <<'EOF'
#!/usr/bin/env bash
echo "about to be killed mid-write"
for d in "$TMPDIR"/*/; do
    if [[ -e "$d/1.log" ]]; then
        : > "$d/1.status"
    fi
done
kill -9 $PPID
sleep 5
EOF
run_fake "$DIR6"
if echo "$FAKE_OUT" | grep -q "^=== Summary: 1 passed, 0 skipped, 0 failed ===$"; then
  fail "an empty status file counted as a pass — a killed job reads as green"
elif [[ $FAKE_RC -eq 0 ]]; then
  fail "an empty status file did not fail the run: $FAKE_OUT"
elif echo "$FAKE_OUT" | grep -q "^=== Summary: 0 passed, 0 skipped, 1 failed ===$"; then
  pass "an empty status file is counted as FAILED"
else
  fail "an empty status file failed the run with the wrong summary: $FAKE_OUT"
fi

# --- 7. A job killed outright, leaving no status file at all, is FAILED ---
# The sibling of case 6 and the path the `|| echo 1` fallback exists for. Both
# are asserted because the fix to case 6 must not be a change that only happens
# to work when the file is absent.
echo ""
echo "--- a SIGKILLed job with no status file is reported as FAILED ---"
DIR7="$(make_fake_repo)"
FIXTURES+=("$DIR7")
install_healthy_bats_runner "$DIR7"
add_case "$DIR7" test-killed.sh <<'EOF'
#!/usr/bin/env bash
echo "about to be killed"
kill -9 $PPID
sleep 5
EOF
run_fake "$DIR7"
if [[ $FAKE_RC -eq 0 ]]; then
  fail "a SIGKILLed job did not fail the run: $FAKE_OUT"
elif echo "$FAKE_OUT" | grep -q "^=== Summary: 0 passed, 0 skipped, 1 failed ===$"; then
  pass "a job killed with no status file written is counted as FAILED"
else
  fail "a SIGKILLed job failed the run with the wrong summary: $FAKE_OUT"
fi

# --- 8. A case script with a syntax error is FAILED ---
# Bash exits 2 on a parse error, which is neither 0 nor the skip code -- the
# case that proves the classification is a three-way split and not "0 or not 0".
echo ""
echo "--- a case script that does not parse is reported as FAILED ---"
DIR8="$(make_fake_repo)"
FIXTURES+=("$DIR8")
install_healthy_bats_runner "$DIR8"
add_case "$DIR8" test-syntax.sh <<'EOF'
#!/usr/bin/env bash
if [ 1 -eq 1 ]; then
  echo "never closed"
EOF
run_fake "$DIR8"
if [[ $FAKE_RC -eq 0 ]]; then
  fail "a case script with a syntax error did not fail the run: $FAKE_OUT"
elif echo "$FAKE_OUT" | grep -q "^=== Summary: 0 passed, 0 skipped, 1 failed ===$"; then
  pass "a case script that fails to parse is counted as FAILED"
else
  fail "a syntax error failed the run with the wrong summary: $FAKE_OUT"
fi

# --- 9. exit 1 is FAILED and exit 77 is SKIPPED, in the same run ---
# One fixture holding both so the split is asserted against a single summary
# line: a skip must not be counted as a pass and must not be counted as a
# failure.
echo ""
echo "--- exit 1 is FAILED and exit 77 is SKIPPED ---"
DIR9="$(make_fake_repo)"
FIXTURES+=("$DIR9")
install_healthy_bats_runner "$DIR9"
add_case "$DIR9" test-a-fails.sh <<'EOF'
#!/usr/bin/env bash
echo "nope"
exit 1
EOF
add_case "$DIR9" test-b-skips.sh <<'EOF'
#!/usr/bin/env bash
echo "SKIP: a required binary is missing"
exit 77
EOF
add_case "$DIR9" test-c-passes.sh <<'EOF'
#!/usr/bin/env bash
echo "fine"
EOF
run_fake "$DIR9"
if [[ $FAKE_RC -eq 0 ]]; then
  fail "a case script exiting 1 did not fail the run: $FAKE_OUT"
elif ! echo "$FAKE_OUT" | grep -q "^=== Summary: 1 passed, 1 skipped, 1 failed ===$"; then
  fail "the pass/skip/fail split was miscounted: $FAKE_OUT"
elif ! echo "$FAKE_OUT" | grep -q "^  test-b-skips.sh$"; then
  fail "the skipped script was not named in the skip list: $FAKE_OUT"
elif echo "$FAKE_OUT" | grep -q "^  test-a-fails.sh$"; then
  pass "exit 1 is FAILED, exit 77 is SKIPPED, and both are named in their lists"
else
  fail "the failed script was not named in the failure list: $FAKE_OUT"
fi

# --- 10. --strict turns a skip into a failure, and names the suite AND the reason ---
# Graceful skipping is right for an ad-hoc run and wrong for a gate. At pre-push a
# suite exiting 77 means a dependency README.md's Prerequisites table documents
# as required is missing on the pushing machine -- and pre-commit prints nothing
# at all for a passing hook, so the skip list this script writes to stdout was
# swallowed whole. A vale-less PATH once shipped a green gate having verified
# 15 of the 17 suites that existed then.
#
# The reason is asserted, not just the name: "something was skipped" leaves the
# reader with no idea which binary to install, which is most of why the swallowed
# list was worth so little in the first place. Matched on the SIX-SPACE INDENT the
# report writes, not on the reason text alone -- the suite's own log is echoed
# back verbatim earlier in the same output, so a bare text match passes even with
# the reason capture deleted. Verified: narrowing the capture to the `SKIP:`
# prefix left the loose form green.
echo ""
echo "--- --strict fails the run on a skipped suite and names it with its reason ---"
DIR10="$(make_fake_repo)"
FIXTURES+=("$DIR10")
install_healthy_bats_runner "$DIR10"
add_case "$DIR10" test-needs-a-binary.sh <<'EOF'
#!/usr/bin/env bash
echo "SKIP: frobnicator is not installed — install it from https://example.invalid"
exit 77
EOF
add_case "$DIR10" test-ok.sh <<'EOF'
#!/usr/bin/env bash
echo "fine"
EOF
run_fake "$DIR10" --strict
if [[ $FAKE_RC -eq 0 ]]; then
  fail "--strict passed with a skipped suite — the gate reports green having verified less than it ran: $FAKE_OUT"
elif ! echo "$FAKE_OUT" | grep -q "a skip is a SETUP ERROR"; then
  fail "--strict failed but never said a skip is a setup error: $FAKE_OUT"
elif ! echo "$FAKE_OUT" | grep -q "test-needs-a-binary.sh"; then
  fail "--strict failed without naming the skipped suite: $FAKE_OUT"
elif echo "$FAKE_OUT" | grep -q "^      SKIP: frobnicator is not installed"; then
  pass "--strict fails on a skip, names the suite, and carries through the reason it printed"
else
  fail "--strict named the suite but swallowed its skip reason: $FAKE_OUT"
fi

# --- 10b. RUN_TESTS_STRICT=1 is the same switch. The hook uses the flag because
# it is self-documenting in .pre-commit-config.yaml; the env var exists for a CI
# runner that cannot edit the command line. Both are asserted so one cannot rot.
echo ""
echo "--- RUN_TESTS_STRICT=1 fails the run on a skipped suite ---"
STRICT_ENV_OUT=""
STRICT_ENV_RC=0
STRICT_ENV_PRIV="$(mktemp -d)"
FIXTURES+=("$STRICT_ENV_PRIV")
STRICT_ENV_OUT="$(TMPDIR="$STRICT_ENV_PRIV" TEST_DIR="$DIR10/cases" RUN_TESTS_STRICT=1 \
  bash "$DIR10/tests/run-tests.sh" 2>&1)" || STRICT_ENV_RC=$?
if [[ $STRICT_ENV_RC -eq 0 ]]; then
  fail "RUN_TESTS_STRICT=1 passed with a skipped suite: $STRICT_ENV_OUT"
elif echo "$STRICT_ENV_OUT" | grep -q "a skip is a SETUP ERROR"; then
  pass "RUN_TESTS_STRICT=1 is the same gate as --strict"
else
  fail "RUN_TESTS_STRICT=1 failed for some other reason: $STRICT_ENV_OUT"
fi

# --- 10c. WITHOUT strict, the same fixture still skips gracefully and passes ---
# The control for 10 and 10b, and the half the coordinator asked for explicitly:
# an ad-hoc `bash tests/run-tests.sh` on a laptop missing a dev binary must not
# go red. Without this, "fix the gate" could quietly mean "fail everywhere".
echo ""
echo "--- the same skipped suite passes, still SKIPPED, without strict ---"
run_fake "$DIR10"
if [[ $FAKE_RC -ne 0 ]]; then
  fail "a skipped suite failed a non-strict run — graceful skipping is gone: $FAKE_OUT"
elif ! echo "$FAKE_OUT" | grep -q "^=== Summary: 1 passed, 1 skipped, 0 failed ===$"; then
  fail "a non-strict run miscounted the skip: $FAKE_OUT"
elif echo "$FAKE_OUT" | grep -q "^      SKIP: frobnicator is not installed"; then
  pass "without strict the suite is SKIPPED, the run passes, and the reason is still reported"
else
  fail "a non-strict run passed but dropped the skip reason: $FAKE_OUT"
fi

# --- 10d. --strict does not become a blanket failure ---
# The case that proves 10 and 10b fail for their stated reason. A clean run with
# nothing skipped must be just as green under --strict as without it, otherwise
# the gate is not a gate, it is an outage.
echo ""
echo "--- --strict is still green when nothing skipped ---"
DIR10D="$(make_fake_repo)"
FIXTURES+=("$DIR10D")
install_healthy_bats_runner "$DIR10D"
add_case "$DIR10D" test-ok.sh <<'EOF'
#!/usr/bin/env bash
echo "fine"
EOF
run_fake "$DIR10D" --strict
if [[ $FAKE_RC -ne 0 ]]; then
  fail "--strict failed a run with nothing skipped — it fails unconditionally: $FAKE_OUT"
elif echo "$FAKE_OUT" | grep -q "^=== Summary: 1 passed, 0 skipped, 0 failed ===$"; then
  pass "--strict leaves a run with no skips green"
else
  fail "--strict passed with the wrong summary: $FAKE_OUT"
fi

# --- 10e. An unknown flag is rejected, not ignored ---
# `--strict` reaching this script as a silently-ignored argument is the single
# typo that turns the gate back off while every hook still reports Passed, so the
# arg loop refuses anything it does not know rather than falling through.
echo ""
echo "--- an unrecognised flag fails with usage instead of being ignored ---"
DIR10E="$(make_fake_repo)"
FIXTURES+=("$DIR10E")
install_healthy_bats_runner "$DIR10E"
add_case "$DIR10E" test-ok.sh <<'EOF'
#!/usr/bin/env bash
echo "fine"
EOF
run_fake "$DIR10E" --strickt
if [[ $FAKE_RC -eq 0 ]]; then
  fail "a misspelled flag was ignored and the run passed — a typo silently disarms the gate: $FAKE_OUT"
elif echo "$FAKE_OUT" | grep -q "Usage: .*--bats-only.*--strict"; then
  pass "an unrecognised flag fails the run with usage"
else
  fail "an unrecognised flag failed but not with usage: $FAKE_OUT"
fi

# --- 10f. A skip reason printed on STDERR, with no `SKIP:` prefix, still lands ---
# There is no house format: three suites print `SKIP: <reason>` on stdout and
# tests/test-sync-plugin-content.sh prints `apm not installed -- skipping (...)`
# on stderr. batch-run.sh folds stderr into the same log, so both are reachable,
# but only a fallback chain finds the second one. Without this case the reason
# extraction could be narrowed to the `SKIP:` prefix and the apm suite would fail
# the gate with no indication of what to install.
echo ""
echo "--- a stderr skip reason with no SKIP: prefix is still reported ---"
DIR10F="$(make_fake_repo)"
FIXTURES+=("$DIR10F")
install_healthy_bats_runner "$DIR10F"
add_case "$DIR10F" test-stderr-skip.sh <<'EOF'
#!/usr/bin/env bash
echo "widgetizer not installed -- skipping (see docs)" >&2
exit 77
EOF
run_fake "$DIR10F" --strict
if [[ $FAKE_RC -eq 0 ]]; then
  fail "--strict passed on a suite that skipped via stderr: $FAKE_OUT"
elif echo "$FAKE_OUT" | grep -q "^      widgetizer not installed -- skipping"; then
  pass "a skip reason printed to stderr without a SKIP: prefix is still carried into the failure"
else
  fail "--strict failed but lost the stderr skip reason: $FAKE_OUT"
fi

# --- 10g. An ambient RUN_TESTS_STRICT=1 must not reach a fixture that did not ask
# for it. This suite is discovered and run by run-tests.sh itself, so when the
# outer run is launched with the env-var spelling the variable used to be exported
# into every child here. That was not a hypothetical: `bash tests/run-tests.sh`
# was green while `RUN_TESTS_STRICT=1 bash tests/run-tests.sh` reported this file
# as the one failure, because case 10c's deliberately-non-strict run inherited
# strict and went red.
#
# Two corrections to the record, because both were overstated before:
#   * The blast radius was TWO assertions, not six -- cases 10c and 10g here, and
#     nothing else in the repo reads RUN_TESTS_STRICT.
#   * The pre-push GATE was never red. It runs `bash tests/run-tests.sh --strict`
#     (see .pre-commit-config.yaml), and the flag sets a shell local that is never
#     exported, so the flag spelling never leaked. Only the env-var spelling did.
#
# run-tests.sh now `unset`s the variable immediately after latching it, so the
# leak is closed at its source and the two spellings hand children an identical
# environment (case 10i pins that directly). The `env -u` in run_fake() is kept as
# this suite's own defence-in-depth rather than as the fix.
#
# The variable is exported here rather than passed as a prefix on purpose: a
# prefix (`RUN_TESTS_STRICT=1 run_fake ...`) applies to the function call, and
# `env -u` inside it would strip it either way, so the prefix form cannot tell a
# working scrub from a broken one. Exporting reproduces the real leak -- the
# ambient environment this whole script runs in -- which is the state that broke.
echo ""
echo "--- an ambient RUN_TESTS_STRICT=1 does not leak into a non-strict fixture ---"
export RUN_TESTS_STRICT=1
run_fake "$DIR10"
unset RUN_TESTS_STRICT
if [[ $FAKE_RC -ne 0 ]]; then
  fail "an inherited RUN_TESTS_STRICT=1 turned a non-strict fixture strict — this suite's own result depends on how it was launched: $FAKE_OUT"
elif echo "$FAKE_OUT" | grep -q "a skip is a SETUP ERROR"; then
  fail "the fixture ran strict despite not asking for it: $FAKE_OUT"
elif echo "$FAKE_OUT" | grep -q "^=== Summary: 1 passed, 1 skipped, 0 failed ===$"; then
  pass "an ambient RUN_TESTS_STRICT=1 is scrubbed from fixtures that did not ask for strict"
else
  fail "the scrubbed run produced the wrong summary: $FAKE_OUT"
fi

# --- 10h. Under --strict the skip report goes to STDERR, and the stdout skip list
# is suppressed rather than printed twice ---
# Both halves are deliberate and neither was pinned. The stream matters because
# this block is what a pre-push reader is handed: pre-commit prints nothing for a
# passing hook and shows the failing hook's output, and the runner routes the
# strict diagnostic to stderr so it survives a caller that redirects stdout. The
# suppression matters because without it the same suites are listed twice a few
# lines apart, which is exactly the noise that trains a reader to scroll past the
# list instead of reading it. Every other strict case captures `2>&1`, which
# folds the two streams together and so cannot see either property.
echo ""
echo "--- --strict reports skips on stderr only, without duplicating the stdout list ---"
DIR10H="$(make_fake_repo)"
FIXTURES+=("$DIR10H")
install_healthy_bats_runner "$DIR10H"
add_case "$DIR10H" test-needs-a-binary.sh <<'EOF'
#!/usr/bin/env bash
echo "SKIP: frobnicator is not installed"
exit 77
EOF
H_PRIV="$(mktemp -d)"
FIXTURES+=("$H_PRIV")
H_RC=0
env -u RUN_TESTS_STRICT TMPDIR="$H_PRIV" TEST_DIR="$DIR10H/cases" \
  bash "$DIR10H/tests/run-tests.sh" --strict \
  >"$H_PRIV/stdout" 2>"$H_PRIV/stderr" || H_RC=$?
if [[ $H_RC -eq 0 ]]; then
  fail "--strict passed with a skipped suite"
elif ! grep -q "a skip is a SETUP ERROR" "$H_PRIV/stderr"; then
  fail "the strict skip report is not on stderr — a caller redirecting stdout loses the only explanation of the failure: $(cat "$H_PRIV/stderr")"
elif grep -q "a skip is a SETUP ERROR" "$H_PRIV/stdout"; then
  fail "the strict skip report was also written to stdout"
elif grep -q "^Skipped scripts:$" "$H_PRIV/stdout"; then
  fail "--strict printed the stdout skip list as well as the stderr report — the same suites are named twice"
else
  pass "--strict reports skipped suites on stderr and suppresses the duplicate stdout list"
fi

# --- 10i. The two documented invocations are equivalent in what a CHILD sees ---
# `bash tests/run-tests.sh --strict` and `RUN_TESTS_STRICT=1 bash
# tests/run-tests.sh` are documented as the same switch, and case 10b already
# asserts they produce the same PARENT verdict. That is the weaker half: the two
# differed in the ENVIRONMENT they handed every dispatched test-*.sh, because the
# flag sets a shell local while the env var stayed exported down the whole process
# tree. A dispatched suite could therefore behave differently depending on which
# spelling launched the run above it -- which is how case 10c went red under one
# invocation and green under the other.
#
# So this asks the children directly rather than reading the parent's summary. The
# case script reports whether RUN_TESTS_STRICT is present in its own environment
# AT ALL (`${VAR+set}`, not `${VAR:-}` -- an exported empty value is still a leak),
# and both spellings must report it absent. Equivalence is asserted between the two
# observations, not just against a hardcoded expectation, so the two cannot drift
# apart in some future direction neither case anticipated.
echo ""
echo "--- --strict and RUN_TESTS_STRICT=1 hand children the same environment ---"
DIR10I="$(make_fake_repo)"
FIXTURES+=("$DIR10I")
install_healthy_bats_runner "$DIR10I"
add_case "$DIR10I" test-reports-its-env.sh <<'EOF'
#!/usr/bin/env bash
if [[ -n "${RUN_TESTS_STRICT+set}" ]]; then
  echo "CHILD-SAW-STRICT=[${RUN_TESTS_STRICT}]"
else
  echo "CHILD-SAW-STRICT=<absent>"
fi
EOF
# Flag spelling: run_fake scrubs the ambient variable first, so what the child
# sees here is purely a function of what run-tests.sh itself exports.
run_fake "$DIR10I" --strict
FLAG_CHILD="$(echo "$FAKE_OUT" | grep -o 'CHILD-SAW-STRICT=.*' | head -n 1 || true)"
FLAG_RC=$FAKE_RC
# Env spelling: invoked directly, NOT through run_fake, because run_fake's `env -u`
# would strip the very variable under test.
I_PRIV="$(mktemp -d)"
FIXTURES+=("$I_PRIV")
ENV_RC=0
ENV_OUT="$(TMPDIR="$I_PRIV" TEST_DIR="$DIR10I/cases" RUN_TESTS_STRICT=1 \
  bash "$DIR10I/tests/run-tests.sh" 2>&1)" || ENV_RC=$?
ENV_CHILD="$(echo "$ENV_OUT" | grep -o 'CHILD-SAW-STRICT=.*' | head -n 1 || true)"
if [[ -z "$FLAG_CHILD" || -z "$ENV_CHILD" ]]; then
  fail "the reporting case script never ran under one of the two invocations (flag: '${FLAG_CHILD:-<none>}', env: '${ENV_CHILD:-<none>}')"
elif [[ "$FLAG_CHILD" != "$ENV_CHILD" ]]; then
  fail "the two documented invocations hand children different environments — flag: $FLAG_CHILD, env: $ENV_CHILD"
elif [[ "$ENV_CHILD" != "CHILD-SAW-STRICT=<absent>" ]]; then
  fail "RUN_TESTS_STRICT is still exported to dispatched suites ($ENV_CHILD) — a suite that itself runs run-tests.sh inherits strictness it never asked for"
elif [[ $FLAG_RC -ne 0 || $ENV_RC -ne 0 ]]; then
  fail "a clean fixture failed under one of the two invocations (flag rc=$FLAG_RC, env rc=$ENV_RC): $FAKE_OUT / $ENV_OUT"
else
  pass "--strict and RUN_TESTS_STRICT=1 both dispatch children with RUN_TESTS_STRICT absent"
fi

echo ""
echo "Results: $PASS passed, $FAIL failed"
[[ $FAIL -eq 0 ]]