holocron/tests/test-run-tests.sh

#!/usr/bin/env bash
# Regression test for tests/run-tests.sh, the dispatcher pre-push actually
# invokes. Nothing tested it at all before this file, and two of the holes that
# left open are the same "green either way" defect class the runner one level
# down (tests/run-bats.sh) had already been fixed for:
#
#   * run_bats() was `if [[ -x "$BATS" ]]; then ... fi` with no else. Renaming,
#     moving, or dropping the executable bit off run-bats.sh made the entire bats
#     suite disappear with no diagnostic while the run printed a green summary and
#     exited 0, and turned `--bats-only` into a no-op that printed nothing.
#   * run_bats() then checked only that run-bats.sh was present and executable,
#     never that it PRODUCED anything. `bash` on an empty run-bats.sh exits 0
#     having printed nothing, so the dispatcher printed `=== bats ===` and a green
#     summary. The runner's `N tests, M failures` line is now required, with a
#     non-zero count.
#   * The per-script status was compared with `-eq`, which is arithmetic, and bash
#     evaluates an empty string as 0 there -- so a status file that existed but was
#     empty counted as a pass.
#
# The rest of the cases pin behaviour that already worked, so the two fixes above
# cannot be "fixed" into a blanket failure: a healthy run is still green, a
# non-zero exit is still FAILED, and exit 77 is still SKIPPED rather than either.
set -euo pipefail

REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
RUN_TESTS="$REPO_ROOT/tests/run-tests.sh"
PASS=0
FAIL=0

pass() { echo "  PASS: $1"; PASS=$((PASS + 1)); }
fail() { echo "  FAIL: $1"; FAIL=$((FAIL + 1)); }

FIXTURES=()
cleanup() { [[ ${#FIXTURES[@]} -eq 0 ]] || rm -rf "${FIXTURES[@]}"; }
trap cleanup EXIT

# Builds a throwaway tree that a copy of run-tests.sh resolves as its own
# REPO_ROOT (it derives that from its own location), so these cases drive the real
# script with a stub bats runner and a stub set of test-*.sh scripts. The fixtures
# live under TMPDIR, never inside the repo, so the real suite cannot pick the case
# scripts up as tests of its own.
#
# Prints the directory and does NOT register it for cleanup -- every caller uses
# `$(make_fake_repo)`, so an append made in here would land in the command
# substitution's subshell and be lost. Registration is the caller's job. Same
# convention as tests/test-run-bats.sh.
make_fake_repo() {
  local dir
  dir="$(mktemp -d)"
  mkdir -p "$dir/tests" "$dir/scripts/lib" "$dir/cases"
  cp "$REPO_ROOT/scripts/lib/batch-run.sh" "$dir/scripts/lib/batch-run.sh"
  cp "$RUN_TESTS" "$dir/tests/run-tests.sh"
  echo "$dir"
}

# Writes a stub tests/run-bats.sh from stdin, executable. Every case that is not
# about the bats runner installs the healthy one so the bats leg is a constant.
install_stub_bats_runner() {
  cat > "$1/tests/run-bats.sh"
  chmod +x "$1/tests/run-bats.sh"
}
install_healthy_bats_runner() {
  install_stub_bats_runner "$1" <<'EOF'
#!/usr/bin/env bash
echo "12 tests, 0 failures"
exit 0
EOF
}

# Writes one case script into the fixture's TEST_DIR. Name must match test-*.sh
# or run-tests.sh will not discover it.
add_case() {
  cat > "$1/cases/$2"
}

# Runs the fixture's run-tests.sh over its cases/ directory, capturing output and
# exit code separately. TMPDIR is private per run so a case script can locate the
# scratch directory run-tests.sh mktemp -d's for itself -- see case 6.
FAKE_OUT=""
FAKE_RC=0
run_fake() {
  local dir="$1" priv
  shift
  priv="$(mktemp -d)"
  FIXTURES+=("$priv")
  FAKE_RC=0
  FAKE_OUT="$(TMPDIR="$priv" TEST_DIR="$dir/cases" bash "$dir/tests/run-tests.sh" "$@" 2>&1)" || FAKE_RC=$?
}

# --- 1. A healthy run is green, runs the bats leg, and says so ---
# The control for cases 2 and 3: it proves those fail because the bats runner is
# unusable, not because run_bats() now fails unconditionally.
echo ""
echo "--- a healthy run passes and reports the bats leg ---"
DIR1="$(make_fake_repo)"
FIXTURES+=("$DIR1")
install_healthy_bats_runner "$DIR1"
add_case "$DIR1" test-ok.sh <<'EOF'
#!/usr/bin/env bash
echo "fine"
EOF
run_fake "$DIR1"
if [[ $FAKE_RC -ne 0 ]]; then
  fail "a healthy fixture failed: $FAKE_OUT"
elif ! echo "$FAKE_OUT" | grep -q "^=== bats ===$"; then
  fail "a healthy run never announced the bats leg: $FAKE_OUT"
elif echo "$FAKE_OUT" | grep -q "^=== Summary: 1 passed, 0 skipped, 0 failed ===$"; then
  pass "a passing case script and a healthy bats runner report 1 passed, 0 failed"
else
  fail "a healthy run produced the wrong summary: $FAKE_OUT"
fi

# --- 2. A non-executable run-bats.sh is a hard error ---
# Reproduced on the real repo before the fix: `chmod -x tests/run-bats.sh &&
# bash tests/run-tests.sh` printed "Summary: 1 passed, 0 skipped, 0 failed",
# exited 0, and never mentioned bats -- 166 tests gone with no diagnostic.
echo ""
echo "--- a non-executable run-bats.sh fails the run instead of vanishing ---"
DIR2="$(make_fake_repo)"
FIXTURES+=("$DIR2")
install_healthy_bats_runner "$DIR2"
chmod -x "$DIR2/tests/run-bats.sh"
add_case "$DIR2" test-ok.sh <<'EOF'
#!/usr/bin/env bash
echo "fine"
EOF
run_fake "$DIR2"
if [[ $FAKE_RC -eq 0 ]]; then
  fail "a non-executable run-bats.sh exited 0 — the whole bats suite can vanish silently"
elif echo "$FAKE_OUT" | grep -q "bats runner not found or not executable"; then
  pass "a non-executable run-bats.sh fails the run and names what is missing"
else
  fail "the run failed but not with the missing-runner message: $FAKE_OUT"
fi

# --- 3. An absent run-bats.sh is the same hard error ---
# The likelier spelling of case 2 in practice: the file is renamed or moved
# rather than losing its mode bit.
echo ""
echo "--- an absent run-bats.sh fails the run ---"
DIR3="$(make_fake_repo)"
FIXTURES+=("$DIR3")
add_case "$DIR3" test-ok.sh <<'EOF'
#!/usr/bin/env bash
echo "fine"
EOF
run_fake "$DIR3"
if [[ $FAKE_RC -eq 0 ]]; then
  fail "a missing run-bats.sh exited 0 — a rename deletes the bats suite from the run with no diagnostic"
elif echo "$FAKE_OUT" | grep -q "bats runner not found or not executable"; then
  pass "a missing run-bats.sh fails the run and names what is missing"
else
  fail "the run failed but not with the missing-runner message: $FAKE_OUT"
fi

# --- 4. --bats-only with an unusable runner is a hard error, not a silent no-op ---
# This mode has nothing else to run, so the old code path printed nothing at all
# and exited 0 -- the single most misleading form of the same bug.
echo ""
echo "--- --bats-only fails loudly when the runner is missing ---"
DIR4="$(make_fake_repo)"
FIXTURES+=("$DIR4")
add_case "$DIR4" test-ok.sh <<'EOF'
#!/usr/bin/env bash
echo "fine"
EOF
run_fake "$DIR4" --bats-only
if [[ $FAKE_RC -eq 0 ]]; then
  fail "--bats-only with no runner exited 0 having printed nothing — a total no-op reported as a pass"
elif echo "$FAKE_OUT" | grep -q "bats runner not found or not executable"; then
  pass "--bats-only fails when the runner is missing rather than doing nothing quietly"
else
  fail "--bats-only failed but not with the missing-runner message: $FAKE_OUT"
fi

# --- 5. A failing bats run propagates ---
# run_bats calls the runner under `set -e`, so a red bats suite aborts the whole
# dispatcher. Asserted here so that stays deliberate rather than incidental.
echo ""
echo "--- a failing bats runner fails the whole run ---"
DIR5="$(make_fake_repo)"
FIXTURES+=("$DIR5")
install_stub_bats_runner "$DIR5" <<'EOF'
#!/usr/bin/env bash
echo "3 tests, 1 failures"
exit 1
EOF
add_case "$DIR5" test-ok.sh <<'EOF'
#!/usr/bin/env bash
echo "fine"
EOF
run_fake "$DIR5"
if [[ $FAKE_RC -eq 0 ]]; then
  fail "a bats runner exiting 1 did not fail the dispatcher"
else
  pass "a failing bats runner propagates out of run-tests.sh"
fi

# --- 5b. An EMPTY run-bats.sh is a hard error, not a green no-op ---
# Present and executable was still not "it ran". `bash` on a zero-byte script
# exits 0 having printed nothing, so the dispatcher printed `=== bats ===`, a
# blank line, and `Summary: 1 passed, 0 skipped, 0 failed` with rc=0 -- the whole
# bats suite gone, exactly the defect cases 2-4 close for the other spellings.
# Truncation, a partial write, an editor saving an empty buffer, and a `set -e`
# abort in a run-bats.sh preamble all produce this file.
#
# It only failed on the real repo incidentally, because tests/test-run-bats.sh
# copies run-bats.sh into its own fixtures and blows up there; rename or retire
# that file and the hole is live in the gate pre-push invokes. This asserts it
# directly.
echo ""
echo "--- an empty run-bats.sh fails the run instead of passing silently ---"
DIR5B="$(make_fake_repo)"
FIXTURES+=("$DIR5B")
: > "$DIR5B/tests/run-bats.sh"
chmod +x "$DIR5B/tests/run-bats.sh"
add_case "$DIR5B" test-ok.sh <<'EOF'
#!/usr/bin/env bash
echo "fine"
EOF
run_fake "$DIR5B"
if echo "$FAKE_OUT" | grep -q "^=== Summary: 1 passed, 0 skipped, 0 failed ===$"; then
  fail "an empty run-bats.sh produced a green summary — the bats suite vanished with no diagnostic"
elif [[ $FAKE_RC -eq 0 ]]; then
  fail "an empty run-bats.sh exited 0: $FAKE_OUT"
elif echo "$FAKE_OUT" | grep -q "without reporting an 'N tests, M failures' summary"; then
  pass "an empty run-bats.sh fails the run and says the bats suite was never verified"
else
  fail "the run failed but not with the no-summary message: $FAKE_OUT"
fi

# --- 5c. A runner reporting zero tests is a hard error too ---
# The other half of "ran but produced nothing": the summary line is there and the
# process exits 0, but it accounts for no tests. run-bats.sh has its own guard for
# this one file down; asserting it here means the dispatcher does not depend on
# that guard surviving, and it pins the count as the thing being read rather than
# the mere presence of a line matching the pattern.
echo ""
echo "--- a bats runner reporting 0 tests fails the run ---"
DIR5C="$(make_fake_repo)"
FIXTURES+=("$DIR5C")
install_stub_bats_runner "$DIR5C" <<'EOF'
#!/usr/bin/env bash
echo "0 tests, 0 failures"
exit 0
EOF
add_case "$DIR5C" test-ok.sh <<'EOF'
#!/usr/bin/env bash
echo "fine"
EOF
run_fake "$DIR5C"
if [[ $FAKE_RC -eq 0 ]]; then
  fail "a bats runner reporting 0 tests exited 0 — a suite that executed nothing read as green: $FAKE_OUT"
elif echo "$FAKE_OUT" | grep -q "reported 0 tests"; then
  pass "a bats runner reporting 0 tests fails the run and says the suite executed nothing"
else
  fail "the run failed but not with the zero-tests message: $FAKE_OUT"
fi

# --- 6. An empty status file is FAILED, not a pass ---
# The status is read back with `cat ... || echo 1`, which covers a *missing*
# file. A file that exists but is empty is what a job killed between the `>`
# truncating it and the `echo` completing leaves behind, and what ENOSPC leaves
# behind. Under the arithmetic `-eq` this used to be compared with,
# `[[ "" -eq 0 ]]` is true and the job counted as a pass.
#
# The case script reproduces that state exactly: it truncates its own status file
# and then SIGKILLs the subshell that would have written the real one. It finds
# the scratch directory through the private TMPDIR run_fake sets -- run-tests.sh
# mktemp -d's under it, and with a single case script the index is always 1.
echo ""
echo "--- an empty status file is reported as FAILED ---"
DIR6="$(make_fake_repo)"
FIXTURES+=("$DIR6")
install_healthy_bats_runner "$DIR6"
add_case "$DIR6" test-empty-status.sh <<'EOF'
#!/usr/bin/env bash
echo "about to be killed mid-write"
for d in "$TMPDIR"/*/; do
    if [[ -e "$d/1.log" ]]; then
        : > "$d/1.status"
    fi
done
kill -9 $PPID
sleep 5
EOF
run_fake "$DIR6"
if echo "$FAKE_OUT" | grep -q "^=== Summary: 1 passed, 0 skipped, 0 failed ===$"; then
  fail "an empty status file counted as a pass — a killed job reads as green"
elif [[ $FAKE_RC -eq 0 ]]; then
  fail "an empty status file did not fail the run: $FAKE_OUT"
elif echo "$FAKE_OUT" | grep -q "^=== Summary: 0 passed, 0 skipped, 1 failed ===$"; then
  pass "an empty status file is counted as FAILED"
else
  fail "an empty status file failed the run with the wrong summary: $FAKE_OUT"
fi

# --- 7. A job killed outright, leaving no status file at all, is FAILED ---
# The sibling of case 6 and the path the `|| echo 1` fallback exists for. Both
# are asserted because the fix to case 6 must not be a change that only happens
# to work when the file is absent.
echo ""
echo "--- a SIGKILLed job with no status file is reported as FAILED ---"
DIR7="$(make_fake_repo)"
FIXTURES+=("$DIR7")
install_healthy_bats_runner "$DIR7"
add_case "$DIR7" test-killed.sh <<'EOF'
#!/usr/bin/env bash
echo "about to be killed"
kill -9 $PPID
sleep 5
EOF
run_fake "$DIR7"
if [[ $FAKE_RC -eq 0 ]]; then
  fail "a SIGKILLed job did not fail the run: $FAKE_OUT"
elif echo "$FAKE_OUT" | grep -q "^=== Summary: 0 passed, 0 skipped, 1 failed ===$"; then
  pass "a job killed with no status file written is counted as FAILED"
else
  fail "a SIGKILLed job failed the run with the wrong summary: $FAKE_OUT"
fi

# --- 8. A case script with a syntax error is FAILED ---
# Bash exits 2 on a parse error, which is neither 0 nor the skip code -- the
# case that proves the classification is a three-way split and not "0 or not 0".
echo ""
echo "--- a case script that does not parse is reported as FAILED ---"
DIR8="$(make_fake_repo)"
FIXTURES+=("$DIR8")
install_healthy_bats_runner "$DIR8"
add_case "$DIR8" test-syntax.sh <<'EOF'
#!/usr/bin/env bash
if [ 1 -eq 1 ]; then
  echo "never closed"
EOF
run_fake "$DIR8"
if [[ $FAKE_RC -eq 0 ]]; then
  fail "a case script with a syntax error did not fail the run: $FAKE_OUT"
elif echo "$FAKE_OUT" | grep -q "^=== Summary: 0 passed, 0 skipped, 1 failed ===$"; then
  pass "a case script that fails to parse is counted as FAILED"
else
  fail "a syntax error failed the run with the wrong summary: $FAKE_OUT"
fi

# --- 9. exit 1 is FAILED and exit 77 is SKIPPED, in the same run ---
# One fixture holding both so the split is asserted against a single summary
# line: a skip must not be counted as a pass and must not be counted as a
# failure.
echo ""
echo "--- exit 1 is FAILED and exit 77 is SKIPPED ---"
DIR9="$(make_fake_repo)"
FIXTURES+=("$DIR9")
install_healthy_bats_runner "$DIR9"
add_case "$DIR9" test-a-fails.sh <<'EOF'
#!/usr/bin/env bash
echo "nope"
exit 1
EOF
add_case "$DIR9" test-b-skips.sh <<'EOF'
#!/usr/bin/env bash
echo "SKIP: a required binary is missing"
exit 77
EOF
add_case "$DIR9" test-c-passes.sh <<'EOF'
#!/usr/bin/env bash
echo "fine"
EOF
run_fake "$DIR9"
if [[ $FAKE_RC -eq 0 ]]; then
  fail "a case script exiting 1 did not fail the run: $FAKE_OUT"
elif ! echo "$FAKE_OUT" | grep -q "^=== Summary: 1 passed, 1 skipped, 1 failed ===$"; then
  fail "the pass/skip/fail split was miscounted: $FAKE_OUT"
elif ! echo "$FAKE_OUT" | grep -q "^  test-b-skips.sh$"; then
  fail "the skipped script was not named in the skip list: $FAKE_OUT"
elif echo "$FAKE_OUT" | grep -q "^  test-a-fails.sh$"; then
  pass "exit 1 is FAILED, exit 77 is SKIPPED, and both are named in their lists"
else
  fail "the failed script was not named in the failure list: $FAKE_OUT"
fi

# --- 10. --strict turns a skip into a failure, and names the suite AND the reason ---
# Graceful skipping is right for an ad-hoc run and wrong for a gate. At pre-push a
# suite exiting 77 means a dependency AGENTS.md documents as required is missing
# on the pushing machine -- and pre-commit prints nothing at all for a passing
# hook, so the skip list this script writes to stdout was swallowed whole. A
# vale-less PATH shipped a green gate having verified 15 of 17 suites.
#
# The reason is asserted, not just the name: "something was skipped" leaves the
# reader with no idea which binary to install, which is most of why the swallowed
# list was worth so little in the first place. Matched on the SIX-SPACE INDENT the
# report writes, not on the reason text alone -- the suite's own log is echoed
# back verbatim earlier in the same output, so a bare text match passes even with
# the reason capture deleted. Verified: narrowing the capture to the `SKIP:`
# prefix left the loose form green.
echo ""
echo "--- --strict fails the run on a skipped suite and names it with its reason ---"
DIR10="$(make_fake_repo)"
FIXTURES+=("$DIR10")
install_healthy_bats_runner "$DIR10"
add_case "$DIR10" test-needs-a-binary.sh <<'EOF'
#!/usr/bin/env bash
echo "SKIP: frobnicator is not installed — install it from https://example.invalid"
exit 77
EOF
add_case "$DIR10" test-ok.sh <<'EOF'
#!/usr/bin/env bash
echo "fine"
EOF
run_fake "$DIR10" --strict
if [[ $FAKE_RC -eq 0 ]]; then
  fail "--strict passed with a skipped suite — the gate reports green having verified less than it ran: $FAKE_OUT"
elif ! echo "$FAKE_OUT" | grep -q "a skip is a SETUP ERROR"; then
  fail "--strict failed but never said a skip is a setup error: $FAKE_OUT"
elif ! echo "$FAKE_OUT" | grep -q "test-needs-a-binary.sh"; then
  fail "--strict failed without naming the skipped suite: $FAKE_OUT"
elif echo "$FAKE_OUT" | grep -q "^      SKIP: frobnicator is not installed"; then
  pass "--strict fails on a skip, names the suite, and carries through the reason it printed"
else
  fail "--strict named the suite but swallowed its skip reason: $FAKE_OUT"
fi

# --- 10b. RUN_TESTS_STRICT=1 is the same switch. The hook uses the flag because
# it is self-documenting in .pre-commit-config.yaml; the env var exists for a CI
# runner that cannot edit the command line. Both are asserted so one cannot rot.
echo ""
echo "--- RUN_TESTS_STRICT=1 fails the run on a skipped suite ---"
STRICT_ENV_OUT=""
STRICT_ENV_RC=0
STRICT_ENV_PRIV="$(mktemp -d)"
FIXTURES+=("$STRICT_ENV_PRIV")
STRICT_ENV_OUT="$(TMPDIR="$STRICT_ENV_PRIV" TEST_DIR="$DIR10/cases" RUN_TESTS_STRICT=1 \
  bash "$DIR10/tests/run-tests.sh" 2>&1)" || STRICT_ENV_RC=$?
if [[ $STRICT_ENV_RC -eq 0 ]]; then
  fail "RUN_TESTS_STRICT=1 passed with a skipped suite: $STRICT_ENV_OUT"
elif echo "$STRICT_ENV_OUT" | grep -q "a skip is a SETUP ERROR"; then
  pass "RUN_TESTS_STRICT=1 is the same gate as --strict"
else
  fail "RUN_TESTS_STRICT=1 failed for some other reason: $STRICT_ENV_OUT"
fi

# --- 10c. WITHOUT strict, the same fixture still skips gracefully and passes ---
# The control for 10 and 10b, and the half the coordinator asked for explicitly:
# an ad-hoc `bash tests/run-tests.sh` on a laptop missing a dev binary must not
# go red. Without this, "fix the gate" could quietly mean "fail everywhere".
echo ""
echo "--- the same skipped suite passes, still SKIPPED, without strict ---"
run_fake "$DIR10"
if [[ $FAKE_RC -ne 0 ]]; then
  fail "a skipped suite failed a non-strict run — graceful skipping is gone: $FAKE_OUT"
elif ! echo "$FAKE_OUT" | grep -q "^=== Summary: 1 passed, 1 skipped, 0 failed ===$"; then
  fail "a non-strict run miscounted the skip: $FAKE_OUT"
elif echo "$FAKE_OUT" | grep -q "^      SKIP: frobnicator is not installed"; then
  pass "without strict the suite is SKIPPED, the run passes, and the reason is still reported"
else
  fail "a non-strict run passed but dropped the skip reason: $FAKE_OUT"
fi

# --- 10d. --strict does not become a blanket failure ---
# The case that proves 10 and 10b fail for their stated reason. A clean run with
# nothing skipped must be just as green under --strict as without it, otherwise
# the gate is not a gate, it is an outage.
echo ""
echo "--- --strict is still green when nothing skipped ---"
DIR10D="$(make_fake_repo)"
FIXTURES+=("$DIR10D")
install_healthy_bats_runner "$DIR10D"
add_case "$DIR10D" test-ok.sh <<'EOF'
#!/usr/bin/env bash
echo "fine"
EOF
run_fake "$DIR10D" --strict
if [[ $FAKE_RC -ne 0 ]]; then
  fail "--strict failed a run with nothing skipped — it fails unconditionally: $FAKE_OUT"
elif echo "$FAKE_OUT" | grep -q "^=== Summary: 1 passed, 0 skipped, 0 failed ===$"; then
  pass "--strict leaves a run with no skips green"
else
  fail "--strict passed with the wrong summary: $FAKE_OUT"
fi

# --- 10e. An unknown flag is rejected, not ignored ---
# `--strict` reaching this script as a silently-ignored argument is the single
# typo that turns the gate back off while every hook still reports Passed, so the
# arg loop refuses anything it does not know rather than falling through.
echo ""
echo "--- an unrecognised flag fails with usage instead of being ignored ---"
DIR10E="$(make_fake_repo)"
FIXTURES+=("$DIR10E")
install_healthy_bats_runner "$DIR10E"
add_case "$DIR10E" test-ok.sh <<'EOF'
#!/usr/bin/env bash
echo "fine"
EOF
run_fake "$DIR10E" --strickt
if [[ $FAKE_RC -eq 0 ]]; then
  fail "a misspelled flag was ignored and the run passed — a typo silently disarms the gate: $FAKE_OUT"
elif echo "$FAKE_OUT" | grep -q "Usage: .*--bats-only.*--strict"; then
  pass "an unrecognised flag fails the run with usage"
else
  fail "an unrecognised flag failed but not with usage: $FAKE_OUT"
fi

# --- 10f. A skip reason printed on STDERR, with no `SKIP:` prefix, still lands ---
# There is no house format: three suites print `SKIP: <reason>` on stdout and
# tests/test-sync-plugin-content.sh prints `apm not installed -- skipping (...)`
# on stderr. batch-run.sh folds stderr into the same log, so both are reachable,
# but only a fallback chain finds the second one. Without this case the reason
# extraction could be narrowed to the `SKIP:` prefix and the apm suite would fail
# the gate with no indication of what to install.
echo ""
echo "--- a stderr skip reason with no SKIP: prefix is still reported ---"
DIR10F="$(make_fake_repo)"
FIXTURES+=("$DIR10F")
install_healthy_bats_runner "$DIR10F"
add_case "$DIR10F" test-stderr-skip.sh <<'EOF'
#!/usr/bin/env bash
echo "widgetizer not installed -- skipping (see docs)" >&2
exit 77
EOF
run_fake "$DIR10F" --strict
if [[ $FAKE_RC -eq 0 ]]; then
  fail "--strict passed on a suite that skipped via stderr: $FAKE_OUT"
elif echo "$FAKE_OUT" | grep -q "^      widgetizer not installed -- skipping"; then
  pass "a skip reason printed to stderr without a SKIP: prefix is still carried into the failure"
else
  fail "--strict failed but lost the stderr skip reason: $FAKE_OUT"
fi

echo ""
echo "Results: $PASS passed, $FAIL failed"
[[ $FAIL -eq 0 ]]