holocron/tests/run-bats.sh

#!/usr/bin/env bash
# Run all bats test files in the repo.
# Usage: bash tests/run-bats.sh
set -euo pipefail

REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
BATS="$REPO_ROOT/tests/bats/bin/bats"

if [[ ! -x "$BATS" ]]; then
    echo "bats not found at $BATS — initializing submodules..." >&2
    git -C "$REPO_ROOT" submodule update --init --recursive
fi

if [[ ! -x "$BATS" ]]; then
    echo "Error: bats still not found at $BATS after submodule init" >&2
    exit 1
fi

# Collected with a `while read` loop rather than `mapfile` — macOS ships
# /bin/bash 3.2, which has no `mapfile`. Process substitution (not a pipe)
# keeps the loop in this shell so the appends survive. `sort` is still fed
# newline-delimited output, exactly as before. Same convention as
# tests/run-tests.sh.
#
# apm_modules/ is excluded because `apm install` materializes a full copy of
# every dependency there — including this repo's own plugins, which it consumes
# from the holocron remote. Those copies carry their own .bats files whose
# relative paths (`$BATS_TEST_DIRNAME/../../../../../../`) resolve to the
# dependency's root, not this repo's, so they fail on a missing bats-support
# helper. They are the same tests already discovered under plugins/.
#
# .claude/skills/ is excluded for the same reason, one install step later --
# but the duplication it guards against is not live yet, and the comment here
# claimed it was. Checked: the deployed tree holds no tests/ directory and no
# .bats file at all. The six dependencies resolve from the holocron remote's
# default branch, which still carries the pre-ADR-0024 flat mirror, so apm
# classifies each one as a marketplace_plugin (`package_type` in apm.lock.yaml)
# and deploys from that mirror -- and the mirror's generator stripped `tests/`
# out. The exclusion is therefore forward-looking, and correctly so: once the
# mirror's deletion reaches the default branch and `apm update` runs, apm
# deploys from `.apm/` directly, `tests/` dirs and all. Nothing in apm filters
# them -- its deploy-time copytree ignore callback (`ignore_non_content()` in
# apm_cli/security/gate.py, 0.28.0) drops symlinks and the .apm-pin marker and
# nothing else. Every `<skill>/tests/*.bats` file under plugins/ would then
# land at .claude/skills/<name>/tests/ and be discovered and run a second time,
# which is ADR-0024 consequence 2 arriving here. Keeping the exclusion now is
# what stops that landing as a mystery double-run on the merge that enables it.
#
# build/ is excluded for the same reason again, one layer further out: `apm
# pack` stages a full copy of a package's tree (including its skills' tests/
# directories) under build/<package>-<version>/ before archiving it. Those
# staged .bats files carry the same six-levels-up REPO_ROOT walk-up as any
# other copy, which resolves past this repo's actual root and fails on a
# missing bats-support helper -- the same failure mode apm_modules/ and
# .claude/skills/ above already guard against, just from a different apm
# subcommand. build/ is gitignored and regenerated on demand, so nothing here
# depends on its contents; the exclusion only stops a stray local `apm pack`
# output from being discovered and double-run.
#
# The walk runs from inside REPO_ROOT so the exclusions match paths RELATIVE to
# it, the same universe the `git ls-files` grep below sees. Matched against
# absolute paths, `*/.claude/worktrees/*` excluded every file whenever the
# checkout itself was a Claude worktree (<repo>/.claude/worktrees/<name>/), and
# the run failed with "N of N tracked .bats file(s) were not discovered".
TEST_FILES=()
while IFS= read -r f; do
    TEST_FILES+=("$REPO_ROOT/${f#./}")
done < <(
    cd "$REPO_ROOT" && find . -name "*.bats" \
        -not -path "*/tests/bats/*" \
        -not -path "*/test_helper/*" \
        -not -path "*/.claude/worktrees/*" \
        -not -path "*/apm_modules/*" \
        -not -path "*/.claude/skills/*" \
        -not -path "*/build/*" \
        | sort
)

# The expected set of files is DERIVED from the index, not guessed at with a
# hardcoded floor. This was `BATS_FILE_FLOOR=8` against a real count of 10, and
# two files of slack is not a hypothetical margin -- deleting two .bats files
# (agentsmd-audit/tests/ alone holds three) left the run reporting
# "155 tests, 0 failures" and exiting 0 with 11 tests silently gone.
#
# `git ls-files` gives the exact set for free. It catches a *removal* (a tracked
# file gone from the worktree) and an *addition* the walk above missed (a tracked
# file the `find` exclusions or a moved search root no longer reach), it needs no
# magic number, and it needs no edit when a plugin is added or removed -- a newly
# `git add`ed .bats file joins the expectation immediately, where a floor only
# ever grows more slack as the suite grows.
#
# Direction matters: every tracked file must have been discovered, but a
# discovered file need NOT be tracked. An untracked, not-yet-committed .bats file
# is ordinary work in progress, and a file removed deliberately with `git rm` (or
# a staged deletion) leaves the index, so an intentional removal passes while an
# accidental disappearance fails. The same `-not -path` exclusions are reapplied
# to the index listing so the two sides are compared over the same universe.
#
# The exact-equality check on `--show-toplevel` is what keeps this off the
# fixture repos in tests/test-run-bats.sh and tests/test-run-tests.sh: those are
# mktemp trees holding one or two .bats files by design, and git resolves no
# worktree for them. That degradation is announced rather than silent, and the
# zero-file check below is unconditional, so a non-git checkout still cannot run
# on an empty set.
EXPECTED_FILES=()
DERIVED=false
GIT_TOPLEVEL="$(git -C "$REPO_ROOT" rev-parse --show-toplevel 2>/dev/null || true)"
if [[ -n "$GIT_TOPLEVEL" && "$GIT_TOPLEVEL" == "$REPO_ROOT" ]]; then
    DERIVED=true
    while IFS= read -r f; do
        [[ -n "$f" ]] && EXPECTED_FILES+=("$REPO_ROOT/$f")
    done < <(
        git -C "$REPO_ROOT" ls-files -- '*.bats' \
            | grep -Ev '(^|/)tests/bats/|(^|/)test_helper/|(^|/)\.claude/worktrees/|(^|/)apm_modules/|(^|/)\.claude/skills/|(^|/)build/' \
            | sort || true
    )
else
    echo "Note: $REPO_ROOT is not a git worktree root, so the expected .bats file set could not be derived from the index — only the zero-file check below applies" >&2
fi

if [[ "$DERIVED" == true && ${#EXPECTED_FILES[@]} -gt 0 ]]; then
    MISSING=()
    for expected in ${EXPECTED_FILES[@]+"${EXPECTED_FILES[@]}"}; do
        found=false
        for actual in ${TEST_FILES[@]+"${TEST_FILES[@]}"}; do
            if [[ "$actual" == "$expected" ]]; then
                found=true
                break
            fi
        done
        [[ "$found" == true ]] || MISSING+=("${expected#"$REPO_ROOT"/}")
    done
    if [[ ${#MISSING[@]} -gt 0 ]]; then
        echo "Error: ${#MISSING[@]} of ${#EXPECTED_FILES[@]} tracked .bats file(s) were not discovered under $REPO_ROOT — they were deleted without being removed from the index, or the search path/exclusions above no longer reach them:" >&2
        for m in ${MISSING[@]+"${MISSING[@]}"}; do
            echo "  $m" >&2
        done
        exit 1
    fi
fi

# Unconditional, and separate from the derived check above: a tree with nothing
# tracked (a tarball export, a fresh scaffold) still must not run on an empty set
# and call it green. This was `exit 0` with a note on stderr nobody reads.
if [[ ${#TEST_FILES[@]} -eq 0 ]]; then
    echo "Error: found 0 .bats file(s) under $REPO_ROOT — the search path is wrong or the suite has been gutted" >&2
    exit 1
fi

# Each file gets its own `bats` process, run concurrently (bounded by core
# count) instead of one `bats` invocation working through all files serially.
# A single test file is still serial internally -- this only overlaps the
# fixed per-process startup cost (git/apm subprocess spawns dominate several
# of these suites) across files, which is where the wall-clock actually goes.
# Output is buffered per file so concurrent TAP streams can't interleave, then
# flushed in stable sorted order once every job has finished.
#
# Dispatch and throttling is scripts/lib/batch-run.sh's batch_run -- shared
# with tests/run-tests.sh so a batching bug fix only needs to land once; see
# that file for why this is batched rather than a rolling `wait -n` pool.
SCRATCH_ROOT="$(mktemp -d)"
trap 'rm -rf "$SCRATCH_ROOT"' EXIT
# Repo-root-relative -- see tests/run-tests.sh for why `../scripts/...` does not
# resolve here despite looking right.
# shellcheck source=scripts/lib/batch-run.sh
source "$REPO_ROOT/scripts/lib/batch-run.sh"

declare -a batch_args=()
i=0
for f in ${TEST_FILES[@]+"${TEST_FILES[@]}"}; do
    i=$((i + 1))
    cmd="$(printf '%q %q; echo $? >%q' "$BATS" "$f" "$SCRATCH_ROOT/$i.status")"
    batch_args+=("$i" "$cmd")
done
batch_run "$SCRATCH_ROOT" ${batch_args[@]+"${batch_args[@]}"}

FAIL=0
TOTAL_OK=0
TOTAL_NOT_OK=0
TOTAL_PLANS=0
i=0
for f in ${TEST_FILES[@]+"${TEST_FILES[@]}"}; do
    i=$((i + 1))
    rel="${f#"$REPO_ROOT"/}"
    echo "=== $rel ==="
    cat "$SCRATCH_ROOT/$i.log"
    echo ""
    file_ok="$(grep -c '^ok ' "$SCRATCH_ROOT/$i.log" || true)"
    file_not_ok="$(grep -c '^not ok ' "$SCRATCH_ROOT/$i.log" || true)"
    # The TAP plan line (`1..N`). Counted separately from the results because an
    # empty-but-valid file emits `1..0` and no result lines at all -- that is a
    # file bats really did run, so it has to be distinguishable from a file that
    # produced nothing whatsoever.
    file_plan="$(grep -c '^1\.\.[0-9]' "$SCRATCH_ROOT/$i.log" || true)"
    # The planned count itself, extracted only when the file emitted exactly one
    # plan line. Zero plans is the broken-harness case the aggregate guard below
    # names, and two or more means the log is not one file's TAP stream at all --
    # in neither case does "the planned count" mean anything, so the per-file
    # comparison is skipped and the previous behaviour stands.
    file_planned=""
    if [[ "$file_plan" -eq 1 ]]; then
        file_planned="$(sed -n 's/^1\.\.\([0-9][0-9]*\).*$/\1/p' "$SCRATCH_ROOT/$i.log")"
    fi
    # String-compared below, not `-ne`. `-ne` is arithmetic and bash evaluates an
    # empty string as 0 there -- `[[ "" -ne 0 ]]` is false -- so an *empty* status
    # file read as a clean exit. The `|| echo 1` fallback only covers a *missing*
    # file; an existing-but-empty one is what a job killed between the `>` and the
    # `echo` leaves behind, or what ENOSPC leaves behind.
    status="$(cat "$SCRATCH_ROOT/$i.status" 2>/dev/null || echo 1)"
    TOTAL_OK=$((TOTAL_OK + file_ok))
    TOTAL_NOT_OK=$((TOTAL_NOT_OK + file_not_ok))
    TOTAL_PLANS=$((TOTAL_PLANS + file_plan))
    # Three independent failure signals, deliberately OR-ed: a file can report
    # `not ok` lines while its process still exits 0 (a bats formatter or wrapper
    # that swallows the status), a file can exit non-zero having emitted no `not
    # ok` at all (a crash, a timeout, an unbound variable in setup_file), and a
    # file can emit FEWER results than its own plan line promised. Real bats
    # normally emits all three consistently, so each signal masks the others and
    # dropping any one of them is invisible without tests that produce one without
    # the rest -- tests/test-run-bats.sh has those.
    #
    # The plan is the third signal and it is now enforced, not merely counted. It
    # is the one that survives precisely the wrapper-swallows-the-status case
    # named above: a process printing `1..10`, three `ok` lines and exit 0 used to
    # be counted as "3 tests, 0 failures" and go green with seven tests silently
    # gone, because the plan was computed for the aggregate zero-count guard below
    # and then discarded. Mismatch either way is a failure -- more results than
    # planned is as broken a TAP stream as fewer.
    file_short=false
    if [[ -n "$file_planned" && $((file_ok + file_not_ok)) -ne "$file_planned" ]]; then
        file_short=true
        echo "Error: $rel planned $file_planned test(s) but emitted $((file_ok + file_not_ok)) result line(s) — the run was truncated, or its exit status was swallowed" >&2
    fi
    if [[ "$file_not_ok" -gt 0 || "$status" != "0" || "$file_short" == true ]]; then
        FAIL=1
    fi
done

# Zero counted tests is never a clean run: files were found (zero discovered
# files, and any tracked file that went missing, exit non-zero above), so nothing
# was executed. Without this, a `bats` that emits
# nothing and exits 0 -- a broken binary, a formatter change, or a wholesale
# `@test` removal -- reports "0 tests, 0 failures" and exits green, silently
# turning a total harness failure into a pass.
#
# The two causes get different messages because they are different problems and
# `1..0` is itself valid TAP: no plan lines at all means bats produced no output
# to parse, while plans present with zero results means bats ran fine and the
# files genuinely declare no tests.
if [[ $((TOTAL_OK + TOTAL_NOT_OK)) -eq 0 ]]; then
    if [[ "$TOTAL_PLANS" -eq 0 ]]; then
        echo "Error: ${#TEST_FILES[@]} .bats file(s) ran but produced no TAP output at all — the bats harness is broken" >&2
    else
        echo "Error: ${#TEST_FILES[@]} .bats file(s) declared 0 tests — every @test appears to have been removed" >&2
    fi
    FAIL=1
fi

echo "$((TOTAL_OK + TOTAL_NOT_OK)) tests, $TOTAL_NOT_OK failures"
exit "$FAIL"