#!/usr/bin/env bash # Run all bats test files in the repo. # Usage: bash tests/run-bats.sh set -euo pipefail REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" BATS="$REPO_ROOT/tests/bats/bin/bats" if [[ ! -x "$BATS" ]]; then echo "bats not found at $BATS — initializing submodules..." >&2 git -C "$REPO_ROOT" submodule update --init --recursive fi if [[ ! -x "$BATS" ]]; then echo "Error: bats still not found at $BATS after submodule init" >&2 exit 1 fi # Collected with a `while read` loop rather than `mapfile` — macOS ships # /bin/bash 3.2, which has no `mapfile`. Process substitution (not a pipe) # keeps the loop in this shell so the appends survive. `sort` is still fed # newline-delimited output, exactly as before. Same convention as # tests/run-tests.sh. TEST_FILES=() while IFS= read -r f; do TEST_FILES+=("$f") done < <( find "$REPO_ROOT" -name "*.bats" \ -not -path "*/tests/bats/*" \ -not -path "*/test_helper/*" \ -not -path "*/.claude/worktrees/*" \ | sort ) # The expected set of files is DERIVED from the index, not guessed at with a # hardcoded floor. This was `BATS_FILE_FLOOR=8` against a real count of 10, and # two files of slack is not a hypothetical margin -- deleting two .bats files # (agentsmd-audit/tests/ alone holds three) left the run reporting # "155 tests, 0 failures" and exiting 0 with 11 tests silently gone. # # `git ls-files` gives the exact set for free. It catches a *removal* (a tracked # file gone from the worktree) and an *addition* the walk above missed (a tracked # file the `find` exclusions or a moved search root no longer reach), it needs no # magic number, and it needs no edit when a plugin is added or removed -- a newly # `git add`ed .bats file joins the expectation immediately, where a floor only # ever grows more slack as the suite grows. # # Direction matters: every tracked file must have been discovered, but a # discovered file need NOT be tracked. An untracked, not-yet-committed .bats file # is ordinary work in progress, and a file removed deliberately with `git rm` (or # a staged deletion) leaves the index, so an intentional removal passes while an # accidental disappearance fails. The same `-not -path` exclusions are reapplied # to the index listing so the two sides are compared over the same universe. # # The exact-equality check on `--show-toplevel` is what keeps this off the # fixture repos in tests/test-run-bats.sh and tests/test-run-tests.sh: those are # mktemp trees holding one or two .bats files by design, and git resolves no # worktree for them. That degradation is announced rather than silent, and the # zero-file check below is unconditional, so a non-git checkout still cannot run # on an empty set. EXPECTED_FILES=() DERIVED=false GIT_TOPLEVEL="$(git -C "$REPO_ROOT" rev-parse --show-toplevel 2>/dev/null || true)" if [[ -n "$GIT_TOPLEVEL" && "$GIT_TOPLEVEL" == "$REPO_ROOT" ]]; then DERIVED=true while IFS= read -r f; do [[ -n "$f" ]] && EXPECTED_FILES+=("$REPO_ROOT/$f") done < <( git -C "$REPO_ROOT" ls-files -- '*.bats' \ | grep -Ev '(^|/)tests/bats/|(^|/)test_helper/|(^|/)\.claude/worktrees/' \ | sort || true ) else echo "Note: $REPO_ROOT is not a git worktree root, so the expected .bats file set could not be derived from the index — only the zero-file check below applies" >&2 fi if [[ "$DERIVED" == true && ${#EXPECTED_FILES[@]} -gt 0 ]]; then MISSING=() for expected in ${EXPECTED_FILES[@]+"${EXPECTED_FILES[@]}"}; do found=false for actual in ${TEST_FILES[@]+"${TEST_FILES[@]}"}; do if [[ "$actual" == "$expected" ]]; then found=true break fi done [[ "$found" == true ]] || MISSING+=("${expected#"$REPO_ROOT"/}") done if [[ ${#MISSING[@]} -gt 0 ]]; then echo "Error: ${#MISSING[@]} of ${#EXPECTED_FILES[@]} tracked .bats file(s) were not discovered under $REPO_ROOT — they were deleted without being removed from the index, or the search path/exclusions above no longer reach them:" >&2 for m in ${MISSING[@]+"${MISSING[@]}"}; do echo " $m" >&2 done exit 1 fi fi # Unconditional, and separate from the derived check above: a tree with nothing # tracked (a tarball export, a fresh scaffold) still must not run on an empty set # and call it green. This was `exit 0` with a note on stderr nobody reads. if [[ ${#TEST_FILES[@]} -eq 0 ]]; then echo "Error: found 0 .bats file(s) under $REPO_ROOT — the search path is wrong or the suite has been gutted" >&2 exit 1 fi # Each file gets its own `bats` process, run concurrently (bounded by core # count) instead of one `bats` invocation working through all files serially. # A single test file is still serial internally -- this only overlaps the # fixed per-process startup cost (git/apm subprocess spawns dominate several # of these suites) across files, which is where the wall-clock actually goes. # Output is buffered per file so concurrent TAP streams can't interleave, then # flushed in stable sorted order once every job has finished. # # Dispatch and throttling is scripts/lib/batch-run.sh's batch_run -- shared # with scripts/sync-plugin-content.sh and tests/run-tests.sh so a batching bug # fix only needs to land once; see that file for why this is batched rather # than a rolling `wait -n` pool. SCRATCH_ROOT="$(mktemp -d)" trap 'rm -rf "$SCRATCH_ROOT"' EXIT # Repo-root-relative -- see tests/run-tests.sh for why `../scripts/...` does not # resolve here despite looking right. # shellcheck source=scripts/lib/batch-run.sh source "$REPO_ROOT/scripts/lib/batch-run.sh" declare -a batch_args=() i=0 for f in ${TEST_FILES[@]+"${TEST_FILES[@]}"}; do i=$((i + 1)) cmd="$(printf '%q %q; echo $? >%q' "$BATS" "$f" "$SCRATCH_ROOT/$i.status")" batch_args+=("$i" "$cmd") done batch_run "$SCRATCH_ROOT" ${batch_args[@]+"${batch_args[@]}"} FAIL=0 TOTAL_OK=0 TOTAL_NOT_OK=0 TOTAL_PLANS=0 i=0 for f in ${TEST_FILES[@]+"${TEST_FILES[@]}"}; do i=$((i + 1)) rel="${f#"$REPO_ROOT"/}" echo "=== $rel ===" cat "$SCRATCH_ROOT/$i.log" echo "" file_ok="$(grep -c '^ok ' "$SCRATCH_ROOT/$i.log" || true)" file_not_ok="$(grep -c '^not ok ' "$SCRATCH_ROOT/$i.log" || true)" # The TAP plan line (`1..N`). Counted separately from the results because an # empty-but-valid file emits `1..0` and no result lines at all -- that is a # file bats really did run, so it has to be distinguishable from a file that # produced nothing whatsoever. file_plan="$(grep -c '^1\.\.[0-9]' "$SCRATCH_ROOT/$i.log" || true)" # String-compared below, not `-ne`. `-ne` is arithmetic and bash evaluates an # empty string as 0 there -- `[[ "" -ne 0 ]]` is false -- so an *empty* status # file read as a clean exit. The `|| echo 1` fallback only covers a *missing* # file; an existing-but-empty one is what a job killed between the `>` and the # `echo` leaves behind, or what ENOSPC leaves behind. status="$(cat "$SCRATCH_ROOT/$i.status" 2>/dev/null || echo 1)" TOTAL_OK=$((TOTAL_OK + file_ok)) TOTAL_NOT_OK=$((TOTAL_NOT_OK + file_not_ok)) TOTAL_PLANS=$((TOTAL_PLANS + file_plan)) # Two independent failure signals, deliberately OR-ed: a file can report `not # ok` lines while its process still exits 0 (a bats formatter or wrapper that # swallows the status), and a file can exit non-zero having emitted no `not # ok` at all (a crash, a timeout, an unbound variable in setup_file). Real # bats normally emits both at once, so each signal masks the other and # dropping either half is invisible without tests that produce one without # the other -- tests/test-run-bats.sh has those. if [[ "$file_not_ok" -gt 0 || "$status" != "0" ]]; then FAIL=1 fi done # Zero counted tests is never a clean run: files were found (zero discovered # files, and any tracked file that went missing, exit non-zero above), so nothing # was executed. Without this, a `bats` that emits # nothing and exits 0 -- a broken binary, a formatter change, or a wholesale # `@test` removal -- reports "0 tests, 0 failures" and exits green, silently # turning a total harness failure into a pass. # # The two causes get different messages because they are different problems and # `1..0` is itself valid TAP: no plan lines at all means bats produced no output # to parse, while plans present with zero results means bats ran fine and the # files genuinely declare no tests. if [[ $((TOTAL_OK + TOTAL_NOT_OK)) -eq 0 ]]; then if [[ "$TOTAL_PLANS" -eq 0 ]]; then echo "Error: ${#TEST_FILES[@]} .bats file(s) ran but produced no TAP output at all — the bats harness is broken" >&2 else echo "Error: ${#TEST_FILES[@]} .bats file(s) declared 0 tests — every @test appears to have been removed" >&2 fi FAIL=1 fi echo "$((TOTAL_OK + TOTAL_NOT_OK)) tests, $TOTAL_NOT_OK failures" exit "$FAIL"