holocron/tests/test-check-apm-agents-valid.sh

#!/usr/bin/env bash
set -euo pipefail

# Tests for scripts/check-apm-agents-valid.sh — the gate that runs factory-audit's
# validate.sh over the repo's REAL plugin-scope agent files. Since ADR-0025 merged
# skill-audit and agent-audit, that validate.sh is one auto-detecting entry point;
# every target this gate hands it is an agent file, so every run below takes the
# agent branch and sources lib-checks-agent.sh.
#
# Case 1 runs against the real repo. Every other case runs against a synthetic
# fixture, for the same reason scripts/check-scope-walkup-sync.sh's tests do: the
# RED cases have to mutate an agent file, and mutating the real tree from a test
# is not on.
#
# The point of case 1b is that case 1's exit 0 is EARNED. Exit 0 is also what
# this script would print if it validated nothing at all, which is the exact
# defect it exists to close — so the clean run's own count is asserted against
# the index rather than taken on trust.

REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
SCRIPT="$REPO_ROOT/scripts/check-apm-agents-valid.sh"
PASS=0
FAIL=0

pass() { echo "  PASS: $1"; PASS=$((PASS + 1)); }
fail() { echo "  FAIL: $1"; FAIL=$((FAIL + 1)); }

# validate.sh is a python3 program. Without python3 the script under test fails
# closed by design, which is correct behavior but makes every case here assert
# the same missing-dependency message instead of what it is meant to assert.
if ! command -v python3 >/dev/null 2>&1; then
  echo "SKIP: python3 is not installed — factory-audit's validate.sh cannot run, so these cases would only re-assert the missing-dependency guard"
  exit 77
fi

FIXTURES=()
cleanup() { [[ ${#FIXTURES[@]} -eq 0 ]] || rm -rf "${FIXTURES[@]}"; }
trap cleanup EXIT

# Per-run scratch dir for captured output. tests/run-tests.sh fans test scripts
# out concurrently, so a fixed path under the shared system temp directory is
# mutable state shared between two simultaneous runs.
RUN_TMP="$(mktemp -d)"
FIXTURES+=("$RUN_TMP")

# Builds a minimal REPO_ROOT: factory-audit's validator and the field inventory it
# reads at load time, plus one plugin carrying a valid agent file. The validator
# is no longer self-contained — ADR-0025 replaced the embedded resolver with a
# sourced lib-boundary-resolver.sh and moved the agent checks into
# lib-checks-agent.sh — so the fixture copies the two libraries the agent branch
# sources as well. Copying validate.sh alone would make every case below fail on
# a missing source file rather than on what it is meant to assert.
#
# The inventory is `agent-field-inventory.md`: factory-audit prefixes every
# flow-specific reference file with skill-/agent-, and validate.sh reads this one
# by that name at load time.
#
# The plugin's
# apm.yml needs a top-level `type:` line — that is the marker validate.sh's
# walk-up uses to resolve plugin scope, and without it the fixture would resolve
# to project scope and fail looking for a .github/agents counterpart.
#
# `pwd -P` because the script under test compares its REPO_ROOT against
# `git rev-parse --show-toplevel`, which is always physical. On a platform where
# the system temp dir is a symlink (macOS /tmp -> /private/tmp) a logical path
# would silently fail that equality and take the un-derived branch, quietly
# turning case 4 into a no-op.
make_fixture() {
  local dir
  dir="$(cd "$(mktemp -d)" && pwd -P)"
  local fa="$dir/plugins/kyberforge/.apm/skills/factory-audit"
  local src="$REPO_ROOT/plugins/kyberforge/.apm/skills/factory-audit"
  mkdir -p "$fa/scripts" "$fa/references" "$dir/plugins/lint/.apm/agents"
  cp "$src/scripts/validate.sh" \
     "$src/scripts/lib-boundary-resolver.sh" \
     "$src/scripts/lib-checks-agent.sh" \
     "$fa/scripts/"
  cp "$src/references/agent-field-inventory.md" "$fa/references/"
  cat > "$dir/plugins/lint/apm.yml" <<'YAML'
name: lint
version: 0.0.1
type: hybrid
YAML
  cat > "$dir/plugins/lint/.apm/agents/lint-runner.agent.md" <<'MD'
---
name: lint-runner
description: Runs a linter sweep over a target scope and reports findings back to the caller.
---

Run the linter over the scope the caller names and report what it found.
MD
  echo "$dir"
}

# --- 1. Exits 0 against this repo's real agent files ---
echo ""
echo "--- exits 0 against this repo's real agent files ---"
if bash "$SCRIPT" "$REPO_ROOT" > "$RUN_TMP/clean.out" 2>&1; then
  pass "exits 0 against this repo's four real plugin-scope agent files"
else
  fail "exited non-zero against this repo's real (already-fixed) agent files"
  sed 's/^/    /' "$RUN_TMP/clean.out"
fi

# --- 1b. That exit 0 was earned: the count matches the index, and is non-zero ---
echo ""
echo "--- the clean run's reported count matches git ls-files ---"
TRACKED_COUNT="$(git -C "$REPO_ROOT" ls-files -- 'plugins/*/.apm/agents/*.agent.md' | grep -c . || true)"
REPORTED_COUNT="$(sed -n 's/^APM agent validation passed: \([0-9]\{1,\}\) plugin-scope.*/\1/p' "$RUN_TMP/clean.out")"
if [[ -z "$REPORTED_COUNT" ]]; then
  fail "the clean run printed no 'APM agent validation passed: N ...' summary line — the script's contract with this test is gone"
  sed 's/^/    /' "$RUN_TMP/clean.out"
elif [[ "$TRACKED_COUNT" -eq 0 ]]; then
  fail "git ls-files found 0 tracked agent files — this test's own expectation is broken, not the script's"
elif [[ "$REPORTED_COUNT" -ne "$TRACKED_COUNT" ]]; then
  fail "the clean run validated $REPORTED_COUNT file(s) but $TRACKED_COUNT are tracked"
else
  pass "validated $REPORTED_COUNT file(s), matching the $TRACKED_COUNT tracked in the index"
fi

# --- 2. An invalid agent file fails, and both the file and the reason are named ---
echo ""
echo "--- an invalid agent file fails, naming the file and the reason ---"
FIX2="$(make_fixture)"
FIXTURES+=("$FIX2")
# `tools:` is deliberately absent from field-inventory.md's apm-agent-allowlist:
# its value shape differs per harness and apm compile copies frontmatter verbatim
# to every target (ADR-0016).
python3 - "$FIX2/plugins/lint/.apm/agents/lint-runner.agent.md" <<'PY'
import sys
p = sys.argv[1]
s = open(p).read()
open(p, 'w').write(s.replace('---\n', '---\ntools: Read, Write\n', 1))
PY
if bash "$SCRIPT" "$FIX2" > "$RUN_TMP/invalid.out" 2>&1; then
  fail "an agent file with a non-allowlisted frontmatter field still exited 0"
  sed 's/^/    /' "$RUN_TMP/invalid.out"
elif ! grep -q 'plugins/lint/\.apm/agents/lint-runner\.agent\.md' "$RUN_TMP/invalid.out"; then
  fail "failed as expected but did not name the offending file"
  sed 's/^/    /' "$RUN_TMP/invalid.out"
elif ! grep -q "field 'tools' is not in the vendor-neutral APM agent allowlist" "$RUN_TMP/invalid.out"; then
  fail "failed as expected and named the file but did not carry validate.sh's reason through"
  sed 's/^/    /' "$RUN_TMP/invalid.out"
else
  pass "an invalid agent file exits 1, naming both the file and validate.sh's reason"
fi

# --- 3. Zero discovered files is an error, not a pass ---
echo ""
echo "--- zero discovered agent files is an error ---"
FIX3="$(make_fixture)"
FIXTURES+=("$FIX3")
rm -f "$FIX3/plugins/lint/.apm/agents/lint-runner.agent.md"
if bash "$SCRIPT" "$FIX3" > "$RUN_TMP/empty.out" 2>&1; then
  fail "a tree with zero agent files exited 0 — the floor is gone and the gate is vacuous"
  sed 's/^/    /' "$RUN_TMP/empty.out"
elif ! grep -q 'found 0 plugin-scope agent file' "$RUN_TMP/empty.out"; then
  fail "a tree with zero agent files exited non-zero but not for the zero-file reason"
  sed 's/^/    /' "$RUN_TMP/empty.out"
else
  pass "a tree with zero agent files exits 1 and says so"
fi

# --- 4. A tracked file missing from the worktree fails, derived from the index ---
# This is the check a hardcoded count cannot make: the file is gone but the count
# of what remains would still look plausible.
echo ""
echo "--- a tracked-but-deleted agent file fails against the derived expectation ---"
FIX4="$(make_fixture)"
FIXTURES+=("$FIX4")
cat > "$FIX4/plugins/lint/.apm/agents/second-agent.agent.md" <<'MD'
---
name: second-agent
description: A second agent, present only so its deletion leaves a plausible-looking non-empty set behind.
---

Do the second thing.
MD
git -C "$FIX4" init -q
git -C "$FIX4" add -A
git -C "$FIX4" -c user.email=t@example.invalid -c user.name=t commit -qm "fixture"
rm -f "$FIX4/plugins/lint/.apm/agents/second-agent.agent.md"
if bash "$SCRIPT" "$FIX4" > "$RUN_TMP/missing.out" 2>&1; then
  fail "a tracked agent file deleted from the worktree still exited 0"
  sed 's/^/    /' "$RUN_TMP/missing.out"
elif grep -q 'not a git worktree root' "$RUN_TMP/missing.out"; then
  fail "the fixture did not resolve as its own git worktree root, so the derived check never ran"
  sed 's/^/    /' "$RUN_TMP/missing.out"
elif ! grep -q 'second-agent\.agent\.md' "$RUN_TMP/missing.out"; then
  fail "failed as expected but did not name the tracked file that went missing"
  sed 's/^/    /' "$RUN_TMP/missing.out"
else
  pass "a tracked agent file deleted from the worktree exits 1 and is named"
fi

# --- 5. An untracked agent file is still validated ---
# The derived expectation is one-directional on purpose (tracked ⊆ discovered).
# Work in progress must not fail the gate for being uncommitted — but it must
# still be validated, or the gate would be trivially bypassed by not committing.
echo ""
echo "--- an untracked, invalid agent file still fails the gate ---"
FIX5="$(make_fixture)"
FIXTURES+=("$FIX5")
git -C "$FIX5" init -q
git -C "$FIX5" add -A
git -C "$FIX5" -c user.email=t@example.invalid -c user.name=t commit -qm "fixture"
cat > "$FIX5/plugins/lint/.apm/agents/wip-agent.agent.md" <<'MD'
---
name: wip-agent
tools: Read, Write
description: An uncommitted work-in-progress agent carrying a non-allowlisted field.
---

Do the work-in-progress thing.
MD
if bash "$SCRIPT" "$FIX5" > "$RUN_TMP/untracked.out" 2>&1; then
  fail "an untracked, invalid agent file was not validated — the gate can be bypassed by not committing"
  sed 's/^/    /' "$RUN_TMP/untracked.out"
elif ! grep -q 'wip-agent\.agent\.md' "$RUN_TMP/untracked.out"; then
  fail "failed but did not name the untracked file"
  sed 's/^/    /' "$RUN_TMP/untracked.out"
else
  pass "an untracked, invalid agent file exits 1 and is named"
fi

# --- 5b. An untracked but VALID agent file does not fail ---
echo ""
echo "--- an untracked, valid agent file passes ---"
FIX5B="$(make_fixture)"
FIXTURES+=("$FIX5B")
git -C "$FIX5B" init -q
git -C "$FIX5B" add -A
git -C "$FIX5B" -c user.email=t@example.invalid -c user.name=t commit -qm "fixture"
cat > "$FIX5B/plugins/lint/.apm/agents/wip-ok.agent.md" <<'MD'
---
name: wip-ok
description: An uncommitted work-in-progress agent that is nonetheless entirely valid.
---

Do the valid work-in-progress thing.
MD
if bash "$SCRIPT" "$FIX5B" > "$RUN_TMP/untracked-ok.out" 2>&1; then
  pass "an untracked but valid agent file does not fail the gate"
else
  fail "an untracked but valid agent file failed the gate — uncommitted work must not be an error"
  sed 's/^/    /' "$RUN_TMP/untracked-ok.out"
fi

# --- 6. A nonexistent REPO_ROOT fails loudly ---
echo ""
echo "--- a nonexistent REPO_ROOT fails loudly ---"
if bash "$SCRIPT" "$RUN_TMP/does-not-exist" > "$RUN_TMP/norepo.out" 2>&1; then
  fail "a nonexistent REPO_ROOT exited 0"
  sed 's/^/    /' "$RUN_TMP/norepo.out"
elif ! grep -q 'is not a directory' "$RUN_TMP/norepo.out"; then
  fail "a nonexistent REPO_ROOT failed for the wrong reason"
  sed 's/^/    /' "$RUN_TMP/norepo.out"
else
  pass "a nonexistent REPO_ROOT exits 1 and says which path it was"
fi

# --- 7. A missing validator is a hard failure, never a silent pass ---
# The whole gate is void without validate.sh, and exit 0 here would be
# indistinguishable from a run where every agent passed.
echo ""
echo "--- a missing validate.sh fails rather than validating nothing ---"
FIX7="$(make_fixture)"
FIXTURES+=("$FIX7")
rm -f "$FIX7/plugins/kyberforge/.apm/skills/factory-audit/scripts/validate.sh"
if bash "$SCRIPT" "$FIX7" > "$RUN_TMP/novalidator.out" 2>&1; then
  fail "a missing validate.sh exited 0 — the gate silently validated nothing"
  sed 's/^/    /' "$RUN_TMP/novalidator.out"
elif ! grep -q 'validator not found' "$RUN_TMP/novalidator.out"; then
  fail "a missing validate.sh failed for the wrong reason"
  sed 's/^/    /' "$RUN_TMP/novalidator.out"
else
  pass "a missing validate.sh exits 1 and names the stale path"
fi

echo ""
echo "Results: $PASS passed, $FAIL failed"
[[ $FAIL -eq 0 ]]