#!/usr/bin/env bash set -euo pipefail # Tests for scripts/check-apm-agents-valid.sh — the gate that runs agent-audit's # validate.sh over the repo's REAL plugin-scope agent files. # # Case 1 runs against the real repo. Every other case runs against a synthetic # fixture, for the same reason scripts/check-scope-walkup-sync.sh's tests do: the # RED cases have to mutate an agent file, and mutating the real tree from a test # is not on. # # The point of case 1b is that case 1's exit 0 is EARNED. Exit 0 is also what # this script would print if it validated nothing at all, which is the exact # defect it exists to close — so the clean run's own count is asserted against # the index rather than taken on trust. REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" SCRIPT="$REPO_ROOT/scripts/check-apm-agents-valid.sh" PASS=0 FAIL=0 pass() { echo " PASS: $1"; PASS=$((PASS + 1)); } fail() { echo " FAIL: $1"; FAIL=$((FAIL + 1)); } # validate.sh is a python3 program. Without python3 the script under test fails # closed by design, which is correct behavior but makes every case here assert # the same missing-dependency message instead of what it is meant to assert. if ! command -v python3 >/dev/null 2>&1; then echo "SKIP: python3 is not installed — agent-audit's validate.sh cannot run, so these cases would only re-assert the missing-dependency guard" exit 77 fi FIXTURES=() cleanup() { [[ ${#FIXTURES[@]} -eq 0 ]] || rm -rf "${FIXTURES[@]}"; } trap cleanup EXIT # Per-run scratch dir for captured output. tests/run-tests.sh fans test scripts # out concurrently, so a fixed path under the shared system temp directory is # mutable state shared between two simultaneous runs. RUN_TMP="$(mktemp -d)" FIXTURES+=("$RUN_TMP") # Builds a minimal REPO_ROOT: agent-audit's validator and the field inventory it # reads at load time, plus one plugin carrying a valid agent file. The plugin's # apm.yml needs a top-level `type:` line — that is the marker validate.sh's # walk-up uses to resolve plugin scope, and without it the fixture would resolve # to project scope and fail looking for a .github/agents counterpart. # # `pwd -P` because the script under test compares its REPO_ROOT against # `git rev-parse --show-toplevel`, which is always physical. On a platform where # the system temp dir is a symlink (macOS /tmp -> /private/tmp) a logical path # would silently fail that equality and take the un-derived branch, quietly # turning case 4 into a no-op. make_fixture() { local dir dir="$(cd "$(mktemp -d)" && pwd -P)" local aa="$dir/plugins/kyberforge/.apm/skills/agent-audit" mkdir -p "$aa/scripts" "$aa/references" "$dir/plugins/lint/.apm/agents" cp "$REPO_ROOT/plugins/kyberforge/.apm/skills/agent-audit/scripts/validate.sh" "$aa/scripts/" cp "$REPO_ROOT/plugins/kyberforge/.apm/skills/agent-audit/references/field-inventory.md" "$aa/references/" cat > "$dir/plugins/lint/apm.yml" <<'YAML' name: lint version: 0.0.1 type: hybrid YAML cat > "$dir/plugins/lint/.apm/agents/lint-runner.agent.md" <<'MD' --- name: lint-runner description: Runs a linter sweep over a target scope and reports findings back to the caller. --- Run the linter over the scope the caller names and report what it found. MD echo "$dir" } # --- 1. Exits 0 against this repo's real agent files --- echo "" echo "--- exits 0 against this repo's real agent files ---" if bash "$SCRIPT" "$REPO_ROOT" > "$RUN_TMP/clean.out" 2>&1; then pass "exits 0 against this repo's four real plugin-scope agent files" else fail "exited non-zero against this repo's real (already-fixed) agent files" sed 's/^/ /' "$RUN_TMP/clean.out" fi # --- 1b. That exit 0 was earned: the count matches the index, and is non-zero --- echo "" echo "--- the clean run's reported count matches git ls-files ---" TRACKED_COUNT="$(git -C "$REPO_ROOT" ls-files -- 'plugins/*/.apm/agents/*.agent.md' | grep -c . || true)" REPORTED_COUNT="$(sed -n 's/^APM agent validation passed: \([0-9]\{1,\}\) plugin-scope.*/\1/p' "$RUN_TMP/clean.out")" if [[ -z "$REPORTED_COUNT" ]]; then fail "the clean run printed no 'APM agent validation passed: N ...' summary line — the script's contract with this test is gone" sed 's/^/ /' "$RUN_TMP/clean.out" elif [[ "$TRACKED_COUNT" -eq 0 ]]; then fail "git ls-files found 0 tracked agent files — this test's own expectation is broken, not the script's" elif [[ "$REPORTED_COUNT" -ne "$TRACKED_COUNT" ]]; then fail "the clean run validated $REPORTED_COUNT file(s) but $TRACKED_COUNT are tracked" else pass "validated $REPORTED_COUNT file(s), matching the $TRACKED_COUNT tracked in the index" fi # --- 2. An invalid agent file fails, and both the file and the reason are named --- echo "" echo "--- an invalid agent file fails, naming the file and the reason ---" FIX2="$(make_fixture)" FIXTURES+=("$FIX2") # `tools:` is deliberately absent from field-inventory.md's apm-agent-allowlist: # its value shape differs per harness and apm compile copies frontmatter verbatim # to every target (ADR-0016). python3 - "$FIX2/plugins/lint/.apm/agents/lint-runner.agent.md" <<'PY' import sys p = sys.argv[1] s = open(p).read() open(p, 'w').write(s.replace('---\n', '---\ntools: Read, Write\n', 1)) PY if bash "$SCRIPT" "$FIX2" > "$RUN_TMP/invalid.out" 2>&1; then fail "an agent file with a non-allowlisted frontmatter field still exited 0" sed 's/^/ /' "$RUN_TMP/invalid.out" elif ! grep -q 'plugins/lint/\.apm/agents/lint-runner\.agent\.md' "$RUN_TMP/invalid.out"; then fail "failed as expected but did not name the offending file" sed 's/^/ /' "$RUN_TMP/invalid.out" elif ! grep -q "field 'tools' is not in the vendor-neutral APM agent allowlist" "$RUN_TMP/invalid.out"; then fail "failed as expected and named the file but did not carry validate.sh's reason through" sed 's/^/ /' "$RUN_TMP/invalid.out" else pass "an invalid agent file exits 1, naming both the file and validate.sh's reason" fi # --- 3. Zero discovered files is an error, not a pass --- echo "" echo "--- zero discovered agent files is an error ---" FIX3="$(make_fixture)" FIXTURES+=("$FIX3") rm -f "$FIX3/plugins/lint/.apm/agents/lint-runner.agent.md" if bash "$SCRIPT" "$FIX3" > "$RUN_TMP/empty.out" 2>&1; then fail "a tree with zero agent files exited 0 — the floor is gone and the gate is vacuous" sed 's/^/ /' "$RUN_TMP/empty.out" elif ! grep -q 'found 0 plugin-scope agent file' "$RUN_TMP/empty.out"; then fail "a tree with zero agent files exited non-zero but not for the zero-file reason" sed 's/^/ /' "$RUN_TMP/empty.out" else pass "a tree with zero agent files exits 1 and says so" fi # --- 4. A tracked file missing from the worktree fails, derived from the index --- # This is the check a hardcoded count cannot make: the file is gone but the count # of what remains would still look plausible. echo "" echo "--- a tracked-but-deleted agent file fails against the derived expectation ---" FIX4="$(make_fixture)" FIXTURES+=("$FIX4") cat > "$FIX4/plugins/lint/.apm/agents/second-agent.agent.md" <<'MD' --- name: second-agent description: A second agent, present only so its deletion leaves a plausible-looking non-empty set behind. --- Do the second thing. MD git -C "$FIX4" init -q git -C "$FIX4" add -A git -C "$FIX4" -c user.email=t@example.invalid -c user.name=t commit -qm "fixture" rm -f "$FIX4/plugins/lint/.apm/agents/second-agent.agent.md" if bash "$SCRIPT" "$FIX4" > "$RUN_TMP/missing.out" 2>&1; then fail "a tracked agent file deleted from the worktree still exited 0" sed 's/^/ /' "$RUN_TMP/missing.out" elif grep -q 'not a git worktree root' "$RUN_TMP/missing.out"; then fail "the fixture did not resolve as its own git worktree root, so the derived check never ran" sed 's/^/ /' "$RUN_TMP/missing.out" elif ! grep -q 'second-agent\.agent\.md' "$RUN_TMP/missing.out"; then fail "failed as expected but did not name the tracked file that went missing" sed 's/^/ /' "$RUN_TMP/missing.out" else pass "a tracked agent file deleted from the worktree exits 1 and is named" fi # --- 5. An untracked agent file is still validated --- # The derived expectation is one-directional on purpose (tracked ⊆ discovered). # Work in progress must not fail the gate for being uncommitted — but it must # still be validated, or the gate would be trivially bypassed by not committing. echo "" echo "--- an untracked, invalid agent file still fails the gate ---" FIX5="$(make_fixture)" FIXTURES+=("$FIX5") git -C "$FIX5" init -q git -C "$FIX5" add -A git -C "$FIX5" -c user.email=t@example.invalid -c user.name=t commit -qm "fixture" cat > "$FIX5/plugins/lint/.apm/agents/wip-agent.agent.md" <<'MD' --- name: wip-agent tools: Read, Write description: An uncommitted work-in-progress agent carrying a non-allowlisted field. --- Do the work-in-progress thing. MD if bash "$SCRIPT" "$FIX5" > "$RUN_TMP/untracked.out" 2>&1; then fail "an untracked, invalid agent file was not validated — the gate can be bypassed by not committing" sed 's/^/ /' "$RUN_TMP/untracked.out" elif ! grep -q 'wip-agent\.agent\.md' "$RUN_TMP/untracked.out"; then fail "failed but did not name the untracked file" sed 's/^/ /' "$RUN_TMP/untracked.out" else pass "an untracked, invalid agent file exits 1 and is named" fi # --- 5b. An untracked but VALID agent file does not fail --- echo "" echo "--- an untracked, valid agent file passes ---" FIX5B="$(make_fixture)" FIXTURES+=("$FIX5B") git -C "$FIX5B" init -q git -C "$FIX5B" add -A git -C "$FIX5B" -c user.email=t@example.invalid -c user.name=t commit -qm "fixture" cat > "$FIX5B/plugins/lint/.apm/agents/wip-ok.agent.md" <<'MD' --- name: wip-ok description: An uncommitted work-in-progress agent that is nonetheless entirely valid. --- Do the valid work-in-progress thing. MD if bash "$SCRIPT" "$FIX5B" > "$RUN_TMP/untracked-ok.out" 2>&1; then pass "an untracked but valid agent file does not fail the gate" else fail "an untracked but valid agent file failed the gate — uncommitted work must not be an error" sed 's/^/ /' "$RUN_TMP/untracked-ok.out" fi # --- 6. A nonexistent REPO_ROOT fails loudly --- echo "" echo "--- a nonexistent REPO_ROOT fails loudly ---" if bash "$SCRIPT" "$RUN_TMP/does-not-exist" > "$RUN_TMP/norepo.out" 2>&1; then fail "a nonexistent REPO_ROOT exited 0" sed 's/^/ /' "$RUN_TMP/norepo.out" elif ! grep -q 'is not a directory' "$RUN_TMP/norepo.out"; then fail "a nonexistent REPO_ROOT failed for the wrong reason" sed 's/^/ /' "$RUN_TMP/norepo.out" else pass "a nonexistent REPO_ROOT exits 1 and says which path it was" fi # --- 7. A missing validator is a hard failure, never a silent pass --- # The whole gate is void without validate.sh, and exit 0 here would be # indistinguishable from a run where every agent passed. echo "" echo "--- a missing validate.sh fails rather than validating nothing ---" FIX7="$(make_fixture)" FIXTURES+=("$FIX7") rm -f "$FIX7/plugins/kyberforge/.apm/skills/agent-audit/scripts/validate.sh" if bash "$SCRIPT" "$FIX7" > "$RUN_TMP/novalidator.out" 2>&1; then fail "a missing validate.sh exited 0 — the gate silently validated nothing" sed 's/^/ /' "$RUN_TMP/novalidator.out" elif ! grep -q 'validator not found' "$RUN_TMP/novalidator.out"; then fail "a missing validate.sh failed for the wrong reason" sed 's/^/ /' "$RUN_TMP/novalidator.out" else pass "a missing validate.sh exits 1 and names the stale path" fi echo "" echo "Results: $PASS passed, $FAIL failed" [[ $FAIL -eq 0 ]]