Files
holocron/plugins/kyberforge/skills/agent-audit/tests/validate.bats
Defame1297 044b2d3f08 fix(kyberforge): fix HOME/git scope-walkup false-FAILs in agent-audit
validate.sh's detect_scope() and validate-provenance.sh's
find_plugin_root() disagreed with new-agent.sh's already-correct,
documented walk-up semantics on three points, each causing validate.sh
to false-FAIL a legitimately-scaffolded project-scope agent pair:

- a marker-less directory walked up into $HOME (no .git/apm.yml of its
  own) was classified as user scope instead of project scope
- the .git-boundary branch returned the walked-to .git location instead
  of the conventional scope root, breaking any <root> that is a
  subdirectory of a larger git-tracked tree (monorepo package dirs)
- the new conventional-root arithmetic introduced to fix the above two
  cases had no guard against non-conventional/hand-placed file paths,
  which could point it at the wrong ancestor

Also adds scripts/check-scope-walkup-sync.sh, a behavioral drift-guard
(per ADR-0014's no-cross-skill-path precedent) that cross-checks the
four independently hand-ported walk-up implementations (validate.sh,
validate-provenance.sh, new-agent.sh, new-skill.sh) against real
fixture scaffolds, wired into .pre-commit-config.yaml at pre-push so
future drift between the ports is caught automatically.

Verified via bash tests/run-tests.sh (13/13) and targeted before/after
reproduction of each bug this closes.
2026-08-12 11:35:23 +00:00

642 lines
19 KiB
Bash

#!/usr/bin/env bats
setup() {
REPO_ROOT="$(cd "$BATS_TEST_DIRNAME/../../../../../" && pwd)"
load "$REPO_ROOT/tests/test_helper/bats-support/load"
load "$REPO_ROOT/tests/test_helper/bats-assert/load"
SCRIPT="$(cd "$BATS_TEST_DIRNAME/../scripts" && pwd)/validate.sh"
TMPDIR="$(mktemp -d)"
# Helper: create an APM package root at <root> (apm.yml with a top-level
# type: line, marking it a real package manifest — not marketplace-only)
# plus a single vendor-neutral agent file at
# <root>/.apm/agents/<name>.agent.md. <extra_frontmatter>, if given, is
# inserted as additional raw frontmatter lines (used to inject fields
# under test).
make_apm_agent() {
local root="$1"
local name="$2"
local extra_frontmatter="${3:-}"
mkdir -p "$root/.apm/agents"
cat > "$root/apm.yml" <<EOF
name: test-package
version: 0.1.0
type: skill
EOF
cat > "$root/.apm/agents/${name}.agent.md" <<EOF
---
name: ${name}
description: A valid agent description.
${extra_frontmatter}
---
You are a test agent. When invoked, do the thing.
EOF
}
}
teardown() {
rm -rf "$TMPDIR"
}
# ---------------------------------------------------------------------------
# Passing cases — project/user scope (unchanged)
# ---------------------------------------------------------------------------
@test "passes on a clean project-scope pair (CC file as input)" {
local root="$TMPDIR/project"
mkdir -p "$root/.git" "$root/.claude/agents" "$root/.github/agents"
cat > "$root/.claude/agents/my-agent.md" <<EOF
---
name: my-agent
description: A valid agent description.
---
You are a test agent. When invoked, do the thing.
EOF
cat > "$root/.github/agents/my-agent.agent.md" <<EOF
---
name: my-agent
description: A valid agent description.
---
You are a test agent. When invoked, do the thing.
EOF
run bash "$SCRIPT" "$root/.claude/agents/my-agent.md"
assert_success
refute_output --partial "FAIL"
}
@test "user scope: \$HOME being a dotfiles .git repo does not shadow user scope" {
local fake_home="$TMPDIR/fakehome"
mkdir -p "$fake_home/.git" "$fake_home/.claude/agents" "$fake_home/.copilot/agents"
cat > "$fake_home/.claude/agents/my-agent.md" <<EOF
---
name: my-agent
description: A valid agent description.
---
You are a test agent. When invoked, do the thing.
EOF
cat > "$fake_home/.copilot/agents/my-agent.agent.md" <<EOF
---
name: my-agent
description: A valid agent description.
---
You are a test agent. When invoked, do the thing.
EOF
run env HOME="$fake_home" bash "$SCRIPT" "$fake_home/.claude/agents/my-agent.md"
assert_success
refute_output --partial "FAIL"
}
@test "user scope: agent file directly in \$HOME (start dir IS exactly \$HOME, no walk-up) resolves to user scope" {
local fake_home="$TMPDIR/fakehome-direct"
mkdir -p "$fake_home" "$fake_home/.copilot/agents"
cat > "$fake_home/my-agent.md" <<EOF
---
name: my-agent
description: A valid agent description.
---
You are a test agent. When invoked, do the thing.
EOF
cat > "$fake_home/.copilot/agents/my-agent.agent.md" <<EOF
---
name: my-agent
description: A valid agent description.
---
You are a test agent. When invoked, do the thing.
EOF
run env HOME="$fake_home" bash "$SCRIPT" "$fake_home/my-agent.md"
assert_success
refute_output --partial "FAIL"
refute_output --partial "counterpart"
}
@test "project scope: a nested marker-less directory walked up into \$HOME resolves to project scope, not user scope (live repro of new-agent.sh's stray-directory case)" {
local fake_home="$TMPDIR/fakehome-nested"
local nested="$fake_home/scratch/testdir"
mkdir -p "$nested/.claude/agents" "$nested/.github/agents"
cat > "$nested/.claude/agents/my-agent.md" <<EOF
---
name: my-agent
description: A valid agent description.
---
You are a test agent. When invoked, do the thing.
EOF
cat > "$nested/.github/agents/my-agent.agent.md" <<EOF
---
name: my-agent
description: A valid agent description.
---
You are a test agent. When invoked, do the thing.
EOF
run env HOME="$fake_home" bash "$SCRIPT" "$nested/.claude/agents/my-agent.md"
assert_success
refute_output --partial "FAIL"
refute_output --partial "counterpart"
}
@test "project scope: nested marker-less dir under \$HOME does NOT look for a counterpart under the shared \$HOME/.copilot or \$HOME/.github dirs" {
local fake_home="$TMPDIR/fakehome-nested2"
local nested="$fake_home/scratch/testdir"
mkdir -p "$nested/.claude/agents" "$fake_home/.copilot/agents"
cat > "$nested/.claude/agents/my-agent.md" <<EOF
---
name: my-agent
description: A valid agent description.
---
You are a test agent. When invoked, do the thing.
EOF
# Decoy counterpart at the *user*-scope location — if scope were
# misclassified as 'user' (the pre-fix bug), validate.sh would find this
# unrelated file and (wrongly) pass.
cat > "$fake_home/.copilot/agents/my-agent.agent.md" <<EOF
---
name: my-agent
description: A valid agent description.
---
You are a test agent. When invoked, do the thing.
EOF
run env HOME="$fake_home" bash "$SCRIPT" "$nested/.claude/agents/my-agent.md"
assert_failure
assert_output --partial "counterpart file not found"
}
@test "project scope: filesystem-root fallback (no \$HOME in path, no markers found) resolves to project scope, not user scope" {
local unrelated_home="$TMPDIR/unrelated-home-never-reached"
local root="$TMPDIR/no-home-relation/deep/proj"
mkdir -p "$root/.claude/agents" "$root/.github/agents"
cat > "$root/.claude/agents/my-agent.md" <<EOF
---
name: my-agent
description: A valid agent description.
---
You are a test agent. When invoked, do the thing.
EOF
cat > "$root/.github/agents/my-agent.agent.md" <<EOF
---
name: my-agent
description: A valid agent description.
---
You are a test agent. When invoked, do the thing.
EOF
run env HOME="$unrelated_home" bash "$SCRIPT" "$root/.claude/agents/my-agent.md"
assert_success
refute_output --partial "FAIL"
refute_output --partial "counterpart"
}
@test "project scope: <root> one level below a .git ancestor resolves scope to <root>, not to wherever .git was found (subdirectory of a larger git-tracked tree)" {
local repo="$TMPDIR/repo-with-subdir"
local root="$repo/subdir"
mkdir -p "$repo/.git" "$root/.claude/agents" "$root/.github/agents"
cat > "$root/.claude/agents/my-agent.md" <<EOF
---
name: my-agent
description: A valid agent description.
---
You are a test agent. When invoked, do the thing.
EOF
cat > "$root/.github/agents/my-agent.agent.md" <<EOF
---
name: my-agent
description: A valid agent description.
---
You are a test agent. When invoked, do the thing.
EOF
# new-agent.sh, invoked with <root> as its root argument, would place the
# counterpart at <root>/.github/agents — not at the repo root's
# .github/agents, even though .git lives at the repo root one level up.
run bash "$SCRIPT" "$root/.claude/agents/my-agent.md"
assert_success
refute_output --partial "FAIL"
refute_output --partial "counterpart"
}
@test "project scope: a non-conventional path (agent file not directly under a literal 'agents' dir) falls back to the nearest .git boundary instead of two-segments-up arithmetic" {
local outer="$TMPDIR/outer-repo"
local pkg="$outer/pkgA"
mkdir -p "$pkg/.git" "$pkg/.github/agents" "$pkg/extra"
# Misplaced file: sits two path segments below $outer (pkgA/extra), which
# matches the conventional_root arithmetic by coincidence, but its
# immediate parent dir is "extra", not "agents" — conventional_shape is
# false, so the fix must fall back to the nearest .git boundary (pkgA),
# not trust $outer.
cat > "$pkg/extra/my-agent.md" <<EOF
---
name: my-agent
description: A valid agent description.
---
You are a test agent. When invoked, do the thing.
EOF
# Counterpart at the nearest-.git root (pkgA), not at $outer — if the
# arithmetic were trusted here, validate.sh would look for a counterpart
# at $outer/.github/agents/my-agent.agent.md, which doesn't exist, and
# false-FAIL.
cat > "$pkg/.github/agents/my-agent.agent.md" <<EOF
---
name: my-agent
description: A valid agent description.
---
You are a test agent. When invoked, do the thing.
EOF
run bash "$SCRIPT" "$pkg/extra/my-agent.md"
assert_success
refute_output --partial "counterpart file not found"
}
# ---------------------------------------------------------------------------
# Failing cases — project/user scope: CC/Copilot pair checks
# ---------------------------------------------------------------------------
@test "fails when a CC-only field ('maxTurns') is present in a project-scope Copilot file" {
local root="$TMPDIR/project"
mkdir -p "$root/.git" "$root/.claude/agents" "$root/.github/agents"
cat > "$root/.claude/agents/my-agent.md" <<EOF
---
name: my-agent
description: A valid agent description.
---
You are a test agent. When invoked, do the thing.
EOF
cat > "$root/.github/agents/my-agent.agent.md" <<EOF
---
name: my-agent
description: A valid agent description.
maxTurns: 10
---
You are a test agent. When invoked, do the thing.
EOF
run bash "$SCRIPT" "$root/.claude/agents/my-agent.md"
assert_failure
assert_output --partial "CC-only field"
assert_output --partial "maxTurns"
}
@test "fails when a Copilot-only field ('target') is present in a project-scope CC file" {
local root="$TMPDIR/project"
mkdir -p "$root/.git" "$root/.claude/agents" "$root/.github/agents"
cat > "$root/.claude/agents/my-agent.md" <<EOF
---
name: my-agent
description: A valid agent description.
target: cli
---
You are a test agent. When invoked, do the thing.
EOF
cat > "$root/.github/agents/my-agent.agent.md" <<EOF
---
name: my-agent
description: A valid agent description.
---
You are a test agent. When invoked, do the thing.
EOF
run bash "$SCRIPT" "$root/.claude/agents/my-agent.md"
assert_failure
assert_output --partial "Copilot-only field"
assert_output --partial "target"
}
@test "fails when the Copilot counterpart is missing at project scope" {
local root="$TMPDIR/project"
mkdir -p "$root/.git" "$root/.claude/agents"
cat > "$root/.claude/agents/my-agent.md" <<EOF
---
name: my-agent
description: A valid agent description.
---
You are a test agent. When invoked, do the thing.
EOF
run bash "$SCRIPT" "$root/.claude/agents/my-agent.md"
assert_failure
assert_output --partial "counterpart file not found"
}
@test "--help exits 0 and shows Usage:" {
run bash "$SCRIPT" --help
assert_success
assert_output --partial "Usage:"
}
@test "fails when no arguments are given" {
run bash "$SCRIPT"
assert_failure
}
# ---------------------------------------------------------------------------
# Passing cases — plugin/APM scope
# ---------------------------------------------------------------------------
@test "passes on a clean plugin/APM-scope agent file (name/description only)" {
local root="$TMPDIR/pkg"
make_apm_agent "$root" "my-agent"
run bash "$SCRIPT" "$root/.apm/agents/my-agent.agent.md"
assert_success
refute_output --partial "FAIL"
}
@test "passes on a clean plugin/APM-scope agent file with optional model field" {
local root="$TMPDIR/pkg"
make_apm_agent "$root" "my-agent" "model: claude-opus-4"
run bash "$SCRIPT" "$root/.apm/agents/my-agent.agent.md"
assert_success
refute_output --partial "FAIL"
}
# ---------------------------------------------------------------------------
# Failing cases — plugin/APM scope: allowlist violations
# ---------------------------------------------------------------------------
@test "fails when 'tools' field is present in a plugin/APM-scope agent file" {
local root="$TMPDIR/pkg"
make_apm_agent "$root" "my-agent" "tools: Read Edit"
run bash "$SCRIPT" "$root/.apm/agents/my-agent.agent.md"
assert_failure
assert_output --partial "tools"
}
@test "fails when a Claude-only field ('maxTurns') is present in a plugin/APM-scope agent file" {
local root="$TMPDIR/pkg"
make_apm_agent "$root" "my-agent" "maxTurns: 10"
run bash "$SCRIPT" "$root/.apm/agents/my-agent.agent.md"
assert_failure
assert_output --partial "maxTurns"
}
@test "fails when a Copilot-only field ('target') is present in a plugin/APM-scope agent file" {
local root="$TMPDIR/pkg"
make_apm_agent "$root" "my-agent" "target: cli"
run bash "$SCRIPT" "$root/.apm/agents/my-agent.agent.md"
assert_failure
assert_output --partial "target"
}
@test "fails when 'hooks' is present in a plugin/APM-scope agent file (outside allowlist)" {
local root="$TMPDIR/pkg"
make_apm_agent "$root" "my-agent" "hooks: {}"
run bash "$SCRIPT" "$root/.apm/agents/my-agent.agent.md"
assert_failure
assert_output --partial "hooks"
}
# ---------------------------------------------------------------------------
# Failing cases — plugin/APM scope: structural checks
# ---------------------------------------------------------------------------
@test "fails when name is not kebab-case in a plugin/APM-scope agent file" {
local root="$TMPDIR/pkg"
mkdir -p "$root/.apm/agents"
cat > "$root/apm.yml" <<EOF
name: test-package
version: 0.1.0
type: skill
EOF
cat > "$root/.apm/agents/my-agent.agent.md" <<EOF
---
name: MyAgent
description: A valid agent description.
---
You are a test agent. When invoked, do the thing.
EOF
run bash "$SCRIPT" "$root/.apm/agents/my-agent.agent.md"
assert_failure
assert_output --partial "kebab"
}
@test "fails when name does not match filename stem in a plugin/APM-scope agent file" {
local root="$TMPDIR/pkg"
mkdir -p "$root/.apm/agents"
cat > "$root/apm.yml" <<EOF
name: test-package
version: 0.1.0
type: skill
EOF
cat > "$root/.apm/agents/my-agent.agent.md" <<EOF
---
name: wrong-name
description: A valid agent description.
---
You are a test agent. When invoked, do the thing.
EOF
run bash "$SCRIPT" "$root/.apm/agents/my-agent.agent.md"
assert_failure
assert_output --partial "does not match filename stem"
}
@test "fails when 'name' field is missing from a plugin/APM-scope agent file" {
local root="$TMPDIR/pkg"
mkdir -p "$root/.apm/agents"
cat > "$root/apm.yml" <<EOF
name: test-package
version: 0.1.0
type: skill
EOF
cat > "$root/.apm/agents/my-agent.agent.md" <<EOF
---
description: A valid agent description.
---
You are a test agent. When invoked, do the thing.
EOF
run bash "$SCRIPT" "$root/.apm/agents/my-agent.agent.md"
assert_failure
}
@test "fails when 'description' field is missing from a plugin/APM-scope agent file" {
local root="$TMPDIR/pkg"
mkdir -p "$root/.apm/agents"
cat > "$root/apm.yml" <<EOF
name: test-package
version: 0.1.0
type: skill
EOF
cat > "$root/.apm/agents/my-agent.agent.md" <<EOF
---
name: my-agent
---
You are a test agent. When invoked, do the thing.
EOF
run bash "$SCRIPT" "$root/.apm/agents/my-agent.agent.md"
assert_failure
}
@test "fails when a template HTML comment is left in plugin/APM-scope frontmatter" {
local root="$TMPDIR/pkg"
mkdir -p "$root/.apm/agents"
cat > "$root/apm.yml" <<EOF
name: test-package
version: 0.1.0
type: skill
EOF
cat > "$root/.apm/agents/my-agent.agent.md" <<EOF
---
name: my-agent
description: A valid agent description.
<!-- model: sonnet
Optional. Omit to inherit the runtime default. -->
---
You are a test agent. When invoked, do the thing.
EOF
run bash "$SCRIPT" "$root/.apm/agents/my-agent.agent.md"
assert_failure
assert_output --partial "template HTML comments"
}
@test "fails when body contains unfilled FILL IN: placeholder in a plugin/APM-scope agent file" {
local root="$TMPDIR/pkg"
mkdir -p "$root/.apm/agents"
cat > "$root/apm.yml" <<EOF
name: test-package
version: 0.1.0
type: skill
EOF
cat > "$root/.apm/agents/my-agent.agent.md" <<EOF
---
name: my-agent
description: A valid agent description.
---
FILL IN: replace this with your system prompt.
EOF
run bash "$SCRIPT" "$root/.apm/agents/my-agent.agent.md"
assert_failure
}
# ---------------------------------------------------------------------------
# Plugin/APM scope: no pair, no counterpart concept
# ---------------------------------------------------------------------------
@test "never raises a 'counterpart' FAIL on a clean plugin/APM-scope agent file" {
local root="$TMPDIR/pkg"
make_apm_agent "$root" "my-agent"
run bash "$SCRIPT" "$root/.apm/agents/my-agent.agent.md"
assert_success
refute_output --partial "counterpart"
}
@test "never raises a 'counterpart' FAIL on a failing plugin/APM-scope agent file" {
local root="$TMPDIR/pkg"
make_apm_agent "$root" "my-agent" "tools: Read"
run bash "$SCRIPT" "$root/.apm/agents/my-agent.agent.md"
assert_failure
refute_output --partial "counterpart"
}
# ---------------------------------------------------------------------------
# Scope-detection walk-up
# ---------------------------------------------------------------------------
@test "walk-up skips a type:-less apm.yml and finds a real package root further up" {
local root="$TMPDIR/case"
mkdir -p "$root/.apm/agents/nested/deeper"
cat > "$root/apm.yml" <<EOF
name: real-package
version: 1.0.0
type: skill
EOF
# Closer to the agent file than the real package root, but has no type:
# line — marketplace-only per monorepo-and-repo-shapes.md, must be skipped.
cat > "$root/.apm/agents/nested/apm.yml" <<EOF
name: not-a-package-manifest
version: 1.0.0
EOF
cat > "$root/.apm/agents/nested/deeper/my-agent.agent.md" <<EOF
---
name: my-agent
description: A valid agent description.
---
You are a test agent. When invoked, do the thing.
EOF
run bash "$SCRIPT" "$root/.apm/agents/nested/deeper/my-agent.agent.md"
assert_success
refute_output --partial "FAIL"
}
@test "type:-less apm.yml is not treated as plugin scope — falls through to project scope" {
local root="$TMPDIR/proj-marketplace"
mkdir -p "$root/.git" "$root/.claude/agents" "$root/.github/agents"
cat > "$root/apm.yml" <<EOF
name: marketplace-root
version: 1.0.0
marketplace:
packages: []
EOF
cat > "$root/.claude/agents/my-agent.md" <<EOF
---
name: my-agent
description: A valid agent description.
---
You are a test agent. When invoked, do the thing.
EOF
cat > "$root/.github/agents/my-agent.agent.md" <<EOF
---
name: my-agent
description: A valid agent description.
---
You are a test agent. When invoked, do the thing.
EOF
run bash "$SCRIPT" "$root/.claude/agents/my-agent.md"
assert_success
refute_output --partial "FAIL"
}
@test "a bare plugin.json with no apm.yml is no longer plugin scope — falls through to project scope" {
local root="$TMPDIR/proj-legacy-plugin-json"
mkdir -p "$root/.git" "$root/.claude/agents" "$root/.github/agents"
echo '{}' > "$root/plugin.json"
# 'hooks' is plugin-silently-ignored only at (old) plugin scope; at
# project scope it's a legitimate CC field. If this directory were
# mis-detected as plugin scope (old plugin.json-based logic), this would
# FAIL with a plugin-silently-ignored-fields finding on 'hooks'.
cat > "$root/.claude/agents/my-agent.md" <<EOF
---
name: my-agent
description: A valid agent description.
hooks:
PostToolUse:
- match: ".*"
command: "echo done"
---
You are a test agent. When invoked, do the thing.
EOF
cat > "$root/.github/agents/my-agent.agent.md" <<EOF
---
name: my-agent
description: A valid agent description.
---
You are a test agent. When invoked, do the thing.
EOF
run bash "$SCRIPT" "$root/.claude/agents/my-agent.md"
assert_success
refute_output --partial "hooks"
}