Defame1297
675ba40238
Validates the new single-file .apm/agents/<name>.agent.md shape agent-author now produces at plugin/APM scope: frontmatter allowlist (name/description/ model only, from a new apm-agent-allowlist entry in field-inventory.md), no counterpart derivation, and Pair Consistency dropped from that scope's report entirely (nothing to pair by design). Adds a plugin/APM-scope-only SUGGESTION when an agent's description/body implies a tool restriction or Claude-only behavior the vendor-neutral frontmatter can no longer express (ADR-0016). Scope detection in both validate.sh and validate-provenance.sh switches from a flat plugin.json/.claude-plugin/plugin.json check to a walk-up for the nearest ancestor apm.yml with a top-level type: field, skipping type:-less marketplace-only manifests — full switch, no dual-mode fallback to the old plugin.json signal. validate-provenance.sh's walk-up was fixed to match validate.sh's (it still used the old plugin.json check, and its counterpart-merge logic was rewritten to read a single file's source_keys instead of merging a CC+Copilot pair, since plugin/APM scope has no counterpart). Project/user scope validation is unchanged in both scripts. Refs: #89
355 lines
10 KiB
Bash
355 lines
10 KiB
Bash
#!/usr/bin/env bats
|
|
|
|
setup() {
|
|
REPO_ROOT="$(cd "$BATS_TEST_DIRNAME/../../../../../" && pwd)"
|
|
load "$REPO_ROOT/tests/test_helper/bats-support/load"
|
|
load "$REPO_ROOT/tests/test_helper/bats-assert/load"
|
|
|
|
SCRIPT="$(cd "$BATS_TEST_DIRNAME/../scripts" && pwd)/validate.sh"
|
|
TMPDIR="$(mktemp -d)"
|
|
|
|
# Helper: create an APM package root at <root> (apm.yml with a top-level
|
|
# type: line, marking it a real package manifest — not marketplace-only)
|
|
# plus a single vendor-neutral agent file at
|
|
# <root>/.apm/agents/<name>.agent.md. <extra_frontmatter>, if given, is
|
|
# inserted as additional raw frontmatter lines (used to inject fields
|
|
# under test).
|
|
make_apm_agent() {
|
|
local root="$1"
|
|
local name="$2"
|
|
local extra_frontmatter="${3:-}"
|
|
mkdir -p "$root/.apm/agents"
|
|
cat > "$root/apm.yml" <<EOF
|
|
name: test-package
|
|
version: 0.1.0
|
|
type: skill
|
|
EOF
|
|
cat > "$root/.apm/agents/${name}.agent.md" <<EOF
|
|
---
|
|
name: ${name}
|
|
description: A valid agent description.
|
|
${extra_frontmatter}
|
|
---
|
|
|
|
You are a test agent. When invoked, do the thing.
|
|
EOF
|
|
}
|
|
}
|
|
|
|
teardown() {
|
|
rm -rf "$TMPDIR"
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Passing cases — project/user scope (unchanged)
|
|
# ---------------------------------------------------------------------------
|
|
|
|
@test "passes on a clean project-scope pair (CC file as input)" {
|
|
local root="$TMPDIR/project"
|
|
mkdir -p "$root/.git" "$root/.claude/agents" "$root/.github/agents"
|
|
cat > "$root/.claude/agents/my-agent.md" <<EOF
|
|
---
|
|
name: my-agent
|
|
description: A valid agent description.
|
|
---
|
|
|
|
You are a test agent. When invoked, do the thing.
|
|
EOF
|
|
cat > "$root/.github/agents/my-agent.agent.md" <<EOF
|
|
---
|
|
name: my-agent
|
|
description: A valid agent description.
|
|
---
|
|
|
|
You are a test agent. When invoked, do the thing.
|
|
EOF
|
|
run bash "$SCRIPT" "$root/.claude/agents/my-agent.md"
|
|
assert_success
|
|
refute_output --partial "FAIL"
|
|
}
|
|
|
|
@test "--help exits 0 and shows Usage:" {
|
|
run bash "$SCRIPT" --help
|
|
assert_success
|
|
assert_output --partial "Usage:"
|
|
}
|
|
|
|
@test "fails when no arguments are given" {
|
|
run bash "$SCRIPT"
|
|
assert_failure
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Passing cases — plugin/APM scope
|
|
# ---------------------------------------------------------------------------
|
|
|
|
@test "passes on a clean plugin/APM-scope agent file (name/description only)" {
|
|
local root="$TMPDIR/pkg"
|
|
make_apm_agent "$root" "my-agent"
|
|
run bash "$SCRIPT" "$root/.apm/agents/my-agent.agent.md"
|
|
assert_success
|
|
refute_output --partial "FAIL"
|
|
}
|
|
|
|
@test "passes on a clean plugin/APM-scope agent file with optional model field" {
|
|
local root="$TMPDIR/pkg"
|
|
make_apm_agent "$root" "my-agent" "model: claude-opus-4"
|
|
run bash "$SCRIPT" "$root/.apm/agents/my-agent.agent.md"
|
|
assert_success
|
|
refute_output --partial "FAIL"
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Failing cases — plugin/APM scope: allowlist violations
|
|
# ---------------------------------------------------------------------------
|
|
|
|
@test "fails when 'tools' field is present in a plugin/APM-scope agent file" {
|
|
local root="$TMPDIR/pkg"
|
|
make_apm_agent "$root" "my-agent" "tools: Read Edit"
|
|
run bash "$SCRIPT" "$root/.apm/agents/my-agent.agent.md"
|
|
assert_failure
|
|
assert_output --partial "tools"
|
|
}
|
|
|
|
@test "fails when a Claude-only field ('maxTurns') is present in a plugin/APM-scope agent file" {
|
|
local root="$TMPDIR/pkg"
|
|
make_apm_agent "$root" "my-agent" "maxTurns: 10"
|
|
run bash "$SCRIPT" "$root/.apm/agents/my-agent.agent.md"
|
|
assert_failure
|
|
assert_output --partial "maxTurns"
|
|
}
|
|
|
|
@test "fails when a Copilot-only field ('target') is present in a plugin/APM-scope agent file" {
|
|
local root="$TMPDIR/pkg"
|
|
make_apm_agent "$root" "my-agent" "target: cli"
|
|
run bash "$SCRIPT" "$root/.apm/agents/my-agent.agent.md"
|
|
assert_failure
|
|
assert_output --partial "target"
|
|
}
|
|
|
|
@test "fails when 'hooks' is present in a plugin/APM-scope agent file (outside allowlist)" {
|
|
local root="$TMPDIR/pkg"
|
|
make_apm_agent "$root" "my-agent" "hooks: {}"
|
|
run bash "$SCRIPT" "$root/.apm/agents/my-agent.agent.md"
|
|
assert_failure
|
|
assert_output --partial "hooks"
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Failing cases — plugin/APM scope: structural checks
|
|
# ---------------------------------------------------------------------------
|
|
|
|
@test "fails when name is not kebab-case in a plugin/APM-scope agent file" {
|
|
local root="$TMPDIR/pkg"
|
|
mkdir -p "$root/.apm/agents"
|
|
cat > "$root/apm.yml" <<EOF
|
|
name: test-package
|
|
version: 0.1.0
|
|
type: skill
|
|
EOF
|
|
cat > "$root/.apm/agents/my-agent.agent.md" <<EOF
|
|
---
|
|
name: MyAgent
|
|
description: A valid agent description.
|
|
---
|
|
|
|
You are a test agent. When invoked, do the thing.
|
|
EOF
|
|
run bash "$SCRIPT" "$root/.apm/agents/my-agent.agent.md"
|
|
assert_failure
|
|
assert_output --partial "kebab"
|
|
}
|
|
|
|
@test "fails when name does not match filename stem in a plugin/APM-scope agent file" {
|
|
local root="$TMPDIR/pkg"
|
|
mkdir -p "$root/.apm/agents"
|
|
cat > "$root/apm.yml" <<EOF
|
|
name: test-package
|
|
version: 0.1.0
|
|
type: skill
|
|
EOF
|
|
cat > "$root/.apm/agents/my-agent.agent.md" <<EOF
|
|
---
|
|
name: wrong-name
|
|
description: A valid agent description.
|
|
---
|
|
|
|
You are a test agent. When invoked, do the thing.
|
|
EOF
|
|
run bash "$SCRIPT" "$root/.apm/agents/my-agent.agent.md"
|
|
assert_failure
|
|
assert_output --partial "does not match filename stem"
|
|
}
|
|
|
|
@test "fails when 'name' field is missing from a plugin/APM-scope agent file" {
|
|
local root="$TMPDIR/pkg"
|
|
mkdir -p "$root/.apm/agents"
|
|
cat > "$root/apm.yml" <<EOF
|
|
name: test-package
|
|
version: 0.1.0
|
|
type: skill
|
|
EOF
|
|
cat > "$root/.apm/agents/my-agent.agent.md" <<EOF
|
|
---
|
|
description: A valid agent description.
|
|
---
|
|
|
|
You are a test agent. When invoked, do the thing.
|
|
EOF
|
|
run bash "$SCRIPT" "$root/.apm/agents/my-agent.agent.md"
|
|
assert_failure
|
|
}
|
|
|
|
@test "fails when 'description' field is missing from a plugin/APM-scope agent file" {
|
|
local root="$TMPDIR/pkg"
|
|
mkdir -p "$root/.apm/agents"
|
|
cat > "$root/apm.yml" <<EOF
|
|
name: test-package
|
|
version: 0.1.0
|
|
type: skill
|
|
EOF
|
|
cat > "$root/.apm/agents/my-agent.agent.md" <<EOF
|
|
---
|
|
name: my-agent
|
|
---
|
|
|
|
You are a test agent. When invoked, do the thing.
|
|
EOF
|
|
run bash "$SCRIPT" "$root/.apm/agents/my-agent.agent.md"
|
|
assert_failure
|
|
}
|
|
|
|
@test "fails when body contains unfilled FILL IN: placeholder in a plugin/APM-scope agent file" {
|
|
local root="$TMPDIR/pkg"
|
|
mkdir -p "$root/.apm/agents"
|
|
cat > "$root/apm.yml" <<EOF
|
|
name: test-package
|
|
version: 0.1.0
|
|
type: skill
|
|
EOF
|
|
cat > "$root/.apm/agents/my-agent.agent.md" <<EOF
|
|
---
|
|
name: my-agent
|
|
description: A valid agent description.
|
|
---
|
|
|
|
FILL IN: replace this with your system prompt.
|
|
EOF
|
|
run bash "$SCRIPT" "$root/.apm/agents/my-agent.agent.md"
|
|
assert_failure
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Plugin/APM scope: no pair, no counterpart concept
|
|
# ---------------------------------------------------------------------------
|
|
|
|
@test "never raises a 'counterpart' FAIL on a clean plugin/APM-scope agent file" {
|
|
local root="$TMPDIR/pkg"
|
|
make_apm_agent "$root" "my-agent"
|
|
run bash "$SCRIPT" "$root/.apm/agents/my-agent.agent.md"
|
|
assert_success
|
|
refute_output --partial "counterpart"
|
|
}
|
|
|
|
@test "never raises a 'counterpart' FAIL on a failing plugin/APM-scope agent file" {
|
|
local root="$TMPDIR/pkg"
|
|
make_apm_agent "$root" "my-agent" "tools: Read"
|
|
run bash "$SCRIPT" "$root/.apm/agents/my-agent.agent.md"
|
|
assert_failure
|
|
refute_output --partial "counterpart"
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Scope-detection walk-up
|
|
# ---------------------------------------------------------------------------
|
|
|
|
@test "walk-up skips a type:-less apm.yml and finds a real package root further up" {
|
|
local root="$TMPDIR/case"
|
|
mkdir -p "$root/.apm/agents/nested/deeper"
|
|
cat > "$root/apm.yml" <<EOF
|
|
name: real-package
|
|
version: 1.0.0
|
|
type: skill
|
|
EOF
|
|
# Closer to the agent file than the real package root, but has no type:
|
|
# line — marketplace-only per monorepo-and-repo-shapes.md, must be skipped.
|
|
cat > "$root/.apm/agents/nested/apm.yml" <<EOF
|
|
name: not-a-package-manifest
|
|
version: 1.0.0
|
|
EOF
|
|
cat > "$root/.apm/agents/nested/deeper/my-agent.agent.md" <<EOF
|
|
---
|
|
name: my-agent
|
|
description: A valid agent description.
|
|
---
|
|
|
|
You are a test agent. When invoked, do the thing.
|
|
EOF
|
|
run bash "$SCRIPT" "$root/.apm/agents/nested/deeper/my-agent.agent.md"
|
|
assert_success
|
|
refute_output --partial "FAIL"
|
|
}
|
|
|
|
@test "type:-less apm.yml is not treated as plugin scope — falls through to project scope" {
|
|
local root="$TMPDIR/proj-marketplace"
|
|
mkdir -p "$root/.git" "$root/.claude/agents" "$root/.github/agents"
|
|
cat > "$root/apm.yml" <<EOF
|
|
name: marketplace-root
|
|
version: 1.0.0
|
|
marketplace:
|
|
packages: []
|
|
EOF
|
|
cat > "$root/.claude/agents/my-agent.md" <<EOF
|
|
---
|
|
name: my-agent
|
|
description: A valid agent description.
|
|
---
|
|
|
|
You are a test agent. When invoked, do the thing.
|
|
EOF
|
|
cat > "$root/.github/agents/my-agent.agent.md" <<EOF
|
|
---
|
|
name: my-agent
|
|
description: A valid agent description.
|
|
---
|
|
|
|
You are a test agent. When invoked, do the thing.
|
|
EOF
|
|
run bash "$SCRIPT" "$root/.claude/agents/my-agent.md"
|
|
assert_success
|
|
refute_output --partial "FAIL"
|
|
}
|
|
|
|
@test "a bare plugin.json with no apm.yml is no longer plugin scope — falls through to project scope" {
|
|
local root="$TMPDIR/proj-legacy-plugin-json"
|
|
mkdir -p "$root/.git" "$root/.claude/agents" "$root/.github/agents"
|
|
echo '{}' > "$root/plugin.json"
|
|
# 'hooks' is plugin-silently-ignored only at (old) plugin scope; at
|
|
# project scope it's a legitimate CC field. If this directory were
|
|
# mis-detected as plugin scope (old plugin.json-based logic), this would
|
|
# FAIL with a plugin-silently-ignored-fields finding on 'hooks'.
|
|
cat > "$root/.claude/agents/my-agent.md" <<EOF
|
|
---
|
|
name: my-agent
|
|
description: A valid agent description.
|
|
hooks:
|
|
PostToolUse:
|
|
- match: ".*"
|
|
command: "echo done"
|
|
---
|
|
|
|
You are a test agent. When invoked, do the thing.
|
|
EOF
|
|
cat > "$root/.github/agents/my-agent.agent.md" <<EOF
|
|
---
|
|
name: my-agent
|
|
description: A valid agent description.
|
|
---
|
|
|
|
You are a test agent. When invoked, do the thing.
|
|
EOF
|
|
run bash "$SCRIPT" "$root/.claude/agents/my-agent.md"
|
|
assert_success
|
|
refute_output --partial "hooks"
|
|
}
|