feat(kyberforge): restructure agent-audit for plugin-scope apm agents
Validates the new single-file .apm/agents/<name>.agent.md shape agent-author now produces at plugin/APM scope: frontmatter allowlist (name/description/ model only, from a new apm-agent-allowlist entry in field-inventory.md), no counterpart derivation, and Pair Consistency dropped from that scope's report entirely (nothing to pair by design). Adds a plugin/APM-scope-only SUGGESTION when an agent's description/body implies a tool restriction or Claude-only behavior the vendor-neutral frontmatter can no longer express (ADR-0016). Scope detection in both validate.sh and validate-provenance.sh switches from a flat plugin.json/.claude-plugin/plugin.json check to a walk-up for the nearest ancestor apm.yml with a top-level type: field, skipping type:-less marketplace-only manifests — full switch, no dual-mode fallback to the old plugin.json signal. validate-provenance.sh's walk-up was fixed to match validate.sh's (it still used the old plugin.json check, and its counterpart-merge logic was rewritten to read a single file's source_keys instead of merging a CC+Copilot pair, since plugin/APM scope has no counterpart). Project/user scope validation is unchanged in both scripts. Refs: #89
This commit is contained in:
@@ -8,26 +8,24 @@ setup() {
|
||||
SCRIPT="$(cd "$BATS_TEST_DIRNAME/../scripts" && pwd)/validate-provenance.sh"
|
||||
TMPDIR="$(mktemp -d)"
|
||||
|
||||
# Helper: create a plugin root with plugin.json and an agents/ directory
|
||||
make_plugin() {
|
||||
# Helper: create an APM package root at <root> (apm.yml with a top-level
|
||||
# type: line — a real package manifest, not marketplace-only) plus a
|
||||
# single vendor-neutral agent file at <root>/.apm/agents/<name>.agent.md.
|
||||
make_package() {
|
||||
local root="$1"
|
||||
mkdir -p "$root/agents"
|
||||
echo '{"name":"test-plugin","version":"0.1.0"}' > "$root/plugin.json"
|
||||
mkdir -p "$root/.apm/agents"
|
||||
cat > "$root/apm.yml" <<EOF
|
||||
name: test-package
|
||||
version: 0.1.0
|
||||
type: skill
|
||||
EOF
|
||||
}
|
||||
|
||||
# Helper: create a clean agent pair (no source_keys)
|
||||
make_clean_pair() {
|
||||
# Helper: create a clean agent file (no source_keys)
|
||||
make_clean_agent() {
|
||||
local root="$1"
|
||||
local name="${2:-my-agent}"
|
||||
cat > "$root/agents/${name}.md" <<EOF
|
||||
---
|
||||
name: ${name}
|
||||
description: A valid agent description.
|
||||
---
|
||||
|
||||
You are a test agent.
|
||||
EOF
|
||||
cat > "$root/agents/${name}.agent.md" <<EOF
|
||||
cat > "$root/.apm/agents/${name}.agent.md" <<EOF
|
||||
---
|
||||
name: ${name}
|
||||
description: A valid agent description.
|
||||
@@ -37,12 +35,12 @@ You are a test agent.
|
||||
EOF
|
||||
}
|
||||
|
||||
# Helper: create a CC agent file with source_keys
|
||||
make_cc_with_source_keys() {
|
||||
# Helper: create an agent file with source_keys
|
||||
make_agent_with_source_keys() {
|
||||
local root="$1"
|
||||
local name="${2:-my-agent}"
|
||||
local slug="${3:-my-source}"
|
||||
cat > "$root/agents/${name}.md" <<EOF
|
||||
cat > "$root/.apm/agents/${name}.agent.md" <<EOF
|
||||
---
|
||||
name: ${name}
|
||||
description: A valid agent description.
|
||||
@@ -50,37 +48,6 @@ source_keys:
|
||||
- ${slug}
|
||||
---
|
||||
|
||||
You are a test agent.
|
||||
EOF
|
||||
}
|
||||
|
||||
# Helper: create a Copilot agent file with source_keys
|
||||
make_copilot_with_source_keys() {
|
||||
local root="$1"
|
||||
local name="${2:-my-agent}"
|
||||
local slug="${3:-my-source}"
|
||||
cat > "$root/agents/${name}.agent.md" <<EOF
|
||||
---
|
||||
name: ${name}
|
||||
description: A valid agent description.
|
||||
source_keys:
|
||||
- ${slug}
|
||||
---
|
||||
|
||||
You are a test agent.
|
||||
EOF
|
||||
}
|
||||
|
||||
# Helper: create a minimal Copilot file without source_keys
|
||||
make_copilot_clean() {
|
||||
local root="$1"
|
||||
local name="${2:-my-agent}"
|
||||
cat > "$root/agents/${name}.agent.md" <<EOF
|
||||
---
|
||||
name: ${name}
|
||||
description: A valid agent description.
|
||||
---
|
||||
|
||||
You are a test agent.
|
||||
EOF
|
||||
}
|
||||
@@ -89,7 +56,7 @@ EOF
|
||||
make_sources_md() {
|
||||
local root="$1"
|
||||
local slug="${2:-my-source}"
|
||||
local contrib="${3:-agents/my-agent.md, agents/my-agent.agent.md}"
|
||||
local contrib="${3:-.apm/agents/my-agent.agent.md}"
|
||||
local research="${4:-(none)}"
|
||||
cat > "$root/sources.md" <<EOF
|
||||
# Sources
|
||||
@@ -120,12 +87,56 @@ teardown() {
|
||||
}
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Non-plugin scope → exit 0 silently
|
||||
# Non-plugin/APM scope → exit 0 silently
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
@test "non-plugin scope: no plugin.json in tree → exit 0, no output" {
|
||||
local dir="$TMPDIR/no-plugin"
|
||||
@test "non-plugin scope: no apm.yml in tree → exit 0, no output" {
|
||||
local dir="$TMPDIR/no-package"
|
||||
mkdir -p "$dir/.apm/agents"
|
||||
cat > "$dir/.apm/agents/my-agent.agent.md" <<EOF
|
||||
---
|
||||
name: my-agent
|
||||
description: A valid agent description.
|
||||
source_keys:
|
||||
- my-source
|
||||
---
|
||||
|
||||
You are a test agent.
|
||||
EOF
|
||||
run bash "$SCRIPT" "$dir/.apm/agents/my-agent.agent.md"
|
||||
assert_success
|
||||
assert_output ""
|
||||
}
|
||||
|
||||
@test "non-plugin scope: apm.yml present but type:-less (marketplace-only) → exit 0, no output" {
|
||||
local dir="$TMPDIR/marketplace-only"
|
||||
mkdir -p "$dir/.apm/agents"
|
||||
cat > "$dir/apm.yml" <<EOF
|
||||
name: root-marketplace
|
||||
marketplace:
|
||||
owner: someone
|
||||
packages:
|
||||
- ./packages/plugin-a
|
||||
EOF
|
||||
cat > "$dir/.apm/agents/my-agent.agent.md" <<EOF
|
||||
---
|
||||
name: my-agent
|
||||
description: A valid agent description.
|
||||
source_keys:
|
||||
- my-source
|
||||
---
|
||||
|
||||
You are a test agent.
|
||||
EOF
|
||||
run bash "$SCRIPT" "$dir/.apm/agents/my-agent.agent.md"
|
||||
assert_success
|
||||
assert_output ""
|
||||
}
|
||||
|
||||
@test "non-plugin scope: bare plugin.json (no apm.yml) is no longer a scope signal → exit 0, no output" {
|
||||
local dir="$TMPDIR/old-plugin-json-only"
|
||||
mkdir -p "$dir/agents"
|
||||
echo '{"name":"test-plugin","version":"0.1.0"}' > "$dir/plugin.json"
|
||||
cat > "$dir/agents/my-agent.md" <<EOF
|
||||
---
|
||||
name: my-agent
|
||||
@@ -141,15 +152,70 @@ EOF
|
||||
assert_output ""
|
||||
}
|
||||
|
||||
@test "non-plugin scope: walk-up stops at .git boundary before reaching an ancestor apm.yml" {
|
||||
local dir="$TMPDIR/repo"
|
||||
mkdir -p "$dir/.git" "$dir/.apm/agents"
|
||||
cat > "$dir/apm.yml" <<EOF
|
||||
name: test-package
|
||||
version: 0.1.0
|
||||
type: skill
|
||||
EOF
|
||||
mkdir -p "$dir/sub/.apm/agents"
|
||||
cat > "$dir/sub/.apm/agents/my-agent.agent.md" <<EOF
|
||||
---
|
||||
name: my-agent
|
||||
description: A valid agent description.
|
||||
source_keys:
|
||||
- my-source
|
||||
---
|
||||
|
||||
You are a test agent.
|
||||
EOF
|
||||
# sub/ has no .git and no apm.yml of its own; the real package apm.yml
|
||||
# lives at $dir, but $dir/.git means the walk from sub/ should stop at
|
||||
# sub/ itself only if sub/ had a .git — here .git is at $dir, ABOVE
|
||||
# sub/, so the walk from sub/ reaches $dir/apm.yml before any .git.
|
||||
# This test instead verifies the walk finds that package root correctly
|
||||
# (a positive case) — see the dedicated .git-stops-first test below for
|
||||
# the negative case.
|
||||
run bash "$SCRIPT" "$dir/sub/.apm/agents/my-agent.agent.md"
|
||||
assert_failure
|
||||
assert_output --partial "FAIL"
|
||||
}
|
||||
|
||||
@test "non-plugin scope: .git between the agent file and an ancestor apm.yml stops the walk first" {
|
||||
local dir="$TMPDIR/repo2"
|
||||
mkdir -p "$dir/.apm/agents"
|
||||
cat > "$dir/apm.yml" <<EOF
|
||||
name: test-package
|
||||
version: 0.1.0
|
||||
type: skill
|
||||
EOF
|
||||
mkdir -p "$dir/sub/.git" "$dir/sub/.apm/agents"
|
||||
cat > "$dir/sub/.apm/agents/my-agent.agent.md" <<EOF
|
||||
---
|
||||
name: my-agent
|
||||
description: A valid agent description.
|
||||
source_keys:
|
||||
- my-source
|
||||
---
|
||||
|
||||
You are a test agent.
|
||||
EOF
|
||||
run bash "$SCRIPT" "$dir/sub/.apm/agents/my-agent.agent.md"
|
||||
assert_success
|
||||
assert_output ""
|
||||
}
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Early exit: no sources.md, no source_keys → exit 0, no output
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
@test "clean pass: no sources.md and no source_keys → exit 0, no output" {
|
||||
local root="$TMPDIR/plugin"
|
||||
make_plugin "$root"
|
||||
make_clean_pair "$root"
|
||||
run bash "$SCRIPT" "$root/agents/my-agent.md"
|
||||
local root="$TMPDIR/package"
|
||||
make_package "$root"
|
||||
make_clean_agent "$root"
|
||||
run bash "$SCRIPT" "$root/.apm/agents/my-agent.agent.md"
|
||||
assert_success
|
||||
assert_output ""
|
||||
}
|
||||
@@ -158,22 +224,11 @@ EOF
|
||||
# Check 0: source_keys present but sources.md absent → FAIL
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
@test "FAIL: source_keys in CC file but sources.md absent" {
|
||||
local root="$TMPDIR/plugin"
|
||||
make_plugin "$root"
|
||||
make_cc_with_source_keys "$root"
|
||||
make_copilot_clean "$root"
|
||||
run bash "$SCRIPT" "$root/agents/my-agent.md"
|
||||
assert_failure
|
||||
assert_output --partial "FAIL"
|
||||
}
|
||||
|
||||
@test "FAIL: source_keys in Copilot file but sources.md absent" {
|
||||
local root="$TMPDIR/plugin"
|
||||
make_plugin "$root"
|
||||
make_clean_pair "$root"
|
||||
make_copilot_with_source_keys "$root"
|
||||
run bash "$SCRIPT" "$root/agents/my-agent.agent.md"
|
||||
@test "FAIL: source_keys in agent file but sources.md absent" {
|
||||
local root="$TMPDIR/package"
|
||||
make_package "$root"
|
||||
make_agent_with_source_keys "$root"
|
||||
run bash "$SCRIPT" "$root/.apm/agents/my-agent.agent.md"
|
||||
assert_failure
|
||||
assert_output --partial "FAIL"
|
||||
}
|
||||
@@ -183,10 +238,9 @@ EOF
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
@test "FAIL: FILL IN: placeholder in sources.md" {
|
||||
local root="$TMPDIR/plugin"
|
||||
make_plugin "$root"
|
||||
make_cc_with_source_keys "$root"
|
||||
make_copilot_with_source_keys "$root"
|
||||
local root="$TMPDIR/package"
|
||||
make_package "$root"
|
||||
make_agent_with_source_keys "$root"
|
||||
cat > "$root/sources.md" <<EOF
|
||||
# Sources
|
||||
|
||||
@@ -194,23 +248,22 @@ EOF
|
||||
|
||||
- **URL:** FILL IN: add url
|
||||
- **Description:** A test source.
|
||||
- **Contributing files:** agents/my-agent.md, agents/my-agent.agent.md
|
||||
- **Contributing files:** .apm/agents/my-agent.agent.md
|
||||
- **Research doc:** (none)
|
||||
- **Status:** \`extracted\`
|
||||
EOF
|
||||
run bash "$SCRIPT" "$root/agents/my-agent.md"
|
||||
run bash "$SCRIPT" "$root/.apm/agents/my-agent.agent.md"
|
||||
assert_failure
|
||||
assert_output --partial "FAIL"
|
||||
}
|
||||
|
||||
@test "FILL IN: inside backticks in sources.md does not fail" {
|
||||
local root="$TMPDIR/plugin"
|
||||
make_plugin "$root"
|
||||
make_cc_with_source_keys "$root"
|
||||
make_copilot_with_source_keys "$root"
|
||||
local root="$TMPDIR/package"
|
||||
make_package "$root"
|
||||
make_agent_with_source_keys "$root"
|
||||
make_sources_md "$root"
|
||||
echo "Use \`FILL IN: value\` as example." >> "$root/sources.md"
|
||||
run bash "$SCRIPT" "$root/agents/my-agent.md"
|
||||
run bash "$SCRIPT" "$root/.apm/agents/my-agent.agent.md"
|
||||
assert_success
|
||||
}
|
||||
|
||||
@@ -218,62 +271,47 @@ EOF
|
||||
# Check 2: source_keys slug missing from sources.md → FAIL
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
@test "FAIL: source_keys slug in CC file not present as H2 in sources.md" {
|
||||
local root="$TMPDIR/plugin"
|
||||
make_plugin "$root"
|
||||
make_cc_with_source_keys "$root" "my-agent" "my-source"
|
||||
make_copilot_clean "$root"
|
||||
@test "FAIL: source_keys slug in agent file not present as H2 in sources.md" {
|
||||
local root="$TMPDIR/package"
|
||||
make_package "$root"
|
||||
make_agent_with_source_keys "$root" "my-agent" "my-source"
|
||||
make_sources_md "$root" "different-source" "(none)" "(none)"
|
||||
run bash "$SCRIPT" "$root/agents/my-agent.md"
|
||||
assert_failure
|
||||
assert_output --partial "FAIL"
|
||||
}
|
||||
|
||||
@test "FAIL: source_keys slug in Copilot file not present as H2 in sources.md" {
|
||||
local root="$TMPDIR/plugin"
|
||||
make_plugin "$root"
|
||||
make_clean_pair "$root"
|
||||
make_copilot_with_source_keys "$root" "my-agent" "my-source"
|
||||
make_sources_md "$root" "different-source" "(none)" "(none)"
|
||||
run bash "$SCRIPT" "$root/agents/my-agent.agent.md"
|
||||
run bash "$SCRIPT" "$root/.apm/agents/my-agent.agent.md"
|
||||
assert_failure
|
||||
assert_output --partial "FAIL"
|
||||
}
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Check 4: Contributing file path doesn't exist → FAIL
|
||||
# Check 3: Contributing file path doesn't exist → FAIL
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
@test "FAIL: Contributing file listed in sources.md does not exist" {
|
||||
local root="$TMPDIR/plugin"
|
||||
make_plugin "$root"
|
||||
make_cc_with_source_keys "$root"
|
||||
make_copilot_with_source_keys "$root"
|
||||
make_sources_md "$root" "my-source" "agents/nonexistent.md"
|
||||
run bash "$SCRIPT" "$root/agents/my-agent.md"
|
||||
local root="$TMPDIR/package"
|
||||
make_package "$root"
|
||||
make_agent_with_source_keys "$root"
|
||||
make_sources_md "$root" "my-source" ".apm/agents/nonexistent.agent.md"
|
||||
run bash "$SCRIPT" "$root/.apm/agents/my-agent.agent.md"
|
||||
assert_failure
|
||||
assert_output --partial "FAIL"
|
||||
}
|
||||
|
||||
@test "pass: (none) in Contributing files is skipped" {
|
||||
local root="$TMPDIR/plugin"
|
||||
make_plugin "$root"
|
||||
make_cc_with_source_keys "$root"
|
||||
make_copilot_with_source_keys "$root"
|
||||
local root="$TMPDIR/package"
|
||||
make_package "$root"
|
||||
make_agent_with_source_keys "$root"
|
||||
make_sources_md "$root" "my-source" "(none — not used directly)"
|
||||
run bash "$SCRIPT" "$root/agents/my-agent.md"
|
||||
run bash "$SCRIPT" "$root/.apm/agents/my-agent.agent.md"
|
||||
assert_success
|
||||
}
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Check 6: Research doc field missing or placeholder → FAIL
|
||||
# Check 5: Research doc field missing or placeholder → FAIL
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
@test "FAIL: Research doc field missing from sources.md entry" {
|
||||
local root="$TMPDIR/plugin"
|
||||
make_plugin "$root"
|
||||
make_cc_with_source_keys "$root"
|
||||
make_copilot_with_source_keys "$root"
|
||||
local root="$TMPDIR/package"
|
||||
make_package "$root"
|
||||
make_agent_with_source_keys "$root"
|
||||
cat > "$root/sources.md" <<EOF
|
||||
# Sources
|
||||
|
||||
@@ -281,19 +319,18 @@ EOF
|
||||
|
||||
- **URL:** https://example.com/my-source
|
||||
- **Description:** A test source.
|
||||
- **Contributing files:** agents/my-agent.md, agents/my-agent.agent.md
|
||||
- **Contributing files:** .apm/agents/my-agent.agent.md
|
||||
- **Status:** \`extracted\`
|
||||
EOF
|
||||
run bash "$SCRIPT" "$root/agents/my-agent.md"
|
||||
run bash "$SCRIPT" "$root/.apm/agents/my-agent.agent.md"
|
||||
assert_failure
|
||||
assert_output --partial "FAIL"
|
||||
}
|
||||
|
||||
@test "FAIL: Research doc field is FILL IN: placeholder" {
|
||||
local root="$TMPDIR/plugin"
|
||||
make_plugin "$root"
|
||||
make_cc_with_source_keys "$root"
|
||||
make_copilot_with_source_keys "$root"
|
||||
local root="$TMPDIR/package"
|
||||
make_package "$root"
|
||||
make_agent_with_source_keys "$root"
|
||||
cat > "$root/sources.md" <<EOF
|
||||
# Sources
|
||||
|
||||
@@ -301,24 +338,24 @@ EOF
|
||||
|
||||
- **URL:** https://example.com/my-source
|
||||
- **Description:** A test source.
|
||||
- **Contributing files:** agents/my-agent.md, agents/my-agent.agent.md
|
||||
- **Contributing files:** .apm/agents/my-agent.agent.md
|
||||
- **Research doc:** FILL IN: path to research doc
|
||||
- **Status:** \`extracted\`
|
||||
EOF
|
||||
run bash "$SCRIPT" "$root/agents/my-agent.md"
|
||||
run bash "$SCRIPT" "$root/.apm/agents/my-agent.agent.md"
|
||||
assert_failure
|
||||
assert_output --partial "FAIL"
|
||||
}
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Check 5: Bidirectional — contributing file missing slug in source_keys → FAIL
|
||||
# Check 4: Bidirectional — contributing file missing slug in source_keys → FAIL
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
@test "FAIL: Contributing file exists but does not list parent slug in source_keys" {
|
||||
local root="$TMPDIR/plugin"
|
||||
make_plugin "$root"
|
||||
# CC file has source_keys: other-source (not my-source)
|
||||
cat > "$root/agents/my-agent.md" <<EOF
|
||||
local root="$TMPDIR/package"
|
||||
make_package "$root"
|
||||
# agent file has source_keys: other-source (not my-source)
|
||||
cat > "$root/.apm/agents/my-agent.agent.md" <<EOF
|
||||
---
|
||||
name: my-agent
|
||||
description: A valid agent description.
|
||||
@@ -328,8 +365,8 @@ source_keys:
|
||||
|
||||
You are a test agent.
|
||||
EOF
|
||||
make_copilot_clean "$root"
|
||||
# sources.md says my-agent.md contributed to my-source, but my-agent.md doesn't list my-source
|
||||
# sources.md says my-agent.agent.md contributed to my-source, but
|
||||
# my-agent.agent.md doesn't list my-source
|
||||
cat > "$root/sources.md" <<EOF
|
||||
# Sources
|
||||
|
||||
@@ -337,7 +374,7 @@ EOF
|
||||
|
||||
- **URL:** https://example.com/other-source
|
||||
- **Description:** A test source.
|
||||
- **Contributing files:** agents/my-agent.md
|
||||
- **Contributing files:** .apm/agents/my-agent.agent.md
|
||||
- **Research doc:** (none)
|
||||
- **Status:** \`extracted\`
|
||||
|
||||
@@ -345,39 +382,24 @@ EOF
|
||||
|
||||
- **URL:** https://example.com/my-source
|
||||
- **Description:** Another source.
|
||||
- **Contributing files:** agents/my-agent.md
|
||||
- **Contributing files:** .apm/agents/my-agent.agent.md
|
||||
- **Research doc:** (none)
|
||||
- **Status:** \`extracted\`
|
||||
EOF
|
||||
run bash "$SCRIPT" "$root/agents/my-agent.md"
|
||||
run bash "$SCRIPT" "$root/.apm/agents/my-agent.agent.md"
|
||||
assert_failure
|
||||
assert_output --partial "FAIL"
|
||||
}
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Entry via Copilot file path
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
@test "accepts Copilot file path as entry point" {
|
||||
local root="$TMPDIR/plugin"
|
||||
make_plugin "$root"
|
||||
make_cc_with_source_keys "$root"
|
||||
make_copilot_with_source_keys "$root"
|
||||
make_sources_md "$root"
|
||||
run bash "$SCRIPT" "$root/agents/my-agent.agent.md"
|
||||
assert_success
|
||||
}
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Clean full pass
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
@test "clean full pass: all checks satisfied via CC file" {
|
||||
local root="$TMPDIR/plugin"
|
||||
make_plugin "$root"
|
||||
make_cc_with_source_keys "$root"
|
||||
make_copilot_with_source_keys "$root"
|
||||
@test "clean full pass: all checks satisfied" {
|
||||
local root="$TMPDIR/package"
|
||||
make_package "$root"
|
||||
make_agent_with_source_keys "$root"
|
||||
make_sources_md "$root"
|
||||
run bash "$SCRIPT" "$root/agents/my-agent.md"
|
||||
run bash "$SCRIPT" "$root/.apm/agents/my-agent.agent.md"
|
||||
assert_success
|
||||
}
|
||||
|
||||
@@ -8,24 +8,27 @@ setup() {
|
||||
SCRIPT="$(cd "$BATS_TEST_DIRNAME/../scripts" && pwd)/validate.sh"
|
||||
TMPDIR="$(mktemp -d)"
|
||||
|
||||
# Helper: create a plugin-scope pair in <dir> with given <name>
|
||||
make_plugin_pair() {
|
||||
local dir="$1"
|
||||
# Helper: create an APM package root at <root> (apm.yml with a top-level
|
||||
# type: line, marking it a real package manifest — not marketplace-only)
|
||||
# plus a single vendor-neutral agent file at
|
||||
# <root>/.apm/agents/<name>.agent.md. <extra_frontmatter>, if given, is
|
||||
# inserted as additional raw frontmatter lines (used to inject fields
|
||||
# under test).
|
||||
make_apm_agent() {
|
||||
local root="$1"
|
||||
local name="$2"
|
||||
mkdir -p "$dir"
|
||||
echo '{}' > "$dir/plugin.json"
|
||||
cat > "$dir/${name}.md" <<EOF
|
||||
---
|
||||
name: ${name}
|
||||
description: A valid agent description.
|
||||
---
|
||||
|
||||
You are a test agent. When invoked, do the thing.
|
||||
local extra_frontmatter="${3:-}"
|
||||
mkdir -p "$root/.apm/agents"
|
||||
cat > "$root/apm.yml" <<EOF
|
||||
name: test-package
|
||||
version: 0.1.0
|
||||
type: skill
|
||||
EOF
|
||||
cat > "$dir/${name}.agent.md" <<EOF
|
||||
cat > "$root/.apm/agents/${name}.agent.md" <<EOF
|
||||
---
|
||||
name: ${name}
|
||||
description: A valid agent description.
|
||||
${extra_frontmatter}
|
||||
---
|
||||
|
||||
You are a test agent. When invoked, do the thing.
|
||||
@@ -38,17 +41,9 @@ teardown() {
|
||||
}
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Passing cases
|
||||
# Passing cases — project/user scope (unchanged)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
@test "passes on a clean plugin-scope pair (CC file as input)" {
|
||||
local dir="$TMPDIR/agents"
|
||||
make_plugin_pair "$dir" "my-agent"
|
||||
run bash "$SCRIPT" "$dir/my-agent.md"
|
||||
assert_success
|
||||
refute_output --partial "FAIL"
|
||||
}
|
||||
|
||||
@test "passes on a clean project-scope pair (CC file as input)" {
|
||||
local root="$TMPDIR/project"
|
||||
mkdir -p "$root/.git" "$root/.claude/agents" "$root/.github/agents"
|
||||
@@ -79,40 +74,261 @@ EOF
|
||||
assert_output --partial "Usage:"
|
||||
}
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Failing cases
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
@test "fails when Copilot counterpart is missing" {
|
||||
local dir="$TMPDIR/agents"
|
||||
make_plugin_pair "$dir" "my-agent"
|
||||
rm "$dir/my-agent.agent.md"
|
||||
run bash "$SCRIPT" "$dir/my-agent.md"
|
||||
@test "fails when no arguments are given" {
|
||||
run bash "$SCRIPT"
|
||||
assert_failure
|
||||
assert_output --partial "counterpart"
|
||||
}
|
||||
|
||||
@test "fails when CC-only field 'maxTurns' is in Copilot file" {
|
||||
local dir="$TMPDIR/agents"
|
||||
make_plugin_pair "$dir" "my-agent"
|
||||
cat > "$dir/my-agent.agent.md" <<EOF
|
||||
---
|
||||
name: my-agent
|
||||
description: A valid agent description.
|
||||
maxTurns: 10
|
||||
---
|
||||
# ---------------------------------------------------------------------------
|
||||
# Passing cases — plugin/APM scope
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
You are a test agent. When invoked, do the thing.
|
||||
EOF
|
||||
run bash "$SCRIPT" "$dir/my-agent.md"
|
||||
@test "passes on a clean plugin/APM-scope agent file (name/description only)" {
|
||||
local root="$TMPDIR/pkg"
|
||||
make_apm_agent "$root" "my-agent"
|
||||
run bash "$SCRIPT" "$root/.apm/agents/my-agent.agent.md"
|
||||
assert_success
|
||||
refute_output --partial "FAIL"
|
||||
}
|
||||
|
||||
@test "passes on a clean plugin/APM-scope agent file with optional model field" {
|
||||
local root="$TMPDIR/pkg"
|
||||
make_apm_agent "$root" "my-agent" "model: claude-opus-4"
|
||||
run bash "$SCRIPT" "$root/.apm/agents/my-agent.agent.md"
|
||||
assert_success
|
||||
refute_output --partial "FAIL"
|
||||
}
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Failing cases — plugin/APM scope: allowlist violations
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
@test "fails when 'tools' field is present in a plugin/APM-scope agent file" {
|
||||
local root="$TMPDIR/pkg"
|
||||
make_apm_agent "$root" "my-agent" "tools: Read Edit"
|
||||
run bash "$SCRIPT" "$root/.apm/agents/my-agent.agent.md"
|
||||
assert_failure
|
||||
assert_output --partial "tools"
|
||||
}
|
||||
|
||||
@test "fails when a Claude-only field ('maxTurns') is present in a plugin/APM-scope agent file" {
|
||||
local root="$TMPDIR/pkg"
|
||||
make_apm_agent "$root" "my-agent" "maxTurns: 10"
|
||||
run bash "$SCRIPT" "$root/.apm/agents/my-agent.agent.md"
|
||||
assert_failure
|
||||
assert_output --partial "maxTurns"
|
||||
}
|
||||
|
||||
@test "fails when plugin-silently-ignored field 'hooks' is in plugin-scope CC file" {
|
||||
local dir="$TMPDIR/agents"
|
||||
make_plugin_pair "$dir" "my-agent"
|
||||
cat > "$dir/my-agent.md" <<EOF
|
||||
@test "fails when a Copilot-only field ('target') is present in a plugin/APM-scope agent file" {
|
||||
local root="$TMPDIR/pkg"
|
||||
make_apm_agent "$root" "my-agent" "target: cli"
|
||||
run bash "$SCRIPT" "$root/.apm/agents/my-agent.agent.md"
|
||||
assert_failure
|
||||
assert_output --partial "target"
|
||||
}
|
||||
|
||||
@test "fails when 'hooks' is present in a plugin/APM-scope agent file (outside allowlist)" {
|
||||
local root="$TMPDIR/pkg"
|
||||
make_apm_agent "$root" "my-agent" "hooks: {}"
|
||||
run bash "$SCRIPT" "$root/.apm/agents/my-agent.agent.md"
|
||||
assert_failure
|
||||
assert_output --partial "hooks"
|
||||
}
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Failing cases — plugin/APM scope: structural checks
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
@test "fails when name is not kebab-case in a plugin/APM-scope agent file" {
|
||||
local root="$TMPDIR/pkg"
|
||||
mkdir -p "$root/.apm/agents"
|
||||
cat > "$root/apm.yml" <<EOF
|
||||
name: test-package
|
||||
version: 0.1.0
|
||||
type: skill
|
||||
EOF
|
||||
cat > "$root/.apm/agents/my-agent.agent.md" <<EOF
|
||||
---
|
||||
name: MyAgent
|
||||
description: A valid agent description.
|
||||
---
|
||||
|
||||
You are a test agent. When invoked, do the thing.
|
||||
EOF
|
||||
run bash "$SCRIPT" "$root/.apm/agents/my-agent.agent.md"
|
||||
assert_failure
|
||||
assert_output --partial "kebab"
|
||||
}
|
||||
|
||||
@test "fails when name does not match filename stem in a plugin/APM-scope agent file" {
|
||||
local root="$TMPDIR/pkg"
|
||||
mkdir -p "$root/.apm/agents"
|
||||
cat > "$root/apm.yml" <<EOF
|
||||
name: test-package
|
||||
version: 0.1.0
|
||||
type: skill
|
||||
EOF
|
||||
cat > "$root/.apm/agents/my-agent.agent.md" <<EOF
|
||||
---
|
||||
name: wrong-name
|
||||
description: A valid agent description.
|
||||
---
|
||||
|
||||
You are a test agent. When invoked, do the thing.
|
||||
EOF
|
||||
run bash "$SCRIPT" "$root/.apm/agents/my-agent.agent.md"
|
||||
assert_failure
|
||||
assert_output --partial "does not match filename stem"
|
||||
}
|
||||
|
||||
@test "fails when 'name' field is missing from a plugin/APM-scope agent file" {
|
||||
local root="$TMPDIR/pkg"
|
||||
mkdir -p "$root/.apm/agents"
|
||||
cat > "$root/apm.yml" <<EOF
|
||||
name: test-package
|
||||
version: 0.1.0
|
||||
type: skill
|
||||
EOF
|
||||
cat > "$root/.apm/agents/my-agent.agent.md" <<EOF
|
||||
---
|
||||
description: A valid agent description.
|
||||
---
|
||||
|
||||
You are a test agent. When invoked, do the thing.
|
||||
EOF
|
||||
run bash "$SCRIPT" "$root/.apm/agents/my-agent.agent.md"
|
||||
assert_failure
|
||||
}
|
||||
|
||||
@test "fails when 'description' field is missing from a plugin/APM-scope agent file" {
|
||||
local root="$TMPDIR/pkg"
|
||||
mkdir -p "$root/.apm/agents"
|
||||
cat > "$root/apm.yml" <<EOF
|
||||
name: test-package
|
||||
version: 0.1.0
|
||||
type: skill
|
||||
EOF
|
||||
cat > "$root/.apm/agents/my-agent.agent.md" <<EOF
|
||||
---
|
||||
name: my-agent
|
||||
---
|
||||
|
||||
You are a test agent. When invoked, do the thing.
|
||||
EOF
|
||||
run bash "$SCRIPT" "$root/.apm/agents/my-agent.agent.md"
|
||||
assert_failure
|
||||
}
|
||||
|
||||
@test "fails when body contains unfilled FILL IN: placeholder in a plugin/APM-scope agent file" {
|
||||
local root="$TMPDIR/pkg"
|
||||
mkdir -p "$root/.apm/agents"
|
||||
cat > "$root/apm.yml" <<EOF
|
||||
name: test-package
|
||||
version: 0.1.0
|
||||
type: skill
|
||||
EOF
|
||||
cat > "$root/.apm/agents/my-agent.agent.md" <<EOF
|
||||
---
|
||||
name: my-agent
|
||||
description: A valid agent description.
|
||||
---
|
||||
|
||||
FILL IN: replace this with your system prompt.
|
||||
EOF
|
||||
run bash "$SCRIPT" "$root/.apm/agents/my-agent.agent.md"
|
||||
assert_failure
|
||||
}
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Plugin/APM scope: no pair, no counterpart concept
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
@test "never raises a 'counterpart' FAIL on a clean plugin/APM-scope agent file" {
|
||||
local root="$TMPDIR/pkg"
|
||||
make_apm_agent "$root" "my-agent"
|
||||
run bash "$SCRIPT" "$root/.apm/agents/my-agent.agent.md"
|
||||
assert_success
|
||||
refute_output --partial "counterpart"
|
||||
}
|
||||
|
||||
@test "never raises a 'counterpart' FAIL on a failing plugin/APM-scope agent file" {
|
||||
local root="$TMPDIR/pkg"
|
||||
make_apm_agent "$root" "my-agent" "tools: Read"
|
||||
run bash "$SCRIPT" "$root/.apm/agents/my-agent.agent.md"
|
||||
assert_failure
|
||||
refute_output --partial "counterpart"
|
||||
}
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Scope-detection walk-up
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
@test "walk-up skips a type:-less apm.yml and finds a real package root further up" {
|
||||
local root="$TMPDIR/case"
|
||||
mkdir -p "$root/.apm/agents/nested/deeper"
|
||||
cat > "$root/apm.yml" <<EOF
|
||||
name: real-package
|
||||
version: 1.0.0
|
||||
type: skill
|
||||
EOF
|
||||
# Closer to the agent file than the real package root, but has no type:
|
||||
# line — marketplace-only per monorepo-and-repo-shapes.md, must be skipped.
|
||||
cat > "$root/.apm/agents/nested/apm.yml" <<EOF
|
||||
name: not-a-package-manifest
|
||||
version: 1.0.0
|
||||
EOF
|
||||
cat > "$root/.apm/agents/nested/deeper/my-agent.agent.md" <<EOF
|
||||
---
|
||||
name: my-agent
|
||||
description: A valid agent description.
|
||||
---
|
||||
|
||||
You are a test agent. When invoked, do the thing.
|
||||
EOF
|
||||
run bash "$SCRIPT" "$root/.apm/agents/nested/deeper/my-agent.agent.md"
|
||||
assert_success
|
||||
refute_output --partial "FAIL"
|
||||
}
|
||||
|
||||
@test "type:-less apm.yml is not treated as plugin scope — falls through to project scope" {
|
||||
local root="$TMPDIR/proj-marketplace"
|
||||
mkdir -p "$root/.git" "$root/.claude/agents" "$root/.github/agents"
|
||||
cat > "$root/apm.yml" <<EOF
|
||||
name: marketplace-root
|
||||
version: 1.0.0
|
||||
marketplace:
|
||||
packages: []
|
||||
EOF
|
||||
cat > "$root/.claude/agents/my-agent.md" <<EOF
|
||||
---
|
||||
name: my-agent
|
||||
description: A valid agent description.
|
||||
---
|
||||
|
||||
You are a test agent. When invoked, do the thing.
|
||||
EOF
|
||||
cat > "$root/.github/agents/my-agent.agent.md" <<EOF
|
||||
---
|
||||
name: my-agent
|
||||
description: A valid agent description.
|
||||
---
|
||||
|
||||
You are a test agent. When invoked, do the thing.
|
||||
EOF
|
||||
run bash "$SCRIPT" "$root/.claude/agents/my-agent.md"
|
||||
assert_success
|
||||
refute_output --partial "FAIL"
|
||||
}
|
||||
|
||||
@test "a bare plugin.json with no apm.yml is no longer plugin scope — falls through to project scope" {
|
||||
local root="$TMPDIR/proj-legacy-plugin-json"
|
||||
mkdir -p "$root/.git" "$root/.claude/agents" "$root/.github/agents"
|
||||
echo '{}' > "$root/plugin.json"
|
||||
# 'hooks' is plugin-silently-ignored only at (old) plugin scope; at
|
||||
# project scope it's a legitimate CC field. If this directory were
|
||||
# mis-detected as plugin scope (old plugin.json-based logic), this would
|
||||
# FAIL with a plugin-silently-ignored-fields finding on 'hooks'.
|
||||
cat > "$root/.claude/agents/my-agent.md" <<EOF
|
||||
---
|
||||
name: my-agent
|
||||
description: A valid agent description.
|
||||
@@ -124,95 +340,15 @@ hooks:
|
||||
|
||||
You are a test agent. When invoked, do the thing.
|
||||
EOF
|
||||
run bash "$SCRIPT" "$dir/my-agent.md"
|
||||
assert_failure
|
||||
assert_output --partial "hooks"
|
||||
}
|
||||
|
||||
@test "fails when CC file name is not kebab-case" {
|
||||
local dir="$TMPDIR/agents"
|
||||
mkdir -p "$dir"
|
||||
echo '{}' > "$dir/plugin.json"
|
||||
cat > "$dir/my-agent.md" <<EOF
|
||||
---
|
||||
name: MyAgent
|
||||
description: A valid agent description.
|
||||
---
|
||||
|
||||
You are a test agent. When invoked, do the thing.
|
||||
EOF
|
||||
cat > "$dir/my-agent.agent.md" <<EOF
|
||||
---
|
||||
name: MyAgent
|
||||
description: A valid agent description.
|
||||
---
|
||||
|
||||
You are a test agent. When invoked, do the thing.
|
||||
EOF
|
||||
run bash "$SCRIPT" "$dir/my-agent.md"
|
||||
assert_failure
|
||||
assert_output --partial "kebab"
|
||||
}
|
||||
|
||||
@test "fails when 'name' field is missing from CC file" {
|
||||
local dir="$TMPDIR/agents"
|
||||
make_plugin_pair "$dir" "my-agent"
|
||||
cat > "$dir/my-agent.md" <<EOF
|
||||
---
|
||||
description: A valid agent description.
|
||||
---
|
||||
|
||||
You are a test agent. When invoked, do the thing.
|
||||
EOF
|
||||
run bash "$SCRIPT" "$dir/my-agent.md"
|
||||
assert_failure
|
||||
}
|
||||
|
||||
@test "fails when 'description' field is missing from CC file" {
|
||||
local dir="$TMPDIR/agents"
|
||||
make_plugin_pair "$dir" "my-agent"
|
||||
cat > "$dir/my-agent.md" <<EOF
|
||||
---
|
||||
name: my-agent
|
||||
---
|
||||
|
||||
You are a test agent. When invoked, do the thing.
|
||||
EOF
|
||||
run bash "$SCRIPT" "$dir/my-agent.md"
|
||||
assert_failure
|
||||
}
|
||||
|
||||
@test "fails when body contains unfilled FILL IN: placeholder" {
|
||||
local dir="$TMPDIR/agents"
|
||||
make_plugin_pair "$dir" "my-agent"
|
||||
cat > "$dir/my-agent.md" <<EOF
|
||||
cat > "$root/.github/agents/my-agent.agent.md" <<EOF
|
||||
---
|
||||
name: my-agent
|
||||
description: A valid agent description.
|
||||
---
|
||||
|
||||
FILL IN: replace this with your system prompt.
|
||||
EOF
|
||||
run bash "$SCRIPT" "$dir/my-agent.md"
|
||||
assert_failure
|
||||
}
|
||||
|
||||
@test "fails when frontmatter name does not match filename stem" {
|
||||
local dir="$TMPDIR/agents"
|
||||
make_plugin_pair "$dir" "my-agent"
|
||||
cat > "$dir/my-agent.agent.md" <<EOF
|
||||
---
|
||||
name: wrong-name
|
||||
description: A valid agent description.
|
||||
---
|
||||
|
||||
You are a test agent. When invoked, do the thing.
|
||||
EOF
|
||||
run bash "$SCRIPT" "$dir/my-agent.md"
|
||||
assert_failure
|
||||
}
|
||||
|
||||
@test "fails when no arguments are given" {
|
||||
run bash "$SCRIPT"
|
||||
assert_failure
|
||||
run bash "$SCRIPT" "$root/.claude/agents/my-agent.md"
|
||||
assert_success
|
||||
refute_output --partial "hooks"
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user