The previous round taught agent-audit's validator to permit disallowedTools but left the skill that writes agents still forbidding it, in six places. Running agent-author on any of the three fenced orchestrators would have stripped the fence, and nothing would have caught it: the validator's allowlist is a permit list, so an absent field passes. The template was the worst of them, since its comment is copied verbatim into every new plugin-scope agent. Where a list had to be restated it is now a pointer to field-inventory.md's apm-agent-allowlist instead -- the same data validate.sh reads -- because a roster copied into a template goes stale one step further out than the roster itself. Where the text has to teach something it teaches the shape rule rather than the exception: tools is an allowlist whose vocabulary differs per harness, so verbatim copy makes one value wrong on one target; disallowedTools is a denylist, where an unrecognised name denies nothing, so the worst case is a missing fence rather than a wrongly granted capability. ADR-0016's amendment claimed an unrecognised key is inert on Copilot while the same ADR's Context says that behaviour is unconfirmed by research -- asserting as settled the exact thing it flags as unknown, and justifying it with apm's compile-time behaviour, which says nothing about Copilot's runtime. It is rewritten into labelled tiers: confirmed for Claude Code with citations, inferred by analogy for Copilot with the analogy's limits stated, unverified where it is unverified, and the residual risk accepted explicitly with its blast radius. It also no longer claims to restore a write sandbox: the denylist does not deny Bash, which these agents inherit and legitimately need. docs/hooks.md called the old root hooks.json a stale sync artifact -- it was added in the plugin's creating commit and pointed at by main's Copilot manifest -- and claimed both ecosystems now resolve hooks/hooks.json. Copilot does not: its hooks field has no default and no compiled manifest declares one, so it resolves nothing. Recorded as the gap it is, with re-injection noted as a follow-up rather than asserted away. Its event list is marked partial. Also: new-agent.bats asserted a hardcoded four-field allowlist and would have rejected a scaffolded agent carrying the field the ADR now blesses; it reads field-inventory.md too. And ADR-0016's premise that Claude's tools: is space-separated was wrong -- it takes a comma-separated string or a YAML list. The incompatibility with Copilot is the vocabulary, not the punctuation. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01X7GvKuJfy2WrdBmUttV4DT
316 lines
12 KiB
Bash
316 lines
12 KiB
Bash
#!/usr/bin/env bats
|
|
|
|
setup() {
|
|
REPO_ROOT="$(cd "$BATS_TEST_DIRNAME/../../../../../../" && pwd)"
|
|
load "$REPO_ROOT/tests/test_helper/bats-support/load"
|
|
load "$REPO_ROOT/tests/test_helper/bats-assert/load"
|
|
|
|
SCRIPT="$(cd "$BATS_TEST_DIRNAME/../scripts" && pwd)/new-agent.sh"
|
|
ROOT="$(mktemp -d)"
|
|
}
|
|
|
|
teardown() {
|
|
rm -rf "$ROOT"
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Help
|
|
# ---------------------------------------------------------------------------
|
|
|
|
@test "--help exits 0" {
|
|
run bash "$SCRIPT" --help
|
|
assert_success
|
|
assert_output --partial "Usage:"
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Plugin/APM scope (type:-bearing apm.yml at root)
|
|
# ---------------------------------------------------------------------------
|
|
|
|
@test "plugin/APM scope: creates single agent file in .apm/agents/" {
|
|
printf 'name: my-package\ntype: skill\n' > "$ROOT/apm.yml"
|
|
run bash "$SCRIPT" my-agent "$ROOT"
|
|
assert_success
|
|
assert [ -f "$ROOT/.apm/agents/my-agent.agent.md" ]
|
|
}
|
|
|
|
@test "plugin/APM scope: does not create the old dual-file pair" {
|
|
printf 'name: my-package\ntype: skill\n' > "$ROOT/apm.yml"
|
|
run bash "$SCRIPT" my-agent "$ROOT"
|
|
assert_success
|
|
assert [ ! -f "$ROOT/agents/my-agent.md" ]
|
|
assert [ ! -f "$ROOT/agents/my-agent.agent.md" ]
|
|
}
|
|
|
|
@test "plugin/APM scope: creates .apm/agents/ directory if missing" {
|
|
printf 'name: my-package\ntype: skill\n' > "$ROOT/apm.yml"
|
|
run bash "$SCRIPT" my-agent "$ROOT"
|
|
assert_success
|
|
assert [ -d "$ROOT/.apm/agents" ]
|
|
}
|
|
|
|
@test "plugin/APM scope: creates sources.md at package root" {
|
|
printf 'name: my-package\ntype: skill\n' > "$ROOT/apm.yml"
|
|
run bash "$SCRIPT" my-agent "$ROOT"
|
|
assert_success
|
|
assert [ -f "$ROOT/sources.md" ]
|
|
}
|
|
|
|
@test "plugin/APM scope: no-op if agent file already exists" {
|
|
printf 'name: my-package\ntype: skill\n' > "$ROOT/apm.yml"
|
|
mkdir -p "$ROOT/.apm/agents"
|
|
echo "existing" > "$ROOT/.apm/agents/my-agent.agent.md"
|
|
run bash "$SCRIPT" my-agent "$ROOT"
|
|
assert_success
|
|
run grep "existing" "$ROOT/.apm/agents/my-agent.agent.md"
|
|
assert_success
|
|
}
|
|
|
|
@test "plugin/APM scope: frontmatter has no tools/isolation/maxTurns/effort/memory/permissionMode keys" {
|
|
printf 'name: my-package\ntype: skill\n' > "$ROOT/apm.yml"
|
|
bash "$SCRIPT" my-agent "$ROOT"
|
|
file="$ROOT/.apm/agents/my-agent.agent.md"
|
|
fm="$(sed -n '/^---$/,/^---$/p' "$file")"
|
|
# Column-0 key lines only — comment bodies in the template are indented,
|
|
# so this anchor naturally excludes commented-out example fields.
|
|
run grep -E '^(tools|isolation|maxTurns|effort|memory|permissionMode|disallowedTools|skills|color|initialPrompt|background|hooks|mcpServers):' <<< "$fm"
|
|
assert_failure
|
|
}
|
|
|
|
# The permitted set is read from the same data agent-audit's validate.sh reads --
|
|
# the apm-agent-allowlist section of agent-audit's field-inventory.md -- rather than
|
|
# restated here. A hardcoded copy drifts: this assertion listed four fields and went
|
|
# on passing after ADR-0016's amendment added disallowedTools, and would have
|
|
# rejected a scaffolded agent that legitimately carried it.
|
|
@test "plugin/APM scope: frontmatter carries only allowlisted fields" {
|
|
inventory="$BATS_TEST_DIRNAME/../../agent-audit/references/field-inventory.md"
|
|
[ -f "$inventory" ] || fail "field-inventory.md not found at $inventory"
|
|
allowlist="$(awk '
|
|
/^## apm-agent-allowlist$/ { insection = 1; next }
|
|
insection && /^##/ { exit }
|
|
insection && NF && $0 !~ /^#/ && $0 !~ /^---/ { print; exit }
|
|
' "$inventory")"
|
|
[ -n "$allowlist" ] || fail "apm-agent-allowlist section is empty in $inventory"
|
|
|
|
printf 'name: my-package\ntype: skill\n' > "$ROOT/apm.yml"
|
|
bash "$SCRIPT" my-agent "$ROOT"
|
|
file="$ROOT/.apm/agents/my-agent.agent.md"
|
|
fm="$(sed -n '/^---$/,/^---$/p' "$file")"
|
|
keys="$(grep -oE '^[a-zA-Z][a-zA-Z0-9_-]*:' <<< "$fm" | sed 's/:$//' | sort -u)"
|
|
for key in $keys; do
|
|
if ! grep -qw "$key" <<< "$allowlist"; then
|
|
fail "frontmatter key '$key' is not in field-inventory.md's apm-agent-allowlist ($allowlist)"
|
|
fi
|
|
done
|
|
}
|
|
|
|
@test "plugin/APM scope: sources.md contributing-files template mentions the single-file path" {
|
|
printf 'name: my-package\ntype: skill\n' > "$ROOT/apm.yml"
|
|
bash "$SCRIPT" my-agent "$ROOT"
|
|
run grep ".apm/agents/<name>.agent.md" "$ROOT/sources.md"
|
|
assert_success
|
|
}
|
|
|
|
@test "plugin/APM scope: template AGENT_NAME substituted" {
|
|
printf 'name: my-package\ntype: skill\n' > "$ROOT/apm.yml"
|
|
bash "$SCRIPT" my-agent "$ROOT"
|
|
run grep "my-agent" "$ROOT/.apm/agents/my-agent.agent.md"
|
|
assert_success
|
|
}
|
|
|
|
@test "plugin/APM scope: walk-up finds apm.yml at an ancestor directory, not just root arg" {
|
|
printf 'name: my-package\ntype: skill\n' > "$ROOT/apm.yml"
|
|
mkdir -p "$ROOT/nested/subdir"
|
|
run bash "$SCRIPT" my-agent "$ROOT/nested/subdir"
|
|
assert_success
|
|
assert [ -f "$ROOT/.apm/agents/my-agent.agent.md" ]
|
|
}
|
|
|
|
@test "plugin/APM scope: apm.yml without type: is skipped (marketplace-only manifest)" {
|
|
printf 'name: my-marketplace\n' > "$ROOT/apm.yml"
|
|
mkdir -p "$ROOT/.git"
|
|
run bash "$SCRIPT" my-agent "$ROOT"
|
|
assert_success
|
|
assert [ ! -f "$ROOT/.apm/agents/my-agent.agent.md" ]
|
|
assert [ -f "$ROOT/.claude/agents/my-agent.md" ]
|
|
}
|
|
|
|
@test "plugin/APM scope: type:-less apm.yml at leaf falls through to a type:-bearing apm.yml higher up" {
|
|
printf 'name: outer-package\ntype: skill\n' > "$ROOT/apm.yml"
|
|
mkdir -p "$ROOT/inner"
|
|
printf 'name: inner-marketplace\n' > "$ROOT/inner/apm.yml"
|
|
run bash "$SCRIPT" my-agent "$ROOT/inner"
|
|
assert_success
|
|
assert [ -f "$ROOT/.apm/agents/my-agent.agent.md" ]
|
|
}
|
|
|
|
@test "plugin/APM scope: matched-quote type value ('skill') is recognized" {
|
|
printf 'name: my-package\ntype: "skill"\n' > "$ROOT/apm.yml"
|
|
run bash "$SCRIPT" my-agent "$ROOT"
|
|
assert_success
|
|
assert [ -f "$ROOT/.apm/agents/my-agent.agent.md" ]
|
|
}
|
|
|
|
@test "plugin/APM scope: mismatched-quote type value is rejected, falls through to project scope" {
|
|
mkdir -p "$ROOT/.git"
|
|
printf "name: my-package\ntype: \"skill'\n" > "$ROOT/apm.yml"
|
|
run bash "$SCRIPT" my-agent "$ROOT"
|
|
assert_success
|
|
assert [ ! -f "$ROOT/.apm/agents/my-agent.agent.md" ]
|
|
assert [ -f "$ROOT/.claude/agents/my-agent.md" ]
|
|
}
|
|
|
|
@test "plugin/APM scope: type: line is recognized even without a trailing newline on the file" {
|
|
printf 'name: my-package\ntype: skill' > "$ROOT/apm.yml"
|
|
run bash "$SCRIPT" my-agent "$ROOT"
|
|
assert_success
|
|
assert [ -f "$ROOT/.apm/agents/my-agent.agent.md" ]
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Old plugin.json marker is no longer recognized (full switch, no dual-mode)
|
|
# ---------------------------------------------------------------------------
|
|
|
|
@test "bare plugin.json (no apm.yml) is no longer detected as plugin scope — falls through to project scope" {
|
|
touch "$ROOT/plugin.json"
|
|
run bash "$SCRIPT" my-agent "$ROOT"
|
|
assert_success
|
|
assert [ ! -f "$ROOT/agents/my-agent.md" ]
|
|
assert [ ! -f "$ROOT/agents/my-agent.agent.md" ]
|
|
assert [ ! -f "$ROOT/.apm/agents/my-agent.agent.md" ]
|
|
assert [ -f "$ROOT/.claude/agents/my-agent.md" ]
|
|
assert [ -f "$ROOT/.github/agents/my-agent.agent.md" ]
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Non-plugin (project) scope
|
|
# ---------------------------------------------------------------------------
|
|
|
|
@test "non-plugin scope: creates claude code file in .claude/agents/" {
|
|
run bash "$SCRIPT" my-agent "$ROOT"
|
|
assert_success
|
|
assert [ -f "$ROOT/.claude/agents/my-agent.md" ]
|
|
}
|
|
|
|
@test "non-plugin scope: creates copilot file in .github/agents/" {
|
|
run bash "$SCRIPT" my-agent "$ROOT"
|
|
assert_success
|
|
assert [ -f "$ROOT/.github/agents/my-agent.agent.md" ]
|
|
}
|
|
|
|
@test "non-plugin scope: creates .claude/agents/ directory if missing" {
|
|
run bash "$SCRIPT" my-agent "$ROOT"
|
|
assert_success
|
|
assert [ -d "$ROOT/.claude/agents" ]
|
|
}
|
|
|
|
@test "non-plugin scope: creates .github/agents/ directory if missing" {
|
|
run bash "$SCRIPT" my-agent "$ROOT"
|
|
assert_success
|
|
assert [ -d "$ROOT/.github/agents" ]
|
|
}
|
|
|
|
@test "non-plugin scope: no sources.md created" {
|
|
run bash "$SCRIPT" my-agent "$ROOT"
|
|
assert_success
|
|
assert [ ! -f "$ROOT/.claude/agents/sources.md" ]
|
|
assert [ ! -f "$ROOT/.github/agents/sources.md" ]
|
|
}
|
|
|
|
@test "non-plugin scope: no-op if claude code file already exists" {
|
|
mkdir -p "$ROOT/.claude/agents"
|
|
echo "existing" > "$ROOT/.claude/agents/my-agent.md"
|
|
run bash "$SCRIPT" my-agent "$ROOT"
|
|
assert_success
|
|
run grep "existing" "$ROOT/.claude/agents/my-agent.md"
|
|
assert_success
|
|
}
|
|
|
|
@test "project scope: detected via .git present above root, output still relative to root arg" {
|
|
mkdir -p "$ROOT/repo/.git"
|
|
mkdir -p "$ROOT/repo/pkg"
|
|
run bash "$SCRIPT" my-agent "$ROOT/repo/pkg"
|
|
assert_success
|
|
assert [ -f "$ROOT/repo/pkg/.claude/agents/my-agent.md" ]
|
|
assert [ -f "$ROOT/repo/pkg/.github/agents/my-agent.agent.md" ]
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# User scope
|
|
# ---------------------------------------------------------------------------
|
|
|
|
@test "user scope: root exactly \$HOME creates files under ~/.claude and ~/.copilot" {
|
|
FAKE_HOME="$(mktemp -d)"
|
|
run env HOME="$FAKE_HOME" bash "$SCRIPT" my-agent "~"
|
|
assert_success
|
|
assert [ -f "$FAKE_HOME/.claude/agents/my-agent.md" ]
|
|
assert [ -f "$FAKE_HOME/.copilot/agents/my-agent.agent.md" ]
|
|
rm -rf "$FAKE_HOME"
|
|
}
|
|
|
|
@test "user scope: \$HOME being a dotfiles .git repo does not shadow user scope" {
|
|
FAKE_HOME="$(mktemp -d)"
|
|
mkdir "$FAKE_HOME/.git"
|
|
run env HOME="$FAKE_HOME" bash "$SCRIPT" my-agent "~"
|
|
assert_success
|
|
assert [ -f "$FAKE_HOME/.claude/agents/my-agent.md" ]
|
|
assert [ -f "$FAKE_HOME/.copilot/agents/my-agent.agent.md" ]
|
|
refute [ -d "$FAKE_HOME/.github" ]
|
|
rm -rf "$FAKE_HOME"
|
|
}
|
|
|
|
@test "user scope is checked directly at \$HOME, no walk-up: a marker-less subdir under \$HOME resolves to project scope, not user scope" {
|
|
FAKE_HOME="$(mktemp -d)"
|
|
mkdir -p "$FAKE_HOME/scratch/testdir"
|
|
run env HOME="$FAKE_HOME" bash "$SCRIPT" my-agent "$FAKE_HOME/scratch/testdir"
|
|
assert_success
|
|
assert [ -f "$FAKE_HOME/scratch/testdir/.claude/agents/my-agent.md" ]
|
|
assert [ -f "$FAKE_HOME/scratch/testdir/.github/agents/my-agent.agent.md" ]
|
|
refute [ -f "$FAKE_HOME/.claude/agents/my-agent.md" ]
|
|
refute [ -f "$FAKE_HOME/.copilot/agents/my-agent.agent.md" ]
|
|
rm -rf "$FAKE_HOME"
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Name validation
|
|
# ---------------------------------------------------------------------------
|
|
|
|
@test "fails when no arguments given" {
|
|
run bash "$SCRIPT"
|
|
assert_failure
|
|
}
|
|
|
|
@test "fails when agent name contains uppercase" {
|
|
run bash "$SCRIPT" MyAgent "$ROOT"
|
|
assert_failure
|
|
}
|
|
|
|
@test "fails when agent name has consecutive hyphens" {
|
|
run bash "$SCRIPT" my--agent "$ROOT"
|
|
assert_failure
|
|
}
|
|
|
|
@test "fails when agent name has a leading hyphen" {
|
|
run bash "$SCRIPT" -my-agent "$ROOT"
|
|
assert_failure
|
|
}
|
|
|
|
@test "fails when agent name has a trailing hyphen" {
|
|
run bash "$SCRIPT" my-agent- "$ROOT"
|
|
assert_failure
|
|
}
|
|
|
|
@test "agent name with numbers is valid" {
|
|
run bash "$SCRIPT" agent-v2 "$ROOT"
|
|
assert_success
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Root validation
|
|
# ---------------------------------------------------------------------------
|
|
|
|
@test "fails when root directory does not exist" {
|
|
run bash "$SCRIPT" my-agent "/nonexistent/path"
|
|
assert_failure
|
|
}
|