Files
holocron/plugins/kyberforge/.apm/skills/agent-author/tests/new-agent.bats
Defame1297 f4bb1cf4e5 docs: finish reconciling the agent write fence across the toolchain
The previous round taught agent-audit's validator to permit disallowedTools but
left the skill that writes agents still forbidding it, in six places. Running
agent-author on any of the three fenced orchestrators would have stripped the
fence, and nothing would have caught it: the validator's allowlist is a permit
list, so an absent field passes. The template was the worst of them, since its
comment is copied verbatim into every new plugin-scope agent.

Where a list had to be restated it is now a pointer to field-inventory.md's
apm-agent-allowlist instead -- the same data validate.sh reads -- because a
roster copied into a template goes stale one step further out than the roster
itself. Where the text has to teach something it teaches the shape rule rather
than the exception: tools is an allowlist whose vocabulary differs per harness,
so verbatim copy makes one value wrong on one target; disallowedTools is a
denylist, where an unrecognised name denies nothing, so the worst case is a
missing fence rather than a wrongly granted capability.

ADR-0016's amendment claimed an unrecognised key is inert on Copilot while the
same ADR's Context says that behaviour is unconfirmed by research -- asserting
as settled the exact thing it flags as unknown, and justifying it with apm's
compile-time behaviour, which says nothing about Copilot's runtime. It is
rewritten into labelled tiers: confirmed for Claude Code with citations,
inferred by analogy for Copilot with the analogy's limits stated, unverified
where it is unverified, and the residual risk accepted explicitly with its
blast radius. It also no longer claims to restore a write sandbox: the denylist
does not deny Bash, which these agents inherit and legitimately need.

docs/hooks.md called the old root hooks.json a stale sync artifact -- it was
added in the plugin's creating commit and pointed at by main's Copilot manifest
-- and claimed both ecosystems now resolve hooks/hooks.json. Copilot does not:
its hooks field has no default and no compiled manifest declares one, so it
resolves nothing. Recorded as the gap it is, with re-injection noted as a
follow-up rather than asserted away. Its event list is marked partial.

Also: new-agent.bats asserted a hardcoded four-field allowlist and would have
rejected a scaffolded agent carrying the field the ADR now blesses; it reads
field-inventory.md too. And ADR-0016's premise that Claude's tools: is
space-separated was wrong -- it takes a comma-separated string or a YAML list.
The incompatibility with Copilot is the vocabulary, not the punctuation.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01X7GvKuJfy2WrdBmUttV4DT
2026-08-14 12:30:30 +00:00

316 lines
12 KiB
Bash

#!/usr/bin/env bats
setup() {
REPO_ROOT="$(cd "$BATS_TEST_DIRNAME/../../../../../../" && pwd)"
load "$REPO_ROOT/tests/test_helper/bats-support/load"
load "$REPO_ROOT/tests/test_helper/bats-assert/load"
SCRIPT="$(cd "$BATS_TEST_DIRNAME/../scripts" && pwd)/new-agent.sh"
ROOT="$(mktemp -d)"
}
teardown() {
rm -rf "$ROOT"
}
# ---------------------------------------------------------------------------
# Help
# ---------------------------------------------------------------------------
@test "--help exits 0" {
run bash "$SCRIPT" --help
assert_success
assert_output --partial "Usage:"
}
# ---------------------------------------------------------------------------
# Plugin/APM scope (type:-bearing apm.yml at root)
# ---------------------------------------------------------------------------
@test "plugin/APM scope: creates single agent file in .apm/agents/" {
printf 'name: my-package\ntype: skill\n' > "$ROOT/apm.yml"
run bash "$SCRIPT" my-agent "$ROOT"
assert_success
assert [ -f "$ROOT/.apm/agents/my-agent.agent.md" ]
}
@test "plugin/APM scope: does not create the old dual-file pair" {
printf 'name: my-package\ntype: skill\n' > "$ROOT/apm.yml"
run bash "$SCRIPT" my-agent "$ROOT"
assert_success
assert [ ! -f "$ROOT/agents/my-agent.md" ]
assert [ ! -f "$ROOT/agents/my-agent.agent.md" ]
}
@test "plugin/APM scope: creates .apm/agents/ directory if missing" {
printf 'name: my-package\ntype: skill\n' > "$ROOT/apm.yml"
run bash "$SCRIPT" my-agent "$ROOT"
assert_success
assert [ -d "$ROOT/.apm/agents" ]
}
@test "plugin/APM scope: creates sources.md at package root" {
printf 'name: my-package\ntype: skill\n' > "$ROOT/apm.yml"
run bash "$SCRIPT" my-agent "$ROOT"
assert_success
assert [ -f "$ROOT/sources.md" ]
}
@test "plugin/APM scope: no-op if agent file already exists" {
printf 'name: my-package\ntype: skill\n' > "$ROOT/apm.yml"
mkdir -p "$ROOT/.apm/agents"
echo "existing" > "$ROOT/.apm/agents/my-agent.agent.md"
run bash "$SCRIPT" my-agent "$ROOT"
assert_success
run grep "existing" "$ROOT/.apm/agents/my-agent.agent.md"
assert_success
}
@test "plugin/APM scope: frontmatter has no tools/isolation/maxTurns/effort/memory/permissionMode keys" {
printf 'name: my-package\ntype: skill\n' > "$ROOT/apm.yml"
bash "$SCRIPT" my-agent "$ROOT"
file="$ROOT/.apm/agents/my-agent.agent.md"
fm="$(sed -n '/^---$/,/^---$/p' "$file")"
# Column-0 key lines only — comment bodies in the template are indented,
# so this anchor naturally excludes commented-out example fields.
run grep -E '^(tools|isolation|maxTurns|effort|memory|permissionMode|disallowedTools|skills|color|initialPrompt|background|hooks|mcpServers):' <<< "$fm"
assert_failure
}
# The permitted set is read from the same data agent-audit's validate.sh reads --
# the apm-agent-allowlist section of agent-audit's field-inventory.md -- rather than
# restated here. A hardcoded copy drifts: this assertion listed four fields and went
# on passing after ADR-0016's amendment added disallowedTools, and would have
# rejected a scaffolded agent that legitimately carried it.
@test "plugin/APM scope: frontmatter carries only allowlisted fields" {
inventory="$BATS_TEST_DIRNAME/../../agent-audit/references/field-inventory.md"
[ -f "$inventory" ] || fail "field-inventory.md not found at $inventory"
allowlist="$(awk '
/^## apm-agent-allowlist$/ { insection = 1; next }
insection && /^##/ { exit }
insection && NF && $0 !~ /^#/ && $0 !~ /^---/ { print; exit }
' "$inventory")"
[ -n "$allowlist" ] || fail "apm-agent-allowlist section is empty in $inventory"
printf 'name: my-package\ntype: skill\n' > "$ROOT/apm.yml"
bash "$SCRIPT" my-agent "$ROOT"
file="$ROOT/.apm/agents/my-agent.agent.md"
fm="$(sed -n '/^---$/,/^---$/p' "$file")"
keys="$(grep -oE '^[a-zA-Z][a-zA-Z0-9_-]*:' <<< "$fm" | sed 's/:$//' | sort -u)"
for key in $keys; do
if ! grep -qw "$key" <<< "$allowlist"; then
fail "frontmatter key '$key' is not in field-inventory.md's apm-agent-allowlist ($allowlist)"
fi
done
}
@test "plugin/APM scope: sources.md contributing-files template mentions the single-file path" {
printf 'name: my-package\ntype: skill\n' > "$ROOT/apm.yml"
bash "$SCRIPT" my-agent "$ROOT"
run grep ".apm/agents/<name>.agent.md" "$ROOT/sources.md"
assert_success
}
@test "plugin/APM scope: template AGENT_NAME substituted" {
printf 'name: my-package\ntype: skill\n' > "$ROOT/apm.yml"
bash "$SCRIPT" my-agent "$ROOT"
run grep "my-agent" "$ROOT/.apm/agents/my-agent.agent.md"
assert_success
}
@test "plugin/APM scope: walk-up finds apm.yml at an ancestor directory, not just root arg" {
printf 'name: my-package\ntype: skill\n' > "$ROOT/apm.yml"
mkdir -p "$ROOT/nested/subdir"
run bash "$SCRIPT" my-agent "$ROOT/nested/subdir"
assert_success
assert [ -f "$ROOT/.apm/agents/my-agent.agent.md" ]
}
@test "plugin/APM scope: apm.yml without type: is skipped (marketplace-only manifest)" {
printf 'name: my-marketplace\n' > "$ROOT/apm.yml"
mkdir -p "$ROOT/.git"
run bash "$SCRIPT" my-agent "$ROOT"
assert_success
assert [ ! -f "$ROOT/.apm/agents/my-agent.agent.md" ]
assert [ -f "$ROOT/.claude/agents/my-agent.md" ]
}
@test "plugin/APM scope: type:-less apm.yml at leaf falls through to a type:-bearing apm.yml higher up" {
printf 'name: outer-package\ntype: skill\n' > "$ROOT/apm.yml"
mkdir -p "$ROOT/inner"
printf 'name: inner-marketplace\n' > "$ROOT/inner/apm.yml"
run bash "$SCRIPT" my-agent "$ROOT/inner"
assert_success
assert [ -f "$ROOT/.apm/agents/my-agent.agent.md" ]
}
@test "plugin/APM scope: matched-quote type value ('skill') is recognized" {
printf 'name: my-package\ntype: "skill"\n' > "$ROOT/apm.yml"
run bash "$SCRIPT" my-agent "$ROOT"
assert_success
assert [ -f "$ROOT/.apm/agents/my-agent.agent.md" ]
}
@test "plugin/APM scope: mismatched-quote type value is rejected, falls through to project scope" {
mkdir -p "$ROOT/.git"
printf "name: my-package\ntype: \"skill'\n" > "$ROOT/apm.yml"
run bash "$SCRIPT" my-agent "$ROOT"
assert_success
assert [ ! -f "$ROOT/.apm/agents/my-agent.agent.md" ]
assert [ -f "$ROOT/.claude/agents/my-agent.md" ]
}
@test "plugin/APM scope: type: line is recognized even without a trailing newline on the file" {
printf 'name: my-package\ntype: skill' > "$ROOT/apm.yml"
run bash "$SCRIPT" my-agent "$ROOT"
assert_success
assert [ -f "$ROOT/.apm/agents/my-agent.agent.md" ]
}
# ---------------------------------------------------------------------------
# Old plugin.json marker is no longer recognized (full switch, no dual-mode)
# ---------------------------------------------------------------------------
@test "bare plugin.json (no apm.yml) is no longer detected as plugin scope — falls through to project scope" {
touch "$ROOT/plugin.json"
run bash "$SCRIPT" my-agent "$ROOT"
assert_success
assert [ ! -f "$ROOT/agents/my-agent.md" ]
assert [ ! -f "$ROOT/agents/my-agent.agent.md" ]
assert [ ! -f "$ROOT/.apm/agents/my-agent.agent.md" ]
assert [ -f "$ROOT/.claude/agents/my-agent.md" ]
assert [ -f "$ROOT/.github/agents/my-agent.agent.md" ]
}
# ---------------------------------------------------------------------------
# Non-plugin (project) scope
# ---------------------------------------------------------------------------
@test "non-plugin scope: creates claude code file in .claude/agents/" {
run bash "$SCRIPT" my-agent "$ROOT"
assert_success
assert [ -f "$ROOT/.claude/agents/my-agent.md" ]
}
@test "non-plugin scope: creates copilot file in .github/agents/" {
run bash "$SCRIPT" my-agent "$ROOT"
assert_success
assert [ -f "$ROOT/.github/agents/my-agent.agent.md" ]
}
@test "non-plugin scope: creates .claude/agents/ directory if missing" {
run bash "$SCRIPT" my-agent "$ROOT"
assert_success
assert [ -d "$ROOT/.claude/agents" ]
}
@test "non-plugin scope: creates .github/agents/ directory if missing" {
run bash "$SCRIPT" my-agent "$ROOT"
assert_success
assert [ -d "$ROOT/.github/agents" ]
}
@test "non-plugin scope: no sources.md created" {
run bash "$SCRIPT" my-agent "$ROOT"
assert_success
assert [ ! -f "$ROOT/.claude/agents/sources.md" ]
assert [ ! -f "$ROOT/.github/agents/sources.md" ]
}
@test "non-plugin scope: no-op if claude code file already exists" {
mkdir -p "$ROOT/.claude/agents"
echo "existing" > "$ROOT/.claude/agents/my-agent.md"
run bash "$SCRIPT" my-agent "$ROOT"
assert_success
run grep "existing" "$ROOT/.claude/agents/my-agent.md"
assert_success
}
@test "project scope: detected via .git present above root, output still relative to root arg" {
mkdir -p "$ROOT/repo/.git"
mkdir -p "$ROOT/repo/pkg"
run bash "$SCRIPT" my-agent "$ROOT/repo/pkg"
assert_success
assert [ -f "$ROOT/repo/pkg/.claude/agents/my-agent.md" ]
assert [ -f "$ROOT/repo/pkg/.github/agents/my-agent.agent.md" ]
}
# ---------------------------------------------------------------------------
# User scope
# ---------------------------------------------------------------------------
@test "user scope: root exactly \$HOME creates files under ~/.claude and ~/.copilot" {
FAKE_HOME="$(mktemp -d)"
run env HOME="$FAKE_HOME" bash "$SCRIPT" my-agent "~"
assert_success
assert [ -f "$FAKE_HOME/.claude/agents/my-agent.md" ]
assert [ -f "$FAKE_HOME/.copilot/agents/my-agent.agent.md" ]
rm -rf "$FAKE_HOME"
}
@test "user scope: \$HOME being a dotfiles .git repo does not shadow user scope" {
FAKE_HOME="$(mktemp -d)"
mkdir "$FAKE_HOME/.git"
run env HOME="$FAKE_HOME" bash "$SCRIPT" my-agent "~"
assert_success
assert [ -f "$FAKE_HOME/.claude/agents/my-agent.md" ]
assert [ -f "$FAKE_HOME/.copilot/agents/my-agent.agent.md" ]
refute [ -d "$FAKE_HOME/.github" ]
rm -rf "$FAKE_HOME"
}
@test "user scope is checked directly at \$HOME, no walk-up: a marker-less subdir under \$HOME resolves to project scope, not user scope" {
FAKE_HOME="$(mktemp -d)"
mkdir -p "$FAKE_HOME/scratch/testdir"
run env HOME="$FAKE_HOME" bash "$SCRIPT" my-agent "$FAKE_HOME/scratch/testdir"
assert_success
assert [ -f "$FAKE_HOME/scratch/testdir/.claude/agents/my-agent.md" ]
assert [ -f "$FAKE_HOME/scratch/testdir/.github/agents/my-agent.agent.md" ]
refute [ -f "$FAKE_HOME/.claude/agents/my-agent.md" ]
refute [ -f "$FAKE_HOME/.copilot/agents/my-agent.agent.md" ]
rm -rf "$FAKE_HOME"
}
# ---------------------------------------------------------------------------
# Name validation
# ---------------------------------------------------------------------------
@test "fails when no arguments given" {
run bash "$SCRIPT"
assert_failure
}
@test "fails when agent name contains uppercase" {
run bash "$SCRIPT" MyAgent "$ROOT"
assert_failure
}
@test "fails when agent name has consecutive hyphens" {
run bash "$SCRIPT" my--agent "$ROOT"
assert_failure
}
@test "fails when agent name has a leading hyphen" {
run bash "$SCRIPT" -my-agent "$ROOT"
assert_failure
}
@test "fails when agent name has a trailing hyphen" {
run bash "$SCRIPT" my-agent- "$ROOT"
assert_failure
}
@test "agent name with numbers is valid" {
run bash "$SCRIPT" agent-v2 "$ROOT"
assert_success
}
# ---------------------------------------------------------------------------
# Root validation
# ---------------------------------------------------------------------------
@test "fails when root directory does not exist" {
run bash "$SCRIPT" my-agent "/nonexistent/path"
assert_failure
}