Defame1297
b0d6d08239
Every defect fixed in f7cc279 was reachable because nothing asserted against it.
The gate had 43 assertions and none of them covered a consumer repo, a non-string
description, an unclosed fence, or the two spec ceilings. Each case below fails
against the pre-fix code and passes against the current one; every one was proved
non-vacuous by mutating a scratch copy of the script and watching the test go red,
independently twice.
The two that mattered most had no fixture anywhere. A consumer repo WITH .git is
the shape the resolver exists to serve, and only the no-.git case had ever been
tested, which is exactly why the blocker was invisible. And ADR-0020 says the
walk-up runs in two passes specifically so a nested .git cannot beat a plugins/
root further up — no fixture had ever placed a .git inside a plugin.
test-adr0020-differential.sh loses _non_adr_hook_error(). It excluded MAX_LINES and
MAX_WORDS from the cross-script comparison on the untested assumption that awk and
splitlines() agree. They do not, and the divergence stayed invisible for exactly as
long as the exclusion stood. The ceilings are now compared like any other rule.
Two existing assertions were repairs, not additions. The skill-improve probe had
been fixed by this very branch, so its iteration permanently took an
assertion-free SKIP that still counted as a pass; both branches now fail loudly and
each names the other file's pin so the two stay in step. And the yaml-none fixture
emitted `---/---`, which never matched the frontmatter pattern at all — it passed on
the bare word "frontmatter", present in both messages, while never reaching the
branch it was named for. Needles throughout that file now name their branch.
Refs: #99
ADR: 0020
Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_015W3iwF9ncfRZddGBxsMCYi
946 lines
32 KiB
Bash
946 lines
32 KiB
Bash
#!/usr/bin/env bats
|
|
|
|
setup() {
|
|
REPO_ROOT="$(cd "$BATS_TEST_DIRNAME/../../../../../../" && pwd)"
|
|
load "$REPO_ROOT/tests/test_helper/bats-support/load"
|
|
load "$REPO_ROOT/tests/test_helper/bats-assert/load"
|
|
|
|
SCRIPT="$(cd "$BATS_TEST_DIRNAME/../scripts" && pwd)/validate.sh"
|
|
TMPDIR="$(mktemp -d)"
|
|
|
|
# Helper: create an APM package root at <root> (apm.yml with a top-level
|
|
# type: line, marking it a real package manifest — not marketplace-only)
|
|
# plus a single vendor-neutral agent file at
|
|
# <root>/.apm/agents/<name>.agent.md. <extra_frontmatter>, if given, is
|
|
# inserted as additional raw frontmatter lines (used to inject fields
|
|
# under test).
|
|
make_apm_agent() {
|
|
local root="$1"
|
|
local name="$2"
|
|
local extra_frontmatter="${3:-}"
|
|
mkdir -p "$root/.apm/agents"
|
|
cat > "$root/apm.yml" <<EOF
|
|
name: test-package
|
|
version: 0.1.0
|
|
type: skill
|
|
EOF
|
|
cat > "$root/.apm/agents/${name}.agent.md" <<EOF
|
|
---
|
|
name: ${name}
|
|
description: A valid agent description.
|
|
${extra_frontmatter}
|
|
---
|
|
|
|
You are a test agent. When invoked, do the thing.
|
|
EOF
|
|
}
|
|
|
|
# Helper: a description of EXACTLY <n> characters that carries a boundary
|
|
# clause and names no routing target. ADR-0020's missing-boundary-clause
|
|
# SUGGESTION fires on any description without one, so a fixture that omits it
|
|
# is never "otherwise clean" and a test refuting SUGGESTION would be asserting
|
|
# the boundary check's absence instead of the thing it names. The clause is
|
|
# paid for out of the measured budget rather than appended to it, because
|
|
# these tests measure the description LENGTH. "anything else" is not
|
|
# hyphenated, so no routing target comes with it.
|
|
desc_of_length() {
|
|
python3 - "$1" <<'PY'
|
|
import sys
|
|
n = int(sys.argv[1])
|
|
prefix = 'Use when doing the thing. Do not use for anything else. '
|
|
assert n >= len(prefix), 'requested description shorter than the boundary clause'
|
|
print(prefix + 'x' * (n - len(prefix)))
|
|
PY
|
|
}
|
|
|
|
# Helper: same shape as make_apm_agent, but the description is supplied
|
|
# verbatim — used by the ADR-0020 description-budget tests.
|
|
make_apm_agent_with_desc() {
|
|
local root="$1" name="$2" desc="$3"
|
|
mkdir -p "$root/.apm/agents"
|
|
cat > "$root/apm.yml" <<EOF
|
|
name: test-package
|
|
version: 0.1.0
|
|
type: skill
|
|
EOF
|
|
cat > "$root/.apm/agents/${name}.agent.md" <<EOF
|
|
---
|
|
name: ${name}
|
|
description: ${desc}
|
|
---
|
|
|
|
You are a test agent. When invoked, do the thing.
|
|
EOF
|
|
}
|
|
}
|
|
|
|
teardown() {
|
|
rm -rf "$TMPDIR"
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Passing cases — project/user scope (unchanged)
|
|
# ---------------------------------------------------------------------------
|
|
|
|
@test "passes on a clean project-scope pair (CC file as input)" {
|
|
local root="$TMPDIR/project"
|
|
mkdir -p "$root/.git" "$root/.claude/agents" "$root/.github/agents"
|
|
cat > "$root/.claude/agents/my-agent.md" <<EOF
|
|
---
|
|
name: my-agent
|
|
description: A valid agent description.
|
|
---
|
|
|
|
You are a test agent. When invoked, do the thing.
|
|
EOF
|
|
cat > "$root/.github/agents/my-agent.agent.md" <<EOF
|
|
---
|
|
name: my-agent
|
|
description: A valid agent description.
|
|
---
|
|
|
|
You are a test agent. When invoked, do the thing.
|
|
EOF
|
|
run bash "$SCRIPT" "$root/.claude/agents/my-agent.md"
|
|
assert_success
|
|
refute_output --partial "FAIL"
|
|
}
|
|
|
|
@test "user scope: \$HOME being a dotfiles .git repo does not shadow user scope" {
|
|
local fake_home="$TMPDIR/fakehome"
|
|
mkdir -p "$fake_home/.git" "$fake_home/.claude/agents" "$fake_home/.copilot/agents"
|
|
cat > "$fake_home/.claude/agents/my-agent.md" <<EOF
|
|
---
|
|
name: my-agent
|
|
description: A valid agent description.
|
|
---
|
|
|
|
You are a test agent. When invoked, do the thing.
|
|
EOF
|
|
cat > "$fake_home/.copilot/agents/my-agent.agent.md" <<EOF
|
|
---
|
|
name: my-agent
|
|
description: A valid agent description.
|
|
---
|
|
|
|
You are a test agent. When invoked, do the thing.
|
|
EOF
|
|
run env HOME="$fake_home" bash "$SCRIPT" "$fake_home/.claude/agents/my-agent.md"
|
|
assert_success
|
|
refute_output --partial "FAIL"
|
|
}
|
|
|
|
@test "user scope: agent file directly in \$HOME (start dir IS exactly \$HOME, no walk-up) resolves to user scope" {
|
|
local fake_home="$TMPDIR/fakehome-direct"
|
|
mkdir -p "$fake_home" "$fake_home/.copilot/agents"
|
|
cat > "$fake_home/my-agent.md" <<EOF
|
|
---
|
|
name: my-agent
|
|
description: A valid agent description.
|
|
---
|
|
|
|
You are a test agent. When invoked, do the thing.
|
|
EOF
|
|
cat > "$fake_home/.copilot/agents/my-agent.agent.md" <<EOF
|
|
---
|
|
name: my-agent
|
|
description: A valid agent description.
|
|
---
|
|
|
|
You are a test agent. When invoked, do the thing.
|
|
EOF
|
|
run env HOME="$fake_home" bash "$SCRIPT" "$fake_home/my-agent.md"
|
|
assert_success
|
|
refute_output --partial "FAIL"
|
|
refute_output --partial "counterpart"
|
|
}
|
|
|
|
@test "project scope: a nested marker-less directory walked up into \$HOME resolves to project scope, not user scope (live repro of new-agent.sh's stray-directory case)" {
|
|
local fake_home="$TMPDIR/fakehome-nested"
|
|
local nested="$fake_home/scratch/testdir"
|
|
mkdir -p "$nested/.claude/agents" "$nested/.github/agents"
|
|
cat > "$nested/.claude/agents/my-agent.md" <<EOF
|
|
---
|
|
name: my-agent
|
|
description: A valid agent description.
|
|
---
|
|
|
|
You are a test agent. When invoked, do the thing.
|
|
EOF
|
|
cat > "$nested/.github/agents/my-agent.agent.md" <<EOF
|
|
---
|
|
name: my-agent
|
|
description: A valid agent description.
|
|
---
|
|
|
|
You are a test agent. When invoked, do the thing.
|
|
EOF
|
|
run env HOME="$fake_home" bash "$SCRIPT" "$nested/.claude/agents/my-agent.md"
|
|
assert_success
|
|
refute_output --partial "FAIL"
|
|
refute_output --partial "counterpart"
|
|
}
|
|
|
|
@test "project scope: nested marker-less dir under \$HOME does NOT look for a counterpart under the shared \$HOME/.copilot or \$HOME/.github dirs" {
|
|
local fake_home="$TMPDIR/fakehome-nested2"
|
|
local nested="$fake_home/scratch/testdir"
|
|
mkdir -p "$nested/.claude/agents" "$fake_home/.copilot/agents"
|
|
cat > "$nested/.claude/agents/my-agent.md" <<EOF
|
|
---
|
|
name: my-agent
|
|
description: A valid agent description.
|
|
---
|
|
|
|
You are a test agent. When invoked, do the thing.
|
|
EOF
|
|
# Decoy counterpart at the *user*-scope location — if scope were
|
|
# misclassified as 'user' (the pre-fix bug), validate.sh would find this
|
|
# unrelated file and (wrongly) pass.
|
|
cat > "$fake_home/.copilot/agents/my-agent.agent.md" <<EOF
|
|
---
|
|
name: my-agent
|
|
description: A valid agent description.
|
|
---
|
|
|
|
You are a test agent. When invoked, do the thing.
|
|
EOF
|
|
run env HOME="$fake_home" bash "$SCRIPT" "$nested/.claude/agents/my-agent.md"
|
|
assert_failure
|
|
assert_output --partial "counterpart file not found"
|
|
}
|
|
|
|
@test "project scope: filesystem-root fallback (no \$HOME in path, no markers found) resolves to project scope, not user scope" {
|
|
local unrelated_home="$TMPDIR/unrelated-home-never-reached"
|
|
local root="$TMPDIR/no-home-relation/deep/proj"
|
|
mkdir -p "$root/.claude/agents" "$root/.github/agents"
|
|
cat > "$root/.claude/agents/my-agent.md" <<EOF
|
|
---
|
|
name: my-agent
|
|
description: A valid agent description.
|
|
---
|
|
|
|
You are a test agent. When invoked, do the thing.
|
|
EOF
|
|
cat > "$root/.github/agents/my-agent.agent.md" <<EOF
|
|
---
|
|
name: my-agent
|
|
description: A valid agent description.
|
|
---
|
|
|
|
You are a test agent. When invoked, do the thing.
|
|
EOF
|
|
run env HOME="$unrelated_home" bash "$SCRIPT" "$root/.claude/agents/my-agent.md"
|
|
assert_success
|
|
refute_output --partial "FAIL"
|
|
refute_output --partial "counterpart"
|
|
}
|
|
|
|
@test "project scope: <root> one level below a .git ancestor resolves scope to <root>, not to wherever .git was found (subdirectory of a larger git-tracked tree)" {
|
|
local repo="$TMPDIR/repo-with-subdir"
|
|
local root="$repo/subdir"
|
|
mkdir -p "$repo/.git" "$root/.claude/agents" "$root/.github/agents"
|
|
cat > "$root/.claude/agents/my-agent.md" <<EOF
|
|
---
|
|
name: my-agent
|
|
description: A valid agent description.
|
|
---
|
|
|
|
You are a test agent. When invoked, do the thing.
|
|
EOF
|
|
cat > "$root/.github/agents/my-agent.agent.md" <<EOF
|
|
---
|
|
name: my-agent
|
|
description: A valid agent description.
|
|
---
|
|
|
|
You are a test agent. When invoked, do the thing.
|
|
EOF
|
|
# new-agent.sh, invoked with <root> as its root argument, would place the
|
|
# counterpart at <root>/.github/agents — not at the repo root's
|
|
# .github/agents, even though .git lives at the repo root one level up.
|
|
run bash "$SCRIPT" "$root/.claude/agents/my-agent.md"
|
|
assert_success
|
|
refute_output --partial "FAIL"
|
|
refute_output --partial "counterpart"
|
|
}
|
|
|
|
@test "project scope: a non-conventional path (agent file not directly under a literal 'agents' dir) falls back to the nearest .git boundary instead of two-segments-up arithmetic" {
|
|
local outer="$TMPDIR/outer-repo"
|
|
local pkg="$outer/pkgA"
|
|
mkdir -p "$pkg/.git" "$pkg/.github/agents" "$pkg/extra"
|
|
# Misplaced file: sits two path segments below $outer (pkgA/extra), which
|
|
# matches the conventional_root arithmetic by coincidence, but its
|
|
# immediate parent dir is "extra", not "agents" — conventional_shape is
|
|
# false, so the fix must fall back to the nearest .git boundary (pkgA),
|
|
# not trust $outer.
|
|
cat > "$pkg/extra/my-agent.md" <<EOF
|
|
---
|
|
name: my-agent
|
|
description: A valid agent description.
|
|
---
|
|
|
|
You are a test agent. When invoked, do the thing.
|
|
EOF
|
|
# Counterpart at the nearest-.git root (pkgA), not at $outer — if the
|
|
# arithmetic were trusted here, validate.sh would look for a counterpart
|
|
# at $outer/.github/agents/my-agent.agent.md, which doesn't exist, and
|
|
# false-FAIL.
|
|
cat > "$pkg/.github/agents/my-agent.agent.md" <<EOF
|
|
---
|
|
name: my-agent
|
|
description: A valid agent description.
|
|
---
|
|
|
|
You are a test agent. When invoked, do the thing.
|
|
EOF
|
|
run bash "$SCRIPT" "$pkg/extra/my-agent.md"
|
|
assert_success
|
|
refute_output --partial "counterpart file not found"
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Failing cases — project/user scope: CC/Copilot pair checks
|
|
# ---------------------------------------------------------------------------
|
|
|
|
@test "fails when a CC-only field ('maxTurns') is present in a project-scope Copilot file" {
|
|
local root="$TMPDIR/project"
|
|
mkdir -p "$root/.git" "$root/.claude/agents" "$root/.github/agents"
|
|
cat > "$root/.claude/agents/my-agent.md" <<EOF
|
|
---
|
|
name: my-agent
|
|
description: A valid agent description.
|
|
---
|
|
|
|
You are a test agent. When invoked, do the thing.
|
|
EOF
|
|
cat > "$root/.github/agents/my-agent.agent.md" <<EOF
|
|
---
|
|
name: my-agent
|
|
description: A valid agent description.
|
|
maxTurns: 10
|
|
---
|
|
|
|
You are a test agent. When invoked, do the thing.
|
|
EOF
|
|
run bash "$SCRIPT" "$root/.claude/agents/my-agent.md"
|
|
assert_failure
|
|
assert_output --partial "CC-only field"
|
|
assert_output --partial "maxTurns"
|
|
}
|
|
|
|
@test "fails when a Copilot-only field ('target') is present in a project-scope CC file" {
|
|
local root="$TMPDIR/project"
|
|
mkdir -p "$root/.git" "$root/.claude/agents" "$root/.github/agents"
|
|
cat > "$root/.claude/agents/my-agent.md" <<EOF
|
|
---
|
|
name: my-agent
|
|
description: A valid agent description.
|
|
target: cli
|
|
---
|
|
|
|
You are a test agent. When invoked, do the thing.
|
|
EOF
|
|
cat > "$root/.github/agents/my-agent.agent.md" <<EOF
|
|
---
|
|
name: my-agent
|
|
description: A valid agent description.
|
|
---
|
|
|
|
You are a test agent. When invoked, do the thing.
|
|
EOF
|
|
run bash "$SCRIPT" "$root/.claude/agents/my-agent.md"
|
|
assert_failure
|
|
assert_output --partial "Copilot-only field"
|
|
assert_output --partial "target"
|
|
}
|
|
|
|
@test "fails when the Copilot counterpart is missing at project scope" {
|
|
local root="$TMPDIR/project"
|
|
mkdir -p "$root/.git" "$root/.claude/agents"
|
|
cat > "$root/.claude/agents/my-agent.md" <<EOF
|
|
---
|
|
name: my-agent
|
|
description: A valid agent description.
|
|
---
|
|
|
|
You are a test agent. When invoked, do the thing.
|
|
EOF
|
|
run bash "$SCRIPT" "$root/.claude/agents/my-agent.md"
|
|
assert_failure
|
|
assert_output --partial "counterpart file not found"
|
|
}
|
|
|
|
@test "--help exits 0 and shows Usage:" {
|
|
run bash "$SCRIPT" --help
|
|
assert_success
|
|
assert_output --partial "Usage:"
|
|
}
|
|
|
|
@test "fails when no arguments are given" {
|
|
run bash "$SCRIPT"
|
|
assert_failure
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Passing cases — plugin/APM scope
|
|
# ---------------------------------------------------------------------------
|
|
|
|
@test "passes on a clean plugin/APM-scope agent file (name/description only)" {
|
|
local root="$TMPDIR/pkg"
|
|
make_apm_agent "$root" "my-agent"
|
|
run bash "$SCRIPT" "$root/.apm/agents/my-agent.agent.md"
|
|
assert_success
|
|
refute_output --partial "FAIL"
|
|
}
|
|
|
|
@test "passes on a clean plugin/APM-scope agent file with optional model field" {
|
|
local root="$TMPDIR/pkg"
|
|
make_apm_agent "$root" "my-agent" "model: claude-opus-4"
|
|
run bash "$SCRIPT" "$root/.apm/agents/my-agent.agent.md"
|
|
assert_success
|
|
refute_output --partial "FAIL"
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Failing cases — plugin/APM scope: allowlist violations
|
|
# ---------------------------------------------------------------------------
|
|
|
|
@test "fails when 'tools' field is present in a plugin/APM-scope agent file" {
|
|
local root="$TMPDIR/pkg"
|
|
make_apm_agent "$root" "my-agent" "tools: Read Edit"
|
|
run bash "$SCRIPT" "$root/.apm/agents/my-agent.agent.md"
|
|
assert_failure
|
|
assert_output --partial "tools"
|
|
}
|
|
|
|
@test "fails when a Claude-only field ('maxTurns') is present in a plugin/APM-scope agent file" {
|
|
local root="$TMPDIR/pkg"
|
|
make_apm_agent "$root" "my-agent" "maxTurns: 10"
|
|
run bash "$SCRIPT" "$root/.apm/agents/my-agent.agent.md"
|
|
assert_failure
|
|
assert_output --partial "maxTurns"
|
|
}
|
|
|
|
@test "fails when a Copilot-only field ('target') is present in a plugin/APM-scope agent file" {
|
|
local root="$TMPDIR/pkg"
|
|
make_apm_agent "$root" "my-agent" "target: cli"
|
|
run bash "$SCRIPT" "$root/.apm/agents/my-agent.agent.md"
|
|
assert_failure
|
|
assert_output --partial "target"
|
|
}
|
|
|
|
@test "fails when 'hooks' is present in a plugin/APM-scope agent file (outside allowlist)" {
|
|
local root="$TMPDIR/pkg"
|
|
make_apm_agent "$root" "my-agent" "hooks: {}"
|
|
run bash "$SCRIPT" "$root/.apm/agents/my-agent.agent.md"
|
|
assert_failure
|
|
assert_output --partial "hooks"
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Failing cases — plugin/APM scope: structural checks
|
|
# ---------------------------------------------------------------------------
|
|
|
|
@test "fails when name is not kebab-case in a plugin/APM-scope agent file" {
|
|
local root="$TMPDIR/pkg"
|
|
mkdir -p "$root/.apm/agents"
|
|
cat > "$root/apm.yml" <<EOF
|
|
name: test-package
|
|
version: 0.1.0
|
|
type: skill
|
|
EOF
|
|
cat > "$root/.apm/agents/my-agent.agent.md" <<EOF
|
|
---
|
|
name: MyAgent
|
|
description: A valid agent description.
|
|
---
|
|
|
|
You are a test agent. When invoked, do the thing.
|
|
EOF
|
|
run bash "$SCRIPT" "$root/.apm/agents/my-agent.agent.md"
|
|
assert_failure
|
|
assert_output --partial "kebab"
|
|
}
|
|
|
|
@test "fails when name does not match filename stem in a plugin/APM-scope agent file" {
|
|
local root="$TMPDIR/pkg"
|
|
mkdir -p "$root/.apm/agents"
|
|
cat > "$root/apm.yml" <<EOF
|
|
name: test-package
|
|
version: 0.1.0
|
|
type: skill
|
|
EOF
|
|
cat > "$root/.apm/agents/my-agent.agent.md" <<EOF
|
|
---
|
|
name: wrong-name
|
|
description: A valid agent description.
|
|
---
|
|
|
|
You are a test agent. When invoked, do the thing.
|
|
EOF
|
|
run bash "$SCRIPT" "$root/.apm/agents/my-agent.agent.md"
|
|
assert_failure
|
|
assert_output --partial "does not match filename stem"
|
|
}
|
|
|
|
@test "fails when 'name' field is missing from a plugin/APM-scope agent file" {
|
|
local root="$TMPDIR/pkg"
|
|
mkdir -p "$root/.apm/agents"
|
|
cat > "$root/apm.yml" <<EOF
|
|
name: test-package
|
|
version: 0.1.0
|
|
type: skill
|
|
EOF
|
|
cat > "$root/.apm/agents/my-agent.agent.md" <<EOF
|
|
---
|
|
description: A valid agent description.
|
|
---
|
|
|
|
You are a test agent. When invoked, do the thing.
|
|
EOF
|
|
run bash "$SCRIPT" "$root/.apm/agents/my-agent.agent.md"
|
|
assert_failure
|
|
}
|
|
|
|
@test "fails when 'description' field is missing from a plugin/APM-scope agent file" {
|
|
local root="$TMPDIR/pkg"
|
|
mkdir -p "$root/.apm/agents"
|
|
cat > "$root/apm.yml" <<EOF
|
|
name: test-package
|
|
version: 0.1.0
|
|
type: skill
|
|
EOF
|
|
cat > "$root/.apm/agents/my-agent.agent.md" <<EOF
|
|
---
|
|
name: my-agent
|
|
---
|
|
|
|
You are a test agent. When invoked, do the thing.
|
|
EOF
|
|
run bash "$SCRIPT" "$root/.apm/agents/my-agent.agent.md"
|
|
assert_failure
|
|
}
|
|
|
|
@test "fails when a template HTML comment is left in plugin/APM-scope frontmatter" {
|
|
local root="$TMPDIR/pkg"
|
|
mkdir -p "$root/.apm/agents"
|
|
cat > "$root/apm.yml" <<EOF
|
|
name: test-package
|
|
version: 0.1.0
|
|
type: skill
|
|
EOF
|
|
cat > "$root/.apm/agents/my-agent.agent.md" <<EOF
|
|
---
|
|
name: my-agent
|
|
description: A valid agent description.
|
|
<!-- model: sonnet
|
|
Optional. Omit to inherit the runtime default. -->
|
|
---
|
|
|
|
You are a test agent. When invoked, do the thing.
|
|
EOF
|
|
run bash "$SCRIPT" "$root/.apm/agents/my-agent.agent.md"
|
|
assert_failure
|
|
assert_output --partial "template HTML comments"
|
|
}
|
|
|
|
@test "fails when body contains unfilled FILL IN: placeholder in a plugin/APM-scope agent file" {
|
|
local root="$TMPDIR/pkg"
|
|
mkdir -p "$root/.apm/agents"
|
|
cat > "$root/apm.yml" <<EOF
|
|
name: test-package
|
|
version: 0.1.0
|
|
type: skill
|
|
EOF
|
|
cat > "$root/.apm/agents/my-agent.agent.md" <<EOF
|
|
---
|
|
name: my-agent
|
|
description: A valid agent description.
|
|
---
|
|
|
|
FILL IN: replace this with your system prompt.
|
|
EOF
|
|
run bash "$SCRIPT" "$root/.apm/agents/my-agent.agent.md"
|
|
assert_failure
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Plugin/APM scope: no pair, no counterpart concept
|
|
# ---------------------------------------------------------------------------
|
|
|
|
@test "never raises a 'counterpart' FAIL on a clean plugin/APM-scope agent file" {
|
|
local root="$TMPDIR/pkg"
|
|
make_apm_agent "$root" "my-agent"
|
|
run bash "$SCRIPT" "$root/.apm/agents/my-agent.agent.md"
|
|
assert_success
|
|
refute_output --partial "counterpart"
|
|
}
|
|
|
|
@test "never raises a 'counterpart' FAIL on a failing plugin/APM-scope agent file" {
|
|
local root="$TMPDIR/pkg"
|
|
make_apm_agent "$root" "my-agent" "tools: Read"
|
|
run bash "$SCRIPT" "$root/.apm/agents/my-agent.agent.md"
|
|
assert_failure
|
|
refute_output --partial "counterpart"
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Scope-detection walk-up
|
|
# ---------------------------------------------------------------------------
|
|
|
|
@test "walk-up skips a type:-less apm.yml and finds a real package root further up" {
|
|
local root="$TMPDIR/case"
|
|
mkdir -p "$root/.apm/agents/nested/deeper"
|
|
cat > "$root/apm.yml" <<EOF
|
|
name: real-package
|
|
version: 1.0.0
|
|
type: skill
|
|
EOF
|
|
# Closer to the agent file than the real package root, but has no type:
|
|
# line — marketplace-only per monorepo-and-repo-shapes.md, must be skipped.
|
|
cat > "$root/.apm/agents/nested/apm.yml" <<EOF
|
|
name: not-a-package-manifest
|
|
version: 1.0.0
|
|
EOF
|
|
cat > "$root/.apm/agents/nested/deeper/my-agent.agent.md" <<EOF
|
|
---
|
|
name: my-agent
|
|
description: A valid agent description.
|
|
---
|
|
|
|
You are a test agent. When invoked, do the thing.
|
|
EOF
|
|
run bash "$SCRIPT" "$root/.apm/agents/nested/deeper/my-agent.agent.md"
|
|
assert_success
|
|
refute_output --partial "FAIL"
|
|
}
|
|
|
|
@test "type:-less apm.yml is not treated as plugin scope — falls through to project scope" {
|
|
local root="$TMPDIR/proj-marketplace"
|
|
mkdir -p "$root/.git" "$root/.claude/agents" "$root/.github/agents"
|
|
cat > "$root/apm.yml" <<EOF
|
|
name: marketplace-root
|
|
version: 1.0.0
|
|
marketplace:
|
|
packages: []
|
|
EOF
|
|
cat > "$root/.claude/agents/my-agent.md" <<EOF
|
|
---
|
|
name: my-agent
|
|
description: A valid agent description.
|
|
---
|
|
|
|
You are a test agent. When invoked, do the thing.
|
|
EOF
|
|
cat > "$root/.github/agents/my-agent.agent.md" <<EOF
|
|
---
|
|
name: my-agent
|
|
description: A valid agent description.
|
|
---
|
|
|
|
You are a test agent. When invoked, do the thing.
|
|
EOF
|
|
run bash "$SCRIPT" "$root/.claude/agents/my-agent.md"
|
|
assert_success
|
|
refute_output --partial "FAIL"
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# ADR-0020 — description budget (250 SUGGESTION / 400 FAIL)
|
|
#
|
|
# Agents take the SAME description gates as skills: name + description is
|
|
# preloaded into every session identically. Agents take NO body word gate — see
|
|
# the final test in this block, which pins that asymmetry.
|
|
# ---------------------------------------------------------------------------
|
|
|
|
@test "ADR-0020: agent description of exactly 250 chars raises no suggestion" {
|
|
local root="$TMPDIR/pkg"
|
|
make_apm_agent_with_desc "$root" "my-agent" "$(desc_of_length 250)"
|
|
run bash "$SCRIPT" "$root/.apm/agents/my-agent.agent.md"
|
|
assert_success
|
|
refute_output --partial "SUGGESTION"
|
|
}
|
|
|
|
@test "ADR-0020: agent description of 251 chars raises a SUGGESTION and still exits 0" {
|
|
local root="$TMPDIR/pkg"
|
|
make_apm_agent_with_desc "$root" "my-agent" "$(desc_of_length 251)"
|
|
run bash "$SCRIPT" "$root/.apm/agents/my-agent.agent.md"
|
|
assert_success
|
|
assert_output --partial "SUGGESTION"
|
|
assert_output --partial "description is 251 chars"
|
|
}
|
|
|
|
@test "ADR-0020: agent description of exactly 400 chars is a SUGGESTION, not a FAIL" {
|
|
local root="$TMPDIR/pkg"
|
|
make_apm_agent_with_desc "$root" "my-agent" "$(desc_of_length 400)"
|
|
run bash "$SCRIPT" "$root/.apm/agents/my-agent.agent.md"
|
|
assert_success
|
|
assert_output --partial "SUGGESTION"
|
|
}
|
|
|
|
@test "ADR-0020: agent description of 401 chars FAILs and exits non-zero" {
|
|
local root="$TMPDIR/pkg"
|
|
make_apm_agent_with_desc "$root" "my-agent" "$(desc_of_length 401)"
|
|
run bash "$SCRIPT" "$root/.apm/agents/my-agent.agent.md"
|
|
assert_failure
|
|
assert_output --partial "description is 401 chars"
|
|
assert_output --partial "400-character ADR-0020 ceiling"
|
|
}
|
|
|
|
@test "ADR-0020: agent description length is measured after YAML folding is resolved" {
|
|
local root="$TMPDIR/pkg"
|
|
mkdir -p "$root/.apm/agents"
|
|
cat > "$root/apm.yml" <<EOF
|
|
name: test-package
|
|
version: 0.1.0
|
|
type: skill
|
|
EOF
|
|
# 11 folded lines of 40 chars + 10 joining spaces = 450 characters. Read off
|
|
# the raw `description: >` line it is 1 character and passes.
|
|
{
|
|
echo "---"
|
|
echo "name: my-agent"
|
|
echo "description: >"
|
|
python3 -c "print('\n'.join([' ' + 'x' * 40] * 11))"
|
|
echo "---"
|
|
echo ""
|
|
echo "You are a test agent."
|
|
} > "$root/.apm/agents/my-agent.agent.md"
|
|
run bash "$SCRIPT" "$root/.apm/agents/my-agent.agent.md"
|
|
assert_failure
|
|
assert_output --partial "description is 450 chars"
|
|
}
|
|
|
|
@test "ADR-0020: the description gate applies at project scope too" {
|
|
local root="$TMPDIR/project"
|
|
local desc
|
|
desc="$(python3 -c "print('x' * 401)")"
|
|
mkdir -p "$root/.git" "$root/.claude/agents" "$root/.github/agents"
|
|
cat > "$root/.claude/agents/my-agent.md" <<EOF
|
|
---
|
|
name: my-agent
|
|
description: $desc
|
|
---
|
|
|
|
You are a test agent. When invoked, do the thing.
|
|
EOF
|
|
cat > "$root/.github/agents/my-agent.agent.md" <<EOF
|
|
---
|
|
name: my-agent
|
|
description: A valid agent description.
|
|
---
|
|
|
|
You are a test agent. When invoked, do the thing.
|
|
EOF
|
|
run bash "$SCRIPT" "$root/.claude/agents/my-agent.md"
|
|
assert_failure
|
|
assert_output --partial "400-character ADR-0020 ceiling"
|
|
}
|
|
|
|
@test "ADR-0020: agents take NO body word gate — a body far over the 900-word skill ceiling passes" {
|
|
local root="$TMPDIR/pkg"
|
|
mkdir -p "$root/.apm/agents"
|
|
cat > "$root/apm.yml" <<EOF
|
|
name: test-package
|
|
version: 0.1.0
|
|
type: skill
|
|
EOF
|
|
# Deliberate asymmetry, not an oversight: a skill body is loaded into the
|
|
# caller's context and competes with the live conversation, while an agent
|
|
# body becomes the system prompt of a fresh context. ADR-0020 gates the
|
|
# former at 900 words and explicitly declines to gate the latter. If a body
|
|
# word gate is ever added here, it contradicts the ADR.
|
|
#
|
|
# The description carries a boundary clause so the ONLY thing this test can
|
|
# go red on is a body finding. Without one, the missing-boundary-clause
|
|
# SUGGESTION fires and the blanket `refute_output --partial "SUGGESTION"`
|
|
# below trips for a reason that has nothing to do with body length — which
|
|
# would look like the invariant breaking while proving nothing about it.
|
|
# AGENTS.md cites this test as the pin for that invariant, so it has to fail
|
|
# for one reason and one reason only.
|
|
{
|
|
echo "---"
|
|
echo "name: my-agent"
|
|
echo "description: A valid agent description. Do not use for anything else."
|
|
echo "---"
|
|
echo ""
|
|
python3 -c "print(' '.join(['word'] * 1500))"
|
|
} > "$root/.apm/agents/my-agent.agent.md"
|
|
run bash "$SCRIPT" "$root/.apm/agents/my-agent.agent.md"
|
|
assert_success
|
|
refute_output --partial "FAIL"
|
|
# A 1,500-word body is 667% of the skill ceiling. Nothing may be said about
|
|
# it at any tier: not a FAIL, not a SUGGESTION, and not the word-count
|
|
# wording either tier would use if a gate were quietly added later.
|
|
refute_output --partial "SUGGESTION"
|
|
refute_output --partial "1500 words"
|
|
refute_output --partial "900-word"
|
|
refute_output --partial "body is"
|
|
}
|
|
|
|
@test "a bare plugin.json with no apm.yml is no longer plugin scope — falls through to project scope" {
|
|
local root="$TMPDIR/proj-legacy-plugin-json"
|
|
mkdir -p "$root/.git" "$root/.claude/agents" "$root/.github/agents"
|
|
echo '{}' > "$root/plugin.json"
|
|
# 'hooks' is plugin-silently-ignored only at (old) plugin scope; at
|
|
# project scope it's a legitimate CC field. If this directory were
|
|
# mis-detected as plugin scope (old plugin.json-based logic), this would
|
|
# FAIL with a plugin-silently-ignored-fields finding on 'hooks'.
|
|
cat > "$root/.claude/agents/my-agent.md" <<EOF
|
|
---
|
|
name: my-agent
|
|
description: A valid agent description.
|
|
hooks:
|
|
PostToolUse:
|
|
- match: ".*"
|
|
command: "echo done"
|
|
---
|
|
|
|
You are a test agent. When invoked, do the thing.
|
|
EOF
|
|
cat > "$root/.github/agents/my-agent.agent.md" <<EOF
|
|
---
|
|
name: my-agent
|
|
description: A valid agent description.
|
|
---
|
|
|
|
You are a test agent. When invoked, do the thing.
|
|
EOF
|
|
run bash "$SCRIPT" "$root/.claude/agents/my-agent.md"
|
|
assert_success
|
|
refute_output --partial "hooks"
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# tools: — both YAML spellings
|
|
# ---------------------------------------------------------------------------
|
|
# The subagent-unavailable-tool SUGGESTION is read off the `tools` field, and
|
|
# `tools` has two legal spellings: an inline scalar and a block sequence. The
|
|
# field used to be pulled out with a line regex whose capture is newline-bounded
|
|
# on purpose, so a block sequence captured NOTHING and the check silently
|
|
# stopped firing — on the shape Copilot agent files actually use, which is to say
|
|
# on the files it was written for. Both spellings are pinned, and they are pinned
|
|
# together: the inline case alone was green throughout.
|
|
|
|
# make_pair <root> <tools-frontmatter> — a project-scope CC + Copilot pair
|
|
# carrying the same `tools` value in both files. `tools` is on neither the
|
|
# claude-code-only nor the copilot-only list, so it is legal in both and the pair
|
|
# stays otherwise clean; the description carries a boundary clause so the only
|
|
# SUGGESTION that can fire is the one under test.
|
|
make_tools_pair() {
|
|
local root="$1" tools="$2"
|
|
mkdir -p "$root/.git" "$root/.claude/agents" "$root/.github/agents"
|
|
local f
|
|
for f in "$root/.claude/agents/my-agent.md" "$root/.github/agents/my-agent.agent.md"; do
|
|
{
|
|
echo "---"
|
|
echo "name: my-agent"
|
|
echo "description: A valid agent description. Do not use for anything else."
|
|
echo "$tools"
|
|
echo "---"
|
|
echo ""
|
|
echo "You are a test agent. When invoked, do the thing."
|
|
} > "$f"
|
|
done
|
|
}
|
|
|
|
@test "a subagent-unavailable tool in an INLINE tools scalar raises a SUGGESTION" {
|
|
make_tools_pair "$TMPDIR/inline" "tools: Read ExitPlanMode"
|
|
run bash "$SCRIPT" "$TMPDIR/inline/.claude/agents/my-agent.md"
|
|
assert_success
|
|
assert_output --partial "'ExitPlanMode' is listed in tools but is never available to subagents"
|
|
}
|
|
|
|
@test "a subagent-unavailable tool in a BLOCK SEQUENCE tools field raises the same SUGGESTION" {
|
|
make_tools_pair "$TMPDIR/block" "$(printf 'tools:\n - Read\n - ExitPlanMode')"
|
|
run bash "$SCRIPT" "$TMPDIR/block/.claude/agents/my-agent.md"
|
|
assert_success
|
|
assert_output --partial "'ExitPlanMode' is listed in tools but is never available to subagents"
|
|
}
|
|
|
|
@test "a tools list with no subagent-unavailable tool stays silent in both spellings" {
|
|
# The control. Without it both cases above are satisfied by a check that
|
|
# fires on every tools field it can see, which would be the opposite defect.
|
|
make_tools_pair "$TMPDIR/inline-clean" "tools: Read Edit"
|
|
run bash "$SCRIPT" "$TMPDIR/inline-clean/.claude/agents/my-agent.md"
|
|
assert_success
|
|
refute_output --partial "never available to subagents"
|
|
|
|
make_tools_pair "$TMPDIR/block-clean" "$(printf 'tools:\n - Read\n - Edit')"
|
|
run bash "$SCRIPT" "$TMPDIR/block-clean/.claude/agents/my-agent.md"
|
|
assert_success
|
|
refute_output --partial "never available to subagents"
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# A file that cannot be read
|
|
# ---------------------------------------------------------------------------
|
|
# scripts/check-apm-agents-valid.sh derives its expected agent-file set from
|
|
# `git ls-files`, so it hands this script paths that are tracked but absent from
|
|
# the worktree — a real and expected state, not a corner case. That used to exit
|
|
# 1 with a bare FileNotFoundError traceback and no FAIL line at all: non-zero, so
|
|
# the gate blocked, but with an interpreter stack instead of a diagnostic naming
|
|
# the file. Both scope paths are covered because they are separate call sites
|
|
# (check_apm_agent_file and check_file) and each needed its own handler.
|
|
#
|
|
# `is-a-dir.agent.md` is a DIRECTORY rather than a chmod 000 file on purpose:
|
|
# these tests run as root in CI, where mode bits do not deny anything and a
|
|
# permissions fixture would be silently readable and prove nothing.
|
|
|
|
@test "a nonexistent plugin/APM-scope agent file gets a FAIL naming the path, not a traceback" {
|
|
local root="$TMPDIR/pkg"
|
|
mkdir -p "$root/.apm/agents"
|
|
cat > "$root/apm.yml" <<EOF
|
|
name: test-package
|
|
version: 0.1.0
|
|
type: skill
|
|
EOF
|
|
run bash "$SCRIPT" "$root/.apm/agents/absent.agent.md"
|
|
assert_failure
|
|
assert_output --partial "FAIL"
|
|
assert_output --partial "could not be read"
|
|
assert_output --partial "absent.agent.md"
|
|
refute_output --partial "Traceback"
|
|
refute_output --partial "FileNotFoundError"
|
|
}
|
|
|
|
@test "an unreadable plugin/APM-scope agent file gets a FAIL naming the path, not a traceback" {
|
|
local root="$TMPDIR/pkg-dir"
|
|
mkdir -p "$root/.apm/agents/is-a-dir.agent.md"
|
|
cat > "$root/apm.yml" <<EOF
|
|
name: test-package
|
|
version: 0.1.0
|
|
type: skill
|
|
EOF
|
|
run bash "$SCRIPT" "$root/.apm/agents/is-a-dir.agent.md"
|
|
assert_failure
|
|
assert_output --partial "FAIL"
|
|
assert_output --partial "could not be read"
|
|
assert_output --partial "is-a-dir.agent.md"
|
|
refute_output --partial "Traceback"
|
|
refute_output --partial "IsADirectoryError"
|
|
}
|
|
|
|
@test "a nonexistent project-scope agent file gets a FAIL naming the path, not a traceback" {
|
|
# The counterpart is pre-checked before either file is opened, so this
|
|
# exercises the OTHER call site: the counterpart exists, the named file does
|
|
# not, and check_file is what has to report it.
|
|
local root="$TMPDIR/proj-missing"
|
|
mkdir -p "$root/.git" "$root/.claude/agents" "$root/.github/agents"
|
|
cat > "$root/.github/agents/my-agent.agent.md" <<EOF
|
|
---
|
|
name: my-agent
|
|
description: A valid agent description. Do not use for anything else.
|
|
---
|
|
|
|
You are a test agent. When invoked, do the thing.
|
|
EOF
|
|
run bash "$SCRIPT" "$root/.claude/agents/my-agent.md"
|
|
assert_failure
|
|
assert_output --partial "FAIL"
|
|
assert_output --partial "could not be read"
|
|
assert_output --partial "my-agent.md"
|
|
refute_output --partial "Traceback"
|
|
refute_output --partial "FileNotFoundError"
|
|
}
|