Defame1297
0ea3f69dc6
Second clean-context audit found author Must/Should and audit FAIL/SUGGESTION tiers drifting apart, and author Musts the audit never checked. - factory-audit: FAIL on absolute or bare relative hook script paths, an applyTo present but empty, and unbalanced braces/brackets in applyTo; judgment steps for dependency stem collisions, helper .json in hook dirs, unresolvable instruction links, prompt model slugs and second-person bodies; an unmatched glob drops to SUGGESTION; deliberate tier deviations recorded in hook-flow.md; validate.sh --help lists the three new modes; DescriptionOpener message no longer prescribes "Use when". - primitive-author: deprecated routing, extra prompt keys and the prompt description contract become Shoulds; hook Musts gain "contributes an entry", no bare relative paths, and executable-when-run-directly; prompt Must 1 covers hardlinks; Vale prose FAILs resolved at close. - forge: say "hook, instruction or prompt" rather than "apm primitive". Refs #94 Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01KkT7RSDwDbmrM9T34b6sTi
398 lines
16 KiB
Bash
398 lines
16 KiB
Bash
#!/usr/bin/env bats
|
|
|
|
# scripts/validate.sh against the three apm primitives with no container of
|
|
# their own: hooks, instructions and prompts. Each rule under test traces to
|
|
# the Authoring checklist in
|
|
# plugins/kyberforge/docs/research/docs/microsoft-apm/<kind>-primitive-schema.md.
|
|
|
|
setup() {
|
|
REPO_ROOT="$(cd "$BATS_TEST_DIRNAME/../../../../../../" && pwd)"
|
|
load "$REPO_ROOT/tests/test_helper/bats-support/load"
|
|
load "$REPO_ROOT/tests/test_helper/bats-assert/load"
|
|
|
|
SCRIPT="$(cd "$BATS_TEST_DIRNAME/../scripts" && pwd)/validate.sh"
|
|
TMPDIR="$(mktemp -d)"
|
|
PKG="$TMPDIR/pkg"
|
|
mkdir -p "$PKG/.apm/hooks/scripts" "$PKG/.apm/instructions" "$PKG/.apm/prompts"
|
|
cat > "$PKG/apm.yml" <<EOF
|
|
name: test-package
|
|
version: 0.1.0
|
|
type: hybrid
|
|
EOF
|
|
printf '#!/usr/bin/env bash\nexit 0\n' > "$PKG/.apm/hooks/scripts/check.sh"
|
|
chmod +x "$PKG/.apm/hooks/scripts/check.sh"
|
|
|
|
# Helper: write <content> as hook file <name> under .apm/hooks/.
|
|
write_hook() {
|
|
printf '%s\n' "$2" > "$PKG/.apm/hooks/$1"
|
|
}
|
|
|
|
# Helper: write an instruction <stem> with raw <frontmatter> and <body>.
|
|
write_instruction() {
|
|
printf -- '---\n%s\n---\n\n%s\n' "$2" "$3" > "$PKG/.apm/instructions/$1.instructions.md"
|
|
}
|
|
|
|
# Helper: write a prompt <stem> with raw <frontmatter> and <body>.
|
|
write_prompt() {
|
|
printf -- '---\n%s\n---\n\n%s\n' "$2" "$3" > "$PKG/.apm/prompts/$1.prompt.md"
|
|
}
|
|
}
|
|
|
|
teardown() {
|
|
rm -rf "$TMPDIR"
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Dispatch
|
|
# ---------------------------------------------------------------------------
|
|
|
|
@test "dispatch: a .json file outside a hooks/ directory matches no shape (exit 2)" {
|
|
printf '{}\n' > "$PKG/settings.json"
|
|
run bash "$SCRIPT" "$PKG/settings.json"
|
|
assert_equal "$status" 2
|
|
assert_output --partial "matches no auditable shape"
|
|
}
|
|
|
|
@test "dispatch: an *.instructions.md under an agents/ directory takes the instruction flow, not the agent flow" {
|
|
mkdir -p "$PKG/.apm/agents"
|
|
printf -- '---\ndescription: x\n---\n\nbody\n' > "$PKG/.apm/agents/x.instructions.md"
|
|
run bash "$SCRIPT" "$PKG/.apm/agents/x.instructions.md"
|
|
assert_failure
|
|
assert_output --partial "is not directly in a .apm/instructions/ directory"
|
|
refute_output --partial "counterpart"
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Hooks
|
|
# ---------------------------------------------------------------------------
|
|
|
|
@test "hook: canonical nested shape with \${PLUGIN_ROOT} passes clean" {
|
|
write_hook hooks.json '{"hooks":{"PreToolUse":[{"matcher":"Bash","hooks":[{"type":"command","command":"${PLUGIN_ROOT}/.apm/hooks/scripts/check.sh","timeout":10}]}]}}'
|
|
run bash "$SCRIPT" "$PKG/.apm/hooks/hooks.json"
|
|
assert_success
|
|
refute_output --partial "FAIL"
|
|
refute_output --partial "SUGGESTION"
|
|
}
|
|
|
|
@test "hook: invalid JSON is a FAIL" {
|
|
write_hook hooks.json '{"hooks": {'
|
|
run bash "$SCRIPT" "$PKG/.apm/hooks/hooks.json"
|
|
assert_failure
|
|
assert_output --partial "is not valid JSON"
|
|
}
|
|
|
|
@test "hook: an event value that is not a list is a FAIL" {
|
|
write_hook hooks.json '{"hooks":{"PreToolUse":{"hooks":[]}}}'
|
|
run bash "$SCRIPT" "$PKG/.apm/hooks/hooks.json"
|
|
assert_failure
|
|
assert_output --partial "event 'PreToolUse' is not a list"
|
|
}
|
|
|
|
@test "hook: a naked slice with a stray scalar key is a FAIL" {
|
|
write_hook hooks.json '{"description":"x","PreToolUse":[{"hooks":[{"type":"command","command":"true"}]}]}'
|
|
run bash "$SCRIPT" "$PKG/.apm/hooks/hooks.json"
|
|
assert_failure
|
|
assert_output --partial "naked settings-slice shape"
|
|
}
|
|
|
|
@test "hook: an all-lowercase event is a FAIL" {
|
|
write_hook hooks.json '{"hooks":{"stop":[{"hooks":[{"type":"command","command":"true"}]}]}}'
|
|
run bash "$SCRIPT" "$PKG/.apm/hooks/hooks.json"
|
|
assert_failure
|
|
assert_output --partial "event 'stop' is all-lowercase"
|
|
}
|
|
|
|
@test "hook: camelCase userPromptSubmit in a Claude-shaped file is a FAIL; mapped sessionStart is not" {
|
|
write_hook hooks.json '{"hooks":{"userPromptSubmit":[{"hooks":[{"type":"command","command":"true"}]}],"sessionStart":[{"hooks":[{"type":"command","command":"true"}]}]}}'
|
|
run bash "$SCRIPT" "$PKG/.apm/hooks/hooks.json"
|
|
assert_failure
|
|
assert_output --partial "event 'userPromptSubmit' is camelCase"
|
|
refute_output --partial "event 'sessionStart'"
|
|
}
|
|
|
|
@test "hook: a flat Copilot-shaped file may use camelCase events" {
|
|
write_hook hooks.json '{"hooks":{"userPromptSubmit":[{"type":"command","bash":"true","timeoutSec":5}]}}'
|
|
run bash "$SCRIPT" "$PKG/.apm/hooks/hooks.json"
|
|
assert_success
|
|
}
|
|
|
|
@test "hook: a referenced script that does not exist is a FAIL" {
|
|
write_hook hooks.json '{"hooks":{"Stop":[{"hooks":[{"type":"command","command":"${PLUGIN_ROOT}/.apm/hooks/scripts/missing.sh"}]}]}}'
|
|
run bash "$SCRIPT" "$PKG/.apm/hooks/hooks.json"
|
|
assert_failure
|
|
assert_output --partial "script '.apm/hooks/scripts/missing.sh' does not exist"
|
|
}
|
|
|
|
@test "hook: a directly-run script without the executable bit is a FAIL; the same script through an interpreter is not" {
|
|
chmod -x "$PKG/.apm/hooks/scripts/check.sh"
|
|
write_hook hooks.json '{"hooks":{"Stop":[{"hooks":[{"type":"command","command":"./scripts/check.sh"}]}]}}'
|
|
run bash "$SCRIPT" "$PKG/.apm/hooks/hooks.json"
|
|
assert_failure
|
|
assert_output --partial "is run directly but is not executable"
|
|
|
|
write_hook hooks.json '{"hooks":{"Stop":[{"hooks":[{"type":"command","command":"bash ${PLUGIN_ROOT}/.apm/hooks/scripts/check.sh"}]}]}}'
|
|
run bash "$SCRIPT" "$PKG/.apm/hooks/hooks.json"
|
|
assert_success
|
|
}
|
|
|
|
@test "hook: a script path escaping the package is a FAIL" {
|
|
write_hook hooks.json '{"hooks":{"Stop":[{"hooks":[{"type":"command","command":"${PLUGIN_ROOT}/../outside.sh"}]}]}}'
|
|
touch "$TMPDIR/outside.sh"
|
|
run bash "$SCRIPT" "$PKG/.apm/hooks/hooks.json"
|
|
assert_failure
|
|
assert_output --partial "resolves outside the package"
|
|
}
|
|
|
|
@test "hook: \${CLAUDE_PLUGIN_ROOT} is a SUGGESTION, not a FAIL" {
|
|
write_hook hooks.json '{"hooks":{"SessionStart":[{"matcher":"startup","hooks":[{"type":"command","command":"${CLAUDE_PLUGIN_ROOT}/.apm/hooks/scripts/check.sh","timeout":5}]}]}}'
|
|
run bash "$SCRIPT" "$PKG/.apm/hooks/hooks.json"
|
|
assert_success
|
|
assert_output --partial "SUGGESTION uses \${CLAUDE_PLUGIN_ROOT}"
|
|
}
|
|
|
|
@test "hook: a deprecated filename-routing stem is a SUGGESTION" {
|
|
write_hook claude-hooks.json '{"hooks":{"Stop":[{"hooks":[{"type":"command","command":"true"}]}]}}'
|
|
run bash "$SCRIPT" "$PKG/.apm/hooks/claude-hooks.json"
|
|
assert_success
|
|
assert_output --partial "deprecated hook filename routing"
|
|
}
|
|
|
|
@test "hook: a symlinked hook file is a FAIL" {
|
|
write_hook real.json '{"hooks":{"Stop":[{"hooks":[{"type":"command","command":"true"}]}]}}'
|
|
ln -s real.json "$PKG/.apm/hooks/link.json"
|
|
run bash "$SCRIPT" "$PKG/.apm/hooks/link.json"
|
|
assert_failure
|
|
assert_output --partial "is a symlink"
|
|
}
|
|
|
|
@test "hook: a hardlinked hook file is not a FAIL (find_hook_files skips symlinks only)" {
|
|
write_hook real.json '{"hooks":{"Stop":[{"hooks":[{"type":"command","command":"true"}]}]}}'
|
|
ln "$PKG/.apm/hooks/real.json" "$PKG/.apm/hooks/linked.json"
|
|
run bash "$SCRIPT" "$PKG/.apm/hooks/linked.json"
|
|
assert_success
|
|
refute_output --partial "hardlink"
|
|
}
|
|
|
|
@test "hook: an absolute script path is a FAIL; an absolute interpreter path is not" {
|
|
write_hook hooks.json '{"hooks":{"Stop":[{"hooks":[{"type":"command","command":"/usr/local/bin/check.sh","timeout":5}]}]}}'
|
|
run bash "$SCRIPT" "$PKG/.apm/hooks/hooks.json"
|
|
assert_failure
|
|
assert_output --partial "is an absolute path"
|
|
|
|
write_hook hooks.json '{"hooks":{"Stop":[{"hooks":[{"type":"command","command":"/usr/bin/env true","timeout":5}]}]}}'
|
|
run bash "$SCRIPT" "$PKG/.apm/hooks/hooks.json"
|
|
assert_success
|
|
refute_output --partial "absolute path"
|
|
}
|
|
|
|
@test "hook: a bare relative path to a package script is a FAIL; a bare command is not" {
|
|
write_hook hooks.json '{"hooks":{"Stop":[{"hooks":[{"type":"command","command":".apm/hooks/scripts/check.sh","timeout":5}]}]}}'
|
|
run bash "$SCRIPT" "$PKG/.apm/hooks/hooks.json"
|
|
assert_failure
|
|
assert_output --partial "is a bare relative path"
|
|
|
|
write_hook hooks.json '{"hooks":{"Stop":[{"hooks":[{"type":"command","command":"npx some-tool --check","timeout":5}]}]}}'
|
|
run bash "$SCRIPT" "$PKG/.apm/hooks/hooks.json"
|
|
assert_success
|
|
refute_output --partial "bare relative path"
|
|
}
|
|
|
|
@test "hook: a file contributing no entries is a FAIL" {
|
|
write_hook hooks.json '{"hooks":{}}'
|
|
run bash "$SCRIPT" "$PKG/.apm/hooks/hooks.json"
|
|
assert_failure
|
|
assert_output --partial "contributes no hook entries"
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Instructions
|
|
# ---------------------------------------------------------------------------
|
|
|
|
@test "instruction: description, applyTo and a body pass clean" {
|
|
write_instruction python 'description: Python style rules
|
|
applyTo: "**/*.py"' 'Use type hints on public functions.'
|
|
run bash "$SCRIPT" "$PKG/.apm/instructions/python.instructions.md"
|
|
assert_success
|
|
refute_output --partial "FAIL"
|
|
refute_output --partial "SUGGESTION"
|
|
}
|
|
|
|
@test "instruction: missing description is a FAIL" {
|
|
write_instruction python 'applyTo: "**/*.py"' 'Use type hints.'
|
|
run bash "$SCRIPT" "$PKG/.apm/instructions/python.instructions.md"
|
|
assert_failure
|
|
assert_output --partial "'description' is missing or empty"
|
|
}
|
|
|
|
@test "instruction: an empty body is a FAIL" {
|
|
write_instruction python 'description: x
|
|
applyTo: "**/*.py"' ''
|
|
run bash "$SCRIPT" "$PKG/.apm/instructions/python.instructions.md"
|
|
assert_failure
|
|
assert_output --partial "body is empty"
|
|
}
|
|
|
|
@test "instruction: invalid frontmatter YAML is a FAIL" {
|
|
write_instruction python 'description: [unclosed' 'body'
|
|
run bash "$SCRIPT" "$PKG/.apm/instructions/python.instructions.md"
|
|
assert_failure
|
|
assert_output --partial "frontmatter is not valid YAML"
|
|
}
|
|
|
|
@test "instruction: no applyTo is a SUGGESTION (deliberate always-on), not a FAIL" {
|
|
write_instruction general 'description: General rules' 'Be kind.'
|
|
run bash "$SCRIPT" "$PKG/.apm/instructions/general.instructions.md"
|
|
assert_success
|
|
assert_output --partial "SUGGESTION no applyTo"
|
|
}
|
|
|
|
@test "instruction: a YAML-list applyTo and unread keys are SUGGESTIONs" {
|
|
write_instruction python 'description: x
|
|
applyTo:
|
|
- "**/*.py"
|
|
name: python' 'body'
|
|
run bash "$SCRIPT" "$PKG/.apm/instructions/python.instructions.md"
|
|
assert_success
|
|
assert_output --partial "applyTo is a YAML list"
|
|
assert_output --partial "frontmatter key(s) name"
|
|
}
|
|
|
|
@test "instruction: an applyTo that is present but empty is a FAIL" {
|
|
write_instruction empty 'description: x
|
|
applyTo: ""' 'body'
|
|
run bash "$SCRIPT" "$PKG/.apm/instructions/empty.instructions.md"
|
|
assert_failure
|
|
assert_output --partial "applyTo is present but empty"
|
|
refute_output --partial "SUGGESTION no applyTo"
|
|
}
|
|
|
|
@test "instruction: an applyTo glob with unbalanced braces is a FAIL" {
|
|
write_instruction broken 'description: x
|
|
applyTo: "**/*.{py"' 'body'
|
|
run bash "$SCRIPT" "$PKG/.apm/instructions/broken.instructions.md"
|
|
assert_failure
|
|
assert_output --partial "unbalanced braces or brackets"
|
|
}
|
|
|
|
@test "instruction: a top-level comma list with a brace group passes clean" {
|
|
write_instruction multi 'description: x
|
|
applyTo: "**/*.py, **/*.{pyi,pyx}"' 'body'
|
|
run bash "$SCRIPT" "$PKG/.apm/instructions/multi.instructions.md"
|
|
assert_success
|
|
refute_output --partial "applyTo"
|
|
}
|
|
|
|
@test "instruction: the same stem at the package root is a FAIL" {
|
|
write_instruction python 'description: x
|
|
applyTo: "**/*.py"' 'body'
|
|
cp "$PKG/.apm/instructions/python.instructions.md" "$PKG/python.instructions.md"
|
|
run bash "$SCRIPT" "$PKG/.apm/instructions/python.instructions.md"
|
|
assert_failure
|
|
assert_output --partial "also exists at the package root"
|
|
}
|
|
|
|
@test "instruction: a hardlinked instruction file is a FAIL" {
|
|
write_instruction python 'description: x
|
|
applyTo: "**/*.py"' 'body'
|
|
ln "$PKG/.apm/instructions/python.instructions.md" "$TMPDIR/python.instructions.md"
|
|
run bash "$SCRIPT" "$PKG/.apm/instructions/python.instructions.md"
|
|
assert_failure
|
|
assert_output --partial "is a hardlink"
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Prompts
|
|
# ---------------------------------------------------------------------------
|
|
|
|
@test "prompt: declared and used inputs with a plain description pass clean" {
|
|
write_prompt review-pr 'description: Review a pull request with gitea-prs and factory-audit, then summarize.
|
|
input:
|
|
- pr_number: "The PR to review"' 'Review PR ${input:pr_number} with gitea-prs.'
|
|
run bash "$SCRIPT" "$PKG/.apm/prompts/review-pr.prompt.md"
|
|
assert_success
|
|
refute_output --partial "FAIL"
|
|
refute_output --partial "SUGGESTION"
|
|
}
|
|
|
|
@test "prompt: missing description is a FAIL" {
|
|
write_prompt review-pr 'model: sonnet' 'Review the PR.'
|
|
run bash "$SCRIPT" "$PKG/.apm/prompts/review-pr.prompt.md"
|
|
assert_failure
|
|
assert_output --partial "'description' is missing or empty"
|
|
}
|
|
|
|
@test "prompt: the upstream docs' - name: x / description: form is a FAIL" {
|
|
write_prompt review-pr 'description: Review a PR.
|
|
input:
|
|
- name: pr_number
|
|
description: The PR' 'Review ${input:pr_number}.'
|
|
run bash "$SCRIPT" "$PKG/.apm/prompts/review-pr.prompt.md"
|
|
assert_failure
|
|
assert_output --partial "yields arguments [name, description]"
|
|
}
|
|
|
|
@test "prompt: an invalid input name is a FAIL" {
|
|
write_prompt review-pr 'description: Review a PR.
|
|
input: [1pr]' 'Review.'
|
|
run bash "$SCRIPT" "$PKG/.apm/prompts/review-pr.prompt.md"
|
|
assert_failure
|
|
assert_output --partial "input name '1pr' does not match"
|
|
}
|
|
|
|
@test "prompt: an undeclared \${input:x} and an unused input are both FAILs" {
|
|
write_prompt review-pr 'description: Review a PR.
|
|
input: [pr_number]' 'Review ${input:branch}.'
|
|
run bash "$SCRIPT" "$PKG/.apm/prompts/review-pr.prompt.md"
|
|
assert_failure
|
|
assert_output --partial "input: does not declare 'branch'"
|
|
assert_output --partial "input 'pr_number' is declared but the body never uses"
|
|
}
|
|
|
|
@test "prompt: \${input:x} with no input: declared is a FAIL" {
|
|
write_prompt review-pr 'description: Review a PR.' 'Review ${input:pr_number}.'
|
|
run bash "$SCRIPT" "$PKG/.apm/prompts/review-pr.prompt.md"
|
|
assert_failure
|
|
assert_output --partial "but no input: is declared"
|
|
}
|
|
|
|
@test "prompt: a trigger clause, an over-long description, dropped keys and camelCase aliases are SUGGESTIONs" {
|
|
local long
|
|
long="Use when the user wants a PR reviewed. $(printf 'x%.0s' $(seq 1 240))"
|
|
write_prompt review-pr "description: $long
|
|
mode: agent
|
|
allowedTools: Bash" 'Review the PR.'
|
|
run bash "$SCRIPT" "$PKG/.apm/prompts/review-pr.prompt.md"
|
|
assert_success
|
|
assert_output --partial "'Use when' trigger clause"
|
|
assert_output --partial "characters (> 250)"
|
|
assert_output --partial "frontmatter key(s) mode are dropped on Claude"
|
|
assert_output --partial "'allowedTools' — use the kebab-case spelling"
|
|
}
|
|
|
|
@test "prompt: argument-hint alongside input: is a SUGGESTION" {
|
|
write_prompt review-pr 'description: Review a PR.
|
|
argument-hint: <pr>
|
|
input: [pr_number]' 'Review ${input:pr_number}.'
|
|
run bash "$SCRIPT" "$PKG/.apm/prompts/review-pr.prompt.md"
|
|
assert_success
|
|
assert_output --partial "argument-hint is set alongside input:"
|
|
}
|
|
|
|
@test "prompt: no procedure heuristic — a long, stepped body is not a script finding" {
|
|
write_prompt review-pr 'description: Review a PR.' "## Step 1
|
|
$(printf 'line\n%.0s' $(seq 1 80))
|
|
## Gotchas
|
|
- x"
|
|
run bash "$SCRIPT" "$PKG/.apm/prompts/review-pr.prompt.md"
|
|
assert_success
|
|
refute_output --partial "SUGGESTION"
|
|
}
|
|
|
|
@test "prompt: a hardlinked prompt file is a FAIL" {
|
|
write_prompt review-pr 'description: Review a pull request with gitea-prs.' 'Review the PR with gitea-prs.'
|
|
ln "$PKG/.apm/prompts/review-pr.prompt.md" "$TMPDIR/review-pr.prompt.md"
|
|
run bash "$SCRIPT" "$PKG/.apm/prompts/review-pr.prompt.md"
|
|
assert_failure
|
|
assert_output --partial "is a hardlink"
|
|
}
|