Defame1297
70210d6a7e
factory-audit gains three Step 0 rows and flows for the apm primitives
that have no container of their own: a .json file under hooks/, a
*.instructions.md and a *.prompt.md. apm validates almost none of them
(invalid hook JSON is skipped silently, instruction validate() only
warns, input: names are never checked against ${input:x}), so the
deterministic checks live in a new scripts/lib-checks-primitive.sh,
wired into validate.sh's path-shape detection. Each check and tier
traces to the Authoring checklists in the microsoft-apm research docs.
- Hook: JSON/shape/event-list checks mirroring the Copilot payload
validator, never-firing event casing, missing/escaping/non-executable
scripts (FAIL); deprecated filename routing and ${CLAUDE_PLUGIN_ROOT}
(SUGGESTION).
- Instruction: location, frontmatter, description, body, stem clash
(FAIL); missing or list applyTo and unread keys (SUGGESTION).
- Prompt: location/name, frontmatter, description, input names, the
upstream `- name: x` docs bug, declared-vs-used ${input:x} (FAIL);
ADR-0029 description length and trigger clause, dropped keys,
camelCase aliases, argument-hint with input (SUGGESTION). Whether a
prompt carries procedure is judgment in prompt-flow.md, not a script
heuristic.
Vale now lints *.instructions.md and *.prompt.md with the Kyberforge
style; test-vale-wrap.sh gains their probe rows. New
tests/validate-primitive.bats (31 cases). kyberforge 2.0.1 -> 2.1.0 with
the executables.allow key, catalog 0.5.1 -> 0.5.2, marketplace.json
regenerated.
Refs #94
Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01KkT7RSDwDbmrM9T34b6sTi
317 lines
13 KiB
Bash
317 lines
13 KiB
Bash
#!/usr/bin/env bats
|
|
|
|
# scripts/validate.sh against the three apm primitives with no container of
|
|
# their own: hooks, instructions and prompts. Each rule under test traces to
|
|
# the Authoring checklist in
|
|
# plugins/kyberforge/docs/research/docs/microsoft-apm/<kind>-primitive-schema.md.
|
|
|
|
setup() {
|
|
REPO_ROOT="$(cd "$BATS_TEST_DIRNAME/../../../../../../" && pwd)"
|
|
load "$REPO_ROOT/tests/test_helper/bats-support/load"
|
|
load "$REPO_ROOT/tests/test_helper/bats-assert/load"
|
|
|
|
SCRIPT="$(cd "$BATS_TEST_DIRNAME/../scripts" && pwd)/validate.sh"
|
|
TMPDIR="$(mktemp -d)"
|
|
PKG="$TMPDIR/pkg"
|
|
mkdir -p "$PKG/.apm/hooks/scripts" "$PKG/.apm/instructions" "$PKG/.apm/prompts"
|
|
cat > "$PKG/apm.yml" <<EOF
|
|
name: test-package
|
|
version: 0.1.0
|
|
type: hybrid
|
|
EOF
|
|
printf '#!/usr/bin/env bash\nexit 0\n' > "$PKG/.apm/hooks/scripts/check.sh"
|
|
chmod +x "$PKG/.apm/hooks/scripts/check.sh"
|
|
|
|
# Helper: write <content> as hook file <name> under .apm/hooks/.
|
|
write_hook() {
|
|
printf '%s\n' "$2" > "$PKG/.apm/hooks/$1"
|
|
}
|
|
|
|
# Helper: write an instruction <stem> with raw <frontmatter> and <body>.
|
|
write_instruction() {
|
|
printf -- '---\n%s\n---\n\n%s\n' "$2" "$3" > "$PKG/.apm/instructions/$1.instructions.md"
|
|
}
|
|
|
|
# Helper: write a prompt <stem> with raw <frontmatter> and <body>.
|
|
write_prompt() {
|
|
printf -- '---\n%s\n---\n\n%s\n' "$2" "$3" > "$PKG/.apm/prompts/$1.prompt.md"
|
|
}
|
|
}
|
|
|
|
teardown() {
|
|
rm -rf "$TMPDIR"
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Dispatch
|
|
# ---------------------------------------------------------------------------
|
|
|
|
@test "dispatch: a .json file outside a hooks/ directory matches no shape (exit 2)" {
|
|
printf '{}\n' > "$PKG/settings.json"
|
|
run bash "$SCRIPT" "$PKG/settings.json"
|
|
assert_equal "$status" 2
|
|
assert_output --partial "matches no auditable shape"
|
|
}
|
|
|
|
@test "dispatch: an *.instructions.md under an agents/ directory takes the instruction flow, not the agent flow" {
|
|
mkdir -p "$PKG/.apm/agents"
|
|
printf -- '---\ndescription: x\n---\n\nbody\n' > "$PKG/.apm/agents/x.instructions.md"
|
|
run bash "$SCRIPT" "$PKG/.apm/agents/x.instructions.md"
|
|
assert_failure
|
|
assert_output --partial "is not directly in a .apm/instructions/ directory"
|
|
refute_output --partial "counterpart"
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Hooks
|
|
# ---------------------------------------------------------------------------
|
|
|
|
@test "hook: canonical nested shape with \${PLUGIN_ROOT} passes clean" {
|
|
write_hook hooks.json '{"hooks":{"PreToolUse":[{"matcher":"Bash","hooks":[{"type":"command","command":"${PLUGIN_ROOT}/.apm/hooks/scripts/check.sh","timeout":10}]}]}}'
|
|
run bash "$SCRIPT" "$PKG/.apm/hooks/hooks.json"
|
|
assert_success
|
|
refute_output --partial "FAIL"
|
|
refute_output --partial "SUGGESTION"
|
|
}
|
|
|
|
@test "hook: invalid JSON is a FAIL" {
|
|
write_hook hooks.json '{"hooks": {'
|
|
run bash "$SCRIPT" "$PKG/.apm/hooks/hooks.json"
|
|
assert_failure
|
|
assert_output --partial "is not valid JSON"
|
|
}
|
|
|
|
@test "hook: an event value that is not a list is a FAIL" {
|
|
write_hook hooks.json '{"hooks":{"PreToolUse":{"hooks":[]}}}'
|
|
run bash "$SCRIPT" "$PKG/.apm/hooks/hooks.json"
|
|
assert_failure
|
|
assert_output --partial "event 'PreToolUse' is not a list"
|
|
}
|
|
|
|
@test "hook: a naked slice with a stray scalar key is a FAIL" {
|
|
write_hook hooks.json '{"description":"x","PreToolUse":[{"hooks":[{"type":"command","command":"true"}]}]}'
|
|
run bash "$SCRIPT" "$PKG/.apm/hooks/hooks.json"
|
|
assert_failure
|
|
assert_output --partial "naked settings-slice shape"
|
|
}
|
|
|
|
@test "hook: an all-lowercase event is a FAIL" {
|
|
write_hook hooks.json '{"hooks":{"stop":[{"hooks":[{"type":"command","command":"true"}]}]}}'
|
|
run bash "$SCRIPT" "$PKG/.apm/hooks/hooks.json"
|
|
assert_failure
|
|
assert_output --partial "event 'stop' is all-lowercase"
|
|
}
|
|
|
|
@test "hook: camelCase userPromptSubmit in a Claude-shaped file is a FAIL; mapped sessionStart is not" {
|
|
write_hook hooks.json '{"hooks":{"userPromptSubmit":[{"hooks":[{"type":"command","command":"true"}]}],"sessionStart":[{"hooks":[{"type":"command","command":"true"}]}]}}'
|
|
run bash "$SCRIPT" "$PKG/.apm/hooks/hooks.json"
|
|
assert_failure
|
|
assert_output --partial "event 'userPromptSubmit' is camelCase"
|
|
refute_output --partial "event 'sessionStart'"
|
|
}
|
|
|
|
@test "hook: a flat Copilot-shaped file may use camelCase events" {
|
|
write_hook hooks.json '{"hooks":{"userPromptSubmit":[{"type":"command","bash":"true","timeoutSec":5}]}}'
|
|
run bash "$SCRIPT" "$PKG/.apm/hooks/hooks.json"
|
|
assert_success
|
|
}
|
|
|
|
@test "hook: a referenced script that does not exist is a FAIL" {
|
|
write_hook hooks.json '{"hooks":{"Stop":[{"hooks":[{"type":"command","command":"${PLUGIN_ROOT}/.apm/hooks/scripts/missing.sh"}]}]}}'
|
|
run bash "$SCRIPT" "$PKG/.apm/hooks/hooks.json"
|
|
assert_failure
|
|
assert_output --partial "script '.apm/hooks/scripts/missing.sh' does not exist"
|
|
}
|
|
|
|
@test "hook: a directly-run script without the executable bit is a FAIL; the same script through an interpreter is not" {
|
|
chmod -x "$PKG/.apm/hooks/scripts/check.sh"
|
|
write_hook hooks.json '{"hooks":{"Stop":[{"hooks":[{"type":"command","command":"./scripts/check.sh"}]}]}}'
|
|
run bash "$SCRIPT" "$PKG/.apm/hooks/hooks.json"
|
|
assert_failure
|
|
assert_output --partial "is run directly but is not executable"
|
|
|
|
write_hook hooks.json '{"hooks":{"Stop":[{"hooks":[{"type":"command","command":"bash ${PLUGIN_ROOT}/.apm/hooks/scripts/check.sh"}]}]}}'
|
|
run bash "$SCRIPT" "$PKG/.apm/hooks/hooks.json"
|
|
assert_success
|
|
}
|
|
|
|
@test "hook: a script path escaping the package is a FAIL" {
|
|
write_hook hooks.json '{"hooks":{"Stop":[{"hooks":[{"type":"command","command":"${PLUGIN_ROOT}/../outside.sh"}]}]}}'
|
|
touch "$TMPDIR/outside.sh"
|
|
run bash "$SCRIPT" "$PKG/.apm/hooks/hooks.json"
|
|
assert_failure
|
|
assert_output --partial "resolves outside the package"
|
|
}
|
|
|
|
@test "hook: \${CLAUDE_PLUGIN_ROOT} is a SUGGESTION, not a FAIL" {
|
|
write_hook hooks.json '{"hooks":{"SessionStart":[{"matcher":"startup","hooks":[{"type":"command","command":"${CLAUDE_PLUGIN_ROOT}/.apm/hooks/scripts/check.sh","timeout":5}]}]}}'
|
|
run bash "$SCRIPT" "$PKG/.apm/hooks/hooks.json"
|
|
assert_success
|
|
assert_output --partial "SUGGESTION uses \${CLAUDE_PLUGIN_ROOT}"
|
|
}
|
|
|
|
@test "hook: a deprecated filename-routing stem is a SUGGESTION" {
|
|
write_hook claude-hooks.json '{"hooks":{"Stop":[{"hooks":[{"type":"command","command":"true"}]}]}}'
|
|
run bash "$SCRIPT" "$PKG/.apm/hooks/claude-hooks.json"
|
|
assert_success
|
|
assert_output --partial "deprecated hook filename routing"
|
|
}
|
|
|
|
@test "hook: a symlinked hook file is a FAIL" {
|
|
write_hook real.json '{"hooks":{"Stop":[{"hooks":[{"type":"command","command":"true"}]}]}}'
|
|
ln -s real.json "$PKG/.apm/hooks/link.json"
|
|
run bash "$SCRIPT" "$PKG/.apm/hooks/link.json"
|
|
assert_failure
|
|
assert_output --partial "is a symlink"
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Instructions
|
|
# ---------------------------------------------------------------------------
|
|
|
|
@test "instruction: description, applyTo and a body pass clean" {
|
|
write_instruction python 'description: Python style rules
|
|
applyTo: "**/*.py"' 'Use type hints on public functions.'
|
|
run bash "$SCRIPT" "$PKG/.apm/instructions/python.instructions.md"
|
|
assert_success
|
|
refute_output --partial "FAIL"
|
|
refute_output --partial "SUGGESTION"
|
|
}
|
|
|
|
@test "instruction: missing description is a FAIL" {
|
|
write_instruction python 'applyTo: "**/*.py"' 'Use type hints.'
|
|
run bash "$SCRIPT" "$PKG/.apm/instructions/python.instructions.md"
|
|
assert_failure
|
|
assert_output --partial "'description' is missing or empty"
|
|
}
|
|
|
|
@test "instruction: an empty body is a FAIL" {
|
|
write_instruction python 'description: x
|
|
applyTo: "**/*.py"' ''
|
|
run bash "$SCRIPT" "$PKG/.apm/instructions/python.instructions.md"
|
|
assert_failure
|
|
assert_output --partial "body is empty"
|
|
}
|
|
|
|
@test "instruction: invalid frontmatter YAML is a FAIL" {
|
|
write_instruction python 'description: [unclosed' 'body'
|
|
run bash "$SCRIPT" "$PKG/.apm/instructions/python.instructions.md"
|
|
assert_failure
|
|
assert_output --partial "frontmatter is not valid YAML"
|
|
}
|
|
|
|
@test "instruction: no applyTo is a SUGGESTION (deliberate always-on), not a FAIL" {
|
|
write_instruction general 'description: General rules' 'Be kind.'
|
|
run bash "$SCRIPT" "$PKG/.apm/instructions/general.instructions.md"
|
|
assert_success
|
|
assert_output --partial "SUGGESTION no applyTo"
|
|
}
|
|
|
|
@test "instruction: a YAML-list applyTo and unread keys are SUGGESTIONs" {
|
|
write_instruction python 'description: x
|
|
applyTo:
|
|
- "**/*.py"
|
|
name: python' 'body'
|
|
run bash "$SCRIPT" "$PKG/.apm/instructions/python.instructions.md"
|
|
assert_success
|
|
assert_output --partial "applyTo is a YAML list"
|
|
assert_output --partial "frontmatter key(s) name"
|
|
}
|
|
|
|
@test "instruction: the same stem at the package root is a FAIL" {
|
|
write_instruction python 'description: x
|
|
applyTo: "**/*.py"' 'body'
|
|
cp "$PKG/.apm/instructions/python.instructions.md" "$PKG/python.instructions.md"
|
|
run bash "$SCRIPT" "$PKG/.apm/instructions/python.instructions.md"
|
|
assert_failure
|
|
assert_output --partial "also exists at the package root"
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Prompts
|
|
# ---------------------------------------------------------------------------
|
|
|
|
@test "prompt: declared and used inputs with a plain description pass clean" {
|
|
write_prompt review-pr 'description: Review a pull request with gitea-prs and factory-audit, then summarize.
|
|
input:
|
|
- pr_number: "The PR to review"' 'Review PR ${input:pr_number} with gitea-prs.'
|
|
run bash "$SCRIPT" "$PKG/.apm/prompts/review-pr.prompt.md"
|
|
assert_success
|
|
refute_output --partial "FAIL"
|
|
refute_output --partial "SUGGESTION"
|
|
}
|
|
|
|
@test "prompt: missing description is a FAIL" {
|
|
write_prompt review-pr 'model: sonnet' 'Review the PR.'
|
|
run bash "$SCRIPT" "$PKG/.apm/prompts/review-pr.prompt.md"
|
|
assert_failure
|
|
assert_output --partial "'description' is missing or empty"
|
|
}
|
|
|
|
@test "prompt: the upstream docs' - name: x / description: form is a FAIL" {
|
|
write_prompt review-pr 'description: Review a PR.
|
|
input:
|
|
- name: pr_number
|
|
description: The PR' 'Review ${input:pr_number}.'
|
|
run bash "$SCRIPT" "$PKG/.apm/prompts/review-pr.prompt.md"
|
|
assert_failure
|
|
assert_output --partial "yields arguments [name, description]"
|
|
}
|
|
|
|
@test "prompt: an invalid input name is a FAIL" {
|
|
write_prompt review-pr 'description: Review a PR.
|
|
input: [1pr]' 'Review.'
|
|
run bash "$SCRIPT" "$PKG/.apm/prompts/review-pr.prompt.md"
|
|
assert_failure
|
|
assert_output --partial "input name '1pr' does not match"
|
|
}
|
|
|
|
@test "prompt: an undeclared \${input:x} and an unused input are both FAILs" {
|
|
write_prompt review-pr 'description: Review a PR.
|
|
input: [pr_number]' 'Review ${input:branch}.'
|
|
run bash "$SCRIPT" "$PKG/.apm/prompts/review-pr.prompt.md"
|
|
assert_failure
|
|
assert_output --partial "input: does not declare 'branch'"
|
|
assert_output --partial "input 'pr_number' is declared but the body never uses"
|
|
}
|
|
|
|
@test "prompt: \${input:x} with no input: declared is a FAIL" {
|
|
write_prompt review-pr 'description: Review a PR.' 'Review ${input:pr_number}.'
|
|
run bash "$SCRIPT" "$PKG/.apm/prompts/review-pr.prompt.md"
|
|
assert_failure
|
|
assert_output --partial "but no input: is declared"
|
|
}
|
|
|
|
@test "prompt: a trigger clause, an over-long description, dropped keys and camelCase aliases are SUGGESTIONs" {
|
|
local long
|
|
long="Use when the user wants a PR reviewed. $(printf 'x%.0s' $(seq 1 240))"
|
|
write_prompt review-pr "description: $long
|
|
mode: agent
|
|
allowedTools: Bash" 'Review the PR.'
|
|
run bash "$SCRIPT" "$PKG/.apm/prompts/review-pr.prompt.md"
|
|
assert_success
|
|
assert_output --partial "'Use when' trigger clause"
|
|
assert_output --partial "characters (> 250)"
|
|
assert_output --partial "frontmatter key(s) mode are dropped on Claude"
|
|
assert_output --partial "'allowedTools' — use the kebab-case spelling"
|
|
}
|
|
|
|
@test "prompt: argument-hint alongside input: is a SUGGESTION" {
|
|
write_prompt review-pr 'description: Review a PR.
|
|
argument-hint: <pr>
|
|
input: [pr_number]' 'Review ${input:pr_number}.'
|
|
run bash "$SCRIPT" "$PKG/.apm/prompts/review-pr.prompt.md"
|
|
assert_success
|
|
assert_output --partial "argument-hint is set alongside input:"
|
|
}
|
|
|
|
@test "prompt: no procedure heuristic — a long, stepped body is not a script finding" {
|
|
write_prompt review-pr 'description: Review a PR.' "## Step 1
|
|
$(printf 'line\n%.0s' $(seq 1 80))
|
|
## Gotchas
|
|
- x"
|
|
run bash "$SCRIPT" "$PKG/.apm/prompts/review-pr.prompt.md"
|
|
assert_success
|
|
refute_output --partial "SUGGESTION"
|
|
}
|