Defame1297
db5a426416
The routing-target check understood only a single-arrow clause naming a bare skill, so most real boundary prose was silently skipped rather than verified. Two of those silences were fail-open: an unrecognised token following a target dropped that target from the check entirely, and a skill directory with no SKILL.md still resolved as a valid routing target, so a broken route passed. Multi-target arrow clauses now draw a SUGGESTION instead of being ignored, hand-invocation phrasing is carved out so it is not read as a route, and a dotted filename parses into a new `unparsed` status rather than disappearing. Three test fixtures had been relying on the SKILL.md-less directory resolving as a target; they are corrected alongside the check. Addresses #107, #108, #110.
708 lines
33 KiB
Bash
Executable File
708 lines
33 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# Regression test for scripts/skill-size-check.sh, which enforces two
|
|
# independent gate families that must not be conflated:
|
|
#
|
|
# * agentskills.io spec conformance — 500 lines and a 5,000-token ceiling
|
|
# enforced via a word-count proxy (MAX_WORDS, currently 2770) over the
|
|
# WHOLE FILE, frontmatter included.
|
|
# * ADR-0020 context budget — description 250 chars SUGGESTION / 400 FAIL,
|
|
# body-ONLY 600 words SUGGESTION / 900 FAIL, and resolvable boundary-clause
|
|
# routing targets.
|
|
#
|
|
# The constant-agreement block below is the load-bearing part: all three copies
|
|
# (this hook, skill-audit's validate.sh, agent-audit's validate.sh) are
|
|
# hand-duplicated because a cache-installed plugin cannot read outside its own
|
|
# directory, and nothing but these assertions stops them drifting.
|
|
set -euo pipefail
|
|
|
|
REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
|
SCRIPT="$REPO_ROOT/scripts/skill-size-check.sh"
|
|
VALIDATE="$REPO_ROOT/plugins/kyberforge/.apm/skills/skill-audit/scripts/validate.sh"
|
|
AGENT_VALIDATE="$REPO_ROOT/plugins/kyberforge/.apm/skills/agent-audit/scripts/validate.sh"
|
|
PASS=0
|
|
FAIL=0
|
|
|
|
pass() { echo " PASS: $1"; PASS=$((PASS + 1)); }
|
|
fail() { echo " FAIL: $1"; FAIL=$((FAIL + 1)); }
|
|
|
|
TMPDIR="$(mktemp -d)"
|
|
trap 'rm -rf "$TMPDIR"' EXIT
|
|
|
|
make_fixture() {
|
|
local name="$1" lines="$2" words_per_line="$3" file
|
|
file="$TMPDIR/$name.md"
|
|
{
|
|
echo "---"
|
|
echo "name: $name"
|
|
echo "description: Test fixture."
|
|
echo "---"
|
|
for ((i = 1; i <= lines; i++)); do
|
|
w=""
|
|
for ((j = 1; j <= words_per_line; j++)); do
|
|
w="$w word"
|
|
done
|
|
echo "$w"
|
|
done
|
|
} > "$file"
|
|
echo "$file"
|
|
}
|
|
|
|
echo ""
|
|
echo "--- passes a file under both limits ---"
|
|
SMALL="$(make_fixture small 10 5)"
|
|
if "$SCRIPT" "$SMALL"; then
|
|
pass "file under both limits exits 0"
|
|
else
|
|
fail "file under both limits should have exited 0"
|
|
fi
|
|
|
|
echo ""
|
|
echo "--- fails a file over the line limit ---"
|
|
MANY_LINES="$(make_fixture many-lines 600 1)"
|
|
if "$SCRIPT" "$MANY_LINES" 2>/dev/null; then
|
|
fail "file over the 500-line ceiling should have exited non-zero"
|
|
else
|
|
pass "file over the 500-line ceiling exits non-zero"
|
|
fi
|
|
|
|
echo ""
|
|
echo "--- fails a file over the word-count limit ---"
|
|
MANY_WORDS="$(make_fixture many-words 10 600)"
|
|
if "$SCRIPT" "$MANY_WORDS" 2>/dev/null; then
|
|
fail "file over the word ceiling should have exited non-zero"
|
|
else
|
|
pass "file over the word ceiling exits non-zero"
|
|
fi
|
|
|
|
# Boundary-pair tests below read the script's current MAX_WORDS rather than
|
|
# hardcoding it, so they don't silently drift if the threshold changes again.
|
|
MAX_WORDS="$(grep -oE '^MAX_WORDS=[0-9]+' "$SCRIPT" | cut -d= -f2)"
|
|
MAX_LINES="$(grep -oE '^MAX_LINES=[0-9]+' "$SCRIPT" | cut -d= -f2)"
|
|
|
|
# The audit (skill-audit/scripts/validate.sh) duplicates both ceilings, because
|
|
# a cache-installed plugin's scripts cannot read files outside the plugin
|
|
# directory. Nothing but this assertion stops the copies drifting, and drift
|
|
# means a SKILL.md passes its own audit and is then rejected by the commit hook.
|
|
echo ""
|
|
echo "--- the hook and skill-audit's validate.sh agree on both ceilings ---"
|
|
if [[ ! -f "$VALIDATE" ]]; then
|
|
fail "skill-audit validate.sh not found at $VALIDATE"
|
|
else
|
|
V_MAX_WORDS="$(grep -oE '^MAX_WORDS = [0-9]+' "$VALIDATE" | grep -oE '[0-9]+')"
|
|
V_MAX_LINES="$(grep -oE '^MAX_LINES = [0-9]+' "$VALIDATE" | grep -oE '[0-9]+')"
|
|
if [[ "$V_MAX_WORDS" == "$MAX_WORDS" ]]; then
|
|
pass "both enforce MAX_WORDS=$MAX_WORDS"
|
|
else
|
|
fail "MAX_WORDS drift: hook says $MAX_WORDS, validate.sh says ${V_MAX_WORDS:-<unset>}"
|
|
fi
|
|
if [[ "$V_MAX_LINES" == "$MAX_LINES" ]]; then
|
|
pass "both enforce MAX_LINES=$MAX_LINES"
|
|
else
|
|
fail "MAX_LINES drift: hook says $MAX_LINES, validate.sh says ${V_MAX_LINES:-<unset>}"
|
|
fi
|
|
fi
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# ADR-0020 constants
|
|
# ---------------------------------------------------------------------------
|
|
# Three hand-maintained copies, for the same cache-isolation reason as
|
|
# MAX_WORDS/MAX_LINES above. skill-audit carries all four; agent-audit carries
|
|
# only the two description constants, because ADR-0020 deliberately gives
|
|
# agents NO body word gate (a skill body competes with the caller's live
|
|
# conversation; an agent body becomes the system prompt of a fresh context).
|
|
# The absence of BODY_* in agent-audit is asserted below so a well-meaning
|
|
# "consistency" edit that adds them fails here rather than contradicting the
|
|
# ADR silently.
|
|
DESC_SUGGEST_CHARS="$(grep -oE '^DESC_SUGGEST_CHARS=[0-9]+' "$SCRIPT" | cut -d= -f2)"
|
|
DESC_MAX_CHARS="$(grep -oE '^DESC_MAX_CHARS=[0-9]+' "$SCRIPT" | cut -d= -f2)"
|
|
BODY_SUGGEST_WORDS="$(grep -oE '^BODY_SUGGEST_WORDS=[0-9]+' "$SCRIPT" | cut -d= -f2)"
|
|
BODY_MAX_WORDS="$(grep -oE '^BODY_MAX_WORDS=[0-9]+' "$SCRIPT" | cut -d= -f2)"
|
|
|
|
echo ""
|
|
echo "--- the hook declares all four ADR-0020 constants ---"
|
|
for pair in "DESC_SUGGEST_CHARS:$DESC_SUGGEST_CHARS" "DESC_MAX_CHARS:$DESC_MAX_CHARS" \
|
|
"BODY_SUGGEST_WORDS:$BODY_SUGGEST_WORDS" "BODY_MAX_WORDS:$BODY_MAX_WORDS"; do
|
|
if [[ -n "${pair#*:}" ]]; then
|
|
pass "${pair%%:*}=${pair#*:}"
|
|
else
|
|
fail "${pair%%:*} is not declared in $SCRIPT"
|
|
fi
|
|
done
|
|
|
|
echo ""
|
|
echo "--- the hook and skill-audit's validate.sh agree on all four ADR-0020 constants ---"
|
|
for const in DESC_SUGGEST_CHARS DESC_MAX_CHARS BODY_SUGGEST_WORDS BODY_MAX_WORDS; do
|
|
hook_value="$(grep -oE "^${const}=[0-9]+" "$SCRIPT" | cut -d= -f2)"
|
|
audit_value="$(grep -oE "^${const} = [0-9]+" "$VALIDATE" | grep -oE '[0-9]+' || true)"
|
|
if [[ -n "$hook_value" && "$hook_value" == "$audit_value" ]]; then
|
|
pass "both enforce $const=$hook_value"
|
|
else
|
|
fail "$const drift: hook says ${hook_value:-<unset>}, skill-audit validate.sh says ${audit_value:-<unset>}"
|
|
fi
|
|
done
|
|
|
|
echo ""
|
|
echo "--- the hook and agent-audit's validate.sh agree on the description constants ---"
|
|
if [[ ! -f "$AGENT_VALIDATE" ]]; then
|
|
fail "agent-audit validate.sh not found at $AGENT_VALIDATE"
|
|
else
|
|
for const in DESC_SUGGEST_CHARS DESC_MAX_CHARS; do
|
|
hook_value="$(grep -oE "^${const}=[0-9]+" "$SCRIPT" | cut -d= -f2)"
|
|
agent_value="$(grep -oE "^${const} = [0-9]+" "$AGENT_VALIDATE" | grep -oE '[0-9]+' || true)"
|
|
if [[ -n "$hook_value" && "$hook_value" == "$agent_value" ]]; then
|
|
pass "both enforce $const=$hook_value"
|
|
else
|
|
fail "$const drift: hook says ${hook_value:-<unset>}, agent-audit validate.sh says ${agent_value:-<unset>}"
|
|
fi
|
|
done
|
|
echo ""
|
|
echo "--- agent-audit declares NO body word gate (ADR-0020 is explicit about this) ---"
|
|
if grep -qE '^BODY_(SUGGEST|MAX)_WORDS = ' "$AGENT_VALIDATE"; then
|
|
fail "agent-audit validate.sh declares a body word gate — ADR-0020 gives agents the description gates and NO body word gate"
|
|
else
|
|
pass "agent-audit validate.sh declares no BODY_*_WORDS constant"
|
|
fi
|
|
fi
|
|
|
|
# make_line_fixture builds a file with an exact total line count (frontmatter
|
|
# included), independent of word count, for the line-boundary tests.
|
|
make_line_fixture() {
|
|
local name="$1" total_lines="$2" file body_lines
|
|
file="$TMPDIR/$name.md"
|
|
{
|
|
echo "---"
|
|
echo "name: $name"
|
|
echo "description: Test fixture."
|
|
echo "---"
|
|
} > "$file"
|
|
body_lines=$((total_lines - 4))
|
|
for ((i = 1; i <= body_lines; i++)); do
|
|
echo "word"
|
|
done >> "$file"
|
|
echo "$file"
|
|
}
|
|
|
|
# The line ceiling is inclusive of the limit itself, enforced via `>` — so
|
|
# exactly $MAX_LINES must pass and $((MAX_LINES + 1)) must fail. This matches
|
|
# skill-audit/scripts/validate.sh's `line_count <= 500` pass condition; the two
|
|
# previously disagreed at exactly $MAX_LINES lines, so a SKILL.md could pass its
|
|
# own audit and still be blocked by the commit hook.
|
|
echo ""
|
|
echo "--- passes a file at exactly the $MAX_LINES-line boundary ---"
|
|
AT_LINES="$(make_line_fixture at-line-limit "$MAX_LINES")"
|
|
ACTUAL_LINES=$(awk 'END{print NR}' "$AT_LINES")
|
|
if [[ "$ACTUAL_LINES" -ne "$MAX_LINES" ]]; then
|
|
fail "fixture has $ACTUAL_LINES lines, expected exactly $MAX_LINES"
|
|
elif "$SCRIPT" "$AT_LINES"; then
|
|
pass "file at exactly $MAX_LINES lines exits 0"
|
|
else
|
|
fail "file at exactly $MAX_LINES lines should have exited 0 (the off-by-one this test guards against)"
|
|
fi
|
|
|
|
echo ""
|
|
echo "--- fails a file one line over the $MAX_LINES-line boundary ---"
|
|
OVER_LINES="$(make_line_fixture over-line-limit "$((MAX_LINES + 1))")"
|
|
ACTUAL_OVER_LINES=$(awk 'END{print NR}' "$OVER_LINES")
|
|
if [[ "$ACTUAL_OVER_LINES" -ne "$((MAX_LINES + 1))" ]]; then
|
|
fail "fixture has $ACTUAL_OVER_LINES lines, expected exactly $((MAX_LINES + 1))"
|
|
elif "$SCRIPT" "$OVER_LINES" 2>/dev/null; then
|
|
fail "file at $((MAX_LINES + 1)) lines should have exited non-zero"
|
|
else
|
|
pass "file at $((MAX_LINES + 1)) lines exits non-zero"
|
|
fi
|
|
|
|
# make_word_fixture builds a file with an exact total word count (frontmatter
|
|
# words included, since the script's `wc -w` counts the whole file).
|
|
#
|
|
# The padding goes in a frontmatter `notes:` field, NOT in the body, and that
|
|
# placement is the point: MAX_WORDS is a whole-file measurement while ADR-0020's
|
|
# BODY_MAX_WORDS is a body-only one. Padding the body would make a 2,770-word
|
|
# fixture trip the 900-word body ceiling too, and the MAX_WORDS boundary test
|
|
# would stop isolating MAX_WORDS. `notes:` is an unused key — the description
|
|
# stays short, so the description gate stays quiet as well.
|
|
make_word_fixture() {
|
|
local name="$1" target="$2" file frame_words padding
|
|
file="$TMPDIR/$name.md"
|
|
padding=""
|
|
{
|
|
echo "---"
|
|
echo "name: $name"
|
|
echo "description: Test fixture."
|
|
echo "notes:$padding"
|
|
echo "---"
|
|
echo ""
|
|
echo "Body."
|
|
} > "$file"
|
|
frame_words=$(wc -w < "$file")
|
|
for ((i = 1; i <= target - frame_words; i++)); do
|
|
padding="$padding word"
|
|
done
|
|
{
|
|
echo "---"
|
|
echo "name: $name"
|
|
echo "description: Test fixture."
|
|
echo "notes:$padding"
|
|
echo "---"
|
|
echo ""
|
|
echo "Body."
|
|
} > "$file"
|
|
echo "$file"
|
|
}
|
|
|
|
echo ""
|
|
echo "--- passes a file at exactly the $MAX_WORDS-word boundary ---"
|
|
AT_WORDS="$(make_word_fixture at-word-limit "$MAX_WORDS")"
|
|
ACTUAL_WORDS=$(wc -w < "$AT_WORDS")
|
|
if [[ "$ACTUAL_WORDS" -ne "$MAX_WORDS" ]]; then
|
|
fail "fixture has $ACTUAL_WORDS words, expected exactly $MAX_WORDS"
|
|
elif "$SCRIPT" "$AT_WORDS"; then
|
|
pass "file at exactly $MAX_WORDS words exits 0"
|
|
else
|
|
fail "file at exactly $MAX_WORDS words should have exited 0"
|
|
fi
|
|
|
|
echo ""
|
|
echo "--- fails a file one word over the $MAX_WORDS-word boundary ---"
|
|
OVER_WORDS="$(make_word_fixture over-word-limit "$((MAX_WORDS + 1))")"
|
|
ACTUAL_OVER_WORDS=$(wc -w < "$OVER_WORDS")
|
|
if [[ "$ACTUAL_OVER_WORDS" -ne "$((MAX_WORDS + 1))" ]]; then
|
|
fail "fixture has $ACTUAL_OVER_WORDS words, expected exactly $((MAX_WORDS + 1))"
|
|
elif "$SCRIPT" "$OVER_WORDS" 2>/dev/null; then
|
|
fail "file at $((MAX_WORDS + 1)) words should have exited non-zero"
|
|
else
|
|
pass "file at $((MAX_WORDS + 1)) words exits non-zero"
|
|
fi
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# ADR-0020 behaviour
|
|
# ---------------------------------------------------------------------------
|
|
|
|
# make_budget_fixture builds a SKILL.md with a verbatim description and an
|
|
# exact BODY word count (frontmatter words excluded — the ADR-0020 body gate
|
|
# counts the body only).
|
|
make_budget_fixture() {
|
|
local name="$1" desc="$2" body_words="$3" file
|
|
file="$TMPDIR/$name.md"
|
|
{
|
|
echo "---"
|
|
echo "name: $name"
|
|
echo "description: $desc"
|
|
echo "---"
|
|
echo ""
|
|
python3 -c "print(' '.join(['word'] * $body_words))"
|
|
} > "$file"
|
|
echo "$file"
|
|
}
|
|
|
|
# desc_of_length <n> — a description of EXACTLY n characters that carries a
|
|
# boundary clause and names no routing target.
|
|
#
|
|
# ADR-0020's missing-boundary-clause SUGGESTION fires on every description
|
|
# without one, so a fixture that omits it is never "otherwise clean": a test
|
|
# asserting silence would be asserting the boundary check's ABSENCE rather than
|
|
# the length boundary it names. The clause is paid for out of the same budget
|
|
# being measured (padding arithmetic, not a fixed suffix) so the character count
|
|
# stays exact. "anything else" is unhyphenated, so no routing target rides along.
|
|
desc_of_length() {
|
|
python3 - "$1" <<'PY'
|
|
import sys
|
|
n = int(sys.argv[1])
|
|
prefix = 'Use when doing the thing. Do not use for anything else. '
|
|
assert n >= len(prefix), 'requested description shorter than the boundary clause'
|
|
print(prefix + 'x' * (n - len(prefix)))
|
|
PY
|
|
}
|
|
|
|
# make_tree_fixture <label> <desc> <body_words> — a SKILL.md inside a synthetic
|
|
# apm plugin monorepo, so the boundary-target resolver has a universe.
|
|
#
|
|
# Resolution walks up FROM THE TARGET FILE to an authoring root (the nearest
|
|
# ancestor holding plugins/*/.apm/{skills,agents}, falling back to .git); it is
|
|
# never derived from the checker's own location, because deriving it from
|
|
# ${BASH_SOURCE} leaked this repo's 39-skill universe into every consumer repo
|
|
# running the hook. A fixture in a bare mktemp -d therefore has NO universe and
|
|
# correctly reports "DID NOT RUN" — that is not a bug to paper over with a
|
|
# looser assertion, it is why the fixture has to be a real tree:
|
|
#
|
|
# <root>/plugins/subject-plugin/.apm/skills/<label>/SKILL.md <- the subject
|
|
# <root>/plugins/subject-plugin/.apm/skills/sibling-skill/ <- same package
|
|
# <root>/plugins/subject-plugin/.apm/agents/sibling-agent.agent.md
|
|
# <root>/plugins/other-plugin/.apm/skills/cross-plugin-skill/ <- sibling plugin
|
|
#
|
|
# The sibling plugin is what makes "every plugin in the monorepo contributes its
|
|
# names" testable; without it a cross-plugin target and a typo are the same.
|
|
#
|
|
# Both sibling skill directories get a real SKILL.md, and that is load-bearing
|
|
# rather than tidiness: a skill directory is a resolvable name only if it HOLDS
|
|
# a SKILL.md. An empty leftover directory is untracked by git, so counting one
|
|
# made a target resolve on the machine that made it and dangle in a fresh clone
|
|
# — the same install-dependence the deployed-tree rule exists to remove. This
|
|
# fixture used to `mkdir` the two siblings and write nothing into them, so it
|
|
# was itself relying on the behaviour the resolver no longer has.
|
|
make_tree_fixture() {
|
|
local label="$1" desc="$2" body_words="$3" root apm sib
|
|
root="$TMPDIR/tree-$label"
|
|
apm="$root/plugins/subject-plugin/.apm"
|
|
mkdir -p "$apm/skills/$label" "$apm/skills/sibling-skill" "$apm/agents" \
|
|
"$root/plugins/other-plugin/.apm/skills/cross-plugin-skill"
|
|
: > "$apm/agents/sibling-agent.agent.md"
|
|
for sib in "$apm/skills/sibling-skill" \
|
|
"$root/plugins/other-plugin/.apm/skills/cross-plugin-skill"; do
|
|
{
|
|
echo "---"
|
|
echo "name: $(basename "$sib")"
|
|
echo "description: Use when doing the other thing. Do not use for anything else."
|
|
echo "---"
|
|
echo ""
|
|
echo "Do the thing."
|
|
} > "$sib/SKILL.md"
|
|
done
|
|
{
|
|
echo "---"
|
|
echo "name: $label"
|
|
echo "description: $desc"
|
|
echo "---"
|
|
echo ""
|
|
python3 -c "print(' '.join(['word'] * $body_words))"
|
|
} > "$apm/skills/$label/SKILL.md"
|
|
echo "$apm/skills/$label/SKILL.md"
|
|
}
|
|
|
|
# expect_gate <label> <expected: pass|suggest|fail> <file> [needle]
|
|
expect_gate() {
|
|
local label="$1" expected="$2" file="$3" needle="${4:-}" out status
|
|
set +e
|
|
out="$("$SCRIPT" "$file" 2>&1)"
|
|
status=$?
|
|
set -e
|
|
case "$expected" in
|
|
pass)
|
|
if [[ $status -eq 0 && -z "$out" ]]; then
|
|
pass "$label"
|
|
else
|
|
fail "$label (exit $status, output: ${out:-<empty>})"
|
|
fi
|
|
;;
|
|
# The tier and the needle are matched ADJACENTLY — `*"SUGGESTION"*"$needle"*`
|
|
# — not as two independent substring tests. Independently, any output
|
|
# carrying a SUGGESTION anywhere and the needle anywhere satisfied the
|
|
# assertion, so a needle emitted at the WRONG TIER still passed: a finding
|
|
# that moved from SUGGESTION to a blocking ERROR line would be caught only
|
|
# by the exit-status test, and one that moved from SUGGESTION to INFO would
|
|
# not be caught at all. grammar_case's `suggests` branch in
|
|
# tests/test-adr0020-targets.sh has always matched them adjacently; this is
|
|
# the same rule.
|
|
suggest)
|
|
if [[ $status -eq 0 && "$out" == *"SUGGESTION"*"$needle"* ]]; then
|
|
pass "$label"
|
|
else
|
|
fail "$label (exit $status, output: ${out:-<empty>})"
|
|
fi
|
|
;;
|
|
fail)
|
|
if [[ $status -ne 0 && "$out" == *"$needle"* ]]; then
|
|
pass "$label"
|
|
else
|
|
fail "$label (exit $status, output: ${out:-<empty>})"
|
|
fi
|
|
;;
|
|
# A check that DECLINED to run must say so and must not fail the file. The
|
|
# ERROR guard is the point: a declined check that also errored would satisfy
|
|
# a bare "output contains INFO" assertion.
|
|
info)
|
|
if [[ $status -eq 0 && "$out" == *"INFO"* && "$out" == *"$needle"* \
|
|
&& "$out" != *"ERROR"* ]]; then
|
|
pass "$label"
|
|
else
|
|
fail "$label (exit $status, output: ${out:-<empty>})"
|
|
fi
|
|
;;
|
|
esac
|
|
}
|
|
|
|
echo ""
|
|
echo "--- description budget: $DESC_SUGGEST_CHARS SUGGESTION / $DESC_MAX_CHARS FAIL, both inclusive ---"
|
|
D_AT_SUGGEST="$(desc_of_length "$DESC_SUGGEST_CHARS")"
|
|
D_OVER_SUGGEST="$(desc_of_length "$((DESC_SUGGEST_CHARS + 1))")"
|
|
D_AT_MAX="$(desc_of_length "$DESC_MAX_CHARS")"
|
|
D_OVER_MAX="$(desc_of_length "$((DESC_MAX_CHARS + 1))")"
|
|
expect_gate "description at exactly $DESC_SUGGEST_CHARS chars is silent" \
|
|
pass "$(make_budget_fixture desc-at-suggest "$D_AT_SUGGEST" 10)"
|
|
expect_gate "description at $((DESC_SUGGEST_CHARS + 1)) chars suggests and exits 0" \
|
|
suggest "$(make_budget_fixture desc-over-suggest "$D_OVER_SUGGEST" 10)" \
|
|
"description is $((DESC_SUGGEST_CHARS + 1)) characters"
|
|
expect_gate "description at exactly $DESC_MAX_CHARS chars suggests, does not fail" \
|
|
suggest "$(make_budget_fixture desc-at-max "$D_AT_MAX" 10)" \
|
|
"description is $DESC_MAX_CHARS characters"
|
|
expect_gate "description at $((DESC_MAX_CHARS + 1)) chars fails" \
|
|
fail "$(make_budget_fixture desc-over-max "$D_OVER_MAX" 10)" \
|
|
"$DESC_MAX_CHARS-character ceiling"
|
|
|
|
echo ""
|
|
echo "--- description length is measured after YAML folding is resolved ---"
|
|
FOLDED="$TMPDIR/folded.md"
|
|
{
|
|
echo "---"
|
|
echo "name: folded"
|
|
echo "description: >"
|
|
python3 -c "print('\n'.join([' ' + 'x' * 40] * 11))"
|
|
echo "---"
|
|
echo ""
|
|
echo "Do the thing."
|
|
} > "$FOLDED"
|
|
expect_gate "a >-folded 450-char description fails (raw first line would read as 1 char)" \
|
|
fail "$FOLDED" "description is 450 characters"
|
|
|
|
# Every body fixture below carries a boundary clause for the same reason
|
|
# desc_of_length() does: without one the missing-boundary-clause SUGGESTION
|
|
# fires and a body-budget test that asserts silence stops isolating the body
|
|
# budget. It is short, so the description gate stays quiet too.
|
|
CLEAN_DESC="Short valid description. Do not use for anything else."
|
|
echo ""
|
|
echo "--- body budget: $BODY_SUGGEST_WORDS SUGGESTION / $BODY_MAX_WORDS FAIL, body only, both inclusive ---"
|
|
expect_gate "body at exactly $BODY_SUGGEST_WORDS words is silent" \
|
|
pass "$(make_budget_fixture body-at-suggest "$CLEAN_DESC" "$BODY_SUGGEST_WORDS")"
|
|
expect_gate "body at $((BODY_SUGGEST_WORDS + 1)) words suggests and exits 0" \
|
|
suggest "$(make_budget_fixture body-over-suggest "$CLEAN_DESC" "$((BODY_SUGGEST_WORDS + 1))")" \
|
|
"body is $((BODY_SUGGEST_WORDS + 1)) words"
|
|
expect_gate "body at exactly $BODY_MAX_WORDS words suggests, does not fail" \
|
|
suggest "$(make_budget_fixture body-at-max "$CLEAN_DESC" "$BODY_MAX_WORDS")" \
|
|
"body is $BODY_MAX_WORDS words"
|
|
expect_gate "body at $((BODY_MAX_WORDS + 1)) words fails" \
|
|
fail "$(make_budget_fixture body-over-max "$CLEAN_DESC" "$((BODY_MAX_WORDS + 1))")" \
|
|
"$BODY_MAX_WORDS-word ceiling"
|
|
|
|
# The two word gates measure different things and must stay separable: a file
|
|
# whose FRONTMATTER pushes the whole-file count past the body ceiling must not
|
|
# trip the body gate, and a file under MAX_WORDS can still fail the body gate.
|
|
echo ""
|
|
echo "--- the body gate and the whole-file gate are independent measurements ---"
|
|
BODY_ONLY_DESC="$(python3 -c "print(' '.join(['w'] * 100))")"
|
|
# The needle pins the COUNT, not the bare word "words". "words" appears in the
|
|
# whole-file ceiling message, in the body ceiling message and in the body target
|
|
# message alike, so it was satisfied by any of the three — including the one
|
|
# this case exists to prove does NOT fire. Naming the number is what makes the
|
|
# assertion about the body-only measurement.
|
|
expect_gate "frontmatter words do not count toward the $BODY_MAX_WORDS-word body ceiling" \
|
|
suggest "$(make_budget_fixture body-independent "$BODY_ONLY_DESC" "$((BODY_MAX_WORDS - 5))")" \
|
|
"body is $((BODY_MAX_WORDS - 5)) words"
|
|
BIG_BODY="$(make_budget_fixture body-over-not-whole-file "$CLEAN_DESC" "$((BODY_MAX_WORDS + 1))")"
|
|
BIG_BODY_WORDS="$(wc -w < "$BIG_BODY")"
|
|
if [[ "$BIG_BODY_WORDS" -le "$MAX_WORDS" ]]; then
|
|
pass "the body-gate fixture is $BIG_BODY_WORDS whole-file words, well under MAX_WORDS=$MAX_WORDS — it fails on the body gate alone"
|
|
else
|
|
fail "the body-gate fixture is $BIG_BODY_WORDS whole-file words, which also trips MAX_WORDS=$MAX_WORDS — the test no longer isolates the body gate"
|
|
fi
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# ADR-0020's hand-invocation carve-out (issue #108)
|
|
# ---------------------------------------------------------------------------
|
|
# A skill carrying `disable-model-invocation: true` is removed from the
|
|
# model-visible listing entirely — it is not preloaded, and the Skill tool
|
|
# refuses to call it — so its description is never matched against user intent.
|
|
# ADR-0020, skill-author Step 2 and skill-audit's own Step 0 all give it ONE
|
|
# plain human-facing sentence: no trigger list, no boundary clause. No validator
|
|
# knew the field existed, so the boundary-clause SUGGESTION fired on exactly the
|
|
# shape the contract mandates, and its remedy — "so the router knows where NOT
|
|
# to send this skill" — named a router that cannot see the skill at all.
|
|
#
|
|
# The carve-out is NARROW and the half it does not cover is the half worth
|
|
# testing: the body is still loaded on invocation, so the body budget stands,
|
|
# and the 400-character ceiling stands because it is an outlier stop rather than
|
|
# a routing-quality target. Every case below asserts one of those two halves.
|
|
make_hand_invoked_fixture() {
|
|
local name="$1" desc="$2" body_words="$3" file
|
|
file="$TMPDIR/$name.md"
|
|
{
|
|
echo "---"
|
|
echo "name: $name"
|
|
echo "description: $desc"
|
|
echo "disable-model-invocation: true"
|
|
echo "---"
|
|
echo ""
|
|
python3 -c "print(' '.join(['word'] * $body_words))"
|
|
} > "$file"
|
|
echo "$file"
|
|
}
|
|
|
|
# A description over the 250-character target, carrying no boundary clause and
|
|
# no routing target — the exact shape `zoom-out` and `caveman` ship. Built with
|
|
# no hyphens so nothing in it reads as a target.
|
|
HAND_DESC="$(python3 -c "
|
|
prefix = 'Tell the agent to zoom out and give broader context. '
|
|
print(prefix + 'x' * (300 - len(prefix)))")"
|
|
|
|
echo ""
|
|
echo "--- a hand-invoked skill is exempt from the routing rules, and only those ---"
|
|
expect_gate "a hand-invoked skill with a 300-char description and no boundary clause is silent" \
|
|
pass "$(make_hand_invoked_fixture hand-quiet "$HAND_DESC" 10)"
|
|
# The control that makes the case above mean something. Same description, same
|
|
# body, only the frontmatter flag removed: both findings must appear, or the
|
|
# exemption is being credited for silence it did not cause.
|
|
expect_gate "control: the SAME description without the flag is over the 250-char target" \
|
|
suggest "$(make_budget_fixture hand-control "$HAND_DESC" 10)" \
|
|
"description is 300 characters"
|
|
expect_gate "control: the SAME description without the flag has no boundary clause" \
|
|
suggest "$(make_budget_fixture hand-control "$HAND_DESC" 10)" \
|
|
"has no boundary clause"
|
|
|
|
echo ""
|
|
echo "--- the carve-out lifts the routing rules ONLY: both size gates still bite ---"
|
|
# The description ceiling is not a routing budget: a hand-invoked description is
|
|
# still the one line a human reads in the `/` menu, and 400 characters is the
|
|
# outlier stop either way.
|
|
HAND_OVER_MAX="$(python3 -c "
|
|
prefix = 'Tell the agent to zoom out and give broader context. '
|
|
print(prefix + 'x' * (401 - len(prefix)))")"
|
|
expect_gate "a hand-invoked description over $DESC_MAX_CHARS chars still FAILS" \
|
|
fail "$(make_hand_invoked_fixture hand-over-max "$HAND_OVER_MAX" 10)" \
|
|
"$DESC_MAX_CHARS-character ceiling"
|
|
# The body is loaded on invocation like any other body and competes with the
|
|
# caller's live conversation exactly the same way, so neither body tier moves.
|
|
expect_gate "a hand-invoked body over $BODY_MAX_WORDS words still FAILS" \
|
|
fail "$(make_hand_invoked_fixture hand-over-body "$HAND_DESC" "$((BODY_MAX_WORDS + 1))")" \
|
|
"$BODY_MAX_WORDS-word ceiling"
|
|
expect_gate "a hand-invoked body over $BODY_SUGGEST_WORDS words is still suggested" \
|
|
suggest "$(make_hand_invoked_fixture hand-over-body-suggest "$HAND_DESC" "$((BODY_SUGGEST_WORDS + 1))")" \
|
|
"body is $((BODY_SUGGEST_WORDS + 1)) words"
|
|
|
|
echo ""
|
|
echo "--- the flag is read as a BOOLEAN, not as any mention of the key ---"
|
|
# `disable-model-invocation: false` is the model-invoked case written out
|
|
# longhand. Reading the key's presence instead of its value would hand every
|
|
# routing exemption to anyone who typed the field at all.
|
|
HAND_FALSE="$TMPDIR/hand-false.md"
|
|
{
|
|
echo "---"
|
|
echo "name: hand-false"
|
|
echo "description: $HAND_DESC"
|
|
echo "disable-model-invocation: false"
|
|
echo "---"
|
|
echo ""
|
|
echo "Do the thing."
|
|
} > "$HAND_FALSE"
|
|
expect_gate "disable-model-invocation: false is NOT the carve-out" \
|
|
suggest "$HAND_FALSE" "has no boundary clause"
|
|
|
|
echo ""
|
|
echo "--- resolvable boundary targets ---"
|
|
# Resolution is against the AUTHORING SOURCE (plugins/*/.apm/skills/ and
|
|
# plugins/*/.apm/agents/), reached by walking up FROM THE SKILL FILE. These
|
|
# fixtures therefore build their own synthetic monorepo (make_tree_fixture) and
|
|
# name only fixture-local targets: they must not depend on this repo's live
|
|
# skills, or renaming git-commits would break a test about extraction grammar.
|
|
expect_gate "a boundary target naming a sibling skill in the same package resolves" \
|
|
pass "$(make_tree_fixture target-ok \
|
|
"Use when doing the thing. Do not use for commits — use sibling-skill instead." 10)"
|
|
expect_gate "a boundary target naming a skill in a SIBLING PLUGIN resolves (that is what a monorepo means)" \
|
|
pass "$(make_tree_fixture target-cross-plugin \
|
|
"Use when doing the thing. Do not use for the other thing — use cross-plugin-skill instead." 10)"
|
|
expect_gate "a boundary target naming an AGENT resolves (agents are valid targets)" \
|
|
pass "$(make_tree_fixture target-agent-ok \
|
|
"Use when doing the thing. Do not use when the caller is an agent — invoke sibling-agent instead." 10)"
|
|
# CORROBORATED: `sibling-skill` resolves in the same sentence, which is what
|
|
# promotes a prose-form target from "reported" to "blocking". A lone prose-form
|
|
# target is deliberately not fatal — see the case below and the shared resolver's
|
|
# CORROBORATION note.
|
|
expect_gate "a boundary target that resolves to nothing fails when its sentence names one that does" \
|
|
fail "$(make_tree_fixture target-missing \
|
|
"Use when doing the thing. Do not use for improvements — use sibling-skill or no-such-skill-anywhere instead." 10)" \
|
|
"routes to 'no-such-skill-anywhere'"
|
|
# UNCORROBORATED: identical grammar to the case above, and identical grammar to
|
|
# "run `pre-commit` instead". Reported at SUGGESTION tier, exit 0 — a gate that
|
|
# ships hot with no baseline and no suppression mechanism must not block a commit
|
|
# on a token it cannot tell from a tool name.
|
|
expect_gate "a lone boundary target that resolves to nothing is reported, not fatal" \
|
|
suggest "$(make_tree_fixture target-missing-lone \
|
|
"Use when doing the thing. Do not use for improvements — use no-such-lone-skill instead." 10)" \
|
|
"routes to 'no-such-lone-skill'"
|
|
expect_gate "a /slash-command boundary target that resolves to nothing fails" \
|
|
fail "$(make_tree_fixture target-missing-slash \
|
|
"Use when doing the thing. Do not use for improvements — use /no-such-slash-skill instead." 10)" \
|
|
"routes to 'no-such-slash-skill'"
|
|
# False-positive guards. These phrasings are lifted from real descriptions:
|
|
# pc-run says "run pre-commit hooks", diagnose chains "fix -> regression-test",
|
|
# gitea-files says "(use Read/Write/Edit)", gitea-labels-milestones says
|
|
# "through `issue_write`/`pull_request_write`". None of them is a routing
|
|
# target, and reading any of them as one makes the gate untrustworthy.
|
|
#
|
|
# Each carries a boundary clause in a SEPARATE sentence. That is not decoration:
|
|
# target extraction is decided per sentence, so the clause satisfies the
|
|
# missing-boundary-clause SUGGESTION (keeping the expected output empty) while
|
|
# leaving the sentence under test outside a boundary context, which is the exact
|
|
# condition each of these is about. They are built as trees so a universe exists
|
|
# — in a bare temp dir the resolver would decline and the guard would pass
|
|
# vacuously, proving nothing about extraction.
|
|
expect_gate "'run pre-commit hooks' outside a boundary sentence is not a routing target" \
|
|
pass "$(make_tree_fixture fp-precommit \
|
|
"Use when the user wants to run pre-commit hooks or install git hooks. Do not use for anything else." 10)"
|
|
expect_gate "an arrow chain outside a boundary clause is not a routing target" \
|
|
pass "$(make_tree_fixture fp-arrow \
|
|
"Reproduce → minimise → instrument → fix → regression-test. Use when a bug is reported. Do not use for anything else." 10)"
|
|
expect_gate "tool names and MCP tool names are not routing targets" \
|
|
pass "$(make_tree_fixture fp-tools \
|
|
"Use when writing issues. Do not use for local files (use Read/Write/Edit) — that write goes through \`issue_write\`/\`pull_request_write\` instead." 10)"
|
|
|
|
echo ""
|
|
echo "--- with NO authoring root the resolver declines OUT LOUD and does not fail the file ---"
|
|
# The consumer/draft case, and a real one: a SKILL.md in a bare directory with no
|
|
# plugins/*/.apm/ above it and no .git has no universe to resolve against. The
|
|
# required behaviour is neither a false FAIL nor silence — silence is how a whole
|
|
# gate family goes missing unnoticed — so the INFO and the named unchecked target
|
|
# are both asserted, along with exit 0. This is the same path make_tree_fixture
|
|
# exists to escape, kept pinned so a future "just use the repo root" shortcut
|
|
# (the ${BASH_SOURCE} universe leak ADR-0020 removed) fails here.
|
|
expect_gate "a fixture with no authoring root reports DID NOT RUN and exits 0" \
|
|
info "$(make_budget_fixture no-universe \
|
|
"Use when doing the thing. Do not use for improvements — use some-other-skill instead." 10)" \
|
|
"Unchecked target(s): some-other-skill"
|
|
|
|
# NOTE: this section prints no header and runs no assertions any more — see why
|
|
# below. The commentary is kept because it records why probes are removed rather
|
|
# than skipped, which is the rule the next person to touch this file needs.
|
|
#
|
|
# ADR-0020 records the broken routing targets and splits fixing them into its own
|
|
# issue. This asserted the gate actually sees them rather than the check being
|
|
# vacuous in the corpus it was written against.
|
|
#
|
|
# There used to be a third probe here, for `skill-improve` in skill-audit's
|
|
# description. It was already stale: that target was fixed, so the iteration
|
|
# permanently took a `pass "SKIP: ..."` branch — an assertion-free result counted
|
|
# in the totals, which is worse than no probe at all because it makes the suite
|
|
# look one test stronger than it is. It also contradicted
|
|
# tests/test-adr0020-targets.sh, which pins the live dangling set as EXACTLY
|
|
# {neuledge-context}; that file is the authority on the set, this one only
|
|
# checks each member is individually detected.
|
|
#
|
|
# Both SKIP branches are gone with it, for the same reason. A probe whose fixture
|
|
# has been retrofitted is not "still passing" — it is a pin that needs updating,
|
|
# here and in the exact-set assertion in test-adr0020-targets.sh, and it should
|
|
# say so out loud rather than quietly agreeing with whatever it finds.
|
|
# The gitea-labels probe was dropped when the issue #99 retrofit cut the
|
|
# composition sentence whose YAML fold produced that target. Per the rule above
|
|
# it is removed, not skipped.
|
|
#
|
|
# The `neuledge-context` probe — the last one — went the same way in wave 3 of
|
|
# that retrofit, which deleted the boundary clause naming it. **The corpus now
|
|
# has zero dangling targets**, so this loop is removed entirely rather than left
|
|
# to iterate over an empty list.
|
|
#
|
|
# That is deliberate and follows the rule stated above. A loop over no probes
|
|
# produces no assertion while still returning success, which is the vacuous-pass
|
|
# shape this comment block exists to reject — it would make the suite look one
|
|
# test stronger than it is, exactly the complaint levelled at the old
|
|
# `skill-improve` SKIP branch.
|
|
#
|
|
# Nothing is lost. This file only ever checked that each member of the live
|
|
# dangling set is individually detected; tests/test-adr0020-targets.sh remains
|
|
# the authority on the set itself, and now pins it as EMPTY, which is what
|
|
# catches a newly-authored clause naming a target that does not resolve. That
|
|
# file also carries synthetic fixtures built inside a real plugin tree, which
|
|
# exercise the detection path without depending on the corpus staying broken.
|
|
#
|
|
# If a real dangling target ever reappears, add its probe back here.
|
|
|
|
echo ""
|
|
echo "Results: $PASS passed, $FAIL failed"
|
|
[[ $FAIL -eq 0 ]]
|