Defame1297
23b843a311
tests/test-adr0020-targets.sh pinned the corpus dangling set as exactly
{gitea-labels, neuledge-context} and tests/test-skill-size-check.sh probed both
individually. The gitea-issues retrofit cut the composition sentence whose
'>'-folded scalar produced 'gitea-labels- milestones', so that target no longer
exists and both suites went red.
EXPECTED_DANGLING is now {neuledge-context} and the gitea-issues probe is
removed rather than skipped, per the rule the probe file states about itself: a
probe whose fixture has been retrofitted is a pin that needs updating, not an
assertion-free pass counted in the totals.
The exact-set assertion stays. An empty expected set is still valid and still
pins that no new dangling target appeared -- which is what it becomes once
research is retrofitted. Both loops carry a shellcheck SC2043 waiver for the
same reason: one entry is the expected steady state, not bad quoting.
Refs #99
586 lines
26 KiB
Bash
Executable File
586 lines
26 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# Regression test for scripts/skill-size-check.sh, which enforces two
|
|
# independent gate families that must not be conflated:
|
|
#
|
|
# * agentskills.io spec conformance — 500 lines and a 5,000-token ceiling
|
|
# enforced via a word-count proxy (MAX_WORDS, currently 2770) over the
|
|
# WHOLE FILE, frontmatter included.
|
|
# * ADR-0020 context budget — description 250 chars SUGGESTION / 400 FAIL,
|
|
# body-ONLY 600 words SUGGESTION / 900 FAIL, and resolvable boundary-clause
|
|
# routing targets.
|
|
#
|
|
# The constant-agreement block below is the load-bearing part: all three copies
|
|
# (this hook, skill-audit's validate.sh, agent-audit's validate.sh) are
|
|
# hand-duplicated because a cache-installed plugin cannot read outside its own
|
|
# directory, and nothing but these assertions stops them drifting.
|
|
set -euo pipefail
|
|
|
|
REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
|
SCRIPT="$REPO_ROOT/scripts/skill-size-check.sh"
|
|
VALIDATE="$REPO_ROOT/plugins/kyberforge/.apm/skills/skill-audit/scripts/validate.sh"
|
|
AGENT_VALIDATE="$REPO_ROOT/plugins/kyberforge/.apm/skills/agent-audit/scripts/validate.sh"
|
|
PASS=0
|
|
FAIL=0
|
|
|
|
pass() { echo " PASS: $1"; PASS=$((PASS + 1)); }
|
|
fail() { echo " FAIL: $1"; FAIL=$((FAIL + 1)); }
|
|
|
|
TMPDIR="$(mktemp -d)"
|
|
trap 'rm -rf "$TMPDIR"' EXIT
|
|
|
|
make_fixture() {
|
|
local name="$1" lines="$2" words_per_line="$3" file
|
|
file="$TMPDIR/$name.md"
|
|
{
|
|
echo "---"
|
|
echo "name: $name"
|
|
echo "description: Test fixture."
|
|
echo "---"
|
|
for ((i = 1; i <= lines; i++)); do
|
|
w=""
|
|
for ((j = 1; j <= words_per_line; j++)); do
|
|
w="$w word"
|
|
done
|
|
echo "$w"
|
|
done
|
|
} > "$file"
|
|
echo "$file"
|
|
}
|
|
|
|
echo ""
|
|
echo "--- passes a file under both limits ---"
|
|
SMALL="$(make_fixture small 10 5)"
|
|
if "$SCRIPT" "$SMALL"; then
|
|
pass "file under both limits exits 0"
|
|
else
|
|
fail "file under both limits should have exited 0"
|
|
fi
|
|
|
|
echo ""
|
|
echo "--- fails a file over the line limit ---"
|
|
MANY_LINES="$(make_fixture many-lines 600 1)"
|
|
if "$SCRIPT" "$MANY_LINES" 2>/dev/null; then
|
|
fail "file over the 500-line ceiling should have exited non-zero"
|
|
else
|
|
pass "file over the 500-line ceiling exits non-zero"
|
|
fi
|
|
|
|
echo ""
|
|
echo "--- fails a file over the word-count limit ---"
|
|
MANY_WORDS="$(make_fixture many-words 10 600)"
|
|
if "$SCRIPT" "$MANY_WORDS" 2>/dev/null; then
|
|
fail "file over the word ceiling should have exited non-zero"
|
|
else
|
|
pass "file over the word ceiling exits non-zero"
|
|
fi
|
|
|
|
# Boundary-pair tests below read the script's current MAX_WORDS rather than
|
|
# hardcoding it, so they don't silently drift if the threshold changes again.
|
|
MAX_WORDS="$(grep -oE '^MAX_WORDS=[0-9]+' "$SCRIPT" | cut -d= -f2)"
|
|
MAX_LINES="$(grep -oE '^MAX_LINES=[0-9]+' "$SCRIPT" | cut -d= -f2)"
|
|
|
|
# The audit (skill-audit/scripts/validate.sh) duplicates both ceilings, because
|
|
# a cache-installed plugin's scripts cannot read files outside the plugin
|
|
# directory. Nothing but this assertion stops the copies drifting, and drift
|
|
# means a SKILL.md passes its own audit and is then rejected by the commit hook.
|
|
echo ""
|
|
echo "--- the hook and skill-audit's validate.sh agree on both ceilings ---"
|
|
if [[ ! -f "$VALIDATE" ]]; then
|
|
fail "skill-audit validate.sh not found at $VALIDATE"
|
|
else
|
|
V_MAX_WORDS="$(grep -oE '^MAX_WORDS = [0-9]+' "$VALIDATE" | grep -oE '[0-9]+')"
|
|
V_MAX_LINES="$(grep -oE '^MAX_LINES = [0-9]+' "$VALIDATE" | grep -oE '[0-9]+')"
|
|
if [[ "$V_MAX_WORDS" == "$MAX_WORDS" ]]; then
|
|
pass "both enforce MAX_WORDS=$MAX_WORDS"
|
|
else
|
|
fail "MAX_WORDS drift: hook says $MAX_WORDS, validate.sh says ${V_MAX_WORDS:-<unset>}"
|
|
fi
|
|
if [[ "$V_MAX_LINES" == "$MAX_LINES" ]]; then
|
|
pass "both enforce MAX_LINES=$MAX_LINES"
|
|
else
|
|
fail "MAX_LINES drift: hook says $MAX_LINES, validate.sh says ${V_MAX_LINES:-<unset>}"
|
|
fi
|
|
fi
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# ADR-0020 constants
|
|
# ---------------------------------------------------------------------------
|
|
# Three hand-maintained copies, for the same cache-isolation reason as
|
|
# MAX_WORDS/MAX_LINES above. skill-audit carries all four; agent-audit carries
|
|
# only the two description constants, because ADR-0020 deliberately gives
|
|
# agents NO body word gate (a skill body competes with the caller's live
|
|
# conversation; an agent body becomes the system prompt of a fresh context).
|
|
# The absence of BODY_* in agent-audit is asserted below so a well-meaning
|
|
# "consistency" edit that adds them fails here rather than contradicting the
|
|
# ADR silently.
|
|
DESC_SUGGEST_CHARS="$(grep -oE '^DESC_SUGGEST_CHARS=[0-9]+' "$SCRIPT" | cut -d= -f2)"
|
|
DESC_MAX_CHARS="$(grep -oE '^DESC_MAX_CHARS=[0-9]+' "$SCRIPT" | cut -d= -f2)"
|
|
BODY_SUGGEST_WORDS="$(grep -oE '^BODY_SUGGEST_WORDS=[0-9]+' "$SCRIPT" | cut -d= -f2)"
|
|
BODY_MAX_WORDS="$(grep -oE '^BODY_MAX_WORDS=[0-9]+' "$SCRIPT" | cut -d= -f2)"
|
|
|
|
echo ""
|
|
echo "--- the hook declares all four ADR-0020 constants ---"
|
|
for pair in "DESC_SUGGEST_CHARS:$DESC_SUGGEST_CHARS" "DESC_MAX_CHARS:$DESC_MAX_CHARS" \
|
|
"BODY_SUGGEST_WORDS:$BODY_SUGGEST_WORDS" "BODY_MAX_WORDS:$BODY_MAX_WORDS"; do
|
|
if [[ -n "${pair#*:}" ]]; then
|
|
pass "${pair%%:*}=${pair#*:}"
|
|
else
|
|
fail "${pair%%:*} is not declared in $SCRIPT"
|
|
fi
|
|
done
|
|
|
|
echo ""
|
|
echo "--- the hook and skill-audit's validate.sh agree on all four ADR-0020 constants ---"
|
|
for const in DESC_SUGGEST_CHARS DESC_MAX_CHARS BODY_SUGGEST_WORDS BODY_MAX_WORDS; do
|
|
hook_value="$(grep -oE "^${const}=[0-9]+" "$SCRIPT" | cut -d= -f2)"
|
|
audit_value="$(grep -oE "^${const} = [0-9]+" "$VALIDATE" | grep -oE '[0-9]+' || true)"
|
|
if [[ -n "$hook_value" && "$hook_value" == "$audit_value" ]]; then
|
|
pass "both enforce $const=$hook_value"
|
|
else
|
|
fail "$const drift: hook says ${hook_value:-<unset>}, skill-audit validate.sh says ${audit_value:-<unset>}"
|
|
fi
|
|
done
|
|
|
|
echo ""
|
|
echo "--- the hook and agent-audit's validate.sh agree on the description constants ---"
|
|
if [[ ! -f "$AGENT_VALIDATE" ]]; then
|
|
fail "agent-audit validate.sh not found at $AGENT_VALIDATE"
|
|
else
|
|
for const in DESC_SUGGEST_CHARS DESC_MAX_CHARS; do
|
|
hook_value="$(grep -oE "^${const}=[0-9]+" "$SCRIPT" | cut -d= -f2)"
|
|
agent_value="$(grep -oE "^${const} = [0-9]+" "$AGENT_VALIDATE" | grep -oE '[0-9]+' || true)"
|
|
if [[ -n "$hook_value" && "$hook_value" == "$agent_value" ]]; then
|
|
pass "both enforce $const=$hook_value"
|
|
else
|
|
fail "$const drift: hook says ${hook_value:-<unset>}, agent-audit validate.sh says ${agent_value:-<unset>}"
|
|
fi
|
|
done
|
|
echo ""
|
|
echo "--- agent-audit declares NO body word gate (ADR-0020 is explicit about this) ---"
|
|
if grep -qE '^BODY_(SUGGEST|MAX)_WORDS = ' "$AGENT_VALIDATE"; then
|
|
fail "agent-audit validate.sh declares a body word gate — ADR-0020 gives agents the description gates and NO body word gate"
|
|
else
|
|
pass "agent-audit validate.sh declares no BODY_*_WORDS constant"
|
|
fi
|
|
fi
|
|
|
|
# make_line_fixture builds a file with an exact total line count (frontmatter
|
|
# included), independent of word count, for the line-boundary tests.
|
|
make_line_fixture() {
|
|
local name="$1" total_lines="$2" file body_lines
|
|
file="$TMPDIR/$name.md"
|
|
{
|
|
echo "---"
|
|
echo "name: $name"
|
|
echo "description: Test fixture."
|
|
echo "---"
|
|
} > "$file"
|
|
body_lines=$((total_lines - 4))
|
|
for ((i = 1; i <= body_lines; i++)); do
|
|
echo "word"
|
|
done >> "$file"
|
|
echo "$file"
|
|
}
|
|
|
|
# The line ceiling is inclusive of the limit itself, enforced via `>` — so
|
|
# exactly $MAX_LINES must pass and $((MAX_LINES + 1)) must fail. This matches
|
|
# skill-audit/scripts/validate.sh's `line_count <= 500` pass condition; the two
|
|
# previously disagreed at exactly $MAX_LINES lines, so a SKILL.md could pass its
|
|
# own audit and still be blocked by the commit hook.
|
|
echo ""
|
|
echo "--- passes a file at exactly the $MAX_LINES-line boundary ---"
|
|
AT_LINES="$(make_line_fixture at-line-limit "$MAX_LINES")"
|
|
ACTUAL_LINES=$(awk 'END{print NR}' "$AT_LINES")
|
|
if [[ "$ACTUAL_LINES" -ne "$MAX_LINES" ]]; then
|
|
fail "fixture has $ACTUAL_LINES lines, expected exactly $MAX_LINES"
|
|
elif "$SCRIPT" "$AT_LINES"; then
|
|
pass "file at exactly $MAX_LINES lines exits 0"
|
|
else
|
|
fail "file at exactly $MAX_LINES lines should have exited 0 (the off-by-one this test guards against)"
|
|
fi
|
|
|
|
echo ""
|
|
echo "--- fails a file one line over the $MAX_LINES-line boundary ---"
|
|
OVER_LINES="$(make_line_fixture over-line-limit "$((MAX_LINES + 1))")"
|
|
ACTUAL_OVER_LINES=$(awk 'END{print NR}' "$OVER_LINES")
|
|
if [[ "$ACTUAL_OVER_LINES" -ne "$((MAX_LINES + 1))" ]]; then
|
|
fail "fixture has $ACTUAL_OVER_LINES lines, expected exactly $((MAX_LINES + 1))"
|
|
elif "$SCRIPT" "$OVER_LINES" 2>/dev/null; then
|
|
fail "file at $((MAX_LINES + 1)) lines should have exited non-zero"
|
|
else
|
|
pass "file at $((MAX_LINES + 1)) lines exits non-zero"
|
|
fi
|
|
|
|
# make_word_fixture builds a file with an exact total word count (frontmatter
|
|
# words included, since the script's `wc -w` counts the whole file).
|
|
#
|
|
# The padding goes in a frontmatter `notes:` field, NOT in the body, and that
|
|
# placement is the point: MAX_WORDS is a whole-file measurement while ADR-0020's
|
|
# BODY_MAX_WORDS is a body-only one. Padding the body would make a 2,770-word
|
|
# fixture trip the 900-word body ceiling too, and the MAX_WORDS boundary test
|
|
# would stop isolating MAX_WORDS. `notes:` is an unused key — the description
|
|
# stays short, so the description gate stays quiet as well.
|
|
make_word_fixture() {
|
|
local name="$1" target="$2" file frame_words padding
|
|
file="$TMPDIR/$name.md"
|
|
padding=""
|
|
{
|
|
echo "---"
|
|
echo "name: $name"
|
|
echo "description: Test fixture."
|
|
echo "notes:$padding"
|
|
echo "---"
|
|
echo ""
|
|
echo "Body."
|
|
} > "$file"
|
|
frame_words=$(wc -w < "$file")
|
|
for ((i = 1; i <= target - frame_words; i++)); do
|
|
padding="$padding word"
|
|
done
|
|
{
|
|
echo "---"
|
|
echo "name: $name"
|
|
echo "description: Test fixture."
|
|
echo "notes:$padding"
|
|
echo "---"
|
|
echo ""
|
|
echo "Body."
|
|
} > "$file"
|
|
echo "$file"
|
|
}
|
|
|
|
echo ""
|
|
echo "--- passes a file at exactly the $MAX_WORDS-word boundary ---"
|
|
AT_WORDS="$(make_word_fixture at-word-limit "$MAX_WORDS")"
|
|
ACTUAL_WORDS=$(wc -w < "$AT_WORDS")
|
|
if [[ "$ACTUAL_WORDS" -ne "$MAX_WORDS" ]]; then
|
|
fail "fixture has $ACTUAL_WORDS words, expected exactly $MAX_WORDS"
|
|
elif "$SCRIPT" "$AT_WORDS"; then
|
|
pass "file at exactly $MAX_WORDS words exits 0"
|
|
else
|
|
fail "file at exactly $MAX_WORDS words should have exited 0"
|
|
fi
|
|
|
|
echo ""
|
|
echo "--- fails a file one word over the $MAX_WORDS-word boundary ---"
|
|
OVER_WORDS="$(make_word_fixture over-word-limit "$((MAX_WORDS + 1))")"
|
|
ACTUAL_OVER_WORDS=$(wc -w < "$OVER_WORDS")
|
|
if [[ "$ACTUAL_OVER_WORDS" -ne "$((MAX_WORDS + 1))" ]]; then
|
|
fail "fixture has $ACTUAL_OVER_WORDS words, expected exactly $((MAX_WORDS + 1))"
|
|
elif "$SCRIPT" "$OVER_WORDS" 2>/dev/null; then
|
|
fail "file at $((MAX_WORDS + 1)) words should have exited non-zero"
|
|
else
|
|
pass "file at $((MAX_WORDS + 1)) words exits non-zero"
|
|
fi
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# ADR-0020 behaviour
|
|
# ---------------------------------------------------------------------------
|
|
|
|
# make_budget_fixture builds a SKILL.md with a verbatim description and an
|
|
# exact BODY word count (frontmatter words excluded — the ADR-0020 body gate
|
|
# counts the body only).
|
|
make_budget_fixture() {
|
|
local name="$1" desc="$2" body_words="$3" file
|
|
file="$TMPDIR/$name.md"
|
|
{
|
|
echo "---"
|
|
echo "name: $name"
|
|
echo "description: $desc"
|
|
echo "---"
|
|
echo ""
|
|
python3 -c "print(' '.join(['word'] * $body_words))"
|
|
} > "$file"
|
|
echo "$file"
|
|
}
|
|
|
|
# desc_of_length <n> — a description of EXACTLY n characters that carries a
|
|
# boundary clause and names no routing target.
|
|
#
|
|
# ADR-0020's missing-boundary-clause SUGGESTION fires on every description
|
|
# without one, so a fixture that omits it is never "otherwise clean": a test
|
|
# asserting silence would be asserting the boundary check's ABSENCE rather than
|
|
# the length boundary it names. The clause is paid for out of the same budget
|
|
# being measured (padding arithmetic, not a fixed suffix) so the character count
|
|
# stays exact. "anything else" is unhyphenated, so no routing target rides along.
|
|
desc_of_length() {
|
|
python3 - "$1" <<'PY'
|
|
import sys
|
|
n = int(sys.argv[1])
|
|
prefix = 'Use when doing the thing. Do not use for anything else. '
|
|
assert n >= len(prefix), 'requested description shorter than the boundary clause'
|
|
print(prefix + 'x' * (n - len(prefix)))
|
|
PY
|
|
}
|
|
|
|
# make_tree_fixture <label> <desc> <body_words> — a SKILL.md inside a synthetic
|
|
# apm plugin monorepo, so the boundary-target resolver has a universe.
|
|
#
|
|
# Resolution walks up FROM THE TARGET FILE to an authoring root (the nearest
|
|
# ancestor holding plugins/*/.apm/{skills,agents}, falling back to .git); it is
|
|
# never derived from the checker's own location, because deriving it from
|
|
# ${BASH_SOURCE} leaked this repo's 39-skill universe into every consumer repo
|
|
# running the hook. A fixture in a bare mktemp -d therefore has NO universe and
|
|
# correctly reports "DID NOT RUN" — that is not a bug to paper over with a
|
|
# looser assertion, it is why the fixture has to be a real tree:
|
|
#
|
|
# <root>/plugins/subject-plugin/.apm/skills/<label>/SKILL.md <- the subject
|
|
# <root>/plugins/subject-plugin/.apm/skills/sibling-skill/ <- same package
|
|
# <root>/plugins/subject-plugin/.apm/agents/sibling-agent.agent.md
|
|
# <root>/plugins/other-plugin/.apm/skills/cross-plugin-skill/ <- sibling plugin
|
|
#
|
|
# The sibling plugin is what makes "every plugin in the monorepo contributes its
|
|
# names" testable; without it a cross-plugin target and a typo are the same.
|
|
make_tree_fixture() {
|
|
local label="$1" desc="$2" body_words="$3" root apm
|
|
root="$TMPDIR/tree-$label"
|
|
apm="$root/plugins/subject-plugin/.apm"
|
|
mkdir -p "$apm/skills/$label" "$apm/skills/sibling-skill" "$apm/agents" \
|
|
"$root/plugins/other-plugin/.apm/skills/cross-plugin-skill"
|
|
: > "$apm/agents/sibling-agent.agent.md"
|
|
{
|
|
echo "---"
|
|
echo "name: $label"
|
|
echo "description: $desc"
|
|
echo "---"
|
|
echo ""
|
|
python3 -c "print(' '.join(['word'] * $body_words))"
|
|
} > "$apm/skills/$label/SKILL.md"
|
|
echo "$apm/skills/$label/SKILL.md"
|
|
}
|
|
|
|
# expect_gate <label> <expected: pass|suggest|fail> <file> [needle]
|
|
expect_gate() {
|
|
local label="$1" expected="$2" file="$3" needle="${4:-}" out status
|
|
set +e
|
|
out="$("$SCRIPT" "$file" 2>&1)"
|
|
status=$?
|
|
set -e
|
|
case "$expected" in
|
|
pass)
|
|
if [[ $status -eq 0 && -z "$out" ]]; then
|
|
pass "$label"
|
|
else
|
|
fail "$label (exit $status, output: ${out:-<empty>})"
|
|
fi
|
|
;;
|
|
suggest)
|
|
if [[ $status -eq 0 && "$out" == *"SUGGESTION"* && "$out" == *"$needle"* ]]; then
|
|
pass "$label"
|
|
else
|
|
fail "$label (exit $status, output: ${out:-<empty>})"
|
|
fi
|
|
;;
|
|
fail)
|
|
if [[ $status -ne 0 && "$out" == *"$needle"* ]]; then
|
|
pass "$label"
|
|
else
|
|
fail "$label (exit $status, output: ${out:-<empty>})"
|
|
fi
|
|
;;
|
|
# A check that DECLINED to run must say so and must not fail the file. The
|
|
# ERROR guard is the point: a declined check that also errored would satisfy
|
|
# a bare "output contains INFO" assertion.
|
|
info)
|
|
if [[ $status -eq 0 && "$out" == *"INFO"* && "$out" == *"$needle"* \
|
|
&& "$out" != *"ERROR"* ]]; then
|
|
pass "$label"
|
|
else
|
|
fail "$label (exit $status, output: ${out:-<empty>})"
|
|
fi
|
|
;;
|
|
esac
|
|
}
|
|
|
|
echo ""
|
|
echo "--- description budget: $DESC_SUGGEST_CHARS SUGGESTION / $DESC_MAX_CHARS FAIL, both inclusive ---"
|
|
D_AT_SUGGEST="$(desc_of_length "$DESC_SUGGEST_CHARS")"
|
|
D_OVER_SUGGEST="$(desc_of_length "$((DESC_SUGGEST_CHARS + 1))")"
|
|
D_AT_MAX="$(desc_of_length "$DESC_MAX_CHARS")"
|
|
D_OVER_MAX="$(desc_of_length "$((DESC_MAX_CHARS + 1))")"
|
|
expect_gate "description at exactly $DESC_SUGGEST_CHARS chars is silent" \
|
|
pass "$(make_budget_fixture desc-at-suggest "$D_AT_SUGGEST" 10)"
|
|
expect_gate "description at $((DESC_SUGGEST_CHARS + 1)) chars suggests and exits 0" \
|
|
suggest "$(make_budget_fixture desc-over-suggest "$D_OVER_SUGGEST" 10)" \
|
|
"description is $((DESC_SUGGEST_CHARS + 1)) characters"
|
|
expect_gate "description at exactly $DESC_MAX_CHARS chars suggests, does not fail" \
|
|
suggest "$(make_budget_fixture desc-at-max "$D_AT_MAX" 10)" \
|
|
"description is $DESC_MAX_CHARS characters"
|
|
expect_gate "description at $((DESC_MAX_CHARS + 1)) chars fails" \
|
|
fail "$(make_budget_fixture desc-over-max "$D_OVER_MAX" 10)" \
|
|
"$DESC_MAX_CHARS-character ceiling"
|
|
|
|
echo ""
|
|
echo "--- description length is measured after YAML folding is resolved ---"
|
|
FOLDED="$TMPDIR/folded.md"
|
|
{
|
|
echo "---"
|
|
echo "name: folded"
|
|
echo "description: >"
|
|
python3 -c "print('\n'.join([' ' + 'x' * 40] * 11))"
|
|
echo "---"
|
|
echo ""
|
|
echo "Do the thing."
|
|
} > "$FOLDED"
|
|
expect_gate "a >-folded 450-char description fails (raw first line would read as 1 char)" \
|
|
fail "$FOLDED" "description is 450 characters"
|
|
|
|
# Every body fixture below carries a boundary clause for the same reason
|
|
# desc_of_length() does: without one the missing-boundary-clause SUGGESTION
|
|
# fires and a body-budget test that asserts silence stops isolating the body
|
|
# budget. It is short, so the description gate stays quiet too.
|
|
CLEAN_DESC="Short valid description. Do not use for anything else."
|
|
echo ""
|
|
echo "--- body budget: $BODY_SUGGEST_WORDS SUGGESTION / $BODY_MAX_WORDS FAIL, body only, both inclusive ---"
|
|
expect_gate "body at exactly $BODY_SUGGEST_WORDS words is silent" \
|
|
pass "$(make_budget_fixture body-at-suggest "$CLEAN_DESC" "$BODY_SUGGEST_WORDS")"
|
|
expect_gate "body at $((BODY_SUGGEST_WORDS + 1)) words suggests and exits 0" \
|
|
suggest "$(make_budget_fixture body-over-suggest "$CLEAN_DESC" "$((BODY_SUGGEST_WORDS + 1))")" \
|
|
"body is $((BODY_SUGGEST_WORDS + 1)) words"
|
|
expect_gate "body at exactly $BODY_MAX_WORDS words suggests, does not fail" \
|
|
suggest "$(make_budget_fixture body-at-max "$CLEAN_DESC" "$BODY_MAX_WORDS")" \
|
|
"body is $BODY_MAX_WORDS words"
|
|
expect_gate "body at $((BODY_MAX_WORDS + 1)) words fails" \
|
|
fail "$(make_budget_fixture body-over-max "$CLEAN_DESC" "$((BODY_MAX_WORDS + 1))")" \
|
|
"$BODY_MAX_WORDS-word ceiling"
|
|
|
|
# The two word gates measure different things and must stay separable: a file
|
|
# whose FRONTMATTER pushes the whole-file count past the body ceiling must not
|
|
# trip the body gate, and a file under MAX_WORDS can still fail the body gate.
|
|
echo ""
|
|
echo "--- the body gate and the whole-file gate are independent measurements ---"
|
|
BODY_ONLY_DESC="$(python3 -c "print(' '.join(['w'] * 100))")"
|
|
expect_gate "frontmatter words do not count toward the $BODY_MAX_WORDS-word body ceiling" \
|
|
suggest "$(make_budget_fixture body-independent "$BODY_ONLY_DESC" "$((BODY_MAX_WORDS - 5))")" \
|
|
"words"
|
|
BIG_BODY="$(make_budget_fixture body-over-not-whole-file "$CLEAN_DESC" "$((BODY_MAX_WORDS + 1))")"
|
|
BIG_BODY_WORDS="$(wc -w < "$BIG_BODY")"
|
|
if [[ "$BIG_BODY_WORDS" -le "$MAX_WORDS" ]]; then
|
|
pass "the body-gate fixture is $BIG_BODY_WORDS whole-file words, well under MAX_WORDS=$MAX_WORDS — it fails on the body gate alone"
|
|
else
|
|
fail "the body-gate fixture is $BIG_BODY_WORDS whole-file words, which also trips MAX_WORDS=$MAX_WORDS — the test no longer isolates the body gate"
|
|
fi
|
|
|
|
echo ""
|
|
echo "--- resolvable boundary targets ---"
|
|
# Resolution is against the AUTHORING SOURCE (plugins/*/.apm/skills/ and
|
|
# plugins/*/.apm/agents/), reached by walking up FROM THE SKILL FILE. These
|
|
# fixtures therefore build their own synthetic monorepo (make_tree_fixture) and
|
|
# name only fixture-local targets: they must not depend on this repo's live
|
|
# skills, or renaming git-commits would break a test about extraction grammar.
|
|
expect_gate "a boundary target naming a sibling skill in the same package resolves" \
|
|
pass "$(make_tree_fixture target-ok \
|
|
"Use when doing the thing. Do not use for commits — use sibling-skill instead." 10)"
|
|
expect_gate "a boundary target naming a skill in a SIBLING PLUGIN resolves (that is what a monorepo means)" \
|
|
pass "$(make_tree_fixture target-cross-plugin \
|
|
"Use when doing the thing. Do not use for the other thing — use cross-plugin-skill instead." 10)"
|
|
expect_gate "a boundary target naming an AGENT resolves (agents are valid targets)" \
|
|
pass "$(make_tree_fixture target-agent-ok \
|
|
"Use when doing the thing. Do not use when the caller is an agent — invoke sibling-agent instead." 10)"
|
|
# CORROBORATED: `sibling-skill` resolves in the same sentence, which is what
|
|
# promotes a prose-form target from "reported" to "blocking". A lone prose-form
|
|
# target is deliberately not fatal — see the case below and the shared resolver's
|
|
# CORROBORATION note.
|
|
expect_gate "a boundary target that resolves to nothing fails when its sentence names one that does" \
|
|
fail "$(make_tree_fixture target-missing \
|
|
"Use when doing the thing. Do not use for improvements — use sibling-skill or no-such-skill-anywhere instead." 10)" \
|
|
"routes to 'no-such-skill-anywhere'"
|
|
# UNCORROBORATED: identical grammar to the case above, and identical grammar to
|
|
# "run `pre-commit` instead". Reported at SUGGESTION tier, exit 0 — a gate that
|
|
# ships hot with no baseline and no suppression mechanism must not block a commit
|
|
# on a token it cannot tell from a tool name.
|
|
expect_gate "a lone boundary target that resolves to nothing is reported, not fatal" \
|
|
suggest "$(make_tree_fixture target-missing-lone \
|
|
"Use when doing the thing. Do not use for improvements — use no-such-lone-skill instead." 10)" \
|
|
"routes to 'no-such-lone-skill'"
|
|
expect_gate "a /slash-command boundary target that resolves to nothing fails" \
|
|
fail "$(make_tree_fixture target-missing-slash \
|
|
"Use when doing the thing. Do not use for improvements — use /no-such-slash-skill instead." 10)" \
|
|
"routes to 'no-such-slash-skill'"
|
|
# False-positive guards. These phrasings are lifted from real descriptions:
|
|
# pc-run says "run pre-commit hooks", diagnose chains "fix -> regression-test",
|
|
# gitea-files says "(use Read/Write/Edit)", gitea-labels-milestones says
|
|
# "through `issue_write`/`pull_request_write`". None of them is a routing
|
|
# target, and reading any of them as one makes the gate untrustworthy.
|
|
#
|
|
# Each carries a boundary clause in a SEPARATE sentence. That is not decoration:
|
|
# target extraction is decided per sentence, so the clause satisfies the
|
|
# missing-boundary-clause SUGGESTION (keeping the expected output empty) while
|
|
# leaving the sentence under test outside a boundary context, which is the exact
|
|
# condition each of these is about. They are built as trees so a universe exists
|
|
# — in a bare temp dir the resolver would decline and the guard would pass
|
|
# vacuously, proving nothing about extraction.
|
|
expect_gate "'run pre-commit hooks' outside a boundary sentence is not a routing target" \
|
|
pass "$(make_tree_fixture fp-precommit \
|
|
"Use when the user wants to run pre-commit hooks or install git hooks. Do not use for anything else." 10)"
|
|
expect_gate "an arrow chain outside a boundary clause is not a routing target" \
|
|
pass "$(make_tree_fixture fp-arrow \
|
|
"Reproduce → minimise → instrument → fix → regression-test. Use when a bug is reported. Do not use for anything else." 10)"
|
|
expect_gate "tool names and MCP tool names are not routing targets" \
|
|
pass "$(make_tree_fixture fp-tools \
|
|
"Use when writing issues. Do not use for local files (use Read/Write/Edit) — that write goes through \`issue_write\`/\`pull_request_write\` instead." 10)"
|
|
|
|
echo ""
|
|
echo "--- with NO authoring root the resolver declines OUT LOUD and does not fail the file ---"
|
|
# The consumer/draft case, and a real one: a SKILL.md in a bare directory with no
|
|
# plugins/*/.apm/ above it and no .git has no universe to resolve against. The
|
|
# required behaviour is neither a false FAIL nor silence — silence is how a whole
|
|
# gate family goes missing unnoticed — so the INFO and the named unchecked target
|
|
# are both asserted, along with exit 0. This is the same path make_tree_fixture
|
|
# exists to escape, kept pinned so a future "just use the repo root" shortcut
|
|
# (the ${BASH_SOURCE} universe leak ADR-0020 removed) fails here.
|
|
expect_gate "a fixture with no authoring root reports DID NOT RUN and exits 0" \
|
|
info "$(make_budget_fixture no-universe \
|
|
"Use when doing the thing. Do not use for improvements — use some-other-skill instead." 10)" \
|
|
"Unchecked target(s): some-other-skill"
|
|
|
|
echo ""
|
|
echo "--- the live dangling routing targets are caught (issue #100) ---"
|
|
# ADR-0020 records the broken routing targets and splits fixing them into its own
|
|
# issue. This asserts the gate actually sees them rather than the check being
|
|
# vacuous in the corpus it was written against.
|
|
#
|
|
# There used to be a third probe here, for `skill-improve` in skill-audit's
|
|
# description. It was already stale: that target was fixed, so the iteration
|
|
# permanently took a `pass "SKIP: ..."` branch — an assertion-free result counted
|
|
# in the totals, which is worse than no probe at all because it makes the suite
|
|
# look one test stronger than it is. It also contradicted
|
|
# tests/test-adr0020-targets.sh, which pins the live dangling set as EXACTLY
|
|
# {neuledge-context}; that file is the authority on the set, this one only
|
|
# checks each member is individually detected.
|
|
#
|
|
# Both SKIP branches are gone with it, for the same reason. A probe whose fixture
|
|
# has been retrofitted is not "still passing" — it is a pin that needs updating,
|
|
# here and in the exact-set assertion in test-adr0020-targets.sh, and it should
|
|
# say so out loud rather than quietly agreeing with whatever it finds.
|
|
# The gitea-labels probe was dropped when the issue #99 retrofit cut the
|
|
# composition sentence whose YAML fold produced that target. Per the rule above
|
|
# it is removed, not skipped.
|
|
# shellcheck disable=SC2043 # one probe left by design -- the list shrinks as
|
|
# each fixture is retrofitted and reaches zero when `research` lands. Keeping the
|
|
# loop means removing the last entry is a one-line edit, not a restructure.
|
|
for probe in \
|
|
"plugins/bin/.apm/skills/research/SKILL.md:neuledge-context"; do
|
|
probe_file="$REPO_ROOT/${probe%%:*}"
|
|
probe_name="${probe##*:}"
|
|
if [[ ! -f "$probe_file" ]]; then
|
|
fail "the probe fixture ${probe%%:*} no longer exists — this pin has become vacuous; update it and EXPECTED_DANGLING in tests/test-adr0020-targets.sh together"
|
|
continue
|
|
fi
|
|
# Captured, not piped: the script exits non-zero on these files and
|
|
# `set -o pipefail` would make the whole pipeline non-zero regardless of what
|
|
# grep found.
|
|
set +e
|
|
probe_out="$("$SCRIPT" "$probe_file" 2>&1)"
|
|
set -e
|
|
if [[ "$probe_out" == *"routes to '$probe_name'"* ]]; then
|
|
pass "detects the dangling '$probe_name' target in ${probe%%:*}"
|
|
else
|
|
fail "did not detect the dangling '$probe_name' target in ${probe%%:*}. If issue #100 retrofitted it, drop this probe and update EXPECTED_DANGLING in tests/test-adr0020-targets.sh; if a false-positive fix took a true positive with it, that is the regression this asserts."
|
|
fi
|
|
done
|
|
|
|
echo ""
|
|
echo "Results: $PASS passed, $FAIL failed"
|
|
[[ $FAIL -eq 0 ]]
|