#!/usr/bin/env bash # Regression test for scripts/skill-size-check.sh, which enforces two # independent gate families that must not be conflated: # # * agentskills.io spec conformance — 500 lines and a 5,000-token ceiling # enforced via a word-count proxy (MAX_WORDS, currently 2770) over the # WHOLE FILE, frontmatter included. # * ADR-0020 context budget — description 250 chars SUGGESTION / 400 FAIL, # body-ONLY 600 words SUGGESTION / 900 FAIL, and resolvable boundary-clause # routing targets. # # The constant-agreement block below is the load-bearing part: all three copies # (this hook, skill-audit's validate.sh, agent-audit's validate.sh) are # hand-duplicated because a cache-installed plugin cannot read outside its own # directory, and nothing but these assertions stops them drifting. set -euo pipefail REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" SCRIPT="$REPO_ROOT/scripts/skill-size-check.sh" VALIDATE="$REPO_ROOT/plugins/kyberforge/.apm/skills/skill-audit/scripts/validate.sh" AGENT_VALIDATE="$REPO_ROOT/plugins/kyberforge/.apm/skills/agent-audit/scripts/validate.sh" PASS=0 FAIL=0 pass() { echo " PASS: $1"; PASS=$((PASS + 1)); } fail() { echo " FAIL: $1"; FAIL=$((FAIL + 1)); } TMPDIR="$(mktemp -d)" trap 'rm -rf "$TMPDIR"' EXIT make_fixture() { local name="$1" lines="$2" words_per_line="$3" file file="$TMPDIR/$name.md" { echo "---" echo "name: $name" echo "description: Test fixture." echo "metadata:" echo " version: \"1.0.0\"" echo "---" for ((i = 1; i <= lines; i++)); do w="" for ((j = 1; j <= words_per_line; j++)); do w="$w word" done echo "$w" done } > "$file" echo "$file" } echo "" echo "--- passes a file under both limits ---" SMALL="$(make_fixture small 10 5)" if "$SCRIPT" "$SMALL"; then pass "file under both limits exits 0" else fail "file under both limits should have exited 0" fi echo "" echo "--- fails a file over the line limit ---" MANY_LINES="$(make_fixture many-lines 600 1)" if "$SCRIPT" "$MANY_LINES" 2>/dev/null; then fail "file over the 500-line ceiling should have exited non-zero" else pass "file over the 500-line ceiling exits non-zero" fi echo "" echo "--- fails a file over the word-count limit ---" MANY_WORDS="$(make_fixture many-words 10 600)" if "$SCRIPT" "$MANY_WORDS" 2>/dev/null; then fail "file over the word ceiling should have exited non-zero" else pass "file over the word ceiling exits non-zero" fi # Boundary-pair tests below read the script's current MAX_WORDS rather than # hardcoding it, so they don't silently drift if the threshold changes again. MAX_WORDS="$(grep -oE '^MAX_WORDS=[0-9]+' "$SCRIPT" | cut -d= -f2)" MAX_LINES="$(grep -oE '^MAX_LINES=[0-9]+' "$SCRIPT" | cut -d= -f2)" # The audit (skill-audit/scripts/validate.sh) duplicates both ceilings, because # a cache-installed plugin's scripts cannot read files outside the plugin # directory. Nothing but this assertion stops the copies drifting, and drift # means a SKILL.md passes its own audit and is then rejected by the commit hook. echo "" echo "--- the hook and skill-audit's validate.sh agree on both ceilings ---" if [[ ! -f "$VALIDATE" ]]; then fail "skill-audit validate.sh not found at $VALIDATE" else V_MAX_WORDS="$(grep -oE '^MAX_WORDS = [0-9]+' "$VALIDATE" | grep -oE '[0-9]+')" V_MAX_LINES="$(grep -oE '^MAX_LINES = [0-9]+' "$VALIDATE" | grep -oE '[0-9]+')" if [[ "$V_MAX_WORDS" == "$MAX_WORDS" ]]; then pass "both enforce MAX_WORDS=$MAX_WORDS" else fail "MAX_WORDS drift: hook says $MAX_WORDS, validate.sh says ${V_MAX_WORDS:-}" fi if [[ "$V_MAX_LINES" == "$MAX_LINES" ]]; then pass "both enforce MAX_LINES=$MAX_LINES" else fail "MAX_LINES drift: hook says $MAX_LINES, validate.sh says ${V_MAX_LINES:-}" fi fi # --------------------------------------------------------------------------- # ADR-0020 constants # --------------------------------------------------------------------------- # Three hand-maintained copies, for the same cache-isolation reason as # MAX_WORDS/MAX_LINES above. skill-audit carries all four; agent-audit carries # only the two description constants, because ADR-0020 deliberately gives # agents NO body word gate (a skill body competes with the caller's live # conversation; an agent body becomes the system prompt of a fresh context). # The absence of BODY_* in agent-audit is asserted below so a well-meaning # "consistency" edit that adds them fails here rather than contradicting the # ADR silently. DESC_SUGGEST_CHARS="$(grep -oE '^DESC_SUGGEST_CHARS=[0-9]+' "$SCRIPT" | cut -d= -f2)" DESC_MAX_CHARS="$(grep -oE '^DESC_MAX_CHARS=[0-9]+' "$SCRIPT" | cut -d= -f2)" BODY_SUGGEST_WORDS="$(grep -oE '^BODY_SUGGEST_WORDS=[0-9]+' "$SCRIPT" | cut -d= -f2)" BODY_MAX_WORDS="$(grep -oE '^BODY_MAX_WORDS=[0-9]+' "$SCRIPT" | cut -d= -f2)" echo "" echo "--- the hook declares all four ADR-0020 constants ---" for pair in "DESC_SUGGEST_CHARS:$DESC_SUGGEST_CHARS" "DESC_MAX_CHARS:$DESC_MAX_CHARS" \ "BODY_SUGGEST_WORDS:$BODY_SUGGEST_WORDS" "BODY_MAX_WORDS:$BODY_MAX_WORDS"; do if [[ -n "${pair#*:}" ]]; then pass "${pair%%:*}=${pair#*:}" else fail "${pair%%:*} is not declared in $SCRIPT" fi done echo "" echo "--- the hook and skill-audit's validate.sh agree on all four ADR-0020 constants ---" for const in DESC_SUGGEST_CHARS DESC_MAX_CHARS BODY_SUGGEST_WORDS BODY_MAX_WORDS; do hook_value="$(grep -oE "^${const}=[0-9]+" "$SCRIPT" | cut -d= -f2)" audit_value="$(grep -oE "^${const} = [0-9]+" "$VALIDATE" | grep -oE '[0-9]+' || true)" if [[ -n "$hook_value" && "$hook_value" == "$audit_value" ]]; then pass "both enforce $const=$hook_value" else fail "$const drift: hook says ${hook_value:-}, skill-audit validate.sh says ${audit_value:-}" fi done echo "" echo "--- the hook and agent-audit's validate.sh agree on the description constants ---" if [[ ! -f "$AGENT_VALIDATE" ]]; then fail "agent-audit validate.sh not found at $AGENT_VALIDATE" else for const in DESC_SUGGEST_CHARS DESC_MAX_CHARS; do hook_value="$(grep -oE "^${const}=[0-9]+" "$SCRIPT" | cut -d= -f2)" agent_value="$(grep -oE "^${const} = [0-9]+" "$AGENT_VALIDATE" | grep -oE '[0-9]+' || true)" if [[ -n "$hook_value" && "$hook_value" == "$agent_value" ]]; then pass "both enforce $const=$hook_value" else fail "$const drift: hook says ${hook_value:-}, agent-audit validate.sh says ${agent_value:-}" fi done echo "" echo "--- agent-audit declares NO body word gate (ADR-0020 is explicit about this) ---" if grep -qE '^BODY_(SUGGEST|MAX)_WORDS = ' "$AGENT_VALIDATE"; then fail "agent-audit validate.sh declares a body word gate — ADR-0020 gives agents the description gates and NO body word gate" else pass "agent-audit validate.sh declares no BODY_*_WORDS constant" fi fi # make_line_fixture builds a file with an exact total line count (frontmatter # included), independent of word count, for the line-boundary tests. make_line_fixture() { local name="$1" total_lines="$2" file body_lines file="$TMPDIR/$name.md" { echo "---" echo "name: $name" echo "description: Test fixture." echo "metadata:" echo " version: \"1.0.0\"" echo "---" } > "$file" body_lines=$((total_lines - 6)) for ((i = 1; i <= body_lines; i++)); do echo "word" done >> "$file" echo "$file" } # The line ceiling is inclusive of the limit itself, enforced via `>` — so # exactly $MAX_LINES must pass and $((MAX_LINES + 1)) must fail. This matches # skill-audit/scripts/validate.sh's `line_count <= 500` pass condition; the two # previously disagreed at exactly $MAX_LINES lines, so a SKILL.md could pass its # own audit and still be blocked by the commit hook. echo "" echo "--- passes a file at exactly the $MAX_LINES-line boundary ---" AT_LINES="$(make_line_fixture at-line-limit "$MAX_LINES")" ACTUAL_LINES=$(awk 'END{print NR}' "$AT_LINES") if [[ "$ACTUAL_LINES" -ne "$MAX_LINES" ]]; then fail "fixture has $ACTUAL_LINES lines, expected exactly $MAX_LINES" elif "$SCRIPT" "$AT_LINES"; then pass "file at exactly $MAX_LINES lines exits 0" else fail "file at exactly $MAX_LINES lines should have exited 0 (the off-by-one this test guards against)" fi echo "" echo "--- fails a file one line over the $MAX_LINES-line boundary ---" OVER_LINES="$(make_line_fixture over-line-limit "$((MAX_LINES + 1))")" ACTUAL_OVER_LINES=$(awk 'END{print NR}' "$OVER_LINES") if [[ "$ACTUAL_OVER_LINES" -ne "$((MAX_LINES + 1))" ]]; then fail "fixture has $ACTUAL_OVER_LINES lines, expected exactly $((MAX_LINES + 1))" elif "$SCRIPT" "$OVER_LINES" 2>/dev/null; then fail "file at $((MAX_LINES + 1)) lines should have exited non-zero" else pass "file at $((MAX_LINES + 1)) lines exits non-zero" fi # make_word_fixture builds a file with an exact total word count (frontmatter # words included, since the script's `wc -w` counts the whole file). # # The padding goes in a frontmatter `notes:` field, NOT in the body, and that # placement is the point: MAX_WORDS is a whole-file measurement while ADR-0020's # BODY_MAX_WORDS is a body-only one. Padding the body would make a 2,770-word # fixture trip the 900-word body ceiling too, and the MAX_WORDS boundary test # would stop isolating MAX_WORDS. `notes:` is an unused key — the description # stays short, so the description gate stays quiet as well. make_word_fixture() { local name="$1" target="$2" file frame_words padding file="$TMPDIR/$name.md" padding="" { echo "---" echo "name: $name" echo "description: Test fixture." echo "metadata:" echo " version: \"1.0.0\"" echo "notes:$padding" echo "---" echo "" echo "Body." } > "$file" frame_words=$(wc -w < "$file") for ((i = 1; i <= target - frame_words; i++)); do padding="$padding word" done { echo "---" echo "name: $name" echo "description: Test fixture." echo "metadata:" echo " version: \"1.0.0\"" echo "notes:$padding" echo "---" echo "" echo "Body." } > "$file" echo "$file" } echo "" echo "--- passes a file at exactly the $MAX_WORDS-word boundary ---" AT_WORDS="$(make_word_fixture at-word-limit "$MAX_WORDS")" ACTUAL_WORDS=$(wc -w < "$AT_WORDS") if [[ "$ACTUAL_WORDS" -ne "$MAX_WORDS" ]]; then fail "fixture has $ACTUAL_WORDS words, expected exactly $MAX_WORDS" elif "$SCRIPT" "$AT_WORDS"; then pass "file at exactly $MAX_WORDS words exits 0" else fail "file at exactly $MAX_WORDS words should have exited 0" fi echo "" echo "--- fails a file one word over the $MAX_WORDS-word boundary ---" OVER_WORDS="$(make_word_fixture over-word-limit "$((MAX_WORDS + 1))")" ACTUAL_OVER_WORDS=$(wc -w < "$OVER_WORDS") if [[ "$ACTUAL_OVER_WORDS" -ne "$((MAX_WORDS + 1))" ]]; then fail "fixture has $ACTUAL_OVER_WORDS words, expected exactly $((MAX_WORDS + 1))" elif "$SCRIPT" "$OVER_WORDS" 2>/dev/null; then fail "file at $((MAX_WORDS + 1)) words should have exited non-zero" else pass "file at $((MAX_WORDS + 1)) words exits non-zero" fi # --------------------------------------------------------------------------- # ADR-0020 behaviour # --------------------------------------------------------------------------- # make_budget_fixture builds a SKILL.md with a verbatim description and an # exact BODY word count (frontmatter words excluded — the ADR-0020 body gate # counts the body only). make_budget_fixture() { local name="$1" desc="$2" body_words="$3" file file="$TMPDIR/$name.md" { echo "---" echo "name: $name" echo "description: $desc" echo "metadata:" echo " version: \"1.0.0\"" echo "---" echo "" python3 -c "print(' '.join(['word'] * $body_words))" } > "$file" echo "$file" } # desc_of_length — a description of EXACTLY n characters that carries a # boundary clause and names no routing target. # # ADR-0020's missing-boundary-clause SUGGESTION fires on every description # without one, so a fixture that omits it is never "otherwise clean": a test # asserting silence would be asserting the boundary check's ABSENCE rather than # the length boundary it names. The clause is paid for out of the same budget # being measured (padding arithmetic, not a fixed suffix) so the character count # stays exact. "anything else" is unhyphenated, so no routing target rides along. desc_of_length() { python3 - "$1" <<'PY' import sys n = int(sys.argv[1]) prefix = 'Use when doing the thing. Do not use for anything else. ' assert n >= len(prefix), 'requested description shorter than the boundary clause' print(prefix + 'x' * (n - len(prefix))) PY } # make_tree_fixture