holocron/tests/test-skill-size-check.sh

#!/usr/bin/env bash
# Regression test for scripts/skill-size-check.sh: enforces agentskills.io's
# 500-line/5,000-token SKILL.md size ceiling. The token half is enforced via a
# word-count proxy (MAX_WORDS, currently 2770) — 5,000 is the token ceiling,
# 2,770 is the word budget the script derives from it at the corpus's densest
# measured prose.
set -euo pipefail

REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
SCRIPT="$REPO_ROOT/scripts/skill-size-check.sh"
VALIDATE="$REPO_ROOT/plugins/kyberforge/.apm/skills/skill-audit/scripts/validate.sh"
PASS=0
FAIL=0

pass() { echo "  PASS: $1"; PASS=$((PASS + 1)); }
fail() { echo "  FAIL: $1"; FAIL=$((FAIL + 1)); }

TMPDIR="$(mktemp -d)"
trap 'rm -rf "$TMPDIR"' EXIT

make_fixture() {
  local name="$1" lines="$2" words_per_line="$3" file
  file="$TMPDIR/$name.md"
  {
    echo "---"
    echo "name: $name"
    echo "description: Test fixture."
    echo "---"
    for ((i = 1; i <= lines; i++)); do
      w=""
      for ((j = 1; j <= words_per_line; j++)); do
        w="$w word"
      done
      echo "$w"
    done
  } > "$file"
  echo "$file"
}

echo ""
echo "--- passes a file under both limits ---"
SMALL="$(make_fixture small 10 5)"
if "$SCRIPT" "$SMALL"; then
  pass "file under both limits exits 0"
else
  fail "file under both limits should have exited 0"
fi

echo ""
echo "--- fails a file over the line limit ---"
MANY_LINES="$(make_fixture many-lines 600 1)"
if "$SCRIPT" "$MANY_LINES" 2>/dev/null; then
  fail "file over the 500-line ceiling should have exited non-zero"
else
  pass "file over the 500-line ceiling exits non-zero"
fi

echo ""
echo "--- fails a file over the word-count limit ---"
MANY_WORDS="$(make_fixture many-words 10 600)"
if "$SCRIPT" "$MANY_WORDS" 2>/dev/null; then
  fail "file over the word ceiling should have exited non-zero"
else
  pass "file over the word ceiling exits non-zero"
fi

# Boundary-pair tests below read the script's current MAX_WORDS rather than
# hardcoding it, so they don't silently drift if the threshold changes again.
MAX_WORDS="$(grep -oE '^MAX_WORDS=[0-9]+' "$SCRIPT" | cut -d= -f2)"
MAX_LINES="$(grep -oE '^MAX_LINES=[0-9]+' "$SCRIPT" | cut -d= -f2)"

# The audit (skill-audit/scripts/validate.sh) duplicates both ceilings, because
# a cache-installed plugin's scripts cannot read files outside the plugin
# directory. Nothing but this assertion stops the copies drifting, and drift
# means a SKILL.md passes its own audit and is then rejected by the commit hook.
echo ""
echo "--- the hook and skill-audit's validate.sh agree on both ceilings ---"
if [[ ! -f "$VALIDATE" ]]; then
  fail "skill-audit validate.sh not found at $VALIDATE"
else
  V_MAX_WORDS="$(grep -oE '^MAX_WORDS = [0-9]+' "$VALIDATE" | grep -oE '[0-9]+')"
  V_MAX_LINES="$(grep -oE '^MAX_LINES = [0-9]+' "$VALIDATE" | grep -oE '[0-9]+')"
  if [[ "$V_MAX_WORDS" == "$MAX_WORDS" ]]; then
    pass "both enforce MAX_WORDS=$MAX_WORDS"
  else
    fail "MAX_WORDS drift: hook says $MAX_WORDS, validate.sh says ${V_MAX_WORDS:-<unset>}"
  fi
  if [[ "$V_MAX_LINES" == "$MAX_LINES" ]]; then
    pass "both enforce MAX_LINES=$MAX_LINES"
  else
    fail "MAX_LINES drift: hook says $MAX_LINES, validate.sh says ${V_MAX_LINES:-<unset>}"
  fi
fi

# make_line_fixture builds a file with an exact total line count (frontmatter
# included), independent of word count, for the line-boundary tests.
make_line_fixture() {
  local name="$1" total_lines="$2" file body_lines
  file="$TMPDIR/$name.md"
  {
    echo "---"
    echo "name: $name"
    echo "description: Test fixture."
    echo "---"
  } > "$file"
  body_lines=$((total_lines - 4))
  for ((i = 1; i <= body_lines; i++)); do
    echo "word"
  done >> "$file"
  echo "$file"
}

# The line ceiling is inclusive of the limit itself, enforced via `>` — so
# exactly $MAX_LINES must pass and $((MAX_LINES + 1)) must fail. This matches
# skill-audit/scripts/validate.sh's `line_count <= 500` pass condition; the two
# previously disagreed at exactly $MAX_LINES lines, so a SKILL.md could pass its
# own audit and still be blocked by the commit hook.
echo ""
echo "--- passes a file at exactly the $MAX_LINES-line boundary ---"
AT_LINES="$(make_line_fixture at-line-limit "$MAX_LINES")"
ACTUAL_LINES=$(awk 'END{print NR}' "$AT_LINES")
if [[ "$ACTUAL_LINES" -ne "$MAX_LINES" ]]; then
  fail "fixture has $ACTUAL_LINES lines, expected exactly $MAX_LINES"
elif "$SCRIPT" "$AT_LINES"; then
  pass "file at exactly $MAX_LINES lines exits 0"
else
  fail "file at exactly $MAX_LINES lines should have exited 0 (the off-by-one this test guards against)"
fi

echo ""
echo "--- fails a file one line over the $MAX_LINES-line boundary ---"
OVER_LINES="$(make_line_fixture over-line-limit "$((MAX_LINES + 1))")"
ACTUAL_OVER_LINES=$(awk 'END{print NR}' "$OVER_LINES")
if [[ "$ACTUAL_OVER_LINES" -ne "$((MAX_LINES + 1))" ]]; then
  fail "fixture has $ACTUAL_OVER_LINES lines, expected exactly $((MAX_LINES + 1))"
elif "$SCRIPT" "$OVER_LINES" 2>/dev/null; then
  fail "file at $((MAX_LINES + 1)) lines should have exited non-zero"
else
  pass "file at $((MAX_LINES + 1)) lines exits non-zero"
fi

# make_word_fixture builds a file with an exact total word count (frontmatter
# words included, since the script's `wc -w` counts the whole file) by padding
# a body line with just enough "word" tokens to close the gap to the target.
make_word_fixture() {
  local name="$1" target="$2" file cur remaining body
  file="$TMPDIR/$name.md"
  {
    echo "---"
    echo "name: $name"
    echo "description: Test fixture."
    echo "---"
  } > "$file"
  cur=$(wc -w < "$file")
  remaining=$((target - cur))
  body=""
  for ((i = 1; i <= remaining; i++)); do
    body="$body word"
  done
  echo "$body" >> "$file"
  echo "$file"
}

echo ""
echo "--- passes a file at exactly the $MAX_WORDS-word boundary ---"
AT_WORDS="$(make_word_fixture at-word-limit "$MAX_WORDS")"
ACTUAL_WORDS=$(wc -w < "$AT_WORDS")
if [[ "$ACTUAL_WORDS" -ne "$MAX_WORDS" ]]; then
  fail "fixture has $ACTUAL_WORDS words, expected exactly $MAX_WORDS"
elif "$SCRIPT" "$AT_WORDS"; then
  pass "file at exactly $MAX_WORDS words exits 0"
else
  fail "file at exactly $MAX_WORDS words should have exited 0"
fi

echo ""
echo "--- fails a file one word over the $MAX_WORDS-word boundary ---"
OVER_WORDS="$(make_word_fixture over-word-limit "$((MAX_WORDS + 1))")"
ACTUAL_OVER_WORDS=$(wc -w < "$OVER_WORDS")
if [[ "$ACTUAL_OVER_WORDS" -ne "$((MAX_WORDS + 1))" ]]; then
  fail "fixture has $ACTUAL_OVER_WORDS words, expected exactly $((MAX_WORDS + 1))"
elif "$SCRIPT" "$OVER_WORDS" 2>/dev/null; then
  fail "file at $((MAX_WORDS + 1)) words should have exited non-zero"
else
  pass "file at $((MAX_WORDS + 1)) words exits non-zero"
fi

echo ""
echo "Results: $PASS passed, $FAIL failed"
[[ $FAIL -eq 0 ]]