holocron/tests/test-vale-hooks-consumer.sh

#!/usr/bin/env bash
# Integration test for .pre-commit-hooks.yaml as an EXTERNAL hook repo — the
# contract ADR-0014 exists to provide, and the one thing running pre-commit
# inside this repo can never exercise: `repo: local` makes pre-commit's clone
# prefix equal to the consuming repo's root, so a hook entry that only works
# because those two coincide passes here and hard-fails everywhere else.
# (It did: every argument after entry[0] resolves against the CONSUMING repo,
# so a `--config plugins/.../.vale.ini` argument gave external consumers
# `E100 [--config] Runtime error ... does not exist`, exit 2, on both Vale hooks.)
#
# The hook repo is built from the WORKING TREE, not from HEAD, so an uncommitted
# change to the manifest or the wrapper is what gets tested.
set -euo pipefail

REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
PASS=0
FAIL=0

pass() { echo "  PASS: $1"; PASS=$((PASS + 1)); }
fail() { echo "  FAIL: $1"; FAIL=$((FAIL + 1)); }

for bin in pre-commit vale git; do
  if ! command -v "$bin" &>/dev/null; then
    echo "SKIP: $bin is not installed — cannot stand up a consumer repo"
    exit 77
  fi
done

WORK="$(mktemp -d)"
trap 'rm -rf "$WORK"' EXIT

HOOK_REPO="$WORK/hookrepo"
CONSUMER="$WORK/consumer"
export PRE_COMMIT_HOME="$WORK/pc-home"

mkdir -p "$HOOK_REPO/plugins/kyberforge/.apm/skills" "$HOOK_REPO/scripts"
cp "$REPO_ROOT/.pre-commit-hooks.yaml" "$HOOK_REPO/"
cp "$REPO_ROOT/scripts/skill-size-check.sh" "$HOOK_REPO/scripts/"
for skill in skill-audit agent-audit; do
  mkdir -p "$HOOK_REPO/plugins/kyberforge/.apm/skills/$skill"
  cp -R "$REPO_ROOT/plugins/kyberforge/.apm/skills/$skill/scripts" \
    "$REPO_ROOT/plugins/kyberforge/.apm/skills/$skill/assets" \
    "$HOOK_REPO/plugins/kyberforge/.apm/skills/$skill/"
done
git -C "$HOOK_REPO" init -q
git -C "$HOOK_REPO" add -A
git -C "$HOOK_REPO" -c user.email=test@example.invalid -c user.name=test commit -qm "hook repo"
HOOK_REV="$(git -C "$HOOK_REPO" rev-parse HEAD)"

# Every hook scopes by filename, so the consumer needs one file of each shape:
# a hook with nothing to match reports `Skipped` and proves nothing. All three
# hooks .pre-commit-hooks.yaml ships are registered — an unregistered one would
# let a regression (a lost `100755` bit, a bad entry path) reach every external
# consumer while this repo's own `repo: local` runs stayed green.
mkdir -p "$CONSUMER/skills/demo" "$CONSUMER/agents"
git -C "$CONSUMER" init -q
cat > "$CONSUMER/.pre-commit-config.yaml" <<EOF
repos:
  - repo: file://$HOOK_REPO
    rev: $HOOK_REV
    hooks:
      - id: kyberforge-vale-audit-skill
      - id: kyberforge-vale-audit-agent
      - id: kyberforge-skill-size-check
EOF

# The two fixtures carry DIFFERENT flagged tokens so an alert can never be
# credited to the hook that did not raise it. Both bodies land mid-sentence in a
# folded block scalar that still spans two physical lines, which is the
# flattening the wrapper exists to do.
write_fixtures() {
  local skill_body="$1"
  local agent_body="${2:-$1}"
  cat > "$CONSUMER/skills/demo/SKILL.md" <<EOF
---
name: demo
description: >
  Use when the caller wants a demonstration skill $skill_body across two
  physical lines of one folded block scalar.
---

Body.
EOF
  cat > "$CONSUMER/agents/demo.md" <<EOF
---
name: demo
description: >
  Use when the caller wants a demonstration agent $agent_body across two
  physical lines of one folded block scalar.
---

Body.
EOF
  git -C "$CONSUMER" add -A
}

# Vale prints each linted path as its own header line with that file's alerts
# indented beneath it, so an alert belongs to the nearest preceding path line.
# Reads a hook log on stdin and prints only the alert lines filed under `$1`.
# The `sed` strips vale's ANSI colouring, which it emits into pre-commit's pipe
# too, so the header lines compare as plain paths.
alerts_for() {
  sed $'s/\033\\[[0-9;]*m//g' | awk -v want="$1" '
    /^[^[:space:]].*\.md$/ { cur = $0; next }
    /^[[:space:]]*[0-9]+:[0-9]+[[:space:]]/ { if (cur == want) print }
  '
}

# --- 1. Each Vale hook resolves its config and gates its own file shape ---
# Asserted per hook, against that hook's own fixture path and its own token. An
# aggregate alert count over both hooks' combined output does not prove this:
# one fixture description carries every flagged token, so ONE working hook
# already clears a `>= 2` threshold. And a hook whose .vale.ini globs match
# nothing reaches neither of the guards below — it still MATCHES the file via
# its `files:` regex, so pre-commit does not report `Skipped`; vale simply lints
# nothing, prints `0 errors ... in 1 file` and exits 0, and the hook shows
# `Passed`. Attribution is the only thing that catches it.
echo ""
echo "--- each Vale hook flags its own fixture in an external consumer repo ---"
write_fixtures "that helps with things" "that will utilize things"
while IFS='|' read -r HOOK_ID FIXTURE TOKEN; do
  [[ -n "$HOOK_ID" ]] || continue
  LOG="$WORK/$HOOK_ID.log"
  set +e
  (cd "$CONSUMER" && pre-commit run "$HOOK_ID" --all-files > "$LOG" 2>&1)
  RC_HOOK=$?
  set -e
  if grep -q "does not exist" "$LOG"; then
    fail "$HOOK_ID hard-errored on a path resolved against the consumer repo (E100) — the bug this test guards against"
    sed 's/^/    /' "$LOG"
  elif grep -q "Skipped" "$LOG"; then
    fail "$HOOK_ID matched no files, so it proved nothing"
    sed 's/^/    /' "$LOG"
  elif [[ $RC_HOOK -eq 0 ]]; then
    fail "$HOOK_ID passed $FIXTURE despite its flagged '$TOKEN' — a .vale.ini glob matching nothing lints zero files and exits 0"
    sed 's/^/    /' "$LOG"
  elif alerts_for "$FIXTURE" < "$LOG" | grep -qF "'$TOKEN'"; then
    pass "$HOOK_ID flattens $FIXTURE and flags its '$TOKEN' in a consumer repo"
  else
    fail "$HOOK_ID failed, but no alert quoting '$TOKEN' was filed under $FIXTURE"
    sed 's/^/    /' "$LOG"
  fi
done <<'EOF'
kyberforge-vale-audit-skill|skills/demo/SKILL.md|helps with
kyberforge-vale-audit-agent|agents/demo.md|utilize
EOF

# --- 1b. Every rule in the shipped style is asserted to FIRE, not merely to
# exist. A Vale rule can be well-formed, load without a diagnostic, and match
# nothing at all: `extends: existence` CONCATENATES multiple `raw:` entries
# rather than alternating them, so a rule written as a list of alternatives
# silently becomes one impossible expression, lints every file clean and exits
# 0 — indistinguishable from a corpus with no violations. `Kyberforge.CompositionNote`
# was written that way first and passed all 43 skill and agent files before the
# defect was found by hand. Each rule gets its own fixture pass, with the token
# it must quote attributed to the file that raised it, so one rule's alert can
# never stand in for another's.
echo ""
echo "--- each Kyberforge description rule fires through both shipped hooks ---"
write_desc_fixtures() {
  local skill_desc="$1" agent_desc="$2"
  cat > "$CONSUMER/skills/demo/SKILL.md" <<EOF
---
name: demo
description: >
  $skill_desc across two
  physical lines of one folded block scalar.
---

Body.
EOF
  cat > "$CONSUMER/agents/demo.md" <<EOF
---
name: demo
description: >
  $agent_desc across two
  physical lines of one folded block scalar.
---

Body.
EOF
  git -C "$CONSUMER" add -A
}

run_rule_case() {
  local label="$1" hook_id="$2" fixture="$3" token="$4"
  local log="$WORK/rule-$label.log"
  set +e
  (cd "$CONSUMER" && pre-commit run "$hook_id" --all-files > "$log" 2>&1)
  local rc=$?
  set -e
  if grep -q "Skipped" "$log"; then
    fail "$hook_id matched no files for $label, so it proved nothing"
    sed 's/^/    /' "$log"
  elif [[ $rc -eq 0 ]]; then
    fail "$hook_id passed $fixture despite its flagged '$token' — $label matches nothing"
    sed 's/^/    /' "$log"
  elif alerts_for "$fixture" < "$log" | grep -qF "'$token'"; then
    pass "$label fires through $hook_id and quotes '$token' under $fixture"
  else
    fail "$hook_id failed, but no $label alert quoting '$token' was filed under $fixture"
    sed 's/^/    /' "$log"
  fi
}

# CompositionNote: a distinct banned token per file shape.
write_desc_fixtures \
  "Use when the caller wants a demo skill that composes other skills" \
  "Use when the caller wants a cross-cutting demo agent"
run_rule_case "Kyberforge.CompositionNote" kyberforge-vale-audit-skill skills/demo/SKILL.md "composes"
run_rule_case "Kyberforge.CompositionNote" kyberforge-vale-audit-agent agents/demo.md "cross-cutting"

# DescriptionOpener: the widened pattern catches every non-imperative "This..."
# opener, not only the literal "This skill"/"This agent" pair it was anchored to
# before. Both fixtures open with "This is", the form two shipped descriptions
# used mid-sentence and which the old pattern could not express.
write_desc_fixtures \
  "This is a demo skill for callers who want one" \
  "This is a demo agent for callers who want one"
run_rule_case "Kyberforge.DescriptionOpener" kyberforge-vale-audit-skill skills/demo/SKILL.md "This"
run_rule_case "Kyberforge.DescriptionOpener" kyberforge-vale-audit-agent agents/demo.md "This"

# --- 2. Clean files pass — the hooks gate, they don't just always fail ---
echo ""
echo "--- all three hooks pass clean files in an external consumer repo ---"
write_fixtures "of the packaged hook contract"
set +e
(cd "$CONSUMER" && pre-commit run --all-files > "$WORK/clean.log" 2>&1)
RC_CLEAN=$?
set -e
if grep -q "Skipped" "$WORK/clean.log"; then
  fail "a hook matched no files on the clean run, so it proved nothing"
  sed 's/^/    /' "$WORK/clean.log"
elif [[ $RC_CLEAN -eq 0 ]]; then
  pass "all three hooks exit 0 on clean files"
else
  fail "hooks failed on clean files (rc=$RC_CLEAN)"
  sed 's/^/    /' "$WORK/clean.log"
fi

# --- 3. The size hook gates too. It ran clean above, which is what proves it
# is executable and its entry path resolves; this half proves it still fails a
# file that breaks the ceiling rather than passing everything. ---
echo ""
echo "--- kyberforge-skill-size-check fails an oversized SKILL.md in an external consumer repo ---"
mkdir -p "$CONSUMER/skills/oversized"
{
  echo "---"
  echo "name: oversized"
  echo "description: Use when the caller wants an oversized fixture."
  echo "---"
  for ((i = 1; i <= 600; i++)); do
    echo "word"
  done
} > "$CONSUMER/skills/oversized/SKILL.md"
git -C "$CONSUMER" add -A
set +e
(cd "$CONSUMER" && pre-commit run kyberforge-skill-size-check --all-files > "$WORK/size.log" 2>&1)
RC_SIZE=$?
set -e
if [[ $RC_SIZE -ne 0 ]] && grep -q "500-line ceiling" "$WORK/size.log"; then
  pass "kyberforge-skill-size-check exits non-zero and names the ceiling it broke"
else
  fail "kyberforge-skill-size-check did not gate an oversized SKILL.md (rc=$RC_SIZE)"
  sed 's/^/    /' "$WORK/size.log"
fi

echo ""
echo "Results: $PASS passed, $FAIL failed"
[[ $FAIL -eq 0 ]]