holocron/tests/test-check-vale-style-sync.sh

#!/usr/bin/env bash
set -euo pipefail

REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
SCRIPT="$REPO_ROOT/scripts/check-vale-style-sync.sh"
PASS=0
FAIL=0

pass() { echo "  PASS: $1"; PASS=$((PASS + 1)); }
fail() { echo "  FAIL: $1"; FAIL=$((FAIL + 1)); }

# One trap over a registry, rather than rebuilding the trap line per fixture:
# the guard is there because bash 3.2 treats "${arr[@]}" on an empty array as
# unbound under `set -u`.
FIXTURES=()
cleanup() { [[ ${#FIXTURES[@]} -eq 0 ]] || rm -rf "${FIXTURES[@]}"; }
trap cleanup EXIT

# Helper: make a fixture repo with skill-audit/agent-audit's Vale copies, in sync by default.
# The wrapper is a stub — the script only diffs it — but the Vale assets and both
# pre-commit manifests are the repo's real ones, because the .vale.ini checks ask
# vale to apply those globs for real and cross-check them against the shipped
# hooks' `files:` regexes. A synthetic style or manifest would prove nothing, and
# copying the real ones keeps agent-audit's intentional KyberforgeCopilot
# divergence in the fixture instead of a sanitized stand-in for it.
make_fixture() {
  local dir
  dir="$(mktemp -d)"
  local skill_audit="$dir/plugins/kyberforge/.apm/skills/skill-audit"
  local agent_audit="$dir/plugins/kyberforge/.apm/skills/agent-audit"
  mkdir -p "$skill_audit/scripts" "$agent_audit/scripts"

  echo '#!/usr/bin/env bash' > "$skill_audit/scripts/vale-wrap.sh"
  echo 'echo wrap' >> "$skill_audit/scripts/vale-wrap.sh"
  cp "$skill_audit/scripts/vale-wrap.sh" "$agent_audit/scripts/vale-wrap.sh"

  cp -R "$REPO_ROOT/plugins/kyberforge/.apm/skills/skill-audit/assets" "$skill_audit/"
  cp -R "$REPO_ROOT/plugins/kyberforge/.apm/skills/agent-audit/assets" "$agent_audit/"
  cp "$REPO_ROOT/.pre-commit-hooks.yaml" "$REPO_ROOT/.pre-commit-config.yaml" "$dir/"

  echo "$dir"
}

# Helper: rewrite a glob section header in one copy's .vale.ini, leaving every
# other line — StylesPath, BasedOnStyles — intact. This is the shape of the
# typo the check exists to catch: the hook still matches the file via its
# `files:` regex, vale lints nothing, and pre-commit reports `Passed`.
break_glob() {
  local ini="$1" old="$2" new="$3"
  python3 - "$ini" "$old" "$new" <<'PYTHON'
import sys
path, old, new = sys.argv[1], sys.argv[2], sys.argv[3]
with open(path, encoding='utf-8') as fh:
    content = fh.read()
assert old in content, f"{old} not found in {path}"
with open(path, 'w', encoding='utf-8') as fh:
    fh.write(content.replace(old, new))
PYTHON
}

# --- 1. Exits 0 when the two copies are in sync ---
echo ""
echo "--- exits 0 when skill-audit and agent-audit copies are in sync ---"
FIXTURE="$(make_fixture)"
FIXTURES+=("$FIXTURE")
if bash "$SCRIPT" "$FIXTURE" > /dev/null 2>&1; then
  pass "exits 0 when copies are in sync"
else
  fail "exited non-zero against in-sync copies"
  bash "$SCRIPT" "$FIXTURE" 2>&1 | sed 's/^/    /' || true
fi

# --- 2. Exits 1 when vale-wrap.sh differs between the two copies ---
echo ""
echo "--- exits 1 when vale-wrap.sh differs ---"
FIXTURE2="$(make_fixture)"
FIXTURES+=("$FIXTURE2")
echo 'echo different' >> "$FIXTURE2/plugins/kyberforge/.apm/skills/skill-audit/scripts/vale-wrap.sh"
if bash "$SCRIPT" "$FIXTURE2" > /dev/null 2>&1; then
  fail "exited 0 when vale-wrap.sh copies differ — expected exit 1"
else
  pass "exits non-zero when vale-wrap.sh copies differ"
fi

# --- 3. Exits 1 when a style rule differs between the two copies ---
echo ""
echo "--- exits 1 when a Kyberforge style rule differs ---"
FIXTURE3="$(make_fixture)"
FIXTURES+=("$FIXTURE3")
echo '  - divergent token' >> "$FIXTURE3/plugins/kyberforge/.apm/skills/agent-audit/assets/vale/styles/Kyberforge/VagueWording.yml"
if bash "$SCRIPT" "$FIXTURE3" > /dev/null 2>&1; then
  fail "exited 0 when a style rule differs — expected exit 1"
else
  pass "exits non-zero when a Kyberforge style rule differs between copies"
fi

# --- 4. Exits 1 when a rule file exists in only one copy ---
echo ""
echo "--- exits 1 when a rule file is missing from one copy ---"
FIXTURE4="$(make_fixture)"
FIXTURES+=("$FIXTURE4")
cat > "$FIXTURE4/plugins/kyberforge/.apm/skills/agent-audit/assets/vale/styles/Kyberforge/Extra.yml" <<'EOF'
extends: existence
message: "Extra: '%s'"
level: error
tokens:
  - divergent token
EOF
if bash "$SCRIPT" "$FIXTURE4" > /dev/null 2>&1; then
  fail "exited 0 when a rule file exists in only one copy — expected exit 1"
else
  pass "exits non-zero when a rule file is missing from one copy"
fi

# --- 5. Exits 0 (no-op) when kyberforge isn't present in the target repo ---
echo ""
echo "--- exits 0 when kyberforge skills are absent (no-op) ---"
FIXTURE5="$(mktemp -d)"
FIXTURES+=("$FIXTURE5")
if bash "$SCRIPT" "$FIXTURE5" > /dev/null 2>&1; then
  pass "exits 0 as a no-op when skill-audit/agent-audit don't exist"
else
  fail "exited non-zero when skill-audit/agent-audit are simply absent"
fi

# --- 5b. Exits 1 when REPO_ROOT does not exist ---
# A nonexistent path used to fall through to the "neither copy present" no-op
# (test 5 above) and exit 0 — indistinguishable from a real, verified in-sync
# result. That guard is for a repo legitimately missing kyberforge, not a
# typo'd or stale path.
echo ""
echo "--- exits 1 when REPO_ROOT does not exist ---"
if bash "$SCRIPT" "/nonexistent/path/$(date +%s)-$$" > /dev/null 2>&1; then
  fail "exited 0 for a nonexistent REPO_ROOT — expected exit 1"
else
  pass "exits non-zero for a nonexistent REPO_ROOT"
fi

# --- 6. Exits 1 when only one of the two copies is present ---
# The no-op guard used `||`, so a single missing copy also exited 0 — a deleted
# or renamed copy passed the sync check silently.
echo ""
echo "--- exits 1 when only one of the two copies is present ---"
FIXTURE6="$(make_fixture)"
FIXTURE7="$(make_fixture)"
FIXTURES+=("$FIXTURE6" "$FIXTURE7")
rm -rf "$FIXTURE6/plugins/kyberforge/.apm/skills/skill-audit"
rm -rf "$FIXTURE7/plugins/kyberforge/.apm/skills/agent-audit"
if bash "$SCRIPT" "$FIXTURE6" > /dev/null 2>&1; then
  fail "exited 0 when only agent-audit is present — expected exit 1"
else
  pass "exits non-zero when skill-audit's copy is missing but agent-audit's is present"
fi
if bash "$SCRIPT" "$FIXTURE7" > /dev/null 2>&1; then
  fail "exited 0 when only skill-audit is present — expected exit 1"
else
  pass "exits non-zero when agent-audit's canonical copy is missing but skill-audit's is present"
fi

# --- 7. Exits 1 when a .vale.ini is missing entirely ---
# Without it vale falls back to an upward config search and lints the file with
# whatever config it happens to find, which is not a failure anyone sees.
echo ""
echo "--- exits 1 when a .vale.ini is missing ---"
FIXTURE8="$(make_fixture)"
FIXTURES+=("$FIXTURE8")
rm -f "$FIXTURE8/plugins/kyberforge/.apm/skills/skill-audit/assets/vale/.vale.ini"
if bash "$SCRIPT" "$FIXTURE8" > /dev/null 2>&1; then
  fail "exited 0 when skill-audit's .vale.ini is missing — expected exit 1"
else
  pass "exits non-zero when a .vale.ini is missing"
fi

# --- 8. Exits 1 when the shared StylesPath line is dropped from either copy ---
# StylesPath resolves relative to the .vale.ini, which is the only reason the
# bundled styles are found from a consuming repo's clone prefix.
echo ""
echo "--- exits 1 when StylesPath is missing from either .vale.ini ---"
FIXTURE9="$(make_fixture)"
FIXTURE10="$(make_fixture)"
FIXTURES+=("$FIXTURE9" "$FIXTURE10")
break_glob "$FIXTURE9/plugins/kyberforge/.apm/skills/skill-audit/assets/vale/.vale.ini" \
  'StylesPath = styles' 'StylesPath = elsewhere'
break_glob "$FIXTURE10/plugins/kyberforge/.apm/skills/agent-audit/assets/vale/.vale.ini" \
  'StylesPath = styles' 'StylesPath = elsewhere'
if bash "$SCRIPT" "$FIXTURE9" > /dev/null 2>&1; then
  fail "exited 0 when skill-audit's .vale.ini lost StylesPath — expected exit 1"
else
  pass "exits non-zero when skill-audit's .vale.ini lost StylesPath"
fi
if bash "$SCRIPT" "$FIXTURE10" > /dev/null 2>&1; then
  fail "exited 0 when agent-audit's .vale.ini lost StylesPath — expected exit 1"
else
  pass "exits non-zero when agent-audit's .vale.ini lost StylesPath"
fi

# --- 9. Exits 1 when no section's BasedOnStyles names Kyberforge ---
# Every rule the prefilter gates on lives in that style, so a section that keeps
# its glob but loses the style lints the file and reports nothing.
echo ""
echo "--- exits 1 when BasedOnStyles no longer names Kyberforge ---"
FIXTURE11="$(make_fixture)"
FIXTURES+=("$FIXTURE11")
break_glob "$FIXTURE11/plugins/kyberforge/.apm/skills/agent-audit/assets/vale/.vale.ini" \
  'BasedOnStyles = Kyberforge' 'BasedOnStyles = KyberforgeCopilot'
if bash "$SCRIPT" "$FIXTURE11" > /dev/null 2>&1; then
  fail "exited 0 when agent-audit's .vale.ini stopped naming Kyberforge — expected exit 1"
else
  pass "exits non-zero when a .vale.ini no longer names the Kyberforge style"
fi

# --- 10. Exits 1 when a glob section stops matching the shape its hook lints ---
# One case per glob section, because each covers a file shape the others don't:
# agent-audit's [**/*.agent.md] is the only section covering a Copilot agent file
# outside an agents/ directory, so breaking it alone is invisible to the others.
echo ""
echo "--- exits 1 when a .vale.ini glob no longer matches its hook's file shape ---"
FIXTURE12="$(make_fixture)"
FIXTURE13="$(make_fixture)"
FIXTURE14="$(make_fixture)"
FIXTURES+=("$FIXTURE12" "$FIXTURE13" "$FIXTURE14")
break_glob "$FIXTURE12/plugins/kyberforge/.apm/skills/skill-audit/assets/vale/.vale.ini" \
  '[**/SKILL.md]' '[**/NOMATCH.md]'
break_glob "$FIXTURE13/plugins/kyberforge/.apm/skills/agent-audit/assets/vale/.vale.ini" \
  '[**/agents/*.md]' '[**/NOMATCH-agents/*.md]'
break_glob "$FIXTURE14/plugins/kyberforge/.apm/skills/agent-audit/assets/vale/.vale.ini" \
  '[**/*.agent.md]' '[**/*.NOMATCH.md]'
if bash "$SCRIPT" "$FIXTURE12" > /dev/null 2>&1; then
  fail "exited 0 when skill-audit's SKILL.md glob matched nothing — expected exit 1"
else
  pass "exits non-zero when skill-audit's SKILL.md glob matches nothing"
fi
if bash "$SCRIPT" "$FIXTURE13" > /dev/null 2>&1; then
  fail "exited 0 when agent-audit's agents/*.md glob matched nothing — expected exit 1"
else
  pass "exits non-zero when agent-audit's agents/*.md glob matches nothing"
fi
if bash "$SCRIPT" "$FIXTURE14" > /dev/null 2>&1; then
  fail "exited 0 when agent-audit's *.agent.md glob matched nothing — expected exit 1"
else
  pass "exits non-zero when agent-audit's *.agent.md glob matches nothing"
fi

# --- 11. Exits 1 when a probe path falls out of every hook's `files:` regex ---
# The probe paths are hardcoded, so they can silently stop representing anything
# the hooks lint. Rescoping the shipped agent hook away from the `.agent.md`
# shape has to fail here rather than leave a probe testing a shape no hook
# matches any more.
echo ""
echo "--- exits 1 when a probe path matches no hook's files: regex ---"
FIXTURE16="$(make_fixture)"
FIXTURES+=("$FIXTURE16")
break_glob "$FIXTURE16/.pre-commit-hooks.yaml" \
  "files: '(^|/)agents/[^/]+\\.md\$|\\.agent\\.md\$'" "files: '(^|/)agents/[^/]+\\.md\$'"
if bash "$SCRIPT" "$FIXTURE16" > /dev/null 2>&1; then
  fail "exited 0 when the agent hook was rescoped away from .agent.md — expected exit 1"
else
  pass "exits non-zero when a probe path is in no hook's scope any more"
fi

# --- 11b. Exits 1 when the local config's files: regex narrows out of sync
# with the canonical .pre-commit-hooks.yaml regex ---
# hook_file_regexes() used to union the two manifests' `files:` regexes before
# checking probe coverage, so a probe that matched only the old, looser
# .pre-commit-hooks.yaml pattern still passed as "in scope" even after
# .pre-commit-config.yaml's copy of the same hook was narrowed away from it.
# That is exactly the shape of rescoping this repo's own agent hook went
# through (SKILL/agent `.md` -> `.apm/.../*.agent.md`): the local hook quietly
# stopped linting a shape the shipped, external-facing manifest still claims
# to cover, and nothing caught it. Reproduce it directly: narrow only the
# fixture's local config regex (leave .pre-commit-hooks.yaml as shipped) and
# assert the check now flags the disagreement instead of passing silently.
echo ""
echo "--- exits 1 when .pre-commit-config.yaml's files: regex drifts out of sync with .pre-commit-hooks.yaml's ---"
FIXTURE16B="$(make_fixture)"
FIXTURES+=("$FIXTURE16B")
break_glob "$FIXTURE16B/.pre-commit-config.yaml" \
  "files: '^plugins/[^/]+/\\.apm/agents/[^/]+\\.agent\\.md\$'" \
  "files: '^plugins/kyberforge/\\.apm/agents/[^/]+\\.agent\\.md\$'"
if bash "$SCRIPT" "$FIXTURE16B" > /dev/null 2>&1; then
  fail "exited 0 when the local config regex narrowed out of sync with .pre-commit-hooks.yaml — expected exit 1"
else
  pass "exits non-zero when the local config regex narrows out of sync with the canonical .pre-commit-hooks.yaml regex"
fi

# --- 12. The text-level assertions hold on a machine without vale ---
# They are the fallback when the glob probe cannot run. With vale on PATH the
# probe fails on these same mutations, so it would mask them: only masking vale
# proves a clean run here means the text assertions themselves ran.
echo ""
echo "--- the StylesPath / BasedOnStyles assertions still gate with vale masked off PATH ---"
VALE_DIR="$(dirname "$(command -v vale 2>/dev/null || echo /nonexistent/vale)")"
PATH_NO_VALE="$(printf '%s' "$PATH" | tr ':' '\n' | grep -vxF "$VALE_DIR" | paste -sd: -)"
if (PATH="$PATH_NO_VALE"; command -v vale >/dev/null 2>&1); then
  fail "could not mask vale off PATH — the vale-absent fallback was not exercised"
else
  FIXTURE17="$(make_fixture)"
  FIXTURE18="$(make_fixture)"
  FIXTURE19="$(make_fixture)"
  FIXTURES+=("$FIXTURE17" "$FIXTURE18" "$FIXTURE19")
  break_glob "$FIXTURE18/plugins/kyberforge/.apm/skills/skill-audit/assets/vale/.vale.ini" \
    'StylesPath = styles' 'StylesPath = elsewhere'
  break_glob "$FIXTURE19/plugins/kyberforge/.apm/skills/agent-audit/assets/vale/.vale.ini" \
    'BasedOnStyles = Kyberforge' 'BasedOnStyles = KyberforgeCopilot'
  if PATH="$PATH_NO_VALE" bash "$SCRIPT" "$FIXTURE17" > /dev/null 2>&1; then
    pass "exits 0 on in-sync copies with vale unavailable"
  else
    fail "exited non-zero on in-sync copies with vale unavailable — the missing binary must warn, not fail"
  fi
  if PATH="$PATH_NO_VALE" bash "$SCRIPT" "$FIXTURE18" > /dev/null 2>&1; then
    fail "exited 0 on a dropped StylesPath with vale unavailable — expected exit 1"
  else
    pass "exits non-zero on a dropped StylesPath with vale unavailable"
  fi
  if PATH="$PATH_NO_VALE" bash "$SCRIPT" "$FIXTURE19" > /dev/null 2>&1; then
    fail "exited 0 on a BasedOnStyles that dropped Kyberforge with vale unavailable — expected exit 1"
  else
    pass "exits non-zero on a BasedOnStyles that dropped Kyberforge with vale unavailable"
  fi
  # A clean run without vale must say so — silence would read as verified.
  if PATH="$PATH_NO_VALE" bash "$SCRIPT" "$FIXTURE17" 2>&1 | grep -q "vale is not installed"; then
    pass "warns that glob coverage was not verified when vale is unavailable"
  else
    fail "exited clean without vale and said nothing — an unverified run looks identical to a verified one"
  fi
fi

# --- 13. The intentional agent-audit-only divergence is NOT flagged ---
# The two .vale.ini files are deliberately different: agent-audit ships an extra
# [**/*.agent.md] section and the KyberforgeCopilot style. A check that diffed
# them would fail the repo as it stands, so assert the divergence is really in
# the fixture before asserting the check tolerates it — otherwise this case would
# still pass if the fixture had quietly stopped carrying it.
echo ""
echo "--- exits 0 despite agent-audit's KyberforgeCopilot divergence ---"
FIXTURE15="$(make_fixture)"
FIXTURES+=("$FIXTURE15")
AGENT_INI15="$FIXTURE15/plugins/kyberforge/.apm/skills/agent-audit/assets/vale/.vale.ini"
SKILL_INI15="$FIXTURE15/plugins/kyberforge/.apm/skills/skill-audit/assets/vale/.vale.ini"
if ! grep -q "KyberforgeCopilot" "$AGENT_INI15" \
  || grep -q "KyberforgeCopilot" "$SKILL_INI15" \
  || [[ ! -d "$FIXTURE15/plugins/kyberforge/.apm/skills/agent-audit/assets/vale/styles/KyberforgeCopilot" ]]; then
  fail "the fixture no longer carries the agent-audit-only KyberforgeCopilot divergence, so tolerating it proves nothing"
elif bash "$SCRIPT" "$FIXTURE15" > /dev/null 2>&1; then
  pass "exits 0 with agent-audit's extra KyberforgeCopilot section and style present"
else
  fail "flagged the intentional agent-audit-only KyberforgeCopilot divergence — expected exit 0"
  bash "$SCRIPT" "$FIXTURE15" 2>&1 | sed 's/^/    /' || true
fi

echo ""
echo "Results: $PASS passed, $FAIL failed"
[[ $FAIL -eq 0 ]]