feat(lint): wire Vale as deterministic prefilter for skill-audit/agent-audit #85
@@ -10,6 +10,11 @@ set -euo pipefail
|
||||
# only one of the two. Run from repo root or pass REPO_ROOT as arg.
|
||||
|
||||
REPO_ROOT="${1:-$(git rev-parse --show-toplevel 2>/dev/null || pwd)}"
|
||||
# Absolutized because the glob probe below `cd`s into a scratch tree, where a
|
||||
# relative --config path would stop resolving.
|
||||
if [[ -d "$REPO_ROOT" ]]; then
|
||||
REPO_ROOT="$(cd "$REPO_ROOT" && pwd)"
|
||||
fi
|
||||
FAIL=0
|
||||
|
||||
err() { echo " FAIL: $1" >&2; FAIL=$((FAIL + 1)); }
|
||||
@@ -41,7 +46,126 @@ if ! diff -rq "$SKILL_AUDIT/assets/vale/styles/Kyberforge" "$AGENT_AUDIT/assets/
|
||||
err "assets/vale/styles/Kyberforge differs between skill-audit and agent-audit"
|
||||
fi
|
||||
|
||||
# --- .vale.ini coverage ------------------------------------------------------
|
||||
# The two .vale.ini files are deliberately NOT identical — agent-audit's carries
|
||||
# an extra [**/*.agent.md] section and the KyberforgeCopilot style — so they
|
||||
# cannot be diffed like the styles above. Nothing else in the repo read them at
|
||||
# all, and that is what let a one-character glob typo silently disable the
|
||||
# prefilter for a whole file type: the hook still MATCHES the file via its
|
||||
# `files:` regex, so pre-commit reports neither `Skipped` nor an error; vale
|
||||
# lints zero files, prints `0 errors ... in 1 file` and exits 0, and the hook
|
||||
# shows `Passed`. So check the parts that must hold in both, not equality.
|
||||
|
||||
SKILL_INI="$SKILL_AUDIT/assets/vale/.vale.ini"
|
||||
AGENT_INI="$AGENT_AUDIT/assets/vale/.vale.ini"
|
||||
|
||||
for ini in "$SKILL_INI" "$AGENT_INI"; do
|
||||
rel_ini="${ini#"$REPO_ROOT"/}"
|
||||
if [[ ! -f "$ini" ]]; then
|
||||
err "$rel_ini is missing — without it vale falls back to an upward config search and lints with whatever it finds"
|
||||
continue
|
||||
fi
|
||||
# StylesPath is resolved relative to the .vale.ini, which is the only reason
|
||||
# the bundled styles are found from a consuming repo's clone prefix.
|
||||
if ! grep -Eq '^[[:space:]]*StylesPath[[:space:]]*=[[:space:]]*styles[[:space:]]*$' "$ini"; then
|
||||
err "$rel_ini has no 'StylesPath = styles' — the bundled styles/ directory would not be found"
|
||||
fi
|
||||
# Matches `Kyberforge` as a whole name, so `KyberforgeCopilot` alone does not
|
||||
# satisfy it. Avoids \b, which is a GNU grep extension.
|
||||
if ! grep -Eq '^[[:space:]]*BasedOnStyles[[:space:]]*=.*Kyberforge([[:space:],]|$)' "$ini"; then
|
||||
err "$rel_ini has no section whose BasedOnStyles names Kyberforge — every rule the audit prefilters on lives in that style"
|
||||
fi
|
||||
done
|
||||
|
||||
# Prints the `files:` regex of every hook, in either manifest, whose entry is
|
||||
# $1's vale-wrap.sh. Records are delimited by their `- id:` line, so the check
|
||||
# does not depend on `entry:` preceding `files:` within a record.
|
||||
hook_file_regexes() {
|
||||
local skill="$1" manifest raw
|
||||
for manifest in "$REPO_ROOT/.pre-commit-hooks.yaml" "$REPO_ROOT/.pre-commit-config.yaml"; do
|
||||
[[ -f "$manifest" ]] || continue
|
||||
awk -v skill="$skill" '
|
||||
function flush() {
|
||||
if (entry ~ skill "/scripts/vale-wrap.sh" && files != "") print files
|
||||
entry = ""; files = ""
|
||||
}
|
||||
/^[ \t]*-[ \t]*id:/ { flush() }
|
||||
/^[ \t]*entry:/ { entry = $0 }
|
||||
/^[ \t]*files:/ { files = $0; sub(/^[ \t]*files:[ \t]*/, "", files) }
|
||||
END { flush() }
|
||||
' "$manifest"
|
||||
done | while IFS= read -r raw; do
|
||||
# Strip the surrounding YAML quotes; the regex itself never carries them.
|
||||
raw="${raw%\'}"; raw="${raw#\'}"
|
||||
raw="${raw%\"}"; raw="${raw#\"}"
|
||||
printf '%s\n' "$raw"
|
||||
done
|
||||
}
|
||||
|
||||
# Asks vale — the thing that actually applies these globs — whether a config
|
||||
# covers a path, rather than reimplementing doublestar matching. The probe file
|
||||
# carries a description with a token Kyberforge.VagueWording flags, so a config
|
||||
# whose glob matches but whose BasedOnStyles lost Kyberforge fails too: it would
|
||||
# lint the file and report nothing.
|
||||
vale_flags_path() {
|
||||
local cfg="$1" rel="$2" tmp out
|
||||
tmp="$(mktemp -d)"
|
||||
mkdir -p "$tmp/$(dirname "$rel")"
|
||||
{
|
||||
echo "---"
|
||||
echo "name: probe"
|
||||
echo "description: Use when the caller wants a probe that helps with things."
|
||||
echo "---"
|
||||
echo ""
|
||||
echo "Body."
|
||||
} > "$tmp/$rel"
|
||||
out="$(cd "$tmp" && vale --config "$cfg" "$rel" 2>&1)" || true
|
||||
rm -rf "$tmp"
|
||||
printf '%s\n' "$out" | grep -qF "Kyberforge.VagueWording"
|
||||
}
|
||||
|
||||
VALE_AVAILABLE=true
|
||||
if ! command -v vale >/dev/null 2>&1; then
|
||||
VALE_AVAILABLE=false
|
||||
echo " WARNING: vale is not installed — .vale.ini glob coverage was NOT verified. Install it (https://vale.sh/docs/vale-cli/installation/) before trusting a clean run." >&2
|
||||
fi
|
||||
|
||||
# One representative path per file shape the prefilter is supposed to cover. Each
|
||||
# is cross-checked against the shipped hooks' `files:` regexes first, so a path
|
||||
# that goes stale because a hook was rescoped fails loudly here instead of
|
||||
# quietly probing a shape nothing lints any more.
|
||||
while IFS='|' read -r skill rel; do
|
||||
[[ -n "$skill" ]] || continue
|
||||
dir="$REPO_ROOT/plugins/kyberforge/skills/$skill"
|
||||
ini="$dir/assets/vale/.vale.ini"
|
||||
[[ -f "$ini" ]] || continue
|
||||
|
||||
regexes="$(hook_file_regexes "$skill")"
|
||||
if [[ -n "$regexes" ]]; then
|
||||
in_scope=false
|
||||
while IFS= read -r re; do
|
||||
[[ -n "$re" ]] || continue
|
||||
if printf '%s\n' "$rel" | grep -Eq "$re"; then
|
||||
in_scope=true
|
||||
fi
|
||||
done <<EOF_RE
|
||||
$regexes
|
||||
EOF_RE
|
||||
if [[ "$in_scope" == false ]]; then
|
||||
err "$rel matches no 'files:' regex of any $skill hook — the probe path is stale, or the hook was rescoped away from a shape it still needs to lint"
|
||||
fi
|
||||
fi
|
||||
|
||||
if [[ "$VALE_AVAILABLE" == true ]] && ! vale_flags_path "$ini" "$rel"; then
|
||||
err "$skill/assets/vale/.vale.ini raises no Kyberforge alert on $rel — its glob sections do not cover a path its own pre-commit hook is scoped to, so the hook passes that shape without linting it"
|
||||
fi
|
||||
done <<'EOF_PROBE'
|
||||
skill-audit|plugins/demo/skills/demo/SKILL.md
|
||||
agent-audit|plugins/demo/agents/demo.md
|
||||
agent-audit|copilot/demo.agent.md
|
||||
EOF_PROBE
|
||||
|
||||
if [[ $FAIL -gt 0 ]]; then
|
||||
echo "Vale style sync check failed: $FAIL error(s). agent-audit's copy is canonical — run scripts/sync-vale-styles.sh to regenerate skill-audit's copy, then commit both." >&2
|
||||
echo "Vale style sync check failed: $FAIL error(s). For a drifted wrapper or style, agent-audit's copy is canonical — run scripts/sync-vale-styles.sh to regenerate skill-audit's copy, then commit both. A .vale.ini finding is not drift and sync-vale-styles.sh will not fix it: edit that file's own StylesPath, BasedOnStyles or glob sections." >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
@@ -9,41 +9,72 @@ FAIL=0
|
||||
pass() { echo " PASS: $1"; PASS=$((PASS + 1)); }
|
||||
fail() { echo " FAIL: $1"; FAIL=$((FAIL + 1)); }
|
||||
|
||||
# One trap over a registry, rather than rebuilding the trap line per fixture:
|
||||
# the guard is there because bash 3.2 treats "${arr[@]}" on an empty array as
|
||||
# unbound under `set -u`.
|
||||
FIXTURES=()
|
||||
cleanup() { [[ ${#FIXTURES[@]} -eq 0 ]] || rm -rf "${FIXTURES[@]}"; }
|
||||
trap cleanup EXIT
|
||||
|
||||
# Helper: make a fixture repo with skill-audit/agent-audit's Vale copies, in sync by default.
|
||||
# The wrapper is a stub — the script only diffs it — but the Vale assets and both
|
||||
# pre-commit manifests are the repo's real ones, because the .vale.ini checks ask
|
||||
# vale to apply those globs for real and cross-check them against the shipped
|
||||
# hooks' `files:` regexes. A synthetic style or manifest would prove nothing, and
|
||||
# copying the real ones keeps agent-audit's intentional KyberforgeCopilot
|
||||
# divergence in the fixture instead of a sanitized stand-in for it.
|
||||
make_fixture() {
|
||||
local dir
|
||||
dir="$(mktemp -d)"
|
||||
local skill_audit="$dir/plugins/kyberforge/skills/skill-audit"
|
||||
local agent_audit="$dir/plugins/kyberforge/skills/agent-audit"
|
||||
mkdir -p "$skill_audit/scripts" "$skill_audit/assets/vale/styles/Kyberforge"
|
||||
mkdir -p "$agent_audit/scripts" "$agent_audit/assets/vale/styles/Kyberforge"
|
||||
mkdir -p "$skill_audit/scripts" "$agent_audit/scripts"
|
||||
|
||||
echo '#!/usr/bin/env bash' > "$skill_audit/scripts/vale-wrap.sh"
|
||||
echo 'echo wrap' >> "$skill_audit/scripts/vale-wrap.sh"
|
||||
cp "$skill_audit/scripts/vale-wrap.sh" "$agent_audit/scripts/vale-wrap.sh"
|
||||
|
||||
echo 'extends: existence' > "$skill_audit/assets/vale/styles/Kyberforge/Rule.yml"
|
||||
cp "$skill_audit/assets/vale/styles/Kyberforge/Rule.yml" "$agent_audit/assets/vale/styles/Kyberforge/Rule.yml"
|
||||
cp -R "$REPO_ROOT/plugins/kyberforge/skills/skill-audit/assets" "$skill_audit/"
|
||||
cp -R "$REPO_ROOT/plugins/kyberforge/skills/agent-audit/assets" "$agent_audit/"
|
||||
cp "$REPO_ROOT/.pre-commit-hooks.yaml" "$REPO_ROOT/.pre-commit-config.yaml" "$dir/"
|
||||
|
||||
echo "$dir"
|
||||
}
|
||||
|
||||
# Helper: rewrite a glob section header in one copy's .vale.ini, leaving every
|
||||
# other line — StylesPath, BasedOnStyles — intact. This is the shape of the
|
||||
# typo the check exists to catch: the hook still matches the file via its
|
||||
# `files:` regex, vale lints nothing, and pre-commit reports `Passed`.
|
||||
break_glob() {
|
||||
local ini="$1" old="$2" new="$3"
|
||||
python3 - "$ini" "$old" "$new" <<'PYTHON'
|
||||
import sys
|
||||
path, old, new = sys.argv[1], sys.argv[2], sys.argv[3]
|
||||
with open(path, encoding='utf-8') as fh:
|
||||
content = fh.read()
|
||||
assert old in content, f"{old} not found in {path}"
|
||||
with open(path, 'w', encoding='utf-8') as fh:
|
||||
fh.write(content.replace(old, new))
|
||||
PYTHON
|
||||
}
|
||||
|
||||
# --- 1. Exits 0 when the two copies are in sync ---
|
||||
echo ""
|
||||
echo "--- exits 0 when skill-audit and agent-audit copies are in sync ---"
|
||||
FIXTURE="$(make_fixture)"
|
||||
trap 'rm -rf "$FIXTURE"' EXIT
|
||||
FIXTURES+=("$FIXTURE")
|
||||
if bash "$SCRIPT" "$FIXTURE" > /dev/null 2>&1; then
|
||||
pass "exits 0 when copies are in sync"
|
||||
else
|
||||
fail "exited non-zero against in-sync copies"
|
||||
bash "$SCRIPT" "$FIXTURE" 2>&1 | sed 's/^/ /' || true
|
||||
fi
|
||||
|
||||
# --- 2. Exits 1 when vale-wrap.sh differs between the two copies ---
|
||||
echo ""
|
||||
echo "--- exits 1 when vale-wrap.sh differs ---"
|
||||
FIXTURE2="$(make_fixture)"
|
||||
trap 'rm -rf "$FIXTURE" "$FIXTURE2"' EXIT
|
||||
FIXTURES+=("$FIXTURE2")
|
||||
echo 'echo different' >> "$FIXTURE2/plugins/kyberforge/skills/skill-audit/scripts/vale-wrap.sh"
|
||||
if bash "$SCRIPT" "$FIXTURE2" > /dev/null 2>&1; then
|
||||
fail "exited 0 when vale-wrap.sh copies differ — expected exit 1"
|
||||
@@ -55,8 +86,8 @@ fi
|
||||
echo ""
|
||||
echo "--- exits 1 when a Kyberforge style rule differs ---"
|
||||
FIXTURE3="$(make_fixture)"
|
||||
trap 'rm -rf "$FIXTURE" "$FIXTURE2" "$FIXTURE3"' EXIT
|
||||
echo 'level: error' >> "$FIXTURE3/plugins/kyberforge/skills/agent-audit/assets/vale/styles/Kyberforge/Rule.yml"
|
||||
FIXTURES+=("$FIXTURE3")
|
||||
echo ' - divergent token' >> "$FIXTURE3/plugins/kyberforge/skills/agent-audit/assets/vale/styles/Kyberforge/VagueWording.yml"
|
||||
if bash "$SCRIPT" "$FIXTURE3" > /dev/null 2>&1; then
|
||||
fail "exited 0 when a style rule differs — expected exit 1"
|
||||
else
|
||||
@@ -67,8 +98,14 @@ fi
|
||||
echo ""
|
||||
echo "--- exits 1 when a rule file is missing from one copy ---"
|
||||
FIXTURE4="$(make_fixture)"
|
||||
trap 'rm -rf "$FIXTURE" "$FIXTURE2" "$FIXTURE3" "$FIXTURE4"' EXIT
|
||||
echo 'extends: existence' > "$FIXTURE4/plugins/kyberforge/skills/agent-audit/assets/vale/styles/Kyberforge/Extra.yml"
|
||||
FIXTURES+=("$FIXTURE4")
|
||||
cat > "$FIXTURE4/plugins/kyberforge/skills/agent-audit/assets/vale/styles/Kyberforge/Extra.yml" <<'EOF'
|
||||
extends: existence
|
||||
message: "Extra: '%s'"
|
||||
level: error
|
||||
tokens:
|
||||
- divergent token
|
||||
EOF
|
||||
if bash "$SCRIPT" "$FIXTURE4" > /dev/null 2>&1; then
|
||||
fail "exited 0 when a rule file exists in only one copy — expected exit 1"
|
||||
else
|
||||
@@ -79,7 +116,7 @@ fi
|
||||
echo ""
|
||||
echo "--- exits 0 when kyberforge skills are absent (no-op) ---"
|
||||
FIXTURE5="$(mktemp -d)"
|
||||
trap 'rm -rf "$FIXTURE" "$FIXTURE2" "$FIXTURE3" "$FIXTURE4" "$FIXTURE5"' EXIT
|
||||
FIXTURES+=("$FIXTURE5")
|
||||
if bash "$SCRIPT" "$FIXTURE5" > /dev/null 2>&1; then
|
||||
pass "exits 0 as a no-op when skill-audit/agent-audit don't exist"
|
||||
else
|
||||
@@ -93,7 +130,7 @@ echo ""
|
||||
echo "--- exits 1 when only one of the two copies is present ---"
|
||||
FIXTURE6="$(make_fixture)"
|
||||
FIXTURE7="$(make_fixture)"
|
||||
trap 'rm -rf "$FIXTURE" "$FIXTURE2" "$FIXTURE3" "$FIXTURE4" "$FIXTURE5" "$FIXTURE6" "$FIXTURE7"' EXIT
|
||||
FIXTURES+=("$FIXTURE6" "$FIXTURE7")
|
||||
rm -rf "$FIXTURE6/plugins/kyberforge/skills/skill-audit"
|
||||
rm -rf "$FIXTURE7/plugins/kyberforge/skills/agent-audit"
|
||||
if bash "$SCRIPT" "$FIXTURE6" > /dev/null 2>&1; then
|
||||
@@ -107,6 +144,172 @@ else
|
||||
pass "exits non-zero when agent-audit's canonical copy is missing but skill-audit's is present"
|
||||
fi
|
||||
|
||||
# --- 7. Exits 1 when a .vale.ini is missing entirely ---
|
||||
# Without it vale falls back to an upward config search and lints the file with
|
||||
# whatever config it happens to find, which is not a failure anyone sees.
|
||||
echo ""
|
||||
echo "--- exits 1 when a .vale.ini is missing ---"
|
||||
FIXTURE8="$(make_fixture)"
|
||||
FIXTURES+=("$FIXTURE8")
|
||||
rm -f "$FIXTURE8/plugins/kyberforge/skills/skill-audit/assets/vale/.vale.ini"
|
||||
if bash "$SCRIPT" "$FIXTURE8" > /dev/null 2>&1; then
|
||||
fail "exited 0 when skill-audit's .vale.ini is missing — expected exit 1"
|
||||
else
|
||||
pass "exits non-zero when a .vale.ini is missing"
|
||||
fi
|
||||
|
||||
# --- 8. Exits 1 when the shared StylesPath line is dropped from either copy ---
|
||||
# StylesPath resolves relative to the .vale.ini, which is the only reason the
|
||||
# bundled styles are found from a consuming repo's clone prefix.
|
||||
echo ""
|
||||
echo "--- exits 1 when StylesPath is missing from either .vale.ini ---"
|
||||
FIXTURE9="$(make_fixture)"
|
||||
FIXTURE10="$(make_fixture)"
|
||||
FIXTURES+=("$FIXTURE9" "$FIXTURE10")
|
||||
break_glob "$FIXTURE9/plugins/kyberforge/skills/skill-audit/assets/vale/.vale.ini" \
|
||||
'StylesPath = styles' 'StylesPath = elsewhere'
|
||||
break_glob "$FIXTURE10/plugins/kyberforge/skills/agent-audit/assets/vale/.vale.ini" \
|
||||
'StylesPath = styles' 'StylesPath = elsewhere'
|
||||
if bash "$SCRIPT" "$FIXTURE9" > /dev/null 2>&1; then
|
||||
fail "exited 0 when skill-audit's .vale.ini lost StylesPath — expected exit 1"
|
||||
else
|
||||
pass "exits non-zero when skill-audit's .vale.ini lost StylesPath"
|
||||
fi
|
||||
if bash "$SCRIPT" "$FIXTURE10" > /dev/null 2>&1; then
|
||||
fail "exited 0 when agent-audit's .vale.ini lost StylesPath — expected exit 1"
|
||||
else
|
||||
pass "exits non-zero when agent-audit's .vale.ini lost StylesPath"
|
||||
fi
|
||||
|
||||
# --- 9. Exits 1 when no section's BasedOnStyles names Kyberforge ---
|
||||
# Every rule the prefilter gates on lives in that style, so a section that keeps
|
||||
# its glob but loses the style lints the file and reports nothing.
|
||||
echo ""
|
||||
echo "--- exits 1 when BasedOnStyles no longer names Kyberforge ---"
|
||||
FIXTURE11="$(make_fixture)"
|
||||
FIXTURES+=("$FIXTURE11")
|
||||
break_glob "$FIXTURE11/plugins/kyberforge/skills/agent-audit/assets/vale/.vale.ini" \
|
||||
'BasedOnStyles = Kyberforge' 'BasedOnStyles = KyberforgeCopilot'
|
||||
if bash "$SCRIPT" "$FIXTURE11" > /dev/null 2>&1; then
|
||||
fail "exited 0 when agent-audit's .vale.ini stopped naming Kyberforge — expected exit 1"
|
||||
else
|
||||
pass "exits non-zero when a .vale.ini no longer names the Kyberforge style"
|
||||
fi
|
||||
|
||||
# --- 10. Exits 1 when a glob section stops matching the shape its hook lints ---
|
||||
# One case per glob section, because each covers a file shape the others don't:
|
||||
# agent-audit's [**/*.agent.md] is the only section covering a Copilot agent file
|
||||
# outside an agents/ directory, so breaking it alone is invisible to the others.
|
||||
echo ""
|
||||
echo "--- exits 1 when a .vale.ini glob no longer matches its hook's file shape ---"
|
||||
FIXTURE12="$(make_fixture)"
|
||||
FIXTURE13="$(make_fixture)"
|
||||
FIXTURE14="$(make_fixture)"
|
||||
FIXTURES+=("$FIXTURE12" "$FIXTURE13" "$FIXTURE14")
|
||||
break_glob "$FIXTURE12/plugins/kyberforge/skills/skill-audit/assets/vale/.vale.ini" \
|
||||
'[**/SKILL.md]' '[**/NOMATCH.md]'
|
||||
break_glob "$FIXTURE13/plugins/kyberforge/skills/agent-audit/assets/vale/.vale.ini" \
|
||||
'[**/agents/*.md]' '[**/NOMATCH-agents/*.md]'
|
||||
break_glob "$FIXTURE14/plugins/kyberforge/skills/agent-audit/assets/vale/.vale.ini" \
|
||||
'[**/*.agent.md]' '[**/*.NOMATCH.md]'
|
||||
if bash "$SCRIPT" "$FIXTURE12" > /dev/null 2>&1; then
|
||||
fail "exited 0 when skill-audit's SKILL.md glob matched nothing — expected exit 1"
|
||||
else
|
||||
pass "exits non-zero when skill-audit's SKILL.md glob matches nothing"
|
||||
fi
|
||||
if bash "$SCRIPT" "$FIXTURE13" > /dev/null 2>&1; then
|
||||
fail "exited 0 when agent-audit's agents/*.md glob matched nothing — expected exit 1"
|
||||
else
|
||||
pass "exits non-zero when agent-audit's agents/*.md glob matches nothing"
|
||||
fi
|
||||
if bash "$SCRIPT" "$FIXTURE14" > /dev/null 2>&1; then
|
||||
fail "exited 0 when agent-audit's *.agent.md glob matched nothing — expected exit 1"
|
||||
else
|
||||
pass "exits non-zero when agent-audit's *.agent.md glob matches nothing"
|
||||
fi
|
||||
|
||||
# --- 11. Exits 1 when a probe path falls out of every hook's `files:` regex ---
|
||||
# The probe paths are hardcoded, so they can silently stop representing anything
|
||||
# the hooks lint. Rescoping the shipped agent hook away from the `.agent.md`
|
||||
# shape has to fail here rather than leave a probe testing a shape no hook
|
||||
# matches any more.
|
||||
echo ""
|
||||
echo "--- exits 1 when a probe path matches no hook's files: regex ---"
|
||||
FIXTURE16="$(make_fixture)"
|
||||
FIXTURES+=("$FIXTURE16")
|
||||
break_glob "$FIXTURE16/.pre-commit-hooks.yaml" \
|
||||
"files: '(^|/)agents/[^/]+\\.md\$|\\.agent\\.md\$'" "files: '(^|/)agents/[^/]+\\.md\$'"
|
||||
if bash "$SCRIPT" "$FIXTURE16" > /dev/null 2>&1; then
|
||||
fail "exited 0 when the agent hook was rescoped away from .agent.md — expected exit 1"
|
||||
else
|
||||
pass "exits non-zero when a probe path is in no hook's scope any more"
|
||||
fi
|
||||
|
||||
# --- 12. The text-level assertions hold on a machine without vale ---
|
||||
# They are the fallback when the glob probe cannot run. With vale on PATH the
|
||||
# probe fails on these same mutations, so it would mask them: only masking vale
|
||||
# proves a clean run here means the text assertions themselves ran.
|
||||
echo ""
|
||||
echo "--- the StylesPath / BasedOnStyles assertions still gate with vale masked off PATH ---"
|
||||
VALE_DIR="$(dirname "$(command -v vale 2>/dev/null || echo /nonexistent/vale)")"
|
||||
PATH_NO_VALE="$(printf '%s' "$PATH" | tr ':' '\n' | grep -vxF "$VALE_DIR" | paste -sd: -)"
|
||||
if (PATH="$PATH_NO_VALE"; command -v vale >/dev/null 2>&1); then
|
||||
fail "could not mask vale off PATH — the vale-absent fallback was not exercised"
|
||||
else
|
||||
FIXTURE17="$(make_fixture)"
|
||||
FIXTURE18="$(make_fixture)"
|
||||
FIXTURE19="$(make_fixture)"
|
||||
FIXTURES+=("$FIXTURE17" "$FIXTURE18" "$FIXTURE19")
|
||||
break_glob "$FIXTURE18/plugins/kyberforge/skills/skill-audit/assets/vale/.vale.ini" \
|
||||
'StylesPath = styles' 'StylesPath = elsewhere'
|
||||
break_glob "$FIXTURE19/plugins/kyberforge/skills/agent-audit/assets/vale/.vale.ini" \
|
||||
'BasedOnStyles = Kyberforge' 'BasedOnStyles = KyberforgeCopilot'
|
||||
if PATH="$PATH_NO_VALE" bash "$SCRIPT" "$FIXTURE17" > /dev/null 2>&1; then
|
||||
pass "exits 0 on in-sync copies with vale unavailable"
|
||||
else
|
||||
fail "exited non-zero on in-sync copies with vale unavailable — the missing binary must warn, not fail"
|
||||
fi
|
||||
if PATH="$PATH_NO_VALE" bash "$SCRIPT" "$FIXTURE18" > /dev/null 2>&1; then
|
||||
fail "exited 0 on a dropped StylesPath with vale unavailable — expected exit 1"
|
||||
else
|
||||
pass "exits non-zero on a dropped StylesPath with vale unavailable"
|
||||
fi
|
||||
if PATH="$PATH_NO_VALE" bash "$SCRIPT" "$FIXTURE19" > /dev/null 2>&1; then
|
||||
fail "exited 0 on a BasedOnStyles that dropped Kyberforge with vale unavailable — expected exit 1"
|
||||
else
|
||||
pass "exits non-zero on a BasedOnStyles that dropped Kyberforge with vale unavailable"
|
||||
fi
|
||||
# A clean run without vale must say so — silence would read as verified.
|
||||
if PATH="$PATH_NO_VALE" bash "$SCRIPT" "$FIXTURE17" 2>&1 | grep -q "vale is not installed"; then
|
||||
pass "warns that glob coverage was not verified when vale is unavailable"
|
||||
else
|
||||
fail "exited clean without vale and said nothing — an unverified run looks identical to a verified one"
|
||||
fi
|
||||
fi
|
||||
|
||||
# --- 13. The intentional agent-audit-only divergence is NOT flagged ---
|
||||
# The two .vale.ini files are deliberately different: agent-audit ships an extra
|
||||
# [**/*.agent.md] section and the KyberforgeCopilot style. A check that diffed
|
||||
# them would fail the repo as it stands, so assert the divergence is really in
|
||||
# the fixture before asserting the check tolerates it — otherwise this case would
|
||||
# still pass if the fixture had quietly stopped carrying it.
|
||||
echo ""
|
||||
echo "--- exits 0 despite agent-audit's KyberforgeCopilot divergence ---"
|
||||
FIXTURE15="$(make_fixture)"
|
||||
FIXTURES+=("$FIXTURE15")
|
||||
AGENT_INI15="$FIXTURE15/plugins/kyberforge/skills/agent-audit/assets/vale/.vale.ini"
|
||||
SKILL_INI15="$FIXTURE15/plugins/kyberforge/skills/skill-audit/assets/vale/.vale.ini"
|
||||
if ! grep -q "KyberforgeCopilot" "$AGENT_INI15" \
|
||||
|| grep -q "KyberforgeCopilot" "$SKILL_INI15" \
|
||||
|| [[ ! -d "$FIXTURE15/plugins/kyberforge/skills/agent-audit/assets/vale/styles/KyberforgeCopilot" ]]; then
|
||||
fail "the fixture no longer carries the agent-audit-only KyberforgeCopilot divergence, so tolerating it proves nothing"
|
||||
elif bash "$SCRIPT" "$FIXTURE15" > /dev/null 2>&1; then
|
||||
pass "exits 0 with agent-audit's extra KyberforgeCopilot section and style present"
|
||||
else
|
||||
fail "flagged the intentional agent-audit-only KyberforgeCopilot divergence — expected exit 0"
|
||||
bash "$SCRIPT" "$FIXTURE15" 2>&1 | sed 's/^/ /' || true
|
||||
fi
|
||||
|
||||
echo ""
|
||||
echo "Results: $PASS passed, $FAIL failed"
|
||||
[[ $FAIL -eq 0 ]]
|
||||
|
||||
@@ -64,13 +64,18 @@ repos:
|
||||
- id: kyberforge-skill-size-check
|
||||
EOF
|
||||
|
||||
# The two fixtures carry DIFFERENT flagged tokens so an alert can never be
|
||||
# credited to the hook that did not raise it. Both bodies land mid-sentence in a
|
||||
# folded block scalar that still spans two physical lines, which is the
|
||||
# flattening the wrapper exists to do.
|
||||
write_fixtures() {
|
||||
local body="$1"
|
||||
local skill_body="$1"
|
||||
local agent_body="${2:-$1}"
|
||||
cat > "$CONSUMER/skills/demo/SKILL.md" <<EOF
|
||||
---
|
||||
name: demo
|
||||
description: >
|
||||
Use when the caller wants a demonstration skill $body across two
|
||||
Use when the caller wants a demonstration skill $skill_body across two
|
||||
physical lines of one folded block scalar.
|
||||
---
|
||||
|
||||
@@ -80,7 +85,7 @@ EOF
|
||||
---
|
||||
name: demo
|
||||
description: >
|
||||
Use when the caller wants a demonstration agent $body across two
|
||||
Use when the caller wants a demonstration agent $agent_body across two
|
||||
physical lines of one folded block scalar.
|
||||
---
|
||||
|
||||
@@ -89,27 +94,56 @@ EOF
|
||||
git -C "$CONSUMER" add -A
|
||||
}
|
||||
|
||||
run_hooks() {
|
||||
(cd "$CONSUMER" && pre-commit run --all-files 2>&1) || true
|
||||
# Vale prints each linted path as its own header line with that file's alerts
|
||||
# indented beneath it, so an alert belongs to the nearest preceding path line.
|
||||
# Reads a hook log on stdin and prints only the alert lines filed under `$1`.
|
||||
# The `sed` strips vale's ANSI colouring, which it emits into pre-commit's pipe
|
||||
# too, so the header lines compare as plain paths.
|
||||
alerts_for() {
|
||||
sed $'s/\033\\[[0-9;]*m//g' | awk -v want="$1" '
|
||||
/^[^[:space:]].*\.md$/ { cur = $0; next }
|
||||
/^[[:space:]]*[0-9]+:[0-9]+[[:space:]]/ { if (cur == want) print }
|
||||
'
|
||||
}
|
||||
|
||||
# --- 1. Both hooks resolve their config and actually gate on a bad file ---
|
||||
# --- 1. Each Vale hook resolves its config and gates its own file shape ---
|
||||
# Asserted per hook, against that hook's own fixture path and its own token. An
|
||||
# aggregate alert count over both hooks' combined output does not prove this:
|
||||
# one fixture description carries every flagged token, so ONE working hook
|
||||
# already clears a `>= 2` threshold. And a hook whose .vale.ini globs match
|
||||
# nothing reaches neither of the guards below — it still MATCHES the file via
|
||||
# its `files:` regex, so pre-commit does not report `Skipped`; vale simply lints
|
||||
# nothing, prints `0 errors ... in 1 file` and exits 0, and the hook shows
|
||||
# `Passed`. Attribution is the only thing that catches it.
|
||||
echo ""
|
||||
echo "--- both Vale hooks run and fail a bad file in an external consumer repo ---"
|
||||
write_fixtures "that helps with and utilize things"
|
||||
OUT_BAD="$(run_hooks)"
|
||||
if echo "$OUT_BAD" | grep -q "does not exist"; then
|
||||
fail "hooks hard-errored on a path resolved against the consumer repo (E100) — the bug this test guards against"
|
||||
echo "$OUT_BAD" | sed 's/^/ /'
|
||||
elif echo "$OUT_BAD" | grep -q "Skipped"; then
|
||||
fail "a hook matched no files, so it proved nothing"
|
||||
echo "$OUT_BAD" | sed 's/^/ /'
|
||||
elif [[ "$(echo "$OUT_BAD" | grep -c "VagueWording")" -ge 2 ]]; then
|
||||
pass "both hooks flatten and flag the folded description in a consumer repo"
|
||||
else
|
||||
fail "hooks did not flag both fixtures"
|
||||
echo "$OUT_BAD" | sed 's/^/ /'
|
||||
fi
|
||||
echo "--- each Vale hook flags its own fixture in an external consumer repo ---"
|
||||
write_fixtures "that helps with things" "that will utilize things"
|
||||
while IFS='|' read -r HOOK_ID FIXTURE TOKEN; do
|
||||
[[ -n "$HOOK_ID" ]] || continue
|
||||
LOG="$WORK/$HOOK_ID.log"
|
||||
set +e
|
||||
(cd "$CONSUMER" && pre-commit run "$HOOK_ID" --all-files > "$LOG" 2>&1)
|
||||
RC_HOOK=$?
|
||||
set -e
|
||||
if grep -q "does not exist" "$LOG"; then
|
||||
fail "$HOOK_ID hard-errored on a path resolved against the consumer repo (E100) — the bug this test guards against"
|
||||
sed 's/^/ /' "$LOG"
|
||||
elif grep -q "Skipped" "$LOG"; then
|
||||
fail "$HOOK_ID matched no files, so it proved nothing"
|
||||
sed 's/^/ /' "$LOG"
|
||||
elif [[ $RC_HOOK -eq 0 ]]; then
|
||||
fail "$HOOK_ID passed $FIXTURE despite its flagged '$TOKEN' — a .vale.ini glob matching nothing lints zero files and exits 0"
|
||||
sed 's/^/ /' "$LOG"
|
||||
elif alerts_for "$FIXTURE" < "$LOG" | grep -qF "'$TOKEN'"; then
|
||||
pass "$HOOK_ID flattens $FIXTURE and flags its '$TOKEN' in a consumer repo"
|
||||
else
|
||||
fail "$HOOK_ID failed, but no alert quoting '$TOKEN' was filed under $FIXTURE"
|
||||
sed 's/^/ /' "$LOG"
|
||||
fi
|
||||
done <<'EOF'
|
||||
kyberforge-vale-audit-skill|skills/demo/SKILL.md|helps with
|
||||
kyberforge-vale-audit-agent|agents/demo.md|utilize
|
||||
EOF
|
||||
|
||||
# --- 2. Clean files pass — the hooks gate, they don't just always fail ---
|
||||
echo ""
|
||||
|
||||
Reference in New Issue
Block a user