fix(lint): attribute Vale alerts per hook and cover .vale.ini in the sync check
The external-consumer test asserted a combined alert count (>=2) across both shipped Vale hooks, but the SKILL.md fixture alone raises two alerts — so one working hook satisfied the threshold. Retargeting agent-audit's glob to match nothing left the suite reporting "3 passed" under the message "both hooks flatten and flag". The Skipped guard does not catch this: the hook still matches the file, Vale lints nothing, reports 0 errors in 1 file and exits 0, which pre-commit renders as Passed. An assertion aggregating over N subjects proves nothing about any individual subject. Each hook now runs individually and its alerts are attributed to the nearest preceding path header, so an alert is checked by path rather than by presence in the combined blob. The two fixtures carry distinct VagueWording tokens, so one hook's alert cannot be credited to another. Nothing in the repo read either .vale.ini — the sync check diffed only vale-wrap.sh and styles/Kyberforge, so a one-line glob typo silently disabled the prefilter for a whole file type. That was the enabling half of the same defect. The check now asserts the shared lines both copies must carry (StylesPath, a section naming Kyberforge as a whole word) without flagging their intentional divergence, and probes each glob section by asking Vale itself to lint a representative path. Regex-to-glob comparison was rejected as it means reimplementing doublestar semantics in bash; a file-count dry-run was rejected because a section whose glob matches but whose BasedOnStyles lost Kyberforge reports "1 file" with no alerts and would pass it. Every new assertion is bound to a failing case in both directions: breaking the artifact fails the suite, and neutering the assertion fails exactly one case. That reverse sweep exposed two assertions bound to no failing case at all, one masked by a stronger check running first. Refs: #85
This commit is contained in:
@@ -10,6 +10,11 @@ set -euo pipefail
|
|||||||
# only one of the two. Run from repo root or pass REPO_ROOT as arg.
|
# only one of the two. Run from repo root or pass REPO_ROOT as arg.
|
||||||
|
|
||||||
REPO_ROOT="${1:-$(git rev-parse --show-toplevel 2>/dev/null || pwd)}"
|
REPO_ROOT="${1:-$(git rev-parse --show-toplevel 2>/dev/null || pwd)}"
|
||||||
|
# Absolutized because the glob probe below `cd`s into a scratch tree, where a
|
||||||
|
# relative --config path would stop resolving.
|
||||||
|
if [[ -d "$REPO_ROOT" ]]; then
|
||||||
|
REPO_ROOT="$(cd "$REPO_ROOT" && pwd)"
|
||||||
|
fi
|
||||||
FAIL=0
|
FAIL=0
|
||||||
|
|
||||||
err() { echo " FAIL: $1" >&2; FAIL=$((FAIL + 1)); }
|
err() { echo " FAIL: $1" >&2; FAIL=$((FAIL + 1)); }
|
||||||
@@ -41,7 +46,126 @@ if ! diff -rq "$SKILL_AUDIT/assets/vale/styles/Kyberforge" "$AGENT_AUDIT/assets/
|
|||||||
err "assets/vale/styles/Kyberforge differs between skill-audit and agent-audit"
|
err "assets/vale/styles/Kyberforge differs between skill-audit and agent-audit"
|
||||||
fi
|
fi
|
||||||
|
|
||||||
|
# --- .vale.ini coverage ------------------------------------------------------
|
||||||
|
# The two .vale.ini files are deliberately NOT identical — agent-audit's carries
|
||||||
|
# an extra [**/*.agent.md] section and the KyberforgeCopilot style — so they
|
||||||
|
# cannot be diffed like the styles above. Nothing else in the repo read them at
|
||||||
|
# all, and that is what let a one-character glob typo silently disable the
|
||||||
|
# prefilter for a whole file type: the hook still MATCHES the file via its
|
||||||
|
# `files:` regex, so pre-commit reports neither `Skipped` nor an error; vale
|
||||||
|
# lints zero files, prints `0 errors ... in 1 file` and exits 0, and the hook
|
||||||
|
# shows `Passed`. So check the parts that must hold in both, not equality.
|
||||||
|
|
||||||
|
SKILL_INI="$SKILL_AUDIT/assets/vale/.vale.ini"
|
||||||
|
AGENT_INI="$AGENT_AUDIT/assets/vale/.vale.ini"
|
||||||
|
|
||||||
|
for ini in "$SKILL_INI" "$AGENT_INI"; do
|
||||||
|
rel_ini="${ini#"$REPO_ROOT"/}"
|
||||||
|
if [[ ! -f "$ini" ]]; then
|
||||||
|
err "$rel_ini is missing — without it vale falls back to an upward config search and lints with whatever it finds"
|
||||||
|
continue
|
||||||
|
fi
|
||||||
|
# StylesPath is resolved relative to the .vale.ini, which is the only reason
|
||||||
|
# the bundled styles are found from a consuming repo's clone prefix.
|
||||||
|
if ! grep -Eq '^[[:space:]]*StylesPath[[:space:]]*=[[:space:]]*styles[[:space:]]*$' "$ini"; then
|
||||||
|
err "$rel_ini has no 'StylesPath = styles' — the bundled styles/ directory would not be found"
|
||||||
|
fi
|
||||||
|
# Matches `Kyberforge` as a whole name, so `KyberforgeCopilot` alone does not
|
||||||
|
# satisfy it. Avoids \b, which is a GNU grep extension.
|
||||||
|
if ! grep -Eq '^[[:space:]]*BasedOnStyles[[:space:]]*=.*Kyberforge([[:space:],]|$)' "$ini"; then
|
||||||
|
err "$rel_ini has no section whose BasedOnStyles names Kyberforge — every rule the audit prefilters on lives in that style"
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
|
||||||
|
# Prints the `files:` regex of every hook, in either manifest, whose entry is
|
||||||
|
# $1's vale-wrap.sh. Records are delimited by their `- id:` line, so the check
|
||||||
|
# does not depend on `entry:` preceding `files:` within a record.
|
||||||
|
hook_file_regexes() {
|
||||||
|
local skill="$1" manifest raw
|
||||||
|
for manifest in "$REPO_ROOT/.pre-commit-hooks.yaml" "$REPO_ROOT/.pre-commit-config.yaml"; do
|
||||||
|
[[ -f "$manifest" ]] || continue
|
||||||
|
awk -v skill="$skill" '
|
||||||
|
function flush() {
|
||||||
|
if (entry ~ skill "/scripts/vale-wrap.sh" && files != "") print files
|
||||||
|
entry = ""; files = ""
|
||||||
|
}
|
||||||
|
/^[ \t]*-[ \t]*id:/ { flush() }
|
||||||
|
/^[ \t]*entry:/ { entry = $0 }
|
||||||
|
/^[ \t]*files:/ { files = $0; sub(/^[ \t]*files:[ \t]*/, "", files) }
|
||||||
|
END { flush() }
|
||||||
|
' "$manifest"
|
||||||
|
done | while IFS= read -r raw; do
|
||||||
|
# Strip the surrounding YAML quotes; the regex itself never carries them.
|
||||||
|
raw="${raw%\'}"; raw="${raw#\'}"
|
||||||
|
raw="${raw%\"}"; raw="${raw#\"}"
|
||||||
|
printf '%s\n' "$raw"
|
||||||
|
done
|
||||||
|
}
|
||||||
|
|
||||||
|
# Asks vale — the thing that actually applies these globs — whether a config
|
||||||
|
# covers a path, rather than reimplementing doublestar matching. The probe file
|
||||||
|
# carries a description with a token Kyberforge.VagueWording flags, so a config
|
||||||
|
# whose glob matches but whose BasedOnStyles lost Kyberforge fails too: it would
|
||||||
|
# lint the file and report nothing.
|
||||||
|
vale_flags_path() {
|
||||||
|
local cfg="$1" rel="$2" tmp out
|
||||||
|
tmp="$(mktemp -d)"
|
||||||
|
mkdir -p "$tmp/$(dirname "$rel")"
|
||||||
|
{
|
||||||
|
echo "---"
|
||||||
|
echo "name: probe"
|
||||||
|
echo "description: Use when the caller wants a probe that helps with things."
|
||||||
|
echo "---"
|
||||||
|
echo ""
|
||||||
|
echo "Body."
|
||||||
|
} > "$tmp/$rel"
|
||||||
|
out="$(cd "$tmp" && vale --config "$cfg" "$rel" 2>&1)" || true
|
||||||
|
rm -rf "$tmp"
|
||||||
|
printf '%s\n' "$out" | grep -qF "Kyberforge.VagueWording"
|
||||||
|
}
|
||||||
|
|
||||||
|
VALE_AVAILABLE=true
|
||||||
|
if ! command -v vale >/dev/null 2>&1; then
|
||||||
|
VALE_AVAILABLE=false
|
||||||
|
echo " WARNING: vale is not installed — .vale.ini glob coverage was NOT verified. Install it (https://vale.sh/docs/vale-cli/installation/) before trusting a clean run." >&2
|
||||||
|
fi
|
||||||
|
|
||||||
|
# One representative path per file shape the prefilter is supposed to cover. Each
|
||||||
|
# is cross-checked against the shipped hooks' `files:` regexes first, so a path
|
||||||
|
# that goes stale because a hook was rescoped fails loudly here instead of
|
||||||
|
# quietly probing a shape nothing lints any more.
|
||||||
|
while IFS='|' read -r skill rel; do
|
||||||
|
[[ -n "$skill" ]] || continue
|
||||||
|
dir="$REPO_ROOT/plugins/kyberforge/skills/$skill"
|
||||||
|
ini="$dir/assets/vale/.vale.ini"
|
||||||
|
[[ -f "$ini" ]] || continue
|
||||||
|
|
||||||
|
regexes="$(hook_file_regexes "$skill")"
|
||||||
|
if [[ -n "$regexes" ]]; then
|
||||||
|
in_scope=false
|
||||||
|
while IFS= read -r re; do
|
||||||
|
[[ -n "$re" ]] || continue
|
||||||
|
if printf '%s\n' "$rel" | grep -Eq "$re"; then
|
||||||
|
in_scope=true
|
||||||
|
fi
|
||||||
|
done <<EOF_RE
|
||||||
|
$regexes
|
||||||
|
EOF_RE
|
||||||
|
if [[ "$in_scope" == false ]]; then
|
||||||
|
err "$rel matches no 'files:' regex of any $skill hook — the probe path is stale, or the hook was rescoped away from a shape it still needs to lint"
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [[ "$VALE_AVAILABLE" == true ]] && ! vale_flags_path "$ini" "$rel"; then
|
||||||
|
err "$skill/assets/vale/.vale.ini raises no Kyberforge alert on $rel — its glob sections do not cover a path its own pre-commit hook is scoped to, so the hook passes that shape without linting it"
|
||||||
|
fi
|
||||||
|
done <<'EOF_PROBE'
|
||||||
|
skill-audit|plugins/demo/skills/demo/SKILL.md
|
||||||
|
agent-audit|plugins/demo/agents/demo.md
|
||||||
|
agent-audit|copilot/demo.agent.md
|
||||||
|
EOF_PROBE
|
||||||
|
|
||||||
if [[ $FAIL -gt 0 ]]; then
|
if [[ $FAIL -gt 0 ]]; then
|
||||||
echo "Vale style sync check failed: $FAIL error(s). agent-audit's copy is canonical — run scripts/sync-vale-styles.sh to regenerate skill-audit's copy, then commit both." >&2
|
echo "Vale style sync check failed: $FAIL error(s). For a drifted wrapper or style, agent-audit's copy is canonical — run scripts/sync-vale-styles.sh to regenerate skill-audit's copy, then commit both. A .vale.ini finding is not drift and sync-vale-styles.sh will not fix it: edit that file's own StylesPath, BasedOnStyles or glob sections." >&2
|
||||||
exit 1
|
exit 1
|
||||||
fi
|
fi
|
||||||
|
|||||||
@@ -9,41 +9,72 @@ FAIL=0
|
|||||||
pass() { echo " PASS: $1"; PASS=$((PASS + 1)); }
|
pass() { echo " PASS: $1"; PASS=$((PASS + 1)); }
|
||||||
fail() { echo " FAIL: $1"; FAIL=$((FAIL + 1)); }
|
fail() { echo " FAIL: $1"; FAIL=$((FAIL + 1)); }
|
||||||
|
|
||||||
|
# One trap over a registry, rather than rebuilding the trap line per fixture:
|
||||||
|
# the guard is there because bash 3.2 treats "${arr[@]}" on an empty array as
|
||||||
|
# unbound under `set -u`.
|
||||||
|
FIXTURES=()
|
||||||
|
cleanup() { [[ ${#FIXTURES[@]} -eq 0 ]] || rm -rf "${FIXTURES[@]}"; }
|
||||||
|
trap cleanup EXIT
|
||||||
|
|
||||||
# Helper: make a fixture repo with skill-audit/agent-audit's Vale copies, in sync by default.
|
# Helper: make a fixture repo with skill-audit/agent-audit's Vale copies, in sync by default.
|
||||||
|
# The wrapper is a stub — the script only diffs it — but the Vale assets and both
|
||||||
|
# pre-commit manifests are the repo's real ones, because the .vale.ini checks ask
|
||||||
|
# vale to apply those globs for real and cross-check them against the shipped
|
||||||
|
# hooks' `files:` regexes. A synthetic style or manifest would prove nothing, and
|
||||||
|
# copying the real ones keeps agent-audit's intentional KyberforgeCopilot
|
||||||
|
# divergence in the fixture instead of a sanitized stand-in for it.
|
||||||
make_fixture() {
|
make_fixture() {
|
||||||
local dir
|
local dir
|
||||||
dir="$(mktemp -d)"
|
dir="$(mktemp -d)"
|
||||||
local skill_audit="$dir/plugins/kyberforge/skills/skill-audit"
|
local skill_audit="$dir/plugins/kyberforge/skills/skill-audit"
|
||||||
local agent_audit="$dir/plugins/kyberforge/skills/agent-audit"
|
local agent_audit="$dir/plugins/kyberforge/skills/agent-audit"
|
||||||
mkdir -p "$skill_audit/scripts" "$skill_audit/assets/vale/styles/Kyberforge"
|
mkdir -p "$skill_audit/scripts" "$agent_audit/scripts"
|
||||||
mkdir -p "$agent_audit/scripts" "$agent_audit/assets/vale/styles/Kyberforge"
|
|
||||||
|
|
||||||
echo '#!/usr/bin/env bash' > "$skill_audit/scripts/vale-wrap.sh"
|
echo '#!/usr/bin/env bash' > "$skill_audit/scripts/vale-wrap.sh"
|
||||||
echo 'echo wrap' >> "$skill_audit/scripts/vale-wrap.sh"
|
echo 'echo wrap' >> "$skill_audit/scripts/vale-wrap.sh"
|
||||||
cp "$skill_audit/scripts/vale-wrap.sh" "$agent_audit/scripts/vale-wrap.sh"
|
cp "$skill_audit/scripts/vale-wrap.sh" "$agent_audit/scripts/vale-wrap.sh"
|
||||||
|
|
||||||
echo 'extends: existence' > "$skill_audit/assets/vale/styles/Kyberforge/Rule.yml"
|
cp -R "$REPO_ROOT/plugins/kyberforge/skills/skill-audit/assets" "$skill_audit/"
|
||||||
cp "$skill_audit/assets/vale/styles/Kyberforge/Rule.yml" "$agent_audit/assets/vale/styles/Kyberforge/Rule.yml"
|
cp -R "$REPO_ROOT/plugins/kyberforge/skills/agent-audit/assets" "$agent_audit/"
|
||||||
|
cp "$REPO_ROOT/.pre-commit-hooks.yaml" "$REPO_ROOT/.pre-commit-config.yaml" "$dir/"
|
||||||
|
|
||||||
echo "$dir"
|
echo "$dir"
|
||||||
}
|
}
|
||||||
|
|
||||||
|
# Helper: rewrite a glob section header in one copy's .vale.ini, leaving every
|
||||||
|
# other line — StylesPath, BasedOnStyles — intact. This is the shape of the
|
||||||
|
# typo the check exists to catch: the hook still matches the file via its
|
||||||
|
# `files:` regex, vale lints nothing, and pre-commit reports `Passed`.
|
||||||
|
break_glob() {
|
||||||
|
local ini="$1" old="$2" new="$3"
|
||||||
|
python3 - "$ini" "$old" "$new" <<'PYTHON'
|
||||||
|
import sys
|
||||||
|
path, old, new = sys.argv[1], sys.argv[2], sys.argv[3]
|
||||||
|
with open(path, encoding='utf-8') as fh:
|
||||||
|
content = fh.read()
|
||||||
|
assert old in content, f"{old} not found in {path}"
|
||||||
|
with open(path, 'w', encoding='utf-8') as fh:
|
||||||
|
fh.write(content.replace(old, new))
|
||||||
|
PYTHON
|
||||||
|
}
|
||||||
|
|
||||||
# --- 1. Exits 0 when the two copies are in sync ---
|
# --- 1. Exits 0 when the two copies are in sync ---
|
||||||
echo ""
|
echo ""
|
||||||
echo "--- exits 0 when skill-audit and agent-audit copies are in sync ---"
|
echo "--- exits 0 when skill-audit and agent-audit copies are in sync ---"
|
||||||
FIXTURE="$(make_fixture)"
|
FIXTURE="$(make_fixture)"
|
||||||
trap 'rm -rf "$FIXTURE"' EXIT
|
FIXTURES+=("$FIXTURE")
|
||||||
if bash "$SCRIPT" "$FIXTURE" > /dev/null 2>&1; then
|
if bash "$SCRIPT" "$FIXTURE" > /dev/null 2>&1; then
|
||||||
pass "exits 0 when copies are in sync"
|
pass "exits 0 when copies are in sync"
|
||||||
else
|
else
|
||||||
fail "exited non-zero against in-sync copies"
|
fail "exited non-zero against in-sync copies"
|
||||||
|
bash "$SCRIPT" "$FIXTURE" 2>&1 | sed 's/^/ /' || true
|
||||||
fi
|
fi
|
||||||
|
|
||||||
# --- 2. Exits 1 when vale-wrap.sh differs between the two copies ---
|
# --- 2. Exits 1 when vale-wrap.sh differs between the two copies ---
|
||||||
echo ""
|
echo ""
|
||||||
echo "--- exits 1 when vale-wrap.sh differs ---"
|
echo "--- exits 1 when vale-wrap.sh differs ---"
|
||||||
FIXTURE2="$(make_fixture)"
|
FIXTURE2="$(make_fixture)"
|
||||||
trap 'rm -rf "$FIXTURE" "$FIXTURE2"' EXIT
|
FIXTURES+=("$FIXTURE2")
|
||||||
echo 'echo different' >> "$FIXTURE2/plugins/kyberforge/skills/skill-audit/scripts/vale-wrap.sh"
|
echo 'echo different' >> "$FIXTURE2/plugins/kyberforge/skills/skill-audit/scripts/vale-wrap.sh"
|
||||||
if bash "$SCRIPT" "$FIXTURE2" > /dev/null 2>&1; then
|
if bash "$SCRIPT" "$FIXTURE2" > /dev/null 2>&1; then
|
||||||
fail "exited 0 when vale-wrap.sh copies differ — expected exit 1"
|
fail "exited 0 when vale-wrap.sh copies differ — expected exit 1"
|
||||||
@@ -55,8 +86,8 @@ fi
|
|||||||
echo ""
|
echo ""
|
||||||
echo "--- exits 1 when a Kyberforge style rule differs ---"
|
echo "--- exits 1 when a Kyberforge style rule differs ---"
|
||||||
FIXTURE3="$(make_fixture)"
|
FIXTURE3="$(make_fixture)"
|
||||||
trap 'rm -rf "$FIXTURE" "$FIXTURE2" "$FIXTURE3"' EXIT
|
FIXTURES+=("$FIXTURE3")
|
||||||
echo 'level: error' >> "$FIXTURE3/plugins/kyberforge/skills/agent-audit/assets/vale/styles/Kyberforge/Rule.yml"
|
echo ' - divergent token' >> "$FIXTURE3/plugins/kyberforge/skills/agent-audit/assets/vale/styles/Kyberforge/VagueWording.yml"
|
||||||
if bash "$SCRIPT" "$FIXTURE3" > /dev/null 2>&1; then
|
if bash "$SCRIPT" "$FIXTURE3" > /dev/null 2>&1; then
|
||||||
fail "exited 0 when a style rule differs — expected exit 1"
|
fail "exited 0 when a style rule differs — expected exit 1"
|
||||||
else
|
else
|
||||||
@@ -67,8 +98,14 @@ fi
|
|||||||
echo ""
|
echo ""
|
||||||
echo "--- exits 1 when a rule file is missing from one copy ---"
|
echo "--- exits 1 when a rule file is missing from one copy ---"
|
||||||
FIXTURE4="$(make_fixture)"
|
FIXTURE4="$(make_fixture)"
|
||||||
trap 'rm -rf "$FIXTURE" "$FIXTURE2" "$FIXTURE3" "$FIXTURE4"' EXIT
|
FIXTURES+=("$FIXTURE4")
|
||||||
echo 'extends: existence' > "$FIXTURE4/plugins/kyberforge/skills/agent-audit/assets/vale/styles/Kyberforge/Extra.yml"
|
cat > "$FIXTURE4/plugins/kyberforge/skills/agent-audit/assets/vale/styles/Kyberforge/Extra.yml" <<'EOF'
|
||||||
|
extends: existence
|
||||||
|
message: "Extra: '%s'"
|
||||||
|
level: error
|
||||||
|
tokens:
|
||||||
|
- divergent token
|
||||||
|
EOF
|
||||||
if bash "$SCRIPT" "$FIXTURE4" > /dev/null 2>&1; then
|
if bash "$SCRIPT" "$FIXTURE4" > /dev/null 2>&1; then
|
||||||
fail "exited 0 when a rule file exists in only one copy — expected exit 1"
|
fail "exited 0 when a rule file exists in only one copy — expected exit 1"
|
||||||
else
|
else
|
||||||
@@ -79,7 +116,7 @@ fi
|
|||||||
echo ""
|
echo ""
|
||||||
echo "--- exits 0 when kyberforge skills are absent (no-op) ---"
|
echo "--- exits 0 when kyberforge skills are absent (no-op) ---"
|
||||||
FIXTURE5="$(mktemp -d)"
|
FIXTURE5="$(mktemp -d)"
|
||||||
trap 'rm -rf "$FIXTURE" "$FIXTURE2" "$FIXTURE3" "$FIXTURE4" "$FIXTURE5"' EXIT
|
FIXTURES+=("$FIXTURE5")
|
||||||
if bash "$SCRIPT" "$FIXTURE5" > /dev/null 2>&1; then
|
if bash "$SCRIPT" "$FIXTURE5" > /dev/null 2>&1; then
|
||||||
pass "exits 0 as a no-op when skill-audit/agent-audit don't exist"
|
pass "exits 0 as a no-op when skill-audit/agent-audit don't exist"
|
||||||
else
|
else
|
||||||
@@ -93,7 +130,7 @@ echo ""
|
|||||||
echo "--- exits 1 when only one of the two copies is present ---"
|
echo "--- exits 1 when only one of the two copies is present ---"
|
||||||
FIXTURE6="$(make_fixture)"
|
FIXTURE6="$(make_fixture)"
|
||||||
FIXTURE7="$(make_fixture)"
|
FIXTURE7="$(make_fixture)"
|
||||||
trap 'rm -rf "$FIXTURE" "$FIXTURE2" "$FIXTURE3" "$FIXTURE4" "$FIXTURE5" "$FIXTURE6" "$FIXTURE7"' EXIT
|
FIXTURES+=("$FIXTURE6" "$FIXTURE7")
|
||||||
rm -rf "$FIXTURE6/plugins/kyberforge/skills/skill-audit"
|
rm -rf "$FIXTURE6/plugins/kyberforge/skills/skill-audit"
|
||||||
rm -rf "$FIXTURE7/plugins/kyberforge/skills/agent-audit"
|
rm -rf "$FIXTURE7/plugins/kyberforge/skills/agent-audit"
|
||||||
if bash "$SCRIPT" "$FIXTURE6" > /dev/null 2>&1; then
|
if bash "$SCRIPT" "$FIXTURE6" > /dev/null 2>&1; then
|
||||||
@@ -107,6 +144,172 @@ else
|
|||||||
pass "exits non-zero when agent-audit's canonical copy is missing but skill-audit's is present"
|
pass "exits non-zero when agent-audit's canonical copy is missing but skill-audit's is present"
|
||||||
fi
|
fi
|
||||||
|
|
||||||
|
# --- 7. Exits 1 when a .vale.ini is missing entirely ---
|
||||||
|
# Without it vale falls back to an upward config search and lints the file with
|
||||||
|
# whatever config it happens to find, which is not a failure anyone sees.
|
||||||
|
echo ""
|
||||||
|
echo "--- exits 1 when a .vale.ini is missing ---"
|
||||||
|
FIXTURE8="$(make_fixture)"
|
||||||
|
FIXTURES+=("$FIXTURE8")
|
||||||
|
rm -f "$FIXTURE8/plugins/kyberforge/skills/skill-audit/assets/vale/.vale.ini"
|
||||||
|
if bash "$SCRIPT" "$FIXTURE8" > /dev/null 2>&1; then
|
||||||
|
fail "exited 0 when skill-audit's .vale.ini is missing — expected exit 1"
|
||||||
|
else
|
||||||
|
pass "exits non-zero when a .vale.ini is missing"
|
||||||
|
fi
|
||||||
|
|
||||||
|
# --- 8. Exits 1 when the shared StylesPath line is dropped from either copy ---
|
||||||
|
# StylesPath resolves relative to the .vale.ini, which is the only reason the
|
||||||
|
# bundled styles are found from a consuming repo's clone prefix.
|
||||||
|
echo ""
|
||||||
|
echo "--- exits 1 when StylesPath is missing from either .vale.ini ---"
|
||||||
|
FIXTURE9="$(make_fixture)"
|
||||||
|
FIXTURE10="$(make_fixture)"
|
||||||
|
FIXTURES+=("$FIXTURE9" "$FIXTURE10")
|
||||||
|
break_glob "$FIXTURE9/plugins/kyberforge/skills/skill-audit/assets/vale/.vale.ini" \
|
||||||
|
'StylesPath = styles' 'StylesPath = elsewhere'
|
||||||
|
break_glob "$FIXTURE10/plugins/kyberforge/skills/agent-audit/assets/vale/.vale.ini" \
|
||||||
|
'StylesPath = styles' 'StylesPath = elsewhere'
|
||||||
|
if bash "$SCRIPT" "$FIXTURE9" > /dev/null 2>&1; then
|
||||||
|
fail "exited 0 when skill-audit's .vale.ini lost StylesPath — expected exit 1"
|
||||||
|
else
|
||||||
|
pass "exits non-zero when skill-audit's .vale.ini lost StylesPath"
|
||||||
|
fi
|
||||||
|
if bash "$SCRIPT" "$FIXTURE10" > /dev/null 2>&1; then
|
||||||
|
fail "exited 0 when agent-audit's .vale.ini lost StylesPath — expected exit 1"
|
||||||
|
else
|
||||||
|
pass "exits non-zero when agent-audit's .vale.ini lost StylesPath"
|
||||||
|
fi
|
||||||
|
|
||||||
|
# --- 9. Exits 1 when no section's BasedOnStyles names Kyberforge ---
|
||||||
|
# Every rule the prefilter gates on lives in that style, so a section that keeps
|
||||||
|
# its glob but loses the style lints the file and reports nothing.
|
||||||
|
echo ""
|
||||||
|
echo "--- exits 1 when BasedOnStyles no longer names Kyberforge ---"
|
||||||
|
FIXTURE11="$(make_fixture)"
|
||||||
|
FIXTURES+=("$FIXTURE11")
|
||||||
|
break_glob "$FIXTURE11/plugins/kyberforge/skills/agent-audit/assets/vale/.vale.ini" \
|
||||||
|
'BasedOnStyles = Kyberforge' 'BasedOnStyles = KyberforgeCopilot'
|
||||||
|
if bash "$SCRIPT" "$FIXTURE11" > /dev/null 2>&1; then
|
||||||
|
fail "exited 0 when agent-audit's .vale.ini stopped naming Kyberforge — expected exit 1"
|
||||||
|
else
|
||||||
|
pass "exits non-zero when a .vale.ini no longer names the Kyberforge style"
|
||||||
|
fi
|
||||||
|
|
||||||
|
# --- 10. Exits 1 when a glob section stops matching the shape its hook lints ---
|
||||||
|
# One case per glob section, because each covers a file shape the others don't:
|
||||||
|
# agent-audit's [**/*.agent.md] is the only section covering a Copilot agent file
|
||||||
|
# outside an agents/ directory, so breaking it alone is invisible to the others.
|
||||||
|
echo ""
|
||||||
|
echo "--- exits 1 when a .vale.ini glob no longer matches its hook's file shape ---"
|
||||||
|
FIXTURE12="$(make_fixture)"
|
||||||
|
FIXTURE13="$(make_fixture)"
|
||||||
|
FIXTURE14="$(make_fixture)"
|
||||||
|
FIXTURES+=("$FIXTURE12" "$FIXTURE13" "$FIXTURE14")
|
||||||
|
break_glob "$FIXTURE12/plugins/kyberforge/skills/skill-audit/assets/vale/.vale.ini" \
|
||||||
|
'[**/SKILL.md]' '[**/NOMATCH.md]'
|
||||||
|
break_glob "$FIXTURE13/plugins/kyberforge/skills/agent-audit/assets/vale/.vale.ini" \
|
||||||
|
'[**/agents/*.md]' '[**/NOMATCH-agents/*.md]'
|
||||||
|
break_glob "$FIXTURE14/plugins/kyberforge/skills/agent-audit/assets/vale/.vale.ini" \
|
||||||
|
'[**/*.agent.md]' '[**/*.NOMATCH.md]'
|
||||||
|
if bash "$SCRIPT" "$FIXTURE12" > /dev/null 2>&1; then
|
||||||
|
fail "exited 0 when skill-audit's SKILL.md glob matched nothing — expected exit 1"
|
||||||
|
else
|
||||||
|
pass "exits non-zero when skill-audit's SKILL.md glob matches nothing"
|
||||||
|
fi
|
||||||
|
if bash "$SCRIPT" "$FIXTURE13" > /dev/null 2>&1; then
|
||||||
|
fail "exited 0 when agent-audit's agents/*.md glob matched nothing — expected exit 1"
|
||||||
|
else
|
||||||
|
pass "exits non-zero when agent-audit's agents/*.md glob matches nothing"
|
||||||
|
fi
|
||||||
|
if bash "$SCRIPT" "$FIXTURE14" > /dev/null 2>&1; then
|
||||||
|
fail "exited 0 when agent-audit's *.agent.md glob matched nothing — expected exit 1"
|
||||||
|
else
|
||||||
|
pass "exits non-zero when agent-audit's *.agent.md glob matches nothing"
|
||||||
|
fi
|
||||||
|
|
||||||
|
# --- 11. Exits 1 when a probe path falls out of every hook's `files:` regex ---
|
||||||
|
# The probe paths are hardcoded, so they can silently stop representing anything
|
||||||
|
# the hooks lint. Rescoping the shipped agent hook away from the `.agent.md`
|
||||||
|
# shape has to fail here rather than leave a probe testing a shape no hook
|
||||||
|
# matches any more.
|
||||||
|
echo ""
|
||||||
|
echo "--- exits 1 when a probe path matches no hook's files: regex ---"
|
||||||
|
FIXTURE16="$(make_fixture)"
|
||||||
|
FIXTURES+=("$FIXTURE16")
|
||||||
|
break_glob "$FIXTURE16/.pre-commit-hooks.yaml" \
|
||||||
|
"files: '(^|/)agents/[^/]+\\.md\$|\\.agent\\.md\$'" "files: '(^|/)agents/[^/]+\\.md\$'"
|
||||||
|
if bash "$SCRIPT" "$FIXTURE16" > /dev/null 2>&1; then
|
||||||
|
fail "exited 0 when the agent hook was rescoped away from .agent.md — expected exit 1"
|
||||||
|
else
|
||||||
|
pass "exits non-zero when a probe path is in no hook's scope any more"
|
||||||
|
fi
|
||||||
|
|
||||||
|
# --- 12. The text-level assertions hold on a machine without vale ---
|
||||||
|
# They are the fallback when the glob probe cannot run. With vale on PATH the
|
||||||
|
# probe fails on these same mutations, so it would mask them: only masking vale
|
||||||
|
# proves a clean run here means the text assertions themselves ran.
|
||||||
|
echo ""
|
||||||
|
echo "--- the StylesPath / BasedOnStyles assertions still gate with vale masked off PATH ---"
|
||||||
|
VALE_DIR="$(dirname "$(command -v vale 2>/dev/null || echo /nonexistent/vale)")"
|
||||||
|
PATH_NO_VALE="$(printf '%s' "$PATH" | tr ':' '\n' | grep -vxF "$VALE_DIR" | paste -sd: -)"
|
||||||
|
if (PATH="$PATH_NO_VALE"; command -v vale >/dev/null 2>&1); then
|
||||||
|
fail "could not mask vale off PATH — the vale-absent fallback was not exercised"
|
||||||
|
else
|
||||||
|
FIXTURE17="$(make_fixture)"
|
||||||
|
FIXTURE18="$(make_fixture)"
|
||||||
|
FIXTURE19="$(make_fixture)"
|
||||||
|
FIXTURES+=("$FIXTURE17" "$FIXTURE18" "$FIXTURE19")
|
||||||
|
break_glob "$FIXTURE18/plugins/kyberforge/skills/skill-audit/assets/vale/.vale.ini" \
|
||||||
|
'StylesPath = styles' 'StylesPath = elsewhere'
|
||||||
|
break_glob "$FIXTURE19/plugins/kyberforge/skills/agent-audit/assets/vale/.vale.ini" \
|
||||||
|
'BasedOnStyles = Kyberforge' 'BasedOnStyles = KyberforgeCopilot'
|
||||||
|
if PATH="$PATH_NO_VALE" bash "$SCRIPT" "$FIXTURE17" > /dev/null 2>&1; then
|
||||||
|
pass "exits 0 on in-sync copies with vale unavailable"
|
||||||
|
else
|
||||||
|
fail "exited non-zero on in-sync copies with vale unavailable — the missing binary must warn, not fail"
|
||||||
|
fi
|
||||||
|
if PATH="$PATH_NO_VALE" bash "$SCRIPT" "$FIXTURE18" > /dev/null 2>&1; then
|
||||||
|
fail "exited 0 on a dropped StylesPath with vale unavailable — expected exit 1"
|
||||||
|
else
|
||||||
|
pass "exits non-zero on a dropped StylesPath with vale unavailable"
|
||||||
|
fi
|
||||||
|
if PATH="$PATH_NO_VALE" bash "$SCRIPT" "$FIXTURE19" > /dev/null 2>&1; then
|
||||||
|
fail "exited 0 on a BasedOnStyles that dropped Kyberforge with vale unavailable — expected exit 1"
|
||||||
|
else
|
||||||
|
pass "exits non-zero on a BasedOnStyles that dropped Kyberforge with vale unavailable"
|
||||||
|
fi
|
||||||
|
# A clean run without vale must say so — silence would read as verified.
|
||||||
|
if PATH="$PATH_NO_VALE" bash "$SCRIPT" "$FIXTURE17" 2>&1 | grep -q "vale is not installed"; then
|
||||||
|
pass "warns that glob coverage was not verified when vale is unavailable"
|
||||||
|
else
|
||||||
|
fail "exited clean without vale and said nothing — an unverified run looks identical to a verified one"
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
|
||||||
|
# --- 13. The intentional agent-audit-only divergence is NOT flagged ---
|
||||||
|
# The two .vale.ini files are deliberately different: agent-audit ships an extra
|
||||||
|
# [**/*.agent.md] section and the KyberforgeCopilot style. A check that diffed
|
||||||
|
# them would fail the repo as it stands, so assert the divergence is really in
|
||||||
|
# the fixture before asserting the check tolerates it — otherwise this case would
|
||||||
|
# still pass if the fixture had quietly stopped carrying it.
|
||||||
|
echo ""
|
||||||
|
echo "--- exits 0 despite agent-audit's KyberforgeCopilot divergence ---"
|
||||||
|
FIXTURE15="$(make_fixture)"
|
||||||
|
FIXTURES+=("$FIXTURE15")
|
||||||
|
AGENT_INI15="$FIXTURE15/plugins/kyberforge/skills/agent-audit/assets/vale/.vale.ini"
|
||||||
|
SKILL_INI15="$FIXTURE15/plugins/kyberforge/skills/skill-audit/assets/vale/.vale.ini"
|
||||||
|
if ! grep -q "KyberforgeCopilot" "$AGENT_INI15" \
|
||||||
|
|| grep -q "KyberforgeCopilot" "$SKILL_INI15" \
|
||||||
|
|| [[ ! -d "$FIXTURE15/plugins/kyberforge/skills/agent-audit/assets/vale/styles/KyberforgeCopilot" ]]; then
|
||||||
|
fail "the fixture no longer carries the agent-audit-only KyberforgeCopilot divergence, so tolerating it proves nothing"
|
||||||
|
elif bash "$SCRIPT" "$FIXTURE15" > /dev/null 2>&1; then
|
||||||
|
pass "exits 0 with agent-audit's extra KyberforgeCopilot section and style present"
|
||||||
|
else
|
||||||
|
fail "flagged the intentional agent-audit-only KyberforgeCopilot divergence — expected exit 0"
|
||||||
|
bash "$SCRIPT" "$FIXTURE15" 2>&1 | sed 's/^/ /' || true
|
||||||
|
fi
|
||||||
|
|
||||||
echo ""
|
echo ""
|
||||||
echo "Results: $PASS passed, $FAIL failed"
|
echo "Results: $PASS passed, $FAIL failed"
|
||||||
[[ $FAIL -eq 0 ]]
|
[[ $FAIL -eq 0 ]]
|
||||||
|
|||||||
@@ -64,13 +64,18 @@ repos:
|
|||||||
- id: kyberforge-skill-size-check
|
- id: kyberforge-skill-size-check
|
||||||
EOF
|
EOF
|
||||||
|
|
||||||
|
# The two fixtures carry DIFFERENT flagged tokens so an alert can never be
|
||||||
|
# credited to the hook that did not raise it. Both bodies land mid-sentence in a
|
||||||
|
# folded block scalar that still spans two physical lines, which is the
|
||||||
|
# flattening the wrapper exists to do.
|
||||||
write_fixtures() {
|
write_fixtures() {
|
||||||
local body="$1"
|
local skill_body="$1"
|
||||||
|
local agent_body="${2:-$1}"
|
||||||
cat > "$CONSUMER/skills/demo/SKILL.md" <<EOF
|
cat > "$CONSUMER/skills/demo/SKILL.md" <<EOF
|
||||||
---
|
---
|
||||||
name: demo
|
name: demo
|
||||||
description: >
|
description: >
|
||||||
Use when the caller wants a demonstration skill $body across two
|
Use when the caller wants a demonstration skill $skill_body across two
|
||||||
physical lines of one folded block scalar.
|
physical lines of one folded block scalar.
|
||||||
---
|
---
|
||||||
|
|
||||||
@@ -80,7 +85,7 @@ EOF
|
|||||||
---
|
---
|
||||||
name: demo
|
name: demo
|
||||||
description: >
|
description: >
|
||||||
Use when the caller wants a demonstration agent $body across two
|
Use when the caller wants a demonstration agent $agent_body across two
|
||||||
physical lines of one folded block scalar.
|
physical lines of one folded block scalar.
|
||||||
---
|
---
|
||||||
|
|
||||||
@@ -89,27 +94,56 @@ EOF
|
|||||||
git -C "$CONSUMER" add -A
|
git -C "$CONSUMER" add -A
|
||||||
}
|
}
|
||||||
|
|
||||||
run_hooks() {
|
# Vale prints each linted path as its own header line with that file's alerts
|
||||||
(cd "$CONSUMER" && pre-commit run --all-files 2>&1) || true
|
# indented beneath it, so an alert belongs to the nearest preceding path line.
|
||||||
|
# Reads a hook log on stdin and prints only the alert lines filed under `$1`.
|
||||||
|
# The `sed` strips vale's ANSI colouring, which it emits into pre-commit's pipe
|
||||||
|
# too, so the header lines compare as plain paths.
|
||||||
|
alerts_for() {
|
||||||
|
sed $'s/\033\\[[0-9;]*m//g' | awk -v want="$1" '
|
||||||
|
/^[^[:space:]].*\.md$/ { cur = $0; next }
|
||||||
|
/^[[:space:]]*[0-9]+:[0-9]+[[:space:]]/ { if (cur == want) print }
|
||||||
|
'
|
||||||
}
|
}
|
||||||
|
|
||||||
# --- 1. Both hooks resolve their config and actually gate on a bad file ---
|
# --- 1. Each Vale hook resolves its config and gates its own file shape ---
|
||||||
|
# Asserted per hook, against that hook's own fixture path and its own token. An
|
||||||
|
# aggregate alert count over both hooks' combined output does not prove this:
|
||||||
|
# one fixture description carries every flagged token, so ONE working hook
|
||||||
|
# already clears a `>= 2` threshold. And a hook whose .vale.ini globs match
|
||||||
|
# nothing reaches neither of the guards below — it still MATCHES the file via
|
||||||
|
# its `files:` regex, so pre-commit does not report `Skipped`; vale simply lints
|
||||||
|
# nothing, prints `0 errors ... in 1 file` and exits 0, and the hook shows
|
||||||
|
# `Passed`. Attribution is the only thing that catches it.
|
||||||
echo ""
|
echo ""
|
||||||
echo "--- both Vale hooks run and fail a bad file in an external consumer repo ---"
|
echo "--- each Vale hook flags its own fixture in an external consumer repo ---"
|
||||||
write_fixtures "that helps with and utilize things"
|
write_fixtures "that helps with things" "that will utilize things"
|
||||||
OUT_BAD="$(run_hooks)"
|
while IFS='|' read -r HOOK_ID FIXTURE TOKEN; do
|
||||||
if echo "$OUT_BAD" | grep -q "does not exist"; then
|
[[ -n "$HOOK_ID" ]] || continue
|
||||||
fail "hooks hard-errored on a path resolved against the consumer repo (E100) — the bug this test guards against"
|
LOG="$WORK/$HOOK_ID.log"
|
||||||
echo "$OUT_BAD" | sed 's/^/ /'
|
set +e
|
||||||
elif echo "$OUT_BAD" | grep -q "Skipped"; then
|
(cd "$CONSUMER" && pre-commit run "$HOOK_ID" --all-files > "$LOG" 2>&1)
|
||||||
fail "a hook matched no files, so it proved nothing"
|
RC_HOOK=$?
|
||||||
echo "$OUT_BAD" | sed 's/^/ /'
|
set -e
|
||||||
elif [[ "$(echo "$OUT_BAD" | grep -c "VagueWording")" -ge 2 ]]; then
|
if grep -q "does not exist" "$LOG"; then
|
||||||
pass "both hooks flatten and flag the folded description in a consumer repo"
|
fail "$HOOK_ID hard-errored on a path resolved against the consumer repo (E100) — the bug this test guards against"
|
||||||
|
sed 's/^/ /' "$LOG"
|
||||||
|
elif grep -q "Skipped" "$LOG"; then
|
||||||
|
fail "$HOOK_ID matched no files, so it proved nothing"
|
||||||
|
sed 's/^/ /' "$LOG"
|
||||||
|
elif [[ $RC_HOOK -eq 0 ]]; then
|
||||||
|
fail "$HOOK_ID passed $FIXTURE despite its flagged '$TOKEN' — a .vale.ini glob matching nothing lints zero files and exits 0"
|
||||||
|
sed 's/^/ /' "$LOG"
|
||||||
|
elif alerts_for "$FIXTURE" < "$LOG" | grep -qF "'$TOKEN'"; then
|
||||||
|
pass "$HOOK_ID flattens $FIXTURE and flags its '$TOKEN' in a consumer repo"
|
||||||
else
|
else
|
||||||
fail "hooks did not flag both fixtures"
|
fail "$HOOK_ID failed, but no alert quoting '$TOKEN' was filed under $FIXTURE"
|
||||||
echo "$OUT_BAD" | sed 's/^/ /'
|
sed 's/^/ /' "$LOG"
|
||||||
fi
|
fi
|
||||||
|
done <<'EOF'
|
||||||
|
kyberforge-vale-audit-skill|skills/demo/SKILL.md|helps with
|
||||||
|
kyberforge-vale-audit-agent|agents/demo.md|utilize
|
||||||
|
EOF
|
||||||
|
|
||||||
# --- 2. Clean files pass — the hooks gate, they don't just always fail ---
|
# --- 2. Clean files pass — the hooks gate, they don't just always fail ---
|
||||||
echo ""
|
echo ""
|
||||||
|
|||||||
Reference in New Issue
Block a user