The external-consumer test asserted a combined alert count (>=2) across both shipped Vale hooks, but the SKILL.md fixture alone raises two alerts — so one working hook satisfied the threshold. Retargeting agent-audit's glob to match nothing left the suite reporting "3 passed" under the message "both hooks flatten and flag". The Skipped guard does not catch this: the hook still matches the file, Vale lints nothing, reports 0 errors in 1 file and exits 0, which pre-commit renders as Passed. An assertion aggregating over N subjects proves nothing about any individual subject. Each hook now runs individually and its alerts are attributed to the nearest preceding path header, so an alert is checked by path rather than by presence in the combined blob. The two fixtures carry distinct VagueWording tokens, so one hook's alert cannot be credited to another. Nothing in the repo read either .vale.ini — the sync check diffed only vale-wrap.sh and styles/Kyberforge, so a one-line glob typo silently disabled the prefilter for a whole file type. That was the enabling half of the same defect. The check now asserts the shared lines both copies must carry (StylesPath, a section naming Kyberforge as a whole word) without flagging their intentional divergence, and probes each glob section by asking Vale itself to lint a representative path. Regex-to-glob comparison was rejected as it means reimplementing doublestar semantics in bash; a file-count dry-run was rejected because a section whose glob matches but whose BasedOnStyles lost Kyberforge reports "1 file" with no alerts and would pass it. Every new assertion is bound to a failing case in both directions: breaking the artifact fails the suite, and neutering the assertion fails exactly one case. That reverse sweep exposed two assertions bound to no failing case at all, one masked by a stronger check running first. Refs: #85
172 lines
7.6 KiB
Bash
Executable File
172 lines
7.6 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
set -euo pipefail
|
|
|
|
# Kyberforge's Vale prefilter is duplicated into skill-audit and agent-audit's own
|
|
# scripts/assets (per plugins/kyberforge/skills/skill-author/references/deployment-modes.md's
|
|
# no-cross-skill-path rule: a plugin's cache-install copy only includes each skill's own files).
|
|
# agent-audit's copy is canonical — it's the superset (Kyberforge + KyberforgeCopilot) that the
|
|
# repo root's own pre-commit hook and .pre-commit-hooks.yaml both consume. This fails the build
|
|
# if skill-audit's copy has drifted from it, since nothing else would catch a rule fix landing in
|
|
# only one of the two. Run from repo root or pass REPO_ROOT as arg.
|
|
|
|
REPO_ROOT="${1:-$(git rev-parse --show-toplevel 2>/dev/null || pwd)}"
|
|
# Absolutized because the glob probe below `cd`s into a scratch tree, where a
|
|
# relative --config path would stop resolving.
|
|
if [[ -d "$REPO_ROOT" ]]; then
|
|
REPO_ROOT="$(cd "$REPO_ROOT" && pwd)"
|
|
fi
|
|
FAIL=0
|
|
|
|
err() { echo " FAIL: $1" >&2; FAIL=$((FAIL + 1)); }
|
|
|
|
SKILL_AUDIT="$REPO_ROOT/plugins/kyberforge/skills/skill-audit"
|
|
AGENT_AUDIT="$REPO_ROOT/plugins/kyberforge/skills/agent-audit"
|
|
|
|
if [[ ! -d "$SKILL_AUDIT" && ! -d "$AGENT_AUDIT" ]]; then
|
|
exit 0
|
|
fi
|
|
|
|
# Exactly one present is drift, not absence: the missing copy can't be in sync
|
|
# with the surviving one, and treating it as a no-op is how a deleted or
|
|
# renamed copy would slip through silently.
|
|
if [[ ! -d "$SKILL_AUDIT" ]]; then
|
|
echo "Vale style sync check failed: $AGENT_AUDIT exists but $SKILL_AUDIT does not — run scripts/sync-vale-styles.sh to regenerate skill-audit's copy." >&2
|
|
exit 1
|
|
fi
|
|
if [[ ! -d "$AGENT_AUDIT" ]]; then
|
|
echo "Vale style sync check failed: $SKILL_AUDIT exists but $AGENT_AUDIT does not — agent-audit holds the canonical copy, so restore it before syncing." >&2
|
|
exit 1
|
|
fi
|
|
|
|
if ! diff -q "$SKILL_AUDIT/scripts/vale-wrap.sh" "$AGENT_AUDIT/scripts/vale-wrap.sh" >/dev/null 2>&1; then
|
|
err "scripts/vale-wrap.sh differs between skill-audit and agent-audit"
|
|
fi
|
|
|
|
if ! diff -rq "$SKILL_AUDIT/assets/vale/styles/Kyberforge" "$AGENT_AUDIT/assets/vale/styles/Kyberforge" >/dev/null 2>&1; then
|
|
err "assets/vale/styles/Kyberforge differs between skill-audit and agent-audit"
|
|
fi
|
|
|
|
# --- .vale.ini coverage ------------------------------------------------------
|
|
# The two .vale.ini files are deliberately NOT identical — agent-audit's carries
|
|
# an extra [**/*.agent.md] section and the KyberforgeCopilot style — so they
|
|
# cannot be diffed like the styles above. Nothing else in the repo read them at
|
|
# all, and that is what let a one-character glob typo silently disable the
|
|
# prefilter for a whole file type: the hook still MATCHES the file via its
|
|
# `files:` regex, so pre-commit reports neither `Skipped` nor an error; vale
|
|
# lints zero files, prints `0 errors ... in 1 file` and exits 0, and the hook
|
|
# shows `Passed`. So check the parts that must hold in both, not equality.
|
|
|
|
SKILL_INI="$SKILL_AUDIT/assets/vale/.vale.ini"
|
|
AGENT_INI="$AGENT_AUDIT/assets/vale/.vale.ini"
|
|
|
|
for ini in "$SKILL_INI" "$AGENT_INI"; do
|
|
rel_ini="${ini#"$REPO_ROOT"/}"
|
|
if [[ ! -f "$ini" ]]; then
|
|
err "$rel_ini is missing — without it vale falls back to an upward config search and lints with whatever it finds"
|
|
continue
|
|
fi
|
|
# StylesPath is resolved relative to the .vale.ini, which is the only reason
|
|
# the bundled styles are found from a consuming repo's clone prefix.
|
|
if ! grep -Eq '^[[:space:]]*StylesPath[[:space:]]*=[[:space:]]*styles[[:space:]]*$' "$ini"; then
|
|
err "$rel_ini has no 'StylesPath = styles' — the bundled styles/ directory would not be found"
|
|
fi
|
|
# Matches `Kyberforge` as a whole name, so `KyberforgeCopilot` alone does not
|
|
# satisfy it. Avoids \b, which is a GNU grep extension.
|
|
if ! grep -Eq '^[[:space:]]*BasedOnStyles[[:space:]]*=.*Kyberforge([[:space:],]|$)' "$ini"; then
|
|
err "$rel_ini has no section whose BasedOnStyles names Kyberforge — every rule the audit prefilters on lives in that style"
|
|
fi
|
|
done
|
|
|
|
# Prints the `files:` regex of every hook, in either manifest, whose entry is
|
|
# $1's vale-wrap.sh. Records are delimited by their `- id:` line, so the check
|
|
# does not depend on `entry:` preceding `files:` within a record.
|
|
hook_file_regexes() {
|
|
local skill="$1" manifest raw
|
|
for manifest in "$REPO_ROOT/.pre-commit-hooks.yaml" "$REPO_ROOT/.pre-commit-config.yaml"; do
|
|
[[ -f "$manifest" ]] || continue
|
|
awk -v skill="$skill" '
|
|
function flush() {
|
|
if (entry ~ skill "/scripts/vale-wrap.sh" && files != "") print files
|
|
entry = ""; files = ""
|
|
}
|
|
/^[ \t]*-[ \t]*id:/ { flush() }
|
|
/^[ \t]*entry:/ { entry = $0 }
|
|
/^[ \t]*files:/ { files = $0; sub(/^[ \t]*files:[ \t]*/, "", files) }
|
|
END { flush() }
|
|
' "$manifest"
|
|
done | while IFS= read -r raw; do
|
|
# Strip the surrounding YAML quotes; the regex itself never carries them.
|
|
raw="${raw%\'}"; raw="${raw#\'}"
|
|
raw="${raw%\"}"; raw="${raw#\"}"
|
|
printf '%s\n' "$raw"
|
|
done
|
|
}
|
|
|
|
# Asks vale — the thing that actually applies these globs — whether a config
|
|
# covers a path, rather than reimplementing doublestar matching. The probe file
|
|
# carries a description with a token Kyberforge.VagueWording flags, so a config
|
|
# whose glob matches but whose BasedOnStyles lost Kyberforge fails too: it would
|
|
# lint the file and report nothing.
|
|
vale_flags_path() {
|
|
local cfg="$1" rel="$2" tmp out
|
|
tmp="$(mktemp -d)"
|
|
mkdir -p "$tmp/$(dirname "$rel")"
|
|
{
|
|
echo "---"
|
|
echo "name: probe"
|
|
echo "description: Use when the caller wants a probe that helps with things."
|
|
echo "---"
|
|
echo ""
|
|
echo "Body."
|
|
} > "$tmp/$rel"
|
|
out="$(cd "$tmp" && vale --config "$cfg" "$rel" 2>&1)" || true
|
|
rm -rf "$tmp"
|
|
printf '%s\n' "$out" | grep -qF "Kyberforge.VagueWording"
|
|
}
|
|
|
|
VALE_AVAILABLE=true
|
|
if ! command -v vale >/dev/null 2>&1; then
|
|
VALE_AVAILABLE=false
|
|
echo " WARNING: vale is not installed — .vale.ini glob coverage was NOT verified. Install it (https://vale.sh/docs/vale-cli/installation/) before trusting a clean run." >&2
|
|
fi
|
|
|
|
# One representative path per file shape the prefilter is supposed to cover. Each
|
|
# is cross-checked against the shipped hooks' `files:` regexes first, so a path
|
|
# that goes stale because a hook was rescoped fails loudly here instead of
|
|
# quietly probing a shape nothing lints any more.
|
|
while IFS='|' read -r skill rel; do
|
|
[[ -n "$skill" ]] || continue
|
|
dir="$REPO_ROOT/plugins/kyberforge/skills/$skill"
|
|
ini="$dir/assets/vale/.vale.ini"
|
|
[[ -f "$ini" ]] || continue
|
|
|
|
regexes="$(hook_file_regexes "$skill")"
|
|
if [[ -n "$regexes" ]]; then
|
|
in_scope=false
|
|
while IFS= read -r re; do
|
|
[[ -n "$re" ]] || continue
|
|
if printf '%s\n' "$rel" | grep -Eq "$re"; then
|
|
in_scope=true
|
|
fi
|
|
done <<EOF_RE
|
|
$regexes
|
|
EOF_RE
|
|
if [[ "$in_scope" == false ]]; then
|
|
err "$rel matches no 'files:' regex of any $skill hook — the probe path is stale, or the hook was rescoped away from a shape it still needs to lint"
|
|
fi
|
|
fi
|
|
|
|
if [[ "$VALE_AVAILABLE" == true ]] && ! vale_flags_path "$ini" "$rel"; then
|
|
err "$skill/assets/vale/.vale.ini raises no Kyberforge alert on $rel — its glob sections do not cover a path its own pre-commit hook is scoped to, so the hook passes that shape without linting it"
|
|
fi
|
|
done <<'EOF_PROBE'
|
|
skill-audit|plugins/demo/skills/demo/SKILL.md
|
|
agent-audit|plugins/demo/agents/demo.md
|
|
agent-audit|copilot/demo.agent.md
|
|
EOF_PROBE
|
|
|
|
if [[ $FAIL -gt 0 ]]; then
|
|
echo "Vale style sync check failed: $FAIL error(s). For a drifted wrapper or style, agent-audit's copy is canonical — run scripts/sync-vale-styles.sh to regenerate skill-audit's copy, then commit both. A .vale.ini finding is not drift and sync-vale-styles.sh will not fix it: edit that file's own StylesPath, BasedOnStyles or glob sections." >&2
|
|
exit 1
|
|
fi
|