diff --git a/scripts/check-vale-style-sync.sh b/scripts/check-vale-style-sync.sh index 4f9dd1c..f6ede61 100755 --- a/scripts/check-vale-style-sync.sh +++ b/scripts/check-vale-style-sync.sh @@ -10,6 +10,11 @@ set -euo pipefail # only one of the two. Run from repo root or pass REPO_ROOT as arg. REPO_ROOT="${1:-$(git rev-parse --show-toplevel 2>/dev/null || pwd)}" +# Absolutized because the glob probe below `cd`s into a scratch tree, where a +# relative --config path would stop resolving. +if [[ -d "$REPO_ROOT" ]]; then + REPO_ROOT="$(cd "$REPO_ROOT" && pwd)" +fi FAIL=0 err() { echo " FAIL: $1" >&2; FAIL=$((FAIL + 1)); } @@ -41,7 +46,126 @@ if ! diff -rq "$SKILL_AUDIT/assets/vale/styles/Kyberforge" "$AGENT_AUDIT/assets/ err "assets/vale/styles/Kyberforge differs between skill-audit and agent-audit" fi +# --- .vale.ini coverage ------------------------------------------------------ +# The two .vale.ini files are deliberately NOT identical — agent-audit's carries +# an extra [**/*.agent.md] section and the KyberforgeCopilot style — so they +# cannot be diffed like the styles above. Nothing else in the repo read them at +# all, and that is what let a one-character glob typo silently disable the +# prefilter for a whole file type: the hook still MATCHES the file via its +# `files:` regex, so pre-commit reports neither `Skipped` nor an error; vale +# lints zero files, prints `0 errors ... in 1 file` and exits 0, and the hook +# shows `Passed`. So check the parts that must hold in both, not equality. + +SKILL_INI="$SKILL_AUDIT/assets/vale/.vale.ini" +AGENT_INI="$AGENT_AUDIT/assets/vale/.vale.ini" + +for ini in "$SKILL_INI" "$AGENT_INI"; do + rel_ini="${ini#"$REPO_ROOT"/}" + if [[ ! -f "$ini" ]]; then + err "$rel_ini is missing — without it vale falls back to an upward config search and lints with whatever it finds" + continue + fi + # StylesPath is resolved relative to the .vale.ini, which is the only reason + # the bundled styles are found from a consuming repo's clone prefix. + if ! grep -Eq '^[[:space:]]*StylesPath[[:space:]]*=[[:space:]]*styles[[:space:]]*$' "$ini"; then + err "$rel_ini has no 'StylesPath = styles' — the bundled styles/ directory would not be found" + fi + # Matches `Kyberforge` as a whole name, so `KyberforgeCopilot` alone does not + # satisfy it. Avoids \b, which is a GNU grep extension. + if ! grep -Eq '^[[:space:]]*BasedOnStyles[[:space:]]*=.*Kyberforge([[:space:],]|$)' "$ini"; then + err "$rel_ini has no section whose BasedOnStyles names Kyberforge — every rule the audit prefilters on lives in that style" + fi +done + +# Prints the `files:` regex of every hook, in either manifest, whose entry is +# $1's vale-wrap.sh. Records are delimited by their `- id:` line, so the check +# does not depend on `entry:` preceding `files:` within a record. +hook_file_regexes() { + local skill="$1" manifest raw + for manifest in "$REPO_ROOT/.pre-commit-hooks.yaml" "$REPO_ROOT/.pre-commit-config.yaml"; do + [[ -f "$manifest" ]] || continue + awk -v skill="$skill" ' + function flush() { + if (entry ~ skill "/scripts/vale-wrap.sh" && files != "") print files + entry = ""; files = "" + } + /^[ \t]*-[ \t]*id:/ { flush() } + /^[ \t]*entry:/ { entry = $0 } + /^[ \t]*files:/ { files = $0; sub(/^[ \t]*files:[ \t]*/, "", files) } + END { flush() } + ' "$manifest" + done | while IFS= read -r raw; do + # Strip the surrounding YAML quotes; the regex itself never carries them. + raw="${raw%\'}"; raw="${raw#\'}" + raw="${raw%\"}"; raw="${raw#\"}" + printf '%s\n' "$raw" + done +} + +# Asks vale — the thing that actually applies these globs — whether a config +# covers a path, rather than reimplementing doublestar matching. The probe file +# carries a description with a token Kyberforge.VagueWording flags, so a config +# whose glob matches but whose BasedOnStyles lost Kyberforge fails too: it would +# lint the file and report nothing. +vale_flags_path() { + local cfg="$1" rel="$2" tmp out + tmp="$(mktemp -d)" + mkdir -p "$tmp/$(dirname "$rel")" + { + echo "---" + echo "name: probe" + echo "description: Use when the caller wants a probe that helps with things." + echo "---" + echo "" + echo "Body." + } > "$tmp/$rel" + out="$(cd "$tmp" && vale --config "$cfg" "$rel" 2>&1)" || true + rm -rf "$tmp" + printf '%s\n' "$out" | grep -qF "Kyberforge.VagueWording" +} + +VALE_AVAILABLE=true +if ! command -v vale >/dev/null 2>&1; then + VALE_AVAILABLE=false + echo " WARNING: vale is not installed — .vale.ini glob coverage was NOT verified. Install it (https://vale.sh/docs/vale-cli/installation/) before trusting a clean run." >&2 +fi + +# One representative path per file shape the prefilter is supposed to cover. Each +# is cross-checked against the shipped hooks' `files:` regexes first, so a path +# that goes stale because a hook was rescoped fails loudly here instead of +# quietly probing a shape nothing lints any more. +while IFS='|' read -r skill rel; do + [[ -n "$skill" ]] || continue + dir="$REPO_ROOT/plugins/kyberforge/skills/$skill" + ini="$dir/assets/vale/.vale.ini" + [[ -f "$ini" ]] || continue + + regexes="$(hook_file_regexes "$skill")" + if [[ -n "$regexes" ]]; then + in_scope=false + while IFS= read -r re; do + [[ -n "$re" ]] || continue + if printf '%s\n' "$rel" | grep -Eq "$re"; then + in_scope=true + fi + done <&2 + echo "Vale style sync check failed: $FAIL error(s). For a drifted wrapper or style, agent-audit's copy is canonical — run scripts/sync-vale-styles.sh to regenerate skill-audit's copy, then commit both. A .vale.ini finding is not drift and sync-vale-styles.sh will not fix it: edit that file's own StylesPath, BasedOnStyles or glob sections." >&2 exit 1 fi diff --git a/tests/test-check-vale-style-sync.sh b/tests/test-check-vale-style-sync.sh index a0d82b1..c4f88bb 100755 --- a/tests/test-check-vale-style-sync.sh +++ b/tests/test-check-vale-style-sync.sh @@ -9,41 +9,72 @@ FAIL=0 pass() { echo " PASS: $1"; PASS=$((PASS + 1)); } fail() { echo " FAIL: $1"; FAIL=$((FAIL + 1)); } +# One trap over a registry, rather than rebuilding the trap line per fixture: +# the guard is there because bash 3.2 treats "${arr[@]}" on an empty array as +# unbound under `set -u`. +FIXTURES=() +cleanup() { [[ ${#FIXTURES[@]} -eq 0 ]] || rm -rf "${FIXTURES[@]}"; } +trap cleanup EXIT + # Helper: make a fixture repo with skill-audit/agent-audit's Vale copies, in sync by default. +# The wrapper is a stub — the script only diffs it — but the Vale assets and both +# pre-commit manifests are the repo's real ones, because the .vale.ini checks ask +# vale to apply those globs for real and cross-check them against the shipped +# hooks' `files:` regexes. A synthetic style or manifest would prove nothing, and +# copying the real ones keeps agent-audit's intentional KyberforgeCopilot +# divergence in the fixture instead of a sanitized stand-in for it. make_fixture() { local dir dir="$(mktemp -d)" local skill_audit="$dir/plugins/kyberforge/skills/skill-audit" local agent_audit="$dir/plugins/kyberforge/skills/agent-audit" - mkdir -p "$skill_audit/scripts" "$skill_audit/assets/vale/styles/Kyberforge" - mkdir -p "$agent_audit/scripts" "$agent_audit/assets/vale/styles/Kyberforge" + mkdir -p "$skill_audit/scripts" "$agent_audit/scripts" echo '#!/usr/bin/env bash' > "$skill_audit/scripts/vale-wrap.sh" echo 'echo wrap' >> "$skill_audit/scripts/vale-wrap.sh" cp "$skill_audit/scripts/vale-wrap.sh" "$agent_audit/scripts/vale-wrap.sh" - echo 'extends: existence' > "$skill_audit/assets/vale/styles/Kyberforge/Rule.yml" - cp "$skill_audit/assets/vale/styles/Kyberforge/Rule.yml" "$agent_audit/assets/vale/styles/Kyberforge/Rule.yml" + cp -R "$REPO_ROOT/plugins/kyberforge/skills/skill-audit/assets" "$skill_audit/" + cp -R "$REPO_ROOT/plugins/kyberforge/skills/agent-audit/assets" "$agent_audit/" + cp "$REPO_ROOT/.pre-commit-hooks.yaml" "$REPO_ROOT/.pre-commit-config.yaml" "$dir/" echo "$dir" } +# Helper: rewrite a glob section header in one copy's .vale.ini, leaving every +# other line — StylesPath, BasedOnStyles — intact. This is the shape of the +# typo the check exists to catch: the hook still matches the file via its +# `files:` regex, vale lints nothing, and pre-commit reports `Passed`. +break_glob() { + local ini="$1" old="$2" new="$3" + python3 - "$ini" "$old" "$new" <<'PYTHON' +import sys +path, old, new = sys.argv[1], sys.argv[2], sys.argv[3] +with open(path, encoding='utf-8') as fh: + content = fh.read() +assert old in content, f"{old} not found in {path}" +with open(path, 'w', encoding='utf-8') as fh: + fh.write(content.replace(old, new)) +PYTHON +} + # --- 1. Exits 0 when the two copies are in sync --- echo "" echo "--- exits 0 when skill-audit and agent-audit copies are in sync ---" FIXTURE="$(make_fixture)" -trap 'rm -rf "$FIXTURE"' EXIT +FIXTURES+=("$FIXTURE") if bash "$SCRIPT" "$FIXTURE" > /dev/null 2>&1; then pass "exits 0 when copies are in sync" else fail "exited non-zero against in-sync copies" + bash "$SCRIPT" "$FIXTURE" 2>&1 | sed 's/^/ /' || true fi # --- 2. Exits 1 when vale-wrap.sh differs between the two copies --- echo "" echo "--- exits 1 when vale-wrap.sh differs ---" FIXTURE2="$(make_fixture)" -trap 'rm -rf "$FIXTURE" "$FIXTURE2"' EXIT +FIXTURES+=("$FIXTURE2") echo 'echo different' >> "$FIXTURE2/plugins/kyberforge/skills/skill-audit/scripts/vale-wrap.sh" if bash "$SCRIPT" "$FIXTURE2" > /dev/null 2>&1; then fail "exited 0 when vale-wrap.sh copies differ — expected exit 1" @@ -55,8 +86,8 @@ fi echo "" echo "--- exits 1 when a Kyberforge style rule differs ---" FIXTURE3="$(make_fixture)" -trap 'rm -rf "$FIXTURE" "$FIXTURE2" "$FIXTURE3"' EXIT -echo 'level: error' >> "$FIXTURE3/plugins/kyberforge/skills/agent-audit/assets/vale/styles/Kyberforge/Rule.yml" +FIXTURES+=("$FIXTURE3") +echo ' - divergent token' >> "$FIXTURE3/plugins/kyberforge/skills/agent-audit/assets/vale/styles/Kyberforge/VagueWording.yml" if bash "$SCRIPT" "$FIXTURE3" > /dev/null 2>&1; then fail "exited 0 when a style rule differs — expected exit 1" else @@ -67,8 +98,14 @@ fi echo "" echo "--- exits 1 when a rule file is missing from one copy ---" FIXTURE4="$(make_fixture)" -trap 'rm -rf "$FIXTURE" "$FIXTURE2" "$FIXTURE3" "$FIXTURE4"' EXIT -echo 'extends: existence' > "$FIXTURE4/plugins/kyberforge/skills/agent-audit/assets/vale/styles/Kyberforge/Extra.yml" +FIXTURES+=("$FIXTURE4") +cat > "$FIXTURE4/plugins/kyberforge/skills/agent-audit/assets/vale/styles/Kyberforge/Extra.yml" <<'EOF' +extends: existence +message: "Extra: '%s'" +level: error +tokens: + - divergent token +EOF if bash "$SCRIPT" "$FIXTURE4" > /dev/null 2>&1; then fail "exited 0 when a rule file exists in only one copy — expected exit 1" else @@ -79,7 +116,7 @@ fi echo "" echo "--- exits 0 when kyberforge skills are absent (no-op) ---" FIXTURE5="$(mktemp -d)" -trap 'rm -rf "$FIXTURE" "$FIXTURE2" "$FIXTURE3" "$FIXTURE4" "$FIXTURE5"' EXIT +FIXTURES+=("$FIXTURE5") if bash "$SCRIPT" "$FIXTURE5" > /dev/null 2>&1; then pass "exits 0 as a no-op when skill-audit/agent-audit don't exist" else @@ -93,7 +130,7 @@ echo "" echo "--- exits 1 when only one of the two copies is present ---" FIXTURE6="$(make_fixture)" FIXTURE7="$(make_fixture)" -trap 'rm -rf "$FIXTURE" "$FIXTURE2" "$FIXTURE3" "$FIXTURE4" "$FIXTURE5" "$FIXTURE6" "$FIXTURE7"' EXIT +FIXTURES+=("$FIXTURE6" "$FIXTURE7") rm -rf "$FIXTURE6/plugins/kyberforge/skills/skill-audit" rm -rf "$FIXTURE7/plugins/kyberforge/skills/agent-audit" if bash "$SCRIPT" "$FIXTURE6" > /dev/null 2>&1; then @@ -107,6 +144,172 @@ else pass "exits non-zero when agent-audit's canonical copy is missing but skill-audit's is present" fi +# --- 7. Exits 1 when a .vale.ini is missing entirely --- +# Without it vale falls back to an upward config search and lints the file with +# whatever config it happens to find, which is not a failure anyone sees. +echo "" +echo "--- exits 1 when a .vale.ini is missing ---" +FIXTURE8="$(make_fixture)" +FIXTURES+=("$FIXTURE8") +rm -f "$FIXTURE8/plugins/kyberforge/skills/skill-audit/assets/vale/.vale.ini" +if bash "$SCRIPT" "$FIXTURE8" > /dev/null 2>&1; then + fail "exited 0 when skill-audit's .vale.ini is missing — expected exit 1" +else + pass "exits non-zero when a .vale.ini is missing" +fi + +# --- 8. Exits 1 when the shared StylesPath line is dropped from either copy --- +# StylesPath resolves relative to the .vale.ini, which is the only reason the +# bundled styles are found from a consuming repo's clone prefix. +echo "" +echo "--- exits 1 when StylesPath is missing from either .vale.ini ---" +FIXTURE9="$(make_fixture)" +FIXTURE10="$(make_fixture)" +FIXTURES+=("$FIXTURE9" "$FIXTURE10") +break_glob "$FIXTURE9/plugins/kyberforge/skills/skill-audit/assets/vale/.vale.ini" \ + 'StylesPath = styles' 'StylesPath = elsewhere' +break_glob "$FIXTURE10/plugins/kyberforge/skills/agent-audit/assets/vale/.vale.ini" \ + 'StylesPath = styles' 'StylesPath = elsewhere' +if bash "$SCRIPT" "$FIXTURE9" > /dev/null 2>&1; then + fail "exited 0 when skill-audit's .vale.ini lost StylesPath — expected exit 1" +else + pass "exits non-zero when skill-audit's .vale.ini lost StylesPath" +fi +if bash "$SCRIPT" "$FIXTURE10" > /dev/null 2>&1; then + fail "exited 0 when agent-audit's .vale.ini lost StylesPath — expected exit 1" +else + pass "exits non-zero when agent-audit's .vale.ini lost StylesPath" +fi + +# --- 9. Exits 1 when no section's BasedOnStyles names Kyberforge --- +# Every rule the prefilter gates on lives in that style, so a section that keeps +# its glob but loses the style lints the file and reports nothing. +echo "" +echo "--- exits 1 when BasedOnStyles no longer names Kyberforge ---" +FIXTURE11="$(make_fixture)" +FIXTURES+=("$FIXTURE11") +break_glob "$FIXTURE11/plugins/kyberforge/skills/agent-audit/assets/vale/.vale.ini" \ + 'BasedOnStyles = Kyberforge' 'BasedOnStyles = KyberforgeCopilot' +if bash "$SCRIPT" "$FIXTURE11" > /dev/null 2>&1; then + fail "exited 0 when agent-audit's .vale.ini stopped naming Kyberforge — expected exit 1" +else + pass "exits non-zero when a .vale.ini no longer names the Kyberforge style" +fi + +# --- 10. Exits 1 when a glob section stops matching the shape its hook lints --- +# One case per glob section, because each covers a file shape the others don't: +# agent-audit's [**/*.agent.md] is the only section covering a Copilot agent file +# outside an agents/ directory, so breaking it alone is invisible to the others. +echo "" +echo "--- exits 1 when a .vale.ini glob no longer matches its hook's file shape ---" +FIXTURE12="$(make_fixture)" +FIXTURE13="$(make_fixture)" +FIXTURE14="$(make_fixture)" +FIXTURES+=("$FIXTURE12" "$FIXTURE13" "$FIXTURE14") +break_glob "$FIXTURE12/plugins/kyberforge/skills/skill-audit/assets/vale/.vale.ini" \ + '[**/SKILL.md]' '[**/NOMATCH.md]' +break_glob "$FIXTURE13/plugins/kyberforge/skills/agent-audit/assets/vale/.vale.ini" \ + '[**/agents/*.md]' '[**/NOMATCH-agents/*.md]' +break_glob "$FIXTURE14/plugins/kyberforge/skills/agent-audit/assets/vale/.vale.ini" \ + '[**/*.agent.md]' '[**/*.NOMATCH.md]' +if bash "$SCRIPT" "$FIXTURE12" > /dev/null 2>&1; then + fail "exited 0 when skill-audit's SKILL.md glob matched nothing — expected exit 1" +else + pass "exits non-zero when skill-audit's SKILL.md glob matches nothing" +fi +if bash "$SCRIPT" "$FIXTURE13" > /dev/null 2>&1; then + fail "exited 0 when agent-audit's agents/*.md glob matched nothing — expected exit 1" +else + pass "exits non-zero when agent-audit's agents/*.md glob matches nothing" +fi +if bash "$SCRIPT" "$FIXTURE14" > /dev/null 2>&1; then + fail "exited 0 when agent-audit's *.agent.md glob matched nothing — expected exit 1" +else + pass "exits non-zero when agent-audit's *.agent.md glob matches nothing" +fi + +# --- 11. Exits 1 when a probe path falls out of every hook's `files:` regex --- +# The probe paths are hardcoded, so they can silently stop representing anything +# the hooks lint. Rescoping the shipped agent hook away from the `.agent.md` +# shape has to fail here rather than leave a probe testing a shape no hook +# matches any more. +echo "" +echo "--- exits 1 when a probe path matches no hook's files: regex ---" +FIXTURE16="$(make_fixture)" +FIXTURES+=("$FIXTURE16") +break_glob "$FIXTURE16/.pre-commit-hooks.yaml" \ + "files: '(^|/)agents/[^/]+\\.md\$|\\.agent\\.md\$'" "files: '(^|/)agents/[^/]+\\.md\$'" +if bash "$SCRIPT" "$FIXTURE16" > /dev/null 2>&1; then + fail "exited 0 when the agent hook was rescoped away from .agent.md — expected exit 1" +else + pass "exits non-zero when a probe path is in no hook's scope any more" +fi + +# --- 12. The text-level assertions hold on a machine without vale --- +# They are the fallback when the glob probe cannot run. With vale on PATH the +# probe fails on these same mutations, so it would mask them: only masking vale +# proves a clean run here means the text assertions themselves ran. +echo "" +echo "--- the StylesPath / BasedOnStyles assertions still gate with vale masked off PATH ---" +VALE_DIR="$(dirname "$(command -v vale 2>/dev/null || echo /nonexistent/vale)")" +PATH_NO_VALE="$(printf '%s' "$PATH" | tr ':' '\n' | grep -vxF "$VALE_DIR" | paste -sd: -)" +if (PATH="$PATH_NO_VALE"; command -v vale >/dev/null 2>&1); then + fail "could not mask vale off PATH — the vale-absent fallback was not exercised" +else + FIXTURE17="$(make_fixture)" + FIXTURE18="$(make_fixture)" + FIXTURE19="$(make_fixture)" + FIXTURES+=("$FIXTURE17" "$FIXTURE18" "$FIXTURE19") + break_glob "$FIXTURE18/plugins/kyberforge/skills/skill-audit/assets/vale/.vale.ini" \ + 'StylesPath = styles' 'StylesPath = elsewhere' + break_glob "$FIXTURE19/plugins/kyberforge/skills/agent-audit/assets/vale/.vale.ini" \ + 'BasedOnStyles = Kyberforge' 'BasedOnStyles = KyberforgeCopilot' + if PATH="$PATH_NO_VALE" bash "$SCRIPT" "$FIXTURE17" > /dev/null 2>&1; then + pass "exits 0 on in-sync copies with vale unavailable" + else + fail "exited non-zero on in-sync copies with vale unavailable — the missing binary must warn, not fail" + fi + if PATH="$PATH_NO_VALE" bash "$SCRIPT" "$FIXTURE18" > /dev/null 2>&1; then + fail "exited 0 on a dropped StylesPath with vale unavailable — expected exit 1" + else + pass "exits non-zero on a dropped StylesPath with vale unavailable" + fi + if PATH="$PATH_NO_VALE" bash "$SCRIPT" "$FIXTURE19" > /dev/null 2>&1; then + fail "exited 0 on a BasedOnStyles that dropped Kyberforge with vale unavailable — expected exit 1" + else + pass "exits non-zero on a BasedOnStyles that dropped Kyberforge with vale unavailable" + fi + # A clean run without vale must say so — silence would read as verified. + if PATH="$PATH_NO_VALE" bash "$SCRIPT" "$FIXTURE17" 2>&1 | grep -q "vale is not installed"; then + pass "warns that glob coverage was not verified when vale is unavailable" + else + fail "exited clean without vale and said nothing — an unverified run looks identical to a verified one" + fi +fi + +# --- 13. The intentional agent-audit-only divergence is NOT flagged --- +# The two .vale.ini files are deliberately different: agent-audit ships an extra +# [**/*.agent.md] section and the KyberforgeCopilot style. A check that diffed +# them would fail the repo as it stands, so assert the divergence is really in +# the fixture before asserting the check tolerates it — otherwise this case would +# still pass if the fixture had quietly stopped carrying it. +echo "" +echo "--- exits 0 despite agent-audit's KyberforgeCopilot divergence ---" +FIXTURE15="$(make_fixture)" +FIXTURES+=("$FIXTURE15") +AGENT_INI15="$FIXTURE15/plugins/kyberforge/skills/agent-audit/assets/vale/.vale.ini" +SKILL_INI15="$FIXTURE15/plugins/kyberforge/skills/skill-audit/assets/vale/.vale.ini" +if ! grep -q "KyberforgeCopilot" "$AGENT_INI15" \ + || grep -q "KyberforgeCopilot" "$SKILL_INI15" \ + || [[ ! -d "$FIXTURE15/plugins/kyberforge/skills/agent-audit/assets/vale/styles/KyberforgeCopilot" ]]; then + fail "the fixture no longer carries the agent-audit-only KyberforgeCopilot divergence, so tolerating it proves nothing" +elif bash "$SCRIPT" "$FIXTURE15" > /dev/null 2>&1; then + pass "exits 0 with agent-audit's extra KyberforgeCopilot section and style present" +else + fail "flagged the intentional agent-audit-only KyberforgeCopilot divergence — expected exit 0" + bash "$SCRIPT" "$FIXTURE15" 2>&1 | sed 's/^/ /' || true +fi + echo "" echo "Results: $PASS passed, $FAIL failed" [[ $FAIL -eq 0 ]] diff --git a/tests/test-vale-hooks-consumer.sh b/tests/test-vale-hooks-consumer.sh index c0eb6a7..b62d519 100755 --- a/tests/test-vale-hooks-consumer.sh +++ b/tests/test-vale-hooks-consumer.sh @@ -64,13 +64,18 @@ repos: - id: kyberforge-skill-size-check EOF +# The two fixtures carry DIFFERENT flagged tokens so an alert can never be +# credited to the hook that did not raise it. Both bodies land mid-sentence in a +# folded block scalar that still spans two physical lines, which is the +# flattening the wrapper exists to do. write_fixtures() { - local body="$1" + local skill_body="$1" + local agent_body="${2:-$1}" cat > "$CONSUMER/skills/demo/SKILL.md" < - Use when the caller wants a demonstration skill $body across two + Use when the caller wants a demonstration skill $skill_body across two physical lines of one folded block scalar. --- @@ -80,7 +85,7 @@ EOF --- name: demo description: > - Use when the caller wants a demonstration agent $body across two + Use when the caller wants a demonstration agent $agent_body across two physical lines of one folded block scalar. --- @@ -89,27 +94,56 @@ EOF git -C "$CONSUMER" add -A } -run_hooks() { - (cd "$CONSUMER" && pre-commit run --all-files 2>&1) || true +# Vale prints each linted path as its own header line with that file's alerts +# indented beneath it, so an alert belongs to the nearest preceding path line. +# Reads a hook log on stdin and prints only the alert lines filed under `$1`. +# The `sed` strips vale's ANSI colouring, which it emits into pre-commit's pipe +# too, so the header lines compare as plain paths. +alerts_for() { + sed $'s/\033\\[[0-9;]*m//g' | awk -v want="$1" ' + /^[^[:space:]].*\.md$/ { cur = $0; next } + /^[[:space:]]*[0-9]+:[0-9]+[[:space:]]/ { if (cur == want) print } + ' } -# --- 1. Both hooks resolve their config and actually gate on a bad file --- +# --- 1. Each Vale hook resolves its config and gates its own file shape --- +# Asserted per hook, against that hook's own fixture path and its own token. An +# aggregate alert count over both hooks' combined output does not prove this: +# one fixture description carries every flagged token, so ONE working hook +# already clears a `>= 2` threshold. And a hook whose .vale.ini globs match +# nothing reaches neither of the guards below — it still MATCHES the file via +# its `files:` regex, so pre-commit does not report `Skipped`; vale simply lints +# nothing, prints `0 errors ... in 1 file` and exits 0, and the hook shows +# `Passed`. Attribution is the only thing that catches it. echo "" -echo "--- both Vale hooks run and fail a bad file in an external consumer repo ---" -write_fixtures "that helps with and utilize things" -OUT_BAD="$(run_hooks)" -if echo "$OUT_BAD" | grep -q "does not exist"; then - fail "hooks hard-errored on a path resolved against the consumer repo (E100) — the bug this test guards against" - echo "$OUT_BAD" | sed 's/^/ /' -elif echo "$OUT_BAD" | grep -q "Skipped"; then - fail "a hook matched no files, so it proved nothing" - echo "$OUT_BAD" | sed 's/^/ /' -elif [[ "$(echo "$OUT_BAD" | grep -c "VagueWording")" -ge 2 ]]; then - pass "both hooks flatten and flag the folded description in a consumer repo" -else - fail "hooks did not flag both fixtures" - echo "$OUT_BAD" | sed 's/^/ /' -fi +echo "--- each Vale hook flags its own fixture in an external consumer repo ---" +write_fixtures "that helps with things" "that will utilize things" +while IFS='|' read -r HOOK_ID FIXTURE TOKEN; do + [[ -n "$HOOK_ID" ]] || continue + LOG="$WORK/$HOOK_ID.log" + set +e + (cd "$CONSUMER" && pre-commit run "$HOOK_ID" --all-files > "$LOG" 2>&1) + RC_HOOK=$? + set -e + if grep -q "does not exist" "$LOG"; then + fail "$HOOK_ID hard-errored on a path resolved against the consumer repo (E100) — the bug this test guards against" + sed 's/^/ /' "$LOG" + elif grep -q "Skipped" "$LOG"; then + fail "$HOOK_ID matched no files, so it proved nothing" + sed 's/^/ /' "$LOG" + elif [[ $RC_HOOK -eq 0 ]]; then + fail "$HOOK_ID passed $FIXTURE despite its flagged '$TOKEN' — a .vale.ini glob matching nothing lints zero files and exits 0" + sed 's/^/ /' "$LOG" + elif alerts_for "$FIXTURE" < "$LOG" | grep -qF "'$TOKEN'"; then + pass "$HOOK_ID flattens $FIXTURE and flags its '$TOKEN' in a consumer repo" + else + fail "$HOOK_ID failed, but no alert quoting '$TOKEN' was filed under $FIXTURE" + sed 's/^/ /' "$LOG" + fi +done <<'EOF' +kyberforge-vale-audit-skill|skills/demo/SKILL.md|helps with +kyberforge-vale-audit-agent|agents/demo.md|utilize +EOF # --- 2. Clean files pass — the hooks gate, they don't just always fail --- echo ""