#!/usr/bin/env bash set -euo pipefail REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" SCRIPT="$REPO_ROOT/scripts/check-vale-style-sync.sh" PASS=0 FAIL=0 pass() { echo " PASS: $1"; PASS=$((PASS + 1)); } fail() { echo " FAIL: $1"; FAIL=$((FAIL + 1)); } # Same `exit 77` (automake convention; run-tests.sh renders it as SKIPPED) guard # tests/test-vale-wrap.sh and tests/test-sync-plugin-content.sh use for a missing # binary. Without it this suite reported 5 genuine failures on a machine with no # vale, none of which were regressions. # # The suite SKIPPING while the script it tests HARD-FAILS is deliberate, not an # inconsistency. The script is a pre-push gate whose exit 0 is a claim that the # repo was verified, and six of its assertions are vale invocations — it must # never make that claim on a machine where they could not run. This suite makes # no claim about the repo; it claims the script behaves correctly, and most of # its cases (every glob-coverage case, 10/10b/11) cannot be exercised at all # without vale. Reporting those as FAIL would say "a regression landed" when the # truth is "this machine is missing a dev dependency" — noise that competes with # real failures. Note also that the vale-absent behavior is still fully covered # here even so: the masking below constructs that condition deliberately on a # machine that HAS vale, which is the only place it can be asserted against a # known-good baseline. if ! command -v vale &>/dev/null; then echo "SKIP: vale is not installed — the glob-coverage cases cannot run (install it: https://vale.sh/docs/vale-cli/installation/)" exit 77 fi # One trap over a registry, rather than rebuilding the trap line per fixture: # the guard is there because bash 3.2 treats "${arr[@]}" on an empty array as # unbound under `set -u`. FIXTURES=() cleanup() { [[ ${#FIXTURES[@]} -eq 0 ]] || rm -rf "${FIXTURES[@]}"; } trap cleanup EXIT # Helper: make a fixture repo with skill-audit/agent-audit's Vale copies, in sync by default. # The wrapper is a stub — the script only diffs it — but the Vale assets and both # pre-commit manifests are the repo's real ones, because the .vale.ini checks ask # vale to apply those globs for real and cross-check them against the shipped # hooks' `files:` regexes. A synthetic style or manifest would prove nothing, and # copying the real ones keeps agent-audit's intentional KyberforgeCopilot # divergence in the fixture instead of a sanitized stand-in for it. make_fixture() { local dir dir="$(mktemp -d)" local skill_audit="$dir/plugins/kyberforge/.apm/skills/skill-audit" local agent_audit="$dir/plugins/kyberforge/.apm/skills/agent-audit" mkdir -p "$skill_audit/scripts" "$agent_audit/scripts" echo '#!/usr/bin/env bash' > "$skill_audit/scripts/vale-wrap.sh" echo 'echo wrap' >> "$skill_audit/scripts/vale-wrap.sh" cp "$skill_audit/scripts/vale-wrap.sh" "$agent_audit/scripts/vale-wrap.sh" cp -R "$REPO_ROOT/plugins/kyberforge/.apm/skills/skill-audit/assets" "$skill_audit/" cp -R "$REPO_ROOT/plugins/kyberforge/.apm/skills/agent-audit/assets" "$agent_audit/" cp "$REPO_ROOT/.pre-commit-hooks.yaml" "$REPO_ROOT/.pre-commit-config.yaml" "$dir/" echo "$dir" } # Helper: rewrite a glob section header in one copy's .vale.ini, leaving every # other line — StylesPath, BasedOnStyles — intact. This is the shape of the # typo the check exists to catch: the hook still matches the file via its # `files:` regex, vale lints nothing, and pre-commit reports `Passed`. break_glob() { local ini="$1" old="$2" new="$3" python3 - "$ini" "$old" "$new" <<'PYTHON' import sys path, old, new = sys.argv[1], sys.argv[2], sys.argv[3] with open(path, encoding='utf-8') as fh: content = fh.read() assert old in content, f"{old} not found in {path}" with open(path, 'w', encoding='utf-8') as fh: fh.write(content.replace(old, new)) PYTHON } # Vale masking, hoisted so the text-only cases below can use it. Case 12 keeps # its own independent construction and its own loud failure if masking breaks — # it is what proves this mechanism works, so it is not refactored onto this. # # Why: a script run with vale on PATH performs six `vale` invocations (one per # probe path), and they are the suite's entire wall clock. The cases that assert # a text-level finding — StylesPath, BasedOnStyles, per-rule overrides — reach # their verdict through `grep` alone and gain nothing from paying for the # probes. Masking vale is not merely cheaper for them, it is STRICTER: with vale # present a dropped StylesPath also breaks the probe, so such a case would still # exit 1 with the assertion under test deleted. Without vale, only the assertion # under test can produce the failure. # # run_check falls back to an unmasked run rather than skipping when masking is # not safely available, so a machine where this cannot work loses speed, never # coverage. The utility probe matters as much as the vale probe: PATH_NO_VALE # deletes a whole PATH entry, and if that entry also carried grep/diff/awk/cat # the script would fail for an unrelated reason and every negative case below # would pass vacuously. VALE_DIR="$(dirname "$(command -v vale 2>/dev/null || echo /nonexistent/vale)")" PATH_NO_VALE="$(printf '%s' "$PATH" | tr ':' '\n' | grep -vxF "$VALE_DIR" | paste -sd: -)" VALE_MASKED=false if ! PATH="$PATH_NO_VALE" bash -c 'command -v vale' >/dev/null 2>&1 \ && PATH="$PATH_NO_VALE" bash -c \ 'command -v grep && command -v diff && command -v awk && command -v cat' >/dev/null 2>&1; then VALE_MASKED=true fi # Runs the check with vale masked off PATH when that is safe. For text-only # assertions ONLY — never for a case whose verdict depends on a glob probe # actually running. # # CHECK_VALE_STYLE_SYNC_ALLOW_MISSING_VALE=1 is required now that the script # treats a missing vale as a FAIL rather than a warning: without the opt-out # every masked run exits 1 unconditionally and every negative case below would # pass vacuously — the precise vacuity this whole round is closing. The opt-out # restores what masking is for here: the text assertion under test becomes the # only thing that can produce a non-zero exit. Case 12 asserts the un-opted-out # masked run really does hard-fail, so this env var cannot quietly become the # only path anyone exercises. run_check_no_vale() { if [[ "$VALE_MASKED" == true ]]; then PATH="$PATH_NO_VALE" CHECK_VALE_STYLE_SYNC_ALLOW_MISSING_VALE=1 bash "$SCRIPT" "$@" else bash "$SCRIPT" "$@" fi } # --- 1. Exits 0 when the two copies are in sync --- echo "" echo "--- exits 0 when skill-audit and agent-audit copies are in sync ---" FIXTURE="$(make_fixture)" FIXTURES+=("$FIXTURE") if bash "$SCRIPT" "$FIXTURE" > /dev/null 2>&1; then pass "exits 0 when copies are in sync" else fail "exited non-zero against in-sync copies" bash "$SCRIPT" "$FIXTURE" 2>&1 | sed 's/^/ /' || true fi # --- 2. Exits 1 when vale-wrap.sh differs between the two copies --- echo "" echo "--- exits 1 when vale-wrap.sh differs ---" FIXTURE2="$(make_fixture)" FIXTURES+=("$FIXTURE2") echo 'echo different' >> "$FIXTURE2/plugins/kyberforge/.apm/skills/skill-audit/scripts/vale-wrap.sh" if bash "$SCRIPT" "$FIXTURE2" > /dev/null 2>&1; then fail "exited 0 when vale-wrap.sh copies differ — expected exit 1" else pass "exits non-zero when vale-wrap.sh copies differ" fi # --- 3. Exits 1 when a style rule differs between the two copies --- echo "" echo "--- exits 1 when a Kyberforge style rule differs ---" FIXTURE3="$(make_fixture)" FIXTURES+=("$FIXTURE3") echo ' - divergent token' >> "$FIXTURE3/plugins/kyberforge/.apm/skills/agent-audit/assets/vale/styles/Kyberforge/VagueWording.yml" if bash "$SCRIPT" "$FIXTURE3" > /dev/null 2>&1; then fail "exited 0 when a style rule differs — expected exit 1" else pass "exits non-zero when a Kyberforge style rule differs between copies" fi # --- 4. Exits 1 when a rule file exists in only one copy --- echo "" echo "--- exits 1 when a rule file is missing from one copy ---" FIXTURE4="$(make_fixture)" FIXTURES+=("$FIXTURE4") cat > "$FIXTURE4/plugins/kyberforge/.apm/skills/agent-audit/assets/vale/styles/Kyberforge/Extra.yml" <<'EOF' extends: existence message: "Extra: '%s'" level: error tokens: - divergent token EOF if bash "$SCRIPT" "$FIXTURE4" > /dev/null 2>&1; then fail "exited 0 when a rule file exists in only one copy — expected exit 1" else pass "exits non-zero when a rule file is missing from one copy" fi # --- 5. Exits 0 (no-op) ONLY when there is no kyberforge plugin at all --- # The no-op is scoped to a repo that never installed kyberforge. Case 5c below # is its counterpart and the one that matters: `plugins/kyberforge/` present but # the `.apm/` targets under it absent is drift, not absence. echo "" echo "--- exits 0 when there is no plugins/kyberforge at all (no-op) ---" FIXTURE5="$(mktemp -d)" FIXTURES+=("$FIXTURE5") if [[ -e "$FIXTURE5/plugins/kyberforge" ]]; then fail "fixture 5 unexpectedly has a plugins/kyberforge, so it does not exercise the no-kyberforge no-op" elif bash "$SCRIPT" "$FIXTURE5" > /dev/null 2>&1; then pass "exits 0 as a no-op when the repo has no kyberforge plugin" else fail "exited non-zero when the repo simply has no kyberforge plugin" fi # --- 5c. Exits 1, saying so, when plugins/kyberforge exists but its .apm/ # targets do not --- # This script hardcodes plugins/kyberforge/.apm/skills/{skill-audit,agent-audit} # and had no floor under them: `mv plugins/kyberforge/.apm plugins/kyberforge/.apm2` # made both directories absent, which fell into the no-op above and exited 0 — # indistinguishable from a verified in-sync result, and swallowed by pre-commit # as `Passed`. A path rewrite is exactly the edit that produces this, and it is # what this PR did to these paths. # # Exit code alone proves little here (the script exits 1 for a dozen reasons), so # assert the MESSAGE: deleting the floor leaves exit 0, but a floor that fired # for the wrong reason would still be a bug this case must catch. echo "" echo "--- exits 1 and says so when plugins/kyberforge exists but .apm/ does not ---" FIXTURE5C="$(make_fixture)" FIXTURES+=("$FIXTURE5C") mv "$FIXTURE5C/plugins/kyberforge/.apm" "$FIXTURE5C/plugins/kyberforge/.apm2" STALE_OUT="" STALE_RC=0 STALE_OUT="$(bash "$SCRIPT" "$FIXTURE5C" 2>&1)" || STALE_RC=$? if [[ $STALE_RC -eq 0 ]]; then fail "exited 0 when plugins/kyberforge exists but its .apm/ targets are gone — expected exit 1" elif ! printf '%s\n' "$STALE_OUT" | grep -q "\.apm/ paths have gone stale"; then fail "failed for the wrong reason on a stale .apm/ path: $(printf '%s' "$STALE_OUT" | tr '\n' ' ')" else pass "exits non-zero and reports a stale .apm/ path when plugins/kyberforge exists without it" fi # --- 5d/5d2. Exits 1, saying so, when the probe TABLE itself verifies nothing --- # 5d used to relocate `assets/vale/` in both skills, on the belief that doing so # skipped the whole probe table with FAIL still at 0. It does not. Run against # the PRE-guard script that fixture already exited 1 with three errors: the # `.vale.ini` loop errs on both missing files long before the probe loop, and # PROBES_CHECKED can only reach 0 when both files are gone — which necessarily # means FAIL >= 2. So it never exercised the guard as a cause, only checked that # its message showed up beside unrelated failures. # # The guard is still worth having, but its real triggers live in the probe table, # which is part of the script rather than the fixture — so these two cases mutate # a COPY of the script and run that. Both assert `1 error(s)`, which is what makes # them real: with the guard deleted each mutation exits 0, and with it present the # guard is provably the only thing that failed the run. assert_mutated() { if diff -q "$SCRIPT" "$1" >/dev/null 2>&1; then fail "the script mutation changed nothing — the probe table's shape has moved, so this case would pass vacuously" return 1 fi } echo "" echo "--- exits 1 and says so when every probe row names a directory that does not exist ---" FIXTURE5D="$(make_fixture)" FIXTURES+=("$FIXTURE5D") SCRATCH5D="$(mktemp -d)" FIXTURES+=("$SCRATCH5D") sed 's/^skill-audit|/skill-auditX|/; s/^agent-audit|/agent-auditX|/' "$SCRIPT" > "$SCRATCH5D/drifted.sh" NOPROBE_OUT="" NOPROBE_RC=0 if assert_mutated "$SCRATCH5D/drifted.sh"; then NOPROBE_OUT="$(bash "$SCRATCH5D/drifted.sh" "$FIXTURE5D" 2>&1)" || NOPROBE_RC=$? if [[ $NOPROBE_RC -eq 0 ]]; then fail "a probe table naming no existing skill directory exited 0 — the glob-coverage section checked nothing and reported success" elif ! printf '%s\n' "$NOPROBE_OUT" | grep -q "no probe path was checked"; then fail "did not report that zero probe paths were checked: $(printf '%s' "$NOPROBE_OUT" | tr '\n' ' ')" elif ! printf '%s\n' "$NOPROBE_OUT" | grep -q "failed: 1 error(s)"; then fail "drifted probe rows failed for reasons beyond the empty probe table, so this guard is not provably what fired: $(printf '%s' "$NOPROBE_OUT" | tr '\n' ' ')" else pass "a probe table whose rows name no existing skill directory fails with that guard as the sole error" fi fi echo "" echo "--- exits 1 and says so when the probe table is empty ---" FIXTURE5D2="$(make_fixture)" FIXTURES+=("$FIXTURE5D2") SCRATCH5D2="$(mktemp -d)" FIXTURES+=("$SCRATCH5D2") # The other reachable trigger: the heredoc gutted outright by a bad merge or a # truncated edit. `done <<'EOF_PROBE'` with no rows between the delimiters is # valid bash — the loop body simply never runs. awk ' /^done <<.EOF_PROBE.$/ { print; inblk = 1; next } inblk && /^EOF_PROBE$/ { print; inblk = 0; next } inblk { next } { print } ' "$SCRIPT" > "$SCRATCH5D2/gutted.sh" EMPTYTBL_OUT="" EMPTYTBL_RC=0 if assert_mutated "$SCRATCH5D2/gutted.sh"; then EMPTYTBL_OUT="$(bash "$SCRATCH5D2/gutted.sh" "$FIXTURE5D2" 2>&1)" || EMPTYTBL_RC=$? if [[ $EMPTYTBL_RC -eq 0 ]]; then fail "an empty probe table exited 0 — the glob-coverage section verified nothing and reported success" elif ! printf '%s\n' "$EMPTYTBL_OUT" | grep -q "no probe path was checked"; then fail "did not report that zero probe paths were checked: $(printf '%s' "$EMPTYTBL_OUT" | tr '\n' ' ')" elif ! printf '%s\n' "$EMPTYTBL_OUT" | grep -q "failed: 1 error(s)"; then fail "an empty probe table failed for reasons beyond the guard: $(printf '%s' "$EMPTYTBL_OUT" | tr '\n' ' ')" else pass "an emptied probe heredoc fails with that guard as the sole error" fi fi # --- 5e. Positive: the check does real work against THIS repo --- # Every case above runs against a synthetic fixture, so the whole suite could be # green while the script inspected nothing at all in the repo it is wired into at # pre-push. The summary line carries the counts; assert they are non-zero. # # BOTH counts, not just the probe count. The `.vale.ini` half of that line was a # hardcoded `2` in each branch of the summary — true on any clean run, since a # missing or unreadable file errs out before the summary is reached, but a # constant states what the author expected rather than what the run inspected, # and extracting only the probe count left it asserted by nothing. It is computed # now, so the count is worth reading and worth pinning. echo "" echo "--- reports a non-zero number of inspected targets against this repo ---" REAL_OUT="" REAL_RC=0 REAL_OUT="$(bash "$SCRIPT" "$REPO_ROOT" 2>&1)" || REAL_RC=$? REAL_PROBES="$(printf '%s\n' "$REAL_OUT" | sed -n 's/.*checked, \([0-9][0-9]*\) glob probe(s).*/\1/p')" REAL_INIS="$(printf '%s\n' "$REAL_OUT" | sed -n 's/.*: \([0-9][0-9]*\) \.vale\.ini file(s) checked.*/\1/p')" if [[ $REAL_RC -ne 0 ]]; then fail "exited non-zero against this repo's real Vale copies" printf '%s\n' "$REAL_OUT" | sed 's/^/ /' elif [[ -z "$REAL_PROBES" || -z "$REAL_INIS" ]]; then fail "a clean run against this repo reported no inspected-target counts, so 'it checked something' is unverifiable: $(printf '%s' "$REAL_OUT" | tr '\n' ' ')" elif [[ "$REAL_PROBES" -lt 1 ]]; then fail "a clean run against this repo verified $REAL_PROBES glob probes — a pass that inspected nothing" elif [[ "$REAL_INIS" -lt 2 ]]; then fail "a clean run against this repo reported $REAL_INIS .vale.ini file(s) checked — both copies' configs must be inspected" else pass "inspects $REAL_INIS .vale.ini file(s) and $REAL_PROBES glob probe(s) against this repo, and exits 0" fi # --- 5b. Exits 1 when REPO_ROOT does not exist --- # A nonexistent path used to fall through to the "neither copy present" no-op # (test 5 above) and exit 0 — indistinguishable from a real, verified in-sync # result. That guard is for a repo legitimately missing kyberforge, not a # typo'd or stale path. echo "" echo "--- exits 1 when REPO_ROOT does not exist ---" if bash "$SCRIPT" "/nonexistent/path/$(date +%s)-$$" > /dev/null 2>&1; then fail "exited 0 for a nonexistent REPO_ROOT — expected exit 1" else pass "exits non-zero for a nonexistent REPO_ROOT" fi # --- 6. Exits 1 when only one of the two copies is present --- # The no-op guard used `||`, so a single missing copy also exited 0 — a deleted # or renamed copy passed the sync check silently. echo "" echo "--- exits 1 when only one of the two copies is present ---" FIXTURE6="$(make_fixture)" FIXTURE7="$(make_fixture)" FIXTURES+=("$FIXTURE6" "$FIXTURE7") rm -rf "$FIXTURE6/plugins/kyberforge/.apm/skills/skill-audit" rm -rf "$FIXTURE7/plugins/kyberforge/.apm/skills/agent-audit" if bash "$SCRIPT" "$FIXTURE6" > /dev/null 2>&1; then fail "exited 0 when only agent-audit is present — expected exit 1" else pass "exits non-zero when skill-audit's copy is missing but agent-audit's is present" fi if bash "$SCRIPT" "$FIXTURE7" > /dev/null 2>&1; then fail "exited 0 when only skill-audit is present — expected exit 1" else pass "exits non-zero when agent-audit's canonical copy is missing but skill-audit's is present" fi # --- 7. Exits 1 when a .vale.ini is missing entirely --- # Without it vale falls back to an upward config search and lints the file with # whatever config it happens to find, which is not a failure anyone sees. echo "" echo "--- exits 1 when a .vale.ini is missing ---" FIXTURE8="$(make_fixture)" FIXTURES+=("$FIXTURE8") rm -f "$FIXTURE8/plugins/kyberforge/.apm/skills/skill-audit/assets/vale/.vale.ini" if bash "$SCRIPT" "$FIXTURE8" > /dev/null 2>&1; then fail "exited 0 when skill-audit's .vale.ini is missing — expected exit 1" else pass "exits non-zero when a .vale.ini is missing" fi # --- 7b. Exits 1, saying so, when a .vale.ini is present but cannot be read --- # Every assertion in that loop is a grep, and grep exits 2 on a read error: the # two `grep -q` checks then misreport a file whose StylesPath and BasedOnStyles # may be perfectly fine, and the override capture swallows the error into an # empty result that reads as "no findings". So the exit code alone proves # nothing here — the check already exits 1 either way, just with the wrong # reason — and this case asserts the MESSAGE. Deleting the readability guard # leaves the exit code at 1 and the diagnosis wrong, which is exactly the # mutation the assertion below kills. # # The unreadable path is a DIRECTORY, not a mode-000 file, and that is the whole # point of the case: `cat` on a directory fails for every uid, while a mode-000 # file is readable by root, which is what this repo's dev environment and its # pre-push hooks run as. A permission-based fixture would pass or fail depending # on the invoking uid; this one does not. echo "" echo "--- exits 1 and says so when a .vale.ini exists but cannot be read ---" FIXTURE8B="$(make_fixture)" FIXTURES+=("$FIXTURE8B") UNREADABLE_INI="$FIXTURE8B/plugins/kyberforge/.apm/skills/skill-audit/assets/vale/.vale.ini" rm -f "$UNREADABLE_INI" mkdir -p "$UNREADABLE_INI" UNREADABLE_OUT="" UNREADABLE_RC=0 UNREADABLE_OUT="$(bash "$SCRIPT" "$FIXTURE8B" 2>&1)" || UNREADABLE_RC=$? if [[ -e "$UNREADABLE_INI" ]] && cat "$UNREADABLE_INI" >/dev/null 2>&1; then fail "the fixture's .vale.ini is still readable, so this case proves nothing about the unreadable branch" elif [[ $UNREADABLE_RC -eq 0 ]]; then fail "exited 0 when skill-audit's .vale.ini could not be read — expected exit 1" elif ! printf '%s\n' "$UNREADABLE_OUT" | grep -q "could not be read"; then fail "failed for the wrong reason on an unreadable .vale.ini — the readability guard did not fire, so the greps misdiagnosed it: $(printf '%s' "$UNREADABLE_OUT" | tr '\n' ' ')" else pass "exits non-zero and reports an unreadable .vale.ini as unreadable, not as missing or malformed" fi # --- 8. Exits 1 when the shared StylesPath line is dropped from either copy --- # StylesPath resolves relative to the .vale.ini, which is the only reason the # bundled styles are found from a consuming repo's clone prefix. # Run with vale masked: a dropped StylesPath also stops vale finding the styles, # so with vale on PATH the glob probe fails too and this case would still exit 1 # with the StylesPath assertion itself deleted. Masking makes the text assertion # the only thing that can produce the verdict. echo "" echo "--- exits 1 when StylesPath is missing from either .vale.ini ---" FIXTURE9="$(make_fixture)" FIXTURE10="$(make_fixture)" FIXTURES+=("$FIXTURE9" "$FIXTURE10") break_glob "$FIXTURE9/plugins/kyberforge/.apm/skills/skill-audit/assets/vale/.vale.ini" \ 'StylesPath = styles' 'StylesPath = elsewhere' break_glob "$FIXTURE10/plugins/kyberforge/.apm/skills/agent-audit/assets/vale/.vale.ini" \ 'StylesPath = styles' 'StylesPath = elsewhere' if run_check_no_vale "$FIXTURE9" > /dev/null 2>&1; then fail "exited 0 when skill-audit's .vale.ini lost StylesPath — expected exit 1" else pass "exits non-zero when skill-audit's .vale.ini lost StylesPath" fi if run_check_no_vale "$FIXTURE10" > /dev/null 2>&1; then fail "exited 0 when agent-audit's .vale.ini lost StylesPath — expected exit 1" else pass "exits non-zero when agent-audit's .vale.ini lost StylesPath" fi # --- 9. Exits 1 when no section's BasedOnStyles names Kyberforge --- # Every rule the prefilter gates on lives in that style, so a section that keeps # its glob but loses the style lints the file and reports nothing. echo "" echo "--- exits 1 when BasedOnStyles no longer names Kyberforge ---" FIXTURE11="$(make_fixture)" FIXTURES+=("$FIXTURE11") break_glob "$FIXTURE11/plugins/kyberforge/.apm/skills/agent-audit/assets/vale/.vale.ini" \ 'BasedOnStyles = Kyberforge' 'BasedOnStyles = KyberforgeCopilot' if run_check_no_vale "$FIXTURE11" > /dev/null 2>&1; then fail "exited 0 when agent-audit's .vale.ini stopped naming Kyberforge — expected exit 1" else pass "exits non-zero when a .vale.ini no longer names the Kyberforge style" fi # --- 9b. Exits 1 when a per-rule override leaves a rule at anything but error --- # The third way to switch a rule off without touching a style file or a glob. # CONTEXT.md's "Vale audit prefilter" entry: "Every rule is `level: error` and # every alert is a FAIL -- no ignorable tier". Vale's exit code keys on `error` # alerts alone, so any such override leaves the glob intact, the styles # byte-identical, and the run at `0 errors`, exit 0, `Passed`. # # Asserted as an ALLOWLIST because that is vale 3.15.2's own semantic, verified # by enumerating the value space: only the exact tokens `YES` and `error` keep a # rule blocking. `warning`/`suggestion` downgrade it (alert printed, exit 0 -- # invisible, since pre-commit swallows a passing hook's output); EVERY other # value silences it outright, including `false`, `0`, `off`, an empty value, # `garbage`, and lowercase `yes`. That last one is why a blocklist of # `NO|warning|suggestion` was not enough: `= yes` reads as "enabled" to a human # and disables the rule. Case 10's glob probe backstops none of this -- it keys # on one Kyberforge.VagueWording alert, so DescriptionOpener, PaddingPhrase, # SentenceOpenerThereIs and ProactivePhrase can each be retired underneath it, # which is why the cases below deliberately target rules that probe never sees. # # Two cases below are about comment forms, and they are NOT symmetric in vale: # `error # note` (spaced) is stripped by vale and stays live, while `error# note` # (no space) is not stripped and silences the rule. The gate demands a bare # token, so it flags both -- deliberately stricter than vale for the spaced form, # and the only way to catch the no-space form without reimplementing vale's # comment parsing. `Kyberforge.Vague2` covers rule names carrying a digit: such a # rule is genuinely silenced by `= NO`, and an alpha-only name class in the gate # would not even see the line. echo "" echo "--- exits 1 when a .vale.ini overrides a Kyberforge rule to anything but YES/error ---" while IFS= read -r override; do [[ -n "$override" ]] || continue # `` stands in for a bare `Rule =` with no value at all, which the # heredoc cannot carry as a trailing space without a linter eating it. override="${override//}" FIXTURE_OV="$(make_fixture)" FIXTURES+=("$FIXTURE_OV") echo "$override" >> "$FIXTURE_OV/plugins/kyberforge/.apm/skills/skill-audit/assets/vale/.vale.ini" if run_check_no_vale "$FIXTURE_OV" > /dev/null 2>&1; then fail "exited 0 with '$override' in skill-audit's .vale.ini -- expected exit 1" else pass "exits non-zero on '$override'" fi done <<'EOF_OVERRIDES' Kyberforge.SentenceOpenerThereIs = NO Kyberforge.VagueWording = warning Kyberforge.SentenceOpenerThereIs = suggestion Kyberforge.SentenceOpenerThereIs = false Kyberforge.DescriptionOpener = 0 Kyberforge.PaddingPhrase = off Kyberforge.SentenceOpenerThereIs = yes Kyberforge.DescriptionOpener = garbage Kyberforge.PaddingPhrase = Kyberforge.SentenceOpenerThereIs = NO # keep quiet Kyberforge.DescriptionOpener = error# silenced, vale strips no comment without a space Kyberforge.PaddingPhrase = error; silenced too, same no-space rule for ';' Kyberforge.DescriptionOpener = error # stripped by vale, still rejected: bare token required Kyberforge.Vague2 = NO Kyberforge.Vague_2 = NO Kyberforge.Vague-2 = NO EOF_OVERRIDES # Same in agent-audit's copy: the check runs over both .vale.ini files, and a # rule retired in only the canonical copy is the likelier direction. `= false` # on ProactivePhrase is the sharpest shape -- one word off the original defect, # on a KyberforgeCopilot rule no glob probe covers. FIXTURE_OV_AGENT="$(make_fixture)" FIXTURES+=("$FIXTURE_OV_AGENT") echo "KyberforgeCopilot.ProactivePhrase = false" \ >> "$FIXTURE_OV_AGENT/plugins/kyberforge/.apm/skills/agent-audit/assets/vale/.vale.ini" if run_check_no_vale "$FIXTURE_OV_AGENT" > /dev/null 2>&1; then fail "exited 0 with 'KyberforgeCopilot.ProactivePhrase = false' in agent-audit's .vale.ini -- expected exit 1" else pass "exits non-zero when agent-audit's copy retires a KyberforgeCopilot rule" fi # The two allowlisted values must NOT trip the assertion -- otherwise it would # fire on any legitimate explicit enablement. Kept as a positive case so an # over-broad tightening of the regex shows up here rather than in the repo. FIXTURE_OV_OK="$(make_fixture)" FIXTURES+=("$FIXTURE_OV_OK") { echo "Kyberforge.SentenceOpenerThereIs = YES" echo "Kyberforge.VagueWording = error" } >> "$FIXTURE_OV_OK/plugins/kyberforge/.apm/skills/skill-audit/assets/vale/.vale.ini" if bash "$SCRIPT" "$FIXTURE_OV_OK" > /dev/null 2>&1; then pass "an explicit '= YES' / '= error' override is not flagged" else fail "flagged an explicit '= YES' / '= error' override -- those are the two values that keep a rule blocking" bash "$SCRIPT" "$FIXTURE_OV_OK" 2>&1 | sed 's/^/ /' || true fi # --- 9c. Exits 1 when agent-audit ships KyberforgeCopilot but never loads it --- # Case 9 asserts only that Kyberforge is named, because skill-audit's copy # legitimately has no Copilot style. So dropping just `, KyberforgeCopilot` from # agent-audit's [**/*.agent.md] section unloaded the whole style silently: no # glob broke, the styles/ diff stayed clean (the directory is still shipped, # only never loaded), the two .vale.ini files are deliberately unequal so no # equality check applies, and case 10's probe still passed because it keys on a # Kyberforge alert. Verified dead by probing a `.agent.md` carrying # "Use proactively": 0 alerts under the broken config, KyberforgeCopilot. # ProactivePhrase under the shipped one. CONTEXT.md describes the style as # "scoped only to `.agent.md` files for the Copilot-only 'Use proactively has # no effect' check", so shipping it unloaded is drift. echo "" echo "--- exits 1 when the shipped KyberforgeCopilot style is named by no BasedOnStyles ---" FIXTURE11C="$(make_fixture)" FIXTURES+=("$FIXTURE11C") break_glob "$FIXTURE11C/plugins/kyberforge/.apm/skills/agent-audit/assets/vale/.vale.ini" \ 'BasedOnStyles = Kyberforge, KyberforgeCopilot' 'BasedOnStyles = Kyberforge' if run_check_no_vale "$FIXTURE11C" > /dev/null 2>&1; then fail "exited 0 when KyberforgeCopilot was dropped from BasedOnStyles -- expected exit 1" else pass "exits non-zero when a shipped KyberforgeCopilot style is never loaded" fi # The assertion is conditional on the style being shipped: a copy with no # KyberforgeCopilot directory (skill-audit's, by design) must stay clean -- # case 13 below covers the shipped-and-loaded pairing. # --- 10. Exits 1 when a glob section stops matching the shape its hook lints --- # One case per glob section, because each covers a file shape the others don't: # agent-audit's [**/*.agent.md] is the only section covering a Copilot agent file # outside an agents/ directory, so breaking it alone is invisible to the others. echo "" echo "--- exits 1 when a .vale.ini glob no longer matches its hook's file shape ---" FIXTURE12="$(make_fixture)" FIXTURE13="$(make_fixture)" FIXTURE14="$(make_fixture)" FIXTURES+=("$FIXTURE12" "$FIXTURE13" "$FIXTURE14") break_glob "$FIXTURE12/plugins/kyberforge/.apm/skills/skill-audit/assets/vale/.vale.ini" \ '[**/SKILL.md]' '[**/NOMATCH.md]' break_glob "$FIXTURE13/plugins/kyberforge/.apm/skills/agent-audit/assets/vale/.vale.ini" \ '[**/agents/*.md]' '[**/NOMATCH-agents/*.md]' break_glob "$FIXTURE14/plugins/kyberforge/.apm/skills/agent-audit/assets/vale/.vale.ini" \ '[**/*.agent.md]' '[**/*.NOMATCH.md]' if bash "$SCRIPT" "$FIXTURE12" > /dev/null 2>&1; then fail "exited 0 when skill-audit's SKILL.md glob matched nothing — expected exit 1" else pass "exits non-zero when skill-audit's SKILL.md glob matches nothing" fi if bash "$SCRIPT" "$FIXTURE13" > /dev/null 2>&1; then fail "exited 0 when agent-audit's agents/*.md glob matched nothing — expected exit 1" else pass "exits non-zero when agent-audit's agents/*.md glob matches nothing" fi if bash "$SCRIPT" "$FIXTURE14" > /dev/null 2>&1; then fail "exited 0 when agent-audit's *.agent.md glob matched nothing — expected exit 1" else pass "exits non-zero when agent-audit's *.agent.md glob matches nothing" fi # --- 10b. Exits 1 when a glob is narrowed to this repo's own plugins/ layout --- # Every probe path used to start with `plugins/`, so a glob narrowed from a # filename shape to a location (`[**/SKILL.md]` -> `[**/.apm/skills/*/SKILL.md]`) # still matched all of them and the check passed -- while a project-scope # `.claude/skills/foo/SKILL.md` started linting as `0 errors ... in 0 files`, # exit 0, hook `Passed`: the exact failure the script's own header comment says # it exists to catch. CONTEXT.md: "A `SKILL.md` outside `plugins/` (e.g. # project-scope `.claude/skills/foo/SKILL.md`) still matches `[**/SKILL.md]` and # gets linted normally -- the globs constrain filename shape, not location." # These narrowings are still valid glob syntax and break no `plugins/`-shaped # file, so only a non-`plugins/` probe path catches them. echo "" echo "--- exits 1 when a .vale.ini glob is narrowed from a filename shape to a location ---" FIXTURE14B="$(make_fixture)" FIXTURE14C="$(make_fixture)" FIXTURES+=("$FIXTURE14B" "$FIXTURE14C") break_glob "$FIXTURE14B/plugins/kyberforge/.apm/skills/skill-audit/assets/vale/.vale.ini" \ '[**/SKILL.md]' '[**/.apm/skills/*/SKILL.md]' break_glob "$FIXTURE14C/plugins/kyberforge/.apm/skills/agent-audit/assets/vale/.vale.ini" \ '[**/agents/*.md]' '[**/.apm/agents/*.md]' if bash "$SCRIPT" "$FIXTURE14B" > /dev/null 2>&1; then fail "exited 0 when skill-audit's glob stopped covering a SKILL.md outside plugins/ -- expected exit 1" else pass "exits non-zero when skill-audit's glob stops covering a project-scope SKILL.md" fi if bash "$SCRIPT" "$FIXTURE14C" > /dev/null 2>&1; then fail "exited 0 when agent-audit's glob stopped covering an agents/*.md outside plugins/ -- expected exit 1" else pass "exits non-zero when agent-audit's glob stops covering a project-scope agents/*.md" fi # --- 11. Exits 1 when a probe path falls out of every hook's `files:` regex --- # The probe paths are hardcoded, so they can silently stop representing anything # the hooks lint. Rescoping the shipped agent hook away from the `.agent.md` # shape has to fail here rather than leave a probe testing a shape no hook # matches any more. echo "" echo "--- exits 1 when a probe path matches no hook's files: regex ---" FIXTURE16="$(make_fixture)" FIXTURES+=("$FIXTURE16") break_glob "$FIXTURE16/.pre-commit-hooks.yaml" \ "files: '(^|/)agents/[^/]+\\.md\$|\\.agent\\.md\$'" "files: '(^|/)agents/[^/]+\\.md\$'" if bash "$SCRIPT" "$FIXTURE16" > /dev/null 2>&1; then fail "exited 0 when the agent hook was rescoped away from .agent.md — expected exit 1" else pass "exits non-zero when a probe path is in no hook's scope any more" fi # --- 11b. Exits 1 when the local config's files: regex narrows out of sync # with the canonical .pre-commit-hooks.yaml regex --- # hook_file_regexes() used to union the two manifests' `files:` regexes before # checking probe coverage, so a probe that matched only the old, looser # .pre-commit-hooks.yaml pattern still passed as "in scope" even after # .pre-commit-config.yaml's copy of the same hook was narrowed away from it. # That is exactly the shape of rescoping this repo's own agent hook went # through (SKILL/agent `.md` -> `.apm/.../*.agent.md`): the local hook quietly # stopped linting a shape the shipped, external-facing manifest still claims # to cover, and nothing caught it. Reproduce it directly: narrow only the # fixture's local config regex (leave .pre-commit-hooks.yaml as shipped) and # assert the check now flags the disagreement instead of passing silently. echo "" echo "--- exits 1 when .pre-commit-config.yaml's files: regex drifts out of sync with .pre-commit-hooks.yaml's ---" FIXTURE16B="$(make_fixture)" FIXTURES+=("$FIXTURE16B") break_glob "$FIXTURE16B/.pre-commit-config.yaml" \ "files: '^plugins/[^/]+/\\.apm/agents/[^/]+\\.agent\\.md\$'" \ "files: '^plugins/kyberforge/\\.apm/agents/[^/]+\\.agent\\.md\$'" if bash "$SCRIPT" "$FIXTURE16B" > /dev/null 2>&1; then fail "exited 0 when the local config regex narrowed out of sync with .pre-commit-hooks.yaml — expected exit 1" else pass "exits non-zero when the local config regex narrows out of sync with the canonical .pre-commit-hooks.yaml regex" fi # --- 12. The text-level assertions hold on a machine without vale --- # They are the fallback when the glob probe cannot run. With vale on PATH the # probe fails on these same mutations, so it would mask them: only masking vale # proves a clean run here means the text assertions themselves ran. echo "" echo "--- the StylesPath / BasedOnStyles assertions still gate with vale masked off PATH ---" VALE_DIR="$(dirname "$(command -v vale 2>/dev/null || echo /nonexistent/vale)")" PATH_NO_VALE="$(printf '%s' "$PATH" | tr ':' '\n' | grep -vxF "$VALE_DIR" | paste -sd: -)" if (PATH="$PATH_NO_VALE"; command -v vale >/dev/null 2>&1); then fail "could not mask vale off PATH — the vale-absent fallback was not exercised" else FIXTURE17="$(make_fixture)" FIXTURE18="$(make_fixture)" FIXTURE19="$(make_fixture)" FIXTURES+=("$FIXTURE17" "$FIXTURE18" "$FIXTURE19") break_glob "$FIXTURE18/plugins/kyberforge/.apm/skills/skill-audit/assets/vale/.vale.ini" \ 'StylesPath = styles' 'StylesPath = elsewhere' break_glob "$FIXTURE19/plugins/kyberforge/.apm/skills/agent-audit/assets/vale/.vale.ini" \ 'BasedOnStyles = Kyberforge' 'BasedOnStyles = KyberforgeCopilot' # 12a. Missing vale is a HARD FAILURE, not a warning — even on copies that are # otherwise perfectly in sync. It used to be a warning, and a warning made the # six glob probes self-disable on the machine that most needed them: applying # the one-character typo `[**/SKILL.md]` -> `[**/SKILLS.md]` and running with # vale off PATH exited 0, its sole output a stderr line pre-commit swallows, # so the pre-push hook reported `Passed`. That is the exact defect the # glob-coverage section exists to catch, disabled by the absence of the tool # that catches it. Assert the MESSAGE: exit 1 has a dozen causes here and the # fixture is in sync, so the code alone would not distinguish this from any # other finding. NOVALE_OUT="" NOVALE_RC=0 NOVALE_OUT="$(PATH="$PATH_NO_VALE" bash "$SCRIPT" "$FIXTURE17" 2>&1)" || NOVALE_RC=$? if [[ $NOVALE_RC -eq 0 ]]; then fail "exited 0 on in-sync copies with vale unavailable — a run that could not verify glob coverage must not report success" elif ! printf '%s\n' "$NOVALE_OUT" | grep -q "vale is not installed, so none of the .vale.ini glob-coverage probes ran"; then fail "failed without vale for the wrong reason — the missing-binary guard did not fire: $(printf '%s' "$NOVALE_OUT" | tr '\n' ' ')" else pass "hard-fails, saying so, when vale is unavailable and no opt-out is set" fi # 12b. The opt-out is the only way to get a clean exit without vale, and it has # to be set deliberately. Absence of the binary must never imply it. if PATH="$PATH_NO_VALE" CHECK_VALE_STYLE_SYNC_ALLOW_MISSING_VALE=1 \ bash "$SCRIPT" "$FIXTURE17" > /dev/null 2>&1; then pass "exits 0 on in-sync copies with vale unavailable and the explicit opt-out set" else fail "exited non-zero on in-sync copies with vale unavailable and CHECK_VALE_STYLE_SYNC_ALLOW_MISSING_VALE=1 — the opt-out does not work" fi # 12c/12d. The text assertions still gate under the opt-out. This is what the # opt-out has to preserve: masking vale makes the assertion under test the only # thing that can produce the verdict (with vale present, a dropped StylesPath # also breaks the probe, so these cases would still exit 1 with the assertion # itself deleted). if PATH="$PATH_NO_VALE" CHECK_VALE_STYLE_SYNC_ALLOW_MISSING_VALE=1 \ bash "$SCRIPT" "$FIXTURE18" > /dev/null 2>&1; then fail "exited 0 on a dropped StylesPath with vale unavailable — expected exit 1" else pass "exits non-zero on a dropped StylesPath with vale unavailable" fi if PATH="$PATH_NO_VALE" CHECK_VALE_STYLE_SYNC_ALLOW_MISSING_VALE=1 \ bash "$SCRIPT" "$FIXTURE19" > /dev/null 2>&1; then fail "exited 0 on a BasedOnStyles that dropped Kyberforge with vale unavailable — expected exit 1" else pass "exits non-zero on a BasedOnStyles that dropped Kyberforge with vale unavailable" fi # 12e. An opted-out clean run must still say it verified nothing — otherwise # the opt-out just reintroduces the silent vacuous pass under a new name. OPTOUT_OUT="$(PATH="$PATH_NO_VALE" CHECK_VALE_STYLE_SYNC_ALLOW_MISSING_VALE=1 \ bash "$SCRIPT" "$FIXTURE17" 2>&1)" if printf '%s\n' "$OPTOUT_OUT" | grep -q "glob coverage was NOT verified" \ && printf '%s\n' "$OPTOUT_OUT" | grep -q "0 glob probe(s) verified"; then pass "an opted-out clean run reports that glob coverage was not verified" else fail "an opted-out clean run did not say it verified no glob coverage — it looks identical to a verified one: $(printf '%s' "$OPTOUT_OUT" | tr '\n' ' ')" fi # 12f. The typo the whole section exists to catch must fail with vale absent # and the opt-out set, or not at all — never pass. It cannot be caught without # vale, so the opt-out must not turn it into a green run by accident: with the # opt-out this fixture legitimately passes, which is precisely why the opt-out # is gated on an env var and 12a is the default. FIXTURE19B="$(make_fixture)" FIXTURES+=("$FIXTURE19B") break_glob "$FIXTURE19B/plugins/kyberforge/.apm/skills/skill-audit/assets/vale/.vale.ini" \ '[**/SKILL.md]' '[**/SKILLS.md]' if PATH="$PATH_NO_VALE" bash "$SCRIPT" "$FIXTURE19B" > /dev/null 2>&1; then fail "the one-character glob typo exited 0 with vale off PATH — the probe self-disabled on the exact defect it exists to catch" else pass "the one-character glob typo does not exit 0 with vale off PATH" fi fi # --- 13. The intentional agent-audit-only divergence is NOT flagged --- # The two .vale.ini files are deliberately different: agent-audit ships an extra # [**/*.agent.md] section and the KyberforgeCopilot style. A check that diffed # them would fail the repo as it stands, so assert the divergence is really in # the fixture before asserting the check tolerates it — otherwise this case would # still pass if the fixture had quietly stopped carrying it. echo "" echo "--- exits 0 despite agent-audit's KyberforgeCopilot divergence ---" FIXTURE15="$(make_fixture)" FIXTURES+=("$FIXTURE15") AGENT_INI15="$FIXTURE15/plugins/kyberforge/.apm/skills/agent-audit/assets/vale/.vale.ini" SKILL_INI15="$FIXTURE15/plugins/kyberforge/.apm/skills/skill-audit/assets/vale/.vale.ini" if ! grep -q "KyberforgeCopilot" "$AGENT_INI15" \ || grep -q "KyberforgeCopilot" "$SKILL_INI15" \ || [[ ! -d "$FIXTURE15/plugins/kyberforge/.apm/skills/agent-audit/assets/vale/styles/KyberforgeCopilot" ]]; then fail "the fixture no longer carries the agent-audit-only KyberforgeCopilot divergence, so tolerating it proves nothing" elif bash "$SCRIPT" "$FIXTURE15" > /dev/null 2>&1; then pass "exits 0 with agent-audit's extra KyberforgeCopilot section and style present" else fail "flagged the intentional agent-audit-only KyberforgeCopilot divergence — expected exit 0" bash "$SCRIPT" "$FIXTURE15" 2>&1 | sed 's/^/ /' || true fi echo "" echo "Results: $PASS passed, $FAIL failed" [[ $FAIL -eq 0 ]]