#!/usr/bin/env bash # Regression test for the two properties of ADR-0020 boundary-target resolution # that decide whether the gate can be trusted at all. # # 1. MACHINE INDEPENDENCE. The resolution universe is derived by walking up # FROM THE TARGET FILE to an authoring root, and when one is found the # deployed .claude/ and .agents/ trees are deliberately NOT consulted. Those # trees are `apm install` output — gitignored, and present only on a machine # that has run it. Four cross-plugin targets in this repo resolved through # .claude/skills/ alone, so the same commit measured 2 dangling targets on a # developer machine and 6 on a fresh clone. A gate shipping hot with no # baseline cannot give two answers, so this file asserts the verdict is # identical with and without a deployed tree — on a synthetic fixture AND on # the real 39-skill corpus. # # 2. THE BARE-TARGET GRAMMAR RULE. A hyphenated token used as a compound # MODIFIER ("pre-commit hooks", "pull-request template") is prose, not a # route; a terminal one is a real target. Getting this wrong in either # direction is fatal: firing on prose makes the gate untrustworthy and it # gets turned off, while suppressing too much deletes the only two true # positives the corpus has. Both live true positives are BARE, which is why # the rule keys on the FOLLOWER TOKEN rather than on marking, and why they # are pinned by name below — a future false-positive fix must not be able to # quietly take them with it. # # 3. IN-SENTENCE CORROBORATION. Terminal position alone is not evidence of a # route: "run `pre-commit` instead", "see `commit-msg`", "use the clean-up # instead" and "run unit-tests" are all terminal, all prose, and all were # hard FAILs with no suppression mechanism anywhere in the gate. A # prose-form target therefore blocks only when its own sentence names # another target that RESOLVES; otherwise it is reported at SUGGESTION tier # and the commit proceeds. Route NOTATION (`/name`, `-> name`) is exempt # and always blocks. Both halves are asserted below: the prose class must # report-not-block, and the notation and corroborated forms must still # ERROR, or the fix would have eaten the gate rather than narrowed it. set -euo pipefail REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" HOOK="$REPO_ROOT/scripts/skill-size-check.sh" PASS=0 FAIL=0 pass() { echo " PASS: $1"; PASS=$((PASS + 1)); } fail() { echo " FAIL: $1"; FAIL=$((FAIL + 1)); } TMPDIR_T="$(mktemp -d)" trap 'rm -rf "$TMPDIR_T"' EXIT # write_skill write_skill() { mkdir -p "$1" { echo "---" echo "name: $2" echo "description: $3" echo "---" echo "" echo "Do the thing." } > "$1/SKILL.md" } # --------------------------------------------------------------------------- # 1. Machine independence — synthetic fixture # --------------------------------------------------------------------------- # Two trees, identical except that one also carries a deployed .claude/ tree # holding a skill and an agent that exist NOWHERE in plugins/. The subject routes # to one name that lives in a sibling plugin (must resolve in both) and one that # lives only in .claude/ (must DANGLE in both — an authoring root exists, so the # deployed tree is not part of the universe). # # If the deployed tree were consulted, the second target would resolve on the # machine that has run `apm install` and dangle on a fresh clone. That is the # 2-vs-6 defect exactly, at fixture scale. echo "" echo "--- the same file gets the same verdict with and without a deployed .claude/ tree ---" build_tree() { local root="$1" write_skill "$root/plugins/other-plugin/.apm/skills/cross-plugin-skill" cross-plugin-skill \ "Use when doing the other thing. Do not use for anything else." write_skill "$root/plugins/subject-plugin/.apm/skills/my-skill" my-skill \ "Use when doing the thing. Do not use for the other thing — use cross-plugin-skill or deployed-only-skill instead." } build_tree "$TMPDIR_T/no-claude" build_tree "$TMPDIR_T/with-claude" # The deployed tree, present only in the second root. Both a skill and an agent, # because both are valid routing targets and both would leak. mkdir -p "$TMPDIR_T/with-claude/.claude/skills/deployed-only-skill" \ "$TMPDIR_T/with-claude/.claude/agents" : > "$TMPDIR_T/with-claude/.claude/agents/deployed-only-agent.md" run_subject() { local root="$1" out set +e out="$(bash "$HOOK" "$root/plugins/subject-plugin/.apm/skills/my-skill/SKILL.md" 2>&1)" set -e # Normalise the tree root out of the paths so the two runs are comparable. printf '%s\n' "$out" | sed "s#$root##g" } NO_CLAUDE_OUT="$(run_subject "$TMPDIR_T/no-claude")" WITH_CLAUDE_OUT="$(run_subject "$TMPDIR_T/with-claude")" if [[ "$NO_CLAUDE_OUT" == "$WITH_CLAUDE_OUT" ]]; then pass "identical output with and without a deployed .claude/ tree" else fail "the deployed .claude/ tree changed the verdict — without: [$NO_CLAUDE_OUT] with: [$WITH_CLAUDE_OUT]" fi # Identical-but-wrong is still possible (both could resolve everything, or # neither could resolve anything), so the CONTENT is asserted too: the # sibling-plugin name must resolve and the deployed-only name must not. if [[ "$WITH_CLAUDE_OUT" == *"routes to 'deployed-only-skill'"* ]]; then pass "a name that exists only in .claude/ still dangles when an authoring root is present" else fail "the deployed-only target did not dangle — the deployed tree is being read into the universe: $WITH_CLAUDE_OUT" fi if [[ "$WITH_CLAUDE_OUT" != *"routes to 'cross-plugin-skill'"* ]]; then pass "a name in a SIBLING PLUGIN resolves, so the comparison above is not 'nothing resolves'" else fail "the sibling-plugin target dangled — the monorepo universe is not being built: $WITH_CLAUDE_OUT" fi # The other half of the rule: with NO authoring root, deployed trees ARE the # universe. That is the consumer case, and without this the rule above could be # implemented as "never read .claude/", which would leave consumers with no # resolution at all. echo "" echo "--- with no authoring root, a deployed .claude/ tree IS the universe ---" CONSUMER="$TMPDIR_T/consumer" mkdir -p "$CONSUMER/.claude/skills/deployed-only-skill" write_skill "$CONSUMER/.claude/skills/my-skill" my-skill \ "Use when doing the thing. Do not use for the other thing — use deployed-only-skill instead." set +e CONSUMER_OUT="$(bash "$HOOK" "$CONSUMER/.claude/skills/my-skill/SKILL.md" 2>&1)" CONSUMER_RC=$? set -e if [[ $CONSUMER_RC -eq 0 && "$CONSUMER_OUT" != *"routes to"* && "$CONSUMER_OUT" != *"DID NOT RUN"* ]]; then pass "a sibling in a deployed .claude/skills/ tree resolves when there is no authoring root" else fail "the consumer path did not resolve through the deployed tree (exit $CONSUMER_RC): ${CONSUMER_OUT:-}" fi # --------------------------------------------------------------------------- # 1b. Machine independence — the real corpus # --------------------------------------------------------------------------- # The fixture above proves the rule; this proves it at the scale where it broke. # # The A/B is built rather than borrowed. plugins/ is copied TWICE: once bare (a # fresh clone), and once with a synthetic .claude/skills/ tree deployed beside it # holding a directory for every name the corpus currently reports as dangling. If # deployed trees leaked back into the universe, the second copy would resolve # those names and report an empty dangling set while the first reported two — # 2-vs-6, reproduced deterministically. # # Deliberately NOT keyed on whether THIS machine has run `apm install`. Doing that # would make the suite fail on a fresh clone (where there is no .claude/ to # contrast against) — a test of machine independence that is itself # machine-dependent. The live tree is still compared, as a third data point, but # nothing here requires it to be in either state. echo "" echo "--- the real corpus reports the same dangling targets with and without a deployed tree ---" dangling_set() { local -a files=() local f # Collected with a `while read` loop, not `mapfile`: macOS ships bash 3.2, # which has no `mapfile`, and tests/test-vale-wrap.sh scans tests/*.sh for # exactly that hazard. `find` rather than a glob so both roots walk identically. while IFS= read -r f; do files+=("$f") done < <(find "$1" -path '*/.apm/skills/*/SKILL.md' | sort) if [[ ${#files[@]} -eq 0 ]]; then echo "NO-FILES-FOUND" return fi set +e bash "$HOOK" ${files[@]+"${files[@]}"} 2>&1 \ | grep -oE "routes to '[^']+'" \ | sed "s/routes to '//; s/'//" \ | sort -u set -e } LIVE_DANGLING="$(dangling_set "$REPO_ROOT/plugins")" FRESH_ROOT="$TMPDIR_T/fresh-clone" mkdir -p "$FRESH_ROOT" cp -R "$REPO_ROOT/plugins" "$FRESH_ROOT/plugins" [[ -f "$REPO_ROOT/apm.yml" ]] && cp "$REPO_ROOT/apm.yml" "$FRESH_ROOT/apm.yml" FRESH_DANGLING="$(dangling_set "$FRESH_ROOT/plugins")" DEPLOYED_ROOT="$TMPDIR_T/deployed-clone" mkdir -p "$DEPLOYED_ROOT/.claude/skills" "$DEPLOYED_ROOT/.claude/agents" cp -R "$REPO_ROOT/plugins" "$DEPLOYED_ROOT/plugins" [[ -f "$REPO_ROOT/apm.yml" ]] && cp "$REPO_ROOT/apm.yml" "$DEPLOYED_ROOT/apm.yml" # Deploy exactly the names that currently dangle. That is the strongest possible # bait: if the deployed tree were consulted, every one of them would resolve and # the dangling set would collapse to empty. DEPLOY_COUNT=0 while IFS= read -r name; do [[ -n "$name" ]] || continue mkdir -p "$DEPLOYED_ROOT/.claude/skills/$name" DEPLOY_COUNT=$((DEPLOY_COUNT + 1)) done <<< "$FRESH_DANGLING" DEPLOYED_DANGLING="$(dangling_set "$DEPLOYED_ROOT/plugins")" if [[ "$DEPLOY_COUNT" -gt 0 ]]; then pass "precondition: $DEPLOY_COUNT dangling name(s) deployed into the contrast tree's .claude/skills/, so the A/B has something to distinguish" else fail "no dangling names to deploy — the corpus reports none, so this A/B distinguishes nothing. Deploy a known-absent name explicitly instead of deriving one." fi if [[ ! -d "$FRESH_ROOT/.claude" && ! -d "$FRESH_ROOT/.agents" ]]; then pass "precondition: the fresh-clone copy has no deployed tree of its own" else fail "the fresh-clone copy picked up a deployed tree — it is not a fresh-clone fixture" fi if [[ "$FRESH_DANGLING" == "$DEPLOYED_DANGLING" ]]; then pass "deploying every dangling name into .claude/skills/ changes nothing: $(echo "$FRESH_DANGLING" | tr '\n' ' ')" else fail "the corpus verdict depends on whether apm install has been run — fresh clone: [$(echo "$FRESH_DANGLING" | tr '\n' ' ')] with a deployed tree: [$(echo "$DEPLOYED_DANGLING" | tr '\n' ' ')]" fi # Third data point: whatever state THIS machine happens to be in, the live tree # must agree with a bare copy of the same plugins/. No precondition on that state # — see the section header. if [[ "$LIVE_DANGLING" == "$FRESH_DANGLING" ]]; then pass "the live tree agrees with a bare copy (this machine $( [[ -d "$REPO_ROOT/.claude/skills" ]] && echo "HAS" || echo "has no" ) deployed .claude/skills/ tree)" else fail "the live tree disagrees with a bare copy of the same plugins/ — live: [$(echo "$LIVE_DANGLING" | tr '\n' ' ')] fresh clone: [$(echo "$FRESH_DANGLING" | tr '\n' ' ')]" fi # --------------------------------------------------------------------------- # 1c. The two live true positives, pinned by name # --------------------------------------------------------------------------- # ADR-0020 records these as real broken routing targets and splits fixing them # into Gitea issue #100. Until that lands they are the ONLY evidence the dangling # check finds anything at all in real prose, so they are asserted as an exact set # rather than a "contains" — a false-positive fix that suppressed one of them # would otherwise land green. # # `gitea-labels` is the subtler of the two and is worth keeping: it is not # written anywhere as `gitea-labels`. gitea-issues' description says "Composes # `gitea-labels-\n milestones`" in a `>`-folded scalar, and the fold joins the # lines into "gitea-labels- milestones" — the trailing hyphen is what keeps the # token terminal and therefore danglable. # # WHEN ISSUE #100 IS FIXED: update EXPECTED_DANGLING to match. Do not delete the # assertion — an empty expected set is fine and still pins that no NEW dangling # target appeared. echo "" echo "--- the two live dangling targets in the corpus are exactly the two ADR-0020 records ---" EXPECTED_DANGLING="$(printf '%s\n' gitea-labels neuledge-context)" if [[ "$LIVE_DANGLING" == "$EXPECTED_DANGLING" ]]; then pass "the corpus dangling set is exactly {gitea-labels, neuledge-context}" else fail "the corpus dangling set changed — expected [$(echo "$EXPECTED_DANGLING" | tr '\n' ' ')], got [$(echo "$LIVE_DANGLING" | tr '\n' ' ')]. If a retrofit fixed one, update EXPECTED_DANGLING; if a false-positive fix silently deleted one, that is the regression this asserts." fi for probe in \ "plugins/bin/.apm/skills/research/SKILL.md:neuledge-context" \ "plugins/gitea/.apm/skills/gitea-issues/SKILL.md:gitea-labels"; do probe_file="$REPO_ROOT/${probe%%:*}" probe_name="${probe##*:}" if [[ ! -f "$probe_file" ]]; then fail "the true-positive fixture ${probe%%:*} no longer exists — this pin has become vacuous" continue fi set +e probe_out="$(bash "$HOOK" "$probe_file" 2>&1)" set -e if [[ "$probe_out" == *"routes to '$probe_name'"* ]]; then pass "detects the dangling '$probe_name' target in ${probe%%:*}" else fail "did not detect the dangling '$probe_name' target in ${probe%%:*} — a false-positive fix has taken a true positive with it: $probe_out" fi done # --------------------------------------------------------------------------- # 2. The bare-target grammar rule # --------------------------------------------------------------------------- # Every fixture is built inside a real plugin tree. In a bare temp directory the # resolver would decline ("DID NOT RUN") and every must-not-error case would pass # vacuously, proving nothing about extraction. echo "" echo "--- attributive compound modifiers are prose, not routing targets ---" GRAMMAR_ROOT="$TMPDIR_T/grammar" write_skill "$GRAMMAR_ROOT/plugins/p/.apm/skills/sibling-skill" sibling-skill \ "Use when doing the other thing. Do not use for anything else." # grammar_case grammar_case() { local slug="$1" mode="$2" needle="$3" desc="$4" out status=0 write_skill "$GRAMMAR_ROOT/plugins/p/.apm/skills/$slug" "$slug" "$desc" set +e out="$(bash "$HOOK" "$GRAMMAR_ROOT/plugins/p/.apm/skills/$slug/SKILL.md" 2>&1)" status=$? set -e if [[ "$out" == *"DID NOT RUN"* ]]; then fail "\"$desc\" — the resolver declined, so this case asserts nothing about extraction: $out" return fi case "$mode" in silent) if [[ $status -eq 0 && -z "$out" ]]; then pass "not a dangling target: \"$desc\"" else fail "\"$desc\" (exit $status, output: ${out:-})" fi ;; errors) if [[ $status -ne 0 && "$out" == *"$needle"* ]]; then pass "still a dangling target: \"$desc\"" else fail "\"$desc\" should have ERRORed with $needle (exit $status, output: ${out:-})" fi ;; suggests) # Reported, not blocking. Both halves matter: an ERROR here would be the # unsuppressable false positive this tier exists to remove, and silence # would mean the gate stopped noticing the target at all. if [[ $status -eq 0 && "$out" == *"SUGGESTION"*"$needle"* && "$out" != *"ERROR"* ]]; then pass "reported but not blocking: \"$desc\"" else fail "\"$desc\" should have exited 0 with a SUGGESTION naming $needle (exit $status, output: ${out:-})" fi ;; esac } # The four phrasings that were hard dangling FAILs with no suppression. All four # are lifted from real descriptions in this corpus. grammar_case fp-precommit-hooks silent "" \ "Use when running the linter. Use pre-commit hooks instead of ad-hoc scripts." grammar_case fp-pull-request silent "" \ "Use when opening changes. Invoke the pull-request template instead of writing one by hand." grammar_case fp-conventional silent "" \ "Use when writing history. Use conventional-commits formatting rather than free-form messages." grammar_case fp-prepush-backticked silent "" \ "Use when checking a branch. Do not use for local edits — run the \`pre-push\` hooks instead." echo "" echo "--- a lone unresolvable token in terminal position is REPORTED, not blocking ---" # The class this tier was added for. Every one of these is grammatically # identical to a real broken route — "route verb + name + terminal" is also how # prose cites a hook, a linter, a file format or an English compound — and every # one of them was a hard FAIL with no suppression mechanism anywhere in the gate. # The skills most exposed are exactly the ones the ADR-0020 retrofit sends # authors back to rewrite first: pc-run, pc-author, vale-run, vale-config and the # apm-* family are all ABOUT hyphenated tools. grammar_case fp-precommit-terminal suggests "routes to 'pre-commit'" \ "Use when running the linter. Do not use for running hooks — run \`pre-commit\` instead." grammar_case fp-commit-msg suggests "routes to 'commit-msg'" \ "Use when writing history. Do not use for the commit message — see \`commit-msg\`." grammar_case fp-type-check suggests "routes to 'type-check'" \ "Use when compiling. Do not use for type errors — run \`type-check\` first." grammar_case fp-semantic-release suggests "routes to 'semantic-release'" \ "Use when tagging a version. Instead, use \`semantic-release\`." # Single-word tool names are the same defect: `eslint` in terminal position hit # the marked-target path and hard-FAILed just as `pre-commit` did. grammar_case fp-eslint suggests "routes to 'eslint'" \ "Use when linting JS. Do not use for style — run \`eslint\` instead." # Bare English compounds, which the backtick path never sees at all. grammar_case fp-clean-up suggests "routes to 'clean-up'" \ "Use when doing the thing. Do not use for the old flow — use the clean-up instead." grammar_case fp-built-in suggests "routes to 'built-in'" \ "Use when doing the thing. Do not use for the custom path — Instead, prefer the built-in." grammar_case fp-write-up suggests "routes to 'write-up'" \ "Use when doing the thing. Do not use for the summary — see the write-up." grammar_case fp-front-end suggests "routes to 'front-end'" \ "Use when doing the thing. Do not use for the API layer — use the front-end." grammar_case fp-unit-tests suggests "routes to 'unit-tests'" \ "Use when testing. Do not run end-to-end, run unit-tests." echo "" echo "--- terminal targets that resolve to nothing still ERROR ---" # The controls. Without them the cases above are satisfied by a check that never # fires, and the narrowing would have eaten the gate rather than sharpened it. # Three forms, three code paths: # * route NOTATION — `/name` and `-> name` — is exempt from corroboration and # blocks on its own. Nobody writes `/pre-commit` or `-> pre-commit` to mean # the hook, so there is no ambiguity to resolve, and an author who wants a # route checked unconditionally has two ways to say so. # * a PROSE-form target — backticked or bare — blocks when its own sentence # names another target that resolves. `sibling-skill` is that corroborator # here; it is the same shape as both live true positives, which sit beside # `write-docs` and `gitea-labels-milestones` respectively. grammar_case tp-arrow errors "routes to 'no-such-arrow-target'" \ "Use when doing the thing. Not the other thing → no-such-arrow-target." grammar_case tp-slash errors "routes to 'no-such-slash-skill'" \ "Use when doing the thing. Do not use for improvements — use /no-such-slash-skill instead." grammar_case tp-backticked errors "routes to 'no-such-backticked-skill'" \ "Use when doing the thing. Do not use for improvements — use \`sibling-skill\` or \`no-such-backticked-skill\` instead." grammar_case tp-bare-terminal errors "routes to 'no-such-bare-skill'" \ "Use when doing the thing. Do not use for improvements — use sibling-skill or no-such-bare-skill instead." # And the confirming half of the grammar rule: a compound-modifier target is # CONFIRM-ONLY, not ignored. When the name does exist it still counts as a route # — the rule suppresses the ERROR, it does not delete the target. echo "" echo "--- an attributive target that DOES resolve is still a route, not a discarded token ---" write_skill "$GRAMMAR_ROOT/plugins/p/.apm/skills/attributive-subject" attributive-subject \ "Use when doing the thing. Do not use for the other thing — use the sibling-skill helper instead." set +e ATTR_OUT="$(bash "$HOOK" "$GRAMMAR_ROOT/plugins/p/.apm/skills/attributive-subject/SKILL.md" 2>&1)" ATTR_RC=$? set -e if [[ $ATTR_RC -eq 0 && -z "$ATTR_OUT" ]]; then pass "a resolving attributive target neither errors nor is reported" else fail "an attributive target naming a REAL skill produced output (exit $ATTR_RC): $ATTR_OUT" fi echo "" echo "Results: $PASS passed, $FAIL failed" [[ $FAIL -eq 0 ]]