fix(skill-audit): make check 9 reachable, wrap-safe and never silently skipped
Check 9 shipped in #130 to close #118, but three defects meant it could not do the job it was added for. Why: - It is INFO-only, so it always exits 0 — and SKILL.md graded exit 0 "a genuine pass" and said the script "prints nothing on success". Every check-9 INFO was discarded before it reached a report, behind three further doors that only opened on a non-zero exit. - `parse_field_raw()` matched `(.+)`, which does not span newlines, so only the first physical line of a wrapped value was compared. Rewriting only the continuation line of a wrapped Description from a hedge to a confident claim produced no finding at all — verbatim the regression #118 was filed about. The bullet branch had the same shape: a wrapped bullet broke the loop and dropped every later entry. - A `git show` failure at the base ref was treated as "creation, nothing to flag" and skipped the whole skill with no output, collapsing "absent at that ref" with "not tracked under that name". A gitignored `.claude/skills/` copy reported clean while the authoring path reported four changed claims. The script's own usage text promises this is "never a silent skip". Implementation notes: - Exit-code guidance re-keyed on output as well as code: 0-and-silent passes, 0-with-output is INFO-only findings, 1 is FAILs, 2 never ran. - `parse_field_raw()` is line-based and joins continuation lines; `normalize_field_text()`'s docstring is now true rather than aspirational. A reorder deliberately fires: the two fields share one parser, and order-insensitivity would mean splitting a prose Description on commas. - The discarded `show_err` is now surfaced as one whole-check INFO naming both readings. - `--base-ref=` given empty now beats the env var, as the usage text always claimed. `validate.sh` gains an ADR-0022 `metadata.version` check at FAIL tier, because any lower tier lets skill-author Step 4 report done on a file the commit gate then refuses. Its `read` heuristic now skips here-doc bodies — reflowing the one offending line would have cleared the finding and left the cause, since every usage() heredoc is one wrap from putting the English verb in column 0. Impact: provenance tests 73 -> 82, validate tests 64 -> 72. Test 72 previously deleted origin/main before asserting the override, so it proved the flag works with no default rather than that it beats one; it now moves origin/main forward first. Refs: #118 ADR: 0022 Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01EeH8SCbcrCAQrtymkNuhKP
This commit is contained in:
@@ -67,6 +67,25 @@ EOF
|
||||
EOF
|
||||
}
|
||||
|
||||
# Helper: a sources.md whose Description is wrapped across two physical
|
||||
# lines — the shape check 9's parser used to truncate at the first newline.
|
||||
make_wrapped_sources_md() {
|
||||
local dir="$1"
|
||||
mkdir -p "$dir/references"
|
||||
cat > "$dir/references/sources.md" <<'EOF'
|
||||
# Sources
|
||||
|
||||
## my-source
|
||||
|
||||
- **URL:** https://example.com/my-source
|
||||
- **Description:** A test source, informing the dispatch table's shape with
|
||||
no forge-specific content drawn directly from it beyond that.
|
||||
- **Contributing files:** SKILL.md
|
||||
- **Research doc:** (none)
|
||||
- **Status:** `extracted`
|
||||
EOF
|
||||
}
|
||||
|
||||
# Helper: turn dir into a real git repo with one commit of its current
|
||||
# contents, and a refs/remotes/origin/main pointing at that same commit.
|
||||
# Check 9 diffs the skill's references/sources.md against `git merge-base
|
||||
@@ -1629,23 +1648,247 @@ EOF
|
||||
assert_output --partial "Check 9 skipped — no base ref could be resolved"
|
||||
}
|
||||
|
||||
@test "check 9: --base-ref overrides the default origin/main resolution" {
|
||||
@test "check 9: --base-ref overrides a default origin/main that resolves to something else" {
|
||||
local skill="$TMPDIR/my-skill"
|
||||
make_skill_with_source_keys "$skill"
|
||||
make_sources_md "$skill"
|
||||
commit_as_base "$skill"
|
||||
local base_sha
|
||||
base_sha="$(git -C "$skill" rev-parse HEAD)"
|
||||
git -C "$skill" update-ref -d refs/remotes/origin/main >/dev/null 2>&1
|
||||
local old_sha
|
||||
old_sha="$(git -C "$skill" rev-parse HEAD)"
|
||||
|
||||
# A SECOND commit carrying the rewritten claim, with origin/main moved onto
|
||||
# it. The default base ref therefore resolves — to a commit that matches the
|
||||
# working tree — so a silent default run proves there was a default here to
|
||||
# override. Deleting origin/main instead, as this test used to, proved only
|
||||
# that the flag works when nothing else does.
|
||||
sed -i 's/^- \*\*Description:\*\* A test source\.$/- **Description:** A rewritten claim./' \
|
||||
"$skill/references/sources.md"
|
||||
git -C "$skill" -c user.email=test@example.com -c user.name=test commit -aqm rewrite >/dev/null 2>&1
|
||||
git -C "$skill" update-ref refs/remotes/origin/main HEAD >/dev/null 2>&1
|
||||
|
||||
run bash "$SCRIPT" "$skill" "--base-ref=$base_sha"
|
||||
run bash "$SCRIPT" "$skill"
|
||||
assert_success
|
||||
assert_output ""
|
||||
|
||||
run bash "$SCRIPT" "$skill" "--base-ref=$old_sha"
|
||||
assert_success
|
||||
assert_output --partial "'Description' changed for 'my-source'"
|
||||
}
|
||||
|
||||
@test "check 9: VALIDATE_PROVENANCE_BASE_REF sets the base ref when no flag is given" {
|
||||
local skill="$TMPDIR/my-skill"
|
||||
make_skill_with_source_keys "$skill"
|
||||
make_sources_md "$skill"
|
||||
commit_as_base "$skill"
|
||||
local old_sha
|
||||
old_sha="$(git -C "$skill" rev-parse HEAD)"
|
||||
sed -i 's/^- \*\*Description:\*\* A test source\.$/- **Description:** A rewritten claim./' \
|
||||
"$skill/references/sources.md"
|
||||
git -C "$skill" -c user.email=test@example.com -c user.name=test commit -aqm rewrite >/dev/null 2>&1
|
||||
git -C "$skill" update-ref refs/remotes/origin/main HEAD >/dev/null 2>&1
|
||||
|
||||
run env VALIDATE_PROVENANCE_BASE_REF="$old_sha" bash "$SCRIPT" "$skill"
|
||||
assert_success
|
||||
assert_output --partial "'Description' changed for 'my-source'"
|
||||
}
|
||||
|
||||
@test "check 9: the --base-ref flag wins over the environment variable" {
|
||||
local skill="$TMPDIR/my-skill"
|
||||
make_skill_with_source_keys "$skill"
|
||||
make_sources_md "$skill"
|
||||
commit_as_base "$skill"
|
||||
local old_sha
|
||||
old_sha="$(git -C "$skill" rev-parse HEAD)"
|
||||
sed -i 's/^- \*\*Description:\*\* A test source\.$/- **Description:** A rewritten claim./' \
|
||||
"$skill/references/sources.md"
|
||||
git -C "$skill" -c user.email=test@example.com -c user.name=test commit -aqm rewrite >/dev/null 2>&1
|
||||
local new_sha
|
||||
new_sha="$(git -C "$skill" rev-parse HEAD)"
|
||||
|
||||
# The environment names the old commit (which would fire), the flag names
|
||||
# the new one (which would not). The usage text promises the flag wins.
|
||||
run env VALIDATE_PROVENANCE_BASE_REF="$old_sha" bash "$SCRIPT" "$skill" "--base-ref=$new_sha"
|
||||
assert_success
|
||||
assert_output ""
|
||||
}
|
||||
|
||||
@test "check 9: an EMPTY --base-ref is still 'given' and wins over the environment variable" {
|
||||
local skill="$TMPDIR/my-skill"
|
||||
make_skill_with_source_keys "$skill"
|
||||
make_sources_md "$skill"
|
||||
commit_as_base "$skill"
|
||||
local old_sha
|
||||
old_sha="$(git -C "$skill" rev-parse HEAD)"
|
||||
sed -i 's/^- \*\*Description:\*\* A test source\.$/- **Description:** A rewritten claim./' \
|
||||
"$skill/references/sources.md"
|
||||
git -C "$skill" -c user.email=test@example.com -c user.name=test commit -aqm rewrite >/dev/null 2>&1
|
||||
git -C "$skill" update-ref refs/remotes/origin/main HEAD >/dev/null 2>&1
|
||||
|
||||
# `--base-ref=` selects the DEFAULT resolution (origin/main, which now
|
||||
# matches the working tree), so nothing fires. Under the old `:-` spelling
|
||||
# the empty value read as absent and the environment variable won, firing
|
||||
# the INFO and contradicting the usage text.
|
||||
run env VALIDATE_PROVENANCE_BASE_REF="$old_sha" bash "$SCRIPT" "$skill" "--base-ref="
|
||||
assert_success
|
||||
assert_output ""
|
||||
}
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Cycle 29 — Check 9: a WRAPPED field value. The parser compared only the first
|
||||
# physical line, so a rewrite confined to a continuation line — the exact
|
||||
# hedge-to-confident-claim shape #118 exists to catch — produced no finding at
|
||||
# all, while a pure re-wrap produced a false one.
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
@test "check 9: a rewrite confined to a wrapped Description's CONTINUATION line fires an INFO" {
|
||||
local skill="$TMPDIR/my-skill"
|
||||
make_skill_with_source_keys "$skill"
|
||||
make_wrapped_sources_md "$skill"
|
||||
commit_as_base "$skill"
|
||||
|
||||
# Hedge to confident claim, on the second physical line only. This is the
|
||||
# regression check 9 was written for, and the one it could not see.
|
||||
sed -i "s|^ no forge-specific content drawn directly from it beyond that\.\$| it grounds Step 2's dispatch table in full.|" \
|
||||
"$skill/references/sources.md"
|
||||
|
||||
run bash "$SCRIPT" "$skill"
|
||||
assert_success
|
||||
assert_output --partial "'Description' changed for 'my-source'"
|
||||
}
|
||||
|
||||
@test "check 9: re-wrapping a Description with no wording change produces no finding" {
|
||||
local skill="$TMPDIR/my-skill"
|
||||
make_skill_with_source_keys "$skill"
|
||||
make_wrapped_sources_md "$skill"
|
||||
commit_as_base "$skill"
|
||||
|
||||
# Same words, different line breaks. normalize_field_text()'s docstring
|
||||
# promises this is invisible; it was not, because the value was truncated
|
||||
# at its first newline before the whitespace collapse ever ran.
|
||||
python3 - "$skill/references/sources.md" <<'PY'
|
||||
import sys
|
||||
path = sys.argv[1]
|
||||
text = open(path).read()
|
||||
old = ("- **Description:** A test source, informing the dispatch table's shape with\n"
|
||||
" no forge-specific content drawn directly from it beyond that.")
|
||||
new = ("- **Description:** A test source, informing the dispatch\n"
|
||||
" table's shape with no forge-specific content drawn\n"
|
||||
" directly from it beyond that.")
|
||||
assert old in text
|
||||
open(path, 'w').write(text.replace(old, new))
|
||||
PY
|
||||
|
||||
run bash "$SCRIPT" "$skill"
|
||||
assert_success
|
||||
assert_output ""
|
||||
}
|
||||
|
||||
@test "check 9: the bullet form of Contributing files is compared, not skipped" {
|
||||
local skill="$TMPDIR/my-skill"
|
||||
make_skill_with_source_keys "$skill"
|
||||
mkdir -p "$skill/references"
|
||||
cat > "$skill/references/sources.md" <<'EOF'
|
||||
# Sources
|
||||
|
||||
## my-source
|
||||
|
||||
- **URL:** https://example.com/my-source
|
||||
- **Description:** A test source.
|
||||
|
||||
**Contributing files:**
|
||||
- SKILL.md (the dispatch table)
|
||||
- references/other.md (the rubric)
|
||||
|
||||
- **Research doc:** (none)
|
||||
- **Status:** `extracted`
|
||||
EOF
|
||||
printf -- '---\nsource_keys:\n - my-source\n---\n\nnotes\n' > "$skill/references/other.md"
|
||||
commit_as_base "$skill"
|
||||
|
||||
# The change is in the SECOND bullet. The old loop joined bullets in
|
||||
# document order too, but broke on any wrapped one — and no test covered
|
||||
# this branch at all, both existing check-9 tests using the inline form.
|
||||
sed -i 's|^- references/other.md (the rubric)$|- references/other.md (the whole rubric, verbatim)|' \
|
||||
"$skill/references/sources.md"
|
||||
|
||||
run bash "$SCRIPT" "$skill"
|
||||
assert_success
|
||||
assert_output --partial "'Contributing files' changed for 'my-source'"
|
||||
}
|
||||
|
||||
@test "check 9: a wrapped bullet does not silently drop the bullets after it" {
|
||||
local skill="$TMPDIR/my-skill"
|
||||
make_skill_with_source_keys "$skill"
|
||||
mkdir -p "$skill/references"
|
||||
cat > "$skill/references/sources.md" <<'EOF'
|
||||
# Sources
|
||||
|
||||
## my-source
|
||||
|
||||
- **URL:** https://example.com/my-source
|
||||
- **Description:** A test source.
|
||||
|
||||
**Contributing files:**
|
||||
- SKILL.md (the dispatch table, and the gates
|
||||
common to every branch of it)
|
||||
- references/other.md (the rubric)
|
||||
|
||||
- **Research doc:** (none)
|
||||
- **Status:** `extracted`
|
||||
EOF
|
||||
printf -- '---\nsource_keys:\n - my-source\n---\n\nnotes\n' > "$skill/references/other.md"
|
||||
commit_as_base "$skill"
|
||||
|
||||
# The old loop broke at the wrapped continuation line, so everything from
|
||||
# here down was never part of the compared value — a change to the last
|
||||
# bullet was invisible.
|
||||
sed -i 's|^- references/other.md (the rubric)$|- references/other.md (rewritten claim)|' \
|
||||
"$skill/references/sources.md"
|
||||
|
||||
run bash "$SCRIPT" "$skill"
|
||||
assert_success
|
||||
assert_output --partial "'Contributing files' changed for 'my-source'"
|
||||
}
|
||||
|
||||
@test "check 9: a field present at the base ref and deleted since is announced" {
|
||||
local skill="$TMPDIR/my-skill"
|
||||
make_skill_with_source_keys "$skill"
|
||||
make_sources_md "$skill"
|
||||
commit_as_base "$skill"
|
||||
|
||||
# No other check in this script requires a Description, so a deleted one
|
||||
# used to leave no finding anywhere: a claim could be withdrawn as
|
||||
# invisibly as it could be strengthened.
|
||||
sed -i '/^- \*\*Description:\*\*/d' "$skill/references/sources.md"
|
||||
|
||||
run bash "$SCRIPT" "$skill"
|
||||
assert_success
|
||||
assert_output --partial "'Description' removed for 'my-source'"
|
||||
}
|
||||
|
||||
@test "check 9: a sources.md untracked at the base ref is announced, not silently skipped" {
|
||||
local repo="$TMPDIR/repo"
|
||||
local skill="$repo/tracked-skill"
|
||||
mkdir -p "$skill"
|
||||
make_skill_with_source_keys "$skill"
|
||||
make_sources_md "$skill"
|
||||
commit_as_base "$repo"
|
||||
|
||||
# A copy of the same skill at a path git does not know — the everyday case
|
||||
# being an installed, gitignored .claude/skills/ tree. The base ref
|
||||
# resolves fine; `git show <ref>:<path>` does not. Treating that as
|
||||
# "creation, nothing to flag" made the whole check vanish without a word,
|
||||
# so the same directory reported findings at one path and silence at the
|
||||
# other.
|
||||
cp -r "$skill" "$repo/untracked-copy"
|
||||
|
||||
run bash "$SCRIPT" "$repo/untracked-copy"
|
||||
assert_success
|
||||
assert_output --partial "INFO"
|
||||
assert_output --partial "is not tracked at"
|
||||
assert_output --partial "Check 9 did not run for any slug in this skill."
|
||||
}
|
||||
|
||||
@test "check 9: an invalid --base-ref value is reported as unresolvable, not a crash" {
|
||||
local skill="$TMPDIR/my-skill"
|
||||
make_skill_with_source_keys "$skill"
|
||||
|
||||
Reference in New Issue
Block a user