fix(skill-audit): make check 9 reachable, wrap-safe and never silently skipped
Check 9 shipped in #130 to close #118, but three defects meant it could not do the job it was added for. Why: - It is INFO-only, so it always exits 0 — and SKILL.md graded exit 0 "a genuine pass" and said the script "prints nothing on success". Every check-9 INFO was discarded before it reached a report, behind three further doors that only opened on a non-zero exit. - `parse_field_raw()` matched `(.+)`, which does not span newlines, so only the first physical line of a wrapped value was compared. Rewriting only the continuation line of a wrapped Description from a hedge to a confident claim produced no finding at all — verbatim the regression #118 was filed about. The bullet branch had the same shape: a wrapped bullet broke the loop and dropped every later entry. - A `git show` failure at the base ref was treated as "creation, nothing to flag" and skipped the whole skill with no output, collapsing "absent at that ref" with "not tracked under that name". A gitignored `.claude/skills/` copy reported clean while the authoring path reported four changed claims. The script's own usage text promises this is "never a silent skip". Implementation notes: - Exit-code guidance re-keyed on output as well as code: 0-and-silent passes, 0-with-output is INFO-only findings, 1 is FAILs, 2 never ran. - `parse_field_raw()` is line-based and joins continuation lines; `normalize_field_text()`'s docstring is now true rather than aspirational. A reorder deliberately fires: the two fields share one parser, and order-insensitivity would mean splitting a prose Description on commas. - The discarded `show_err` is now surfaced as one whole-check INFO naming both readings. - `--base-ref=` given empty now beats the env var, as the usage text always claimed. `validate.sh` gains an ADR-0022 `metadata.version` check at FAIL tier, because any lower tier lets skill-author Step 4 report done on a file the commit gate then refuses. Its `read` heuristic now skips here-doc bodies — reflowing the one offending line would have cleared the finding and left the cause, since every usage() heredoc is one wrap from putting the English verb in column 0. Impact: provenance tests 73 -> 82, validate tests 64 -> 72. Test 72 previously deleted origin/main before asserting the override, so it proved the flag works with no default rather than that it beats one; it now moves origin/main forward first. Refs: #118 ADR: 0022 Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01EeH8SCbcrCAQrtymkNuhKP
This commit is contained in:
@@ -16,6 +16,10 @@ setup() {
|
||||
# SUGGESTION-freedom would be asserting the boundary check's absence instead
|
||||
# of the thing it names. "anything else" is not hyphenated, so the clause adds
|
||||
# a boundary marker without adding a routing target to resolve.
|
||||
#
|
||||
# metadata.version is equally load-bearing: ADR-0022 makes it mandatory and
|
||||
# validate.sh FAILs without it, so a fixture omitting it would not be
|
||||
# "otherwise clean" either.
|
||||
make_valid_skill() {
|
||||
local dir="$1"
|
||||
local name
|
||||
@@ -25,6 +29,8 @@ setup() {
|
||||
---
|
||||
name: $name
|
||||
description: A valid skill description that is well within the limit. Do not use for anything else.
|
||||
metadata:
|
||||
version: "1.0.0"
|
||||
---
|
||||
|
||||
## Step 1
|
||||
@@ -59,6 +65,8 @@ PY
|
||||
echo "---"
|
||||
echo "name: $name"
|
||||
echo "description: $desc"
|
||||
echo "metadata:"
|
||||
echo ' version: "1.0.0"'
|
||||
echo "---"
|
||||
echo ""
|
||||
python3 -c "print(' '.join(['word'] * $body_words))"
|
||||
@@ -294,6 +302,141 @@ EOF
|
||||
assert_success
|
||||
}
|
||||
|
||||
@test "prose inside a usage() here-doc that wraps onto a line starting with 'read' does not fail" {
|
||||
local skill="$TMPDIR/my-skill"
|
||||
make_valid_skill "$skill"
|
||||
# The exact shape that made skill-audit hard-FAIL on its own
|
||||
# validate-provenance.sh: a usage() heredoc whose wrapped sentence puts the
|
||||
# English verb "read" in column 0.
|
||||
cat > "$skill/scripts/helper.sh" <<'SH'
|
||||
#!/usr/bin/env bash
|
||||
usage() {
|
||||
cat <<EOF
|
||||
Checks performed:
|
||||
4 A Contributing files block this parser cannot
|
||||
read is reported as an INFO, never skipped silently.
|
||||
EOF
|
||||
}
|
||||
usage
|
||||
SH
|
||||
chmod +x "$skill/scripts/helper.sh"
|
||||
run bash "$SCRIPT" "$skill"
|
||||
assert_success
|
||||
}
|
||||
|
||||
@test "a real interactive read AFTER a here-doc is still caught" {
|
||||
local skill="$TMPDIR/my-skill"
|
||||
make_valid_skill "$skill"
|
||||
# Pins that the here-doc exemption ends at its terminator. A body skip that
|
||||
# ran to end-of-file would swallow this read and report the script clean.
|
||||
cat > "$skill/scripts/helper.sh" <<'SH'
|
||||
#!/usr/bin/env bash
|
||||
cat <<EOF
|
||||
read this text
|
||||
EOF
|
||||
read -r ANSWER
|
||||
SH
|
||||
chmod +x "$skill/scripts/helper.sh"
|
||||
run bash "$SCRIPT" "$skill"
|
||||
assert_failure
|
||||
}
|
||||
|
||||
@test "an unterminated here-doc opener does not disarm the check for the rest of the file" {
|
||||
local skill="$TMPDIR/my-skill"
|
||||
make_valid_skill "$skill"
|
||||
# `<<` here is inside a string, not an opener. Treating it as one would skip
|
||||
# every following line — a false negative, the direction this check must
|
||||
# never fail in.
|
||||
cat > "$skill/scripts/helper.sh" <<'SH'
|
||||
#!/usr/bin/env bash
|
||||
echo "shift left with a << b"
|
||||
read -r ANSWER
|
||||
SH
|
||||
chmod +x "$skill/scripts/helper.sh"
|
||||
run bash "$SCRIPT" "$skill"
|
||||
assert_failure
|
||||
}
|
||||
|
||||
@test "a bare input() inside an embedded-python here-doc is still caught" {
|
||||
local skill="$TMPDIR/my-skill"
|
||||
make_valid_skill "$skill"
|
||||
# The here-doc exemption is for the `read` heuristic only: these scripts
|
||||
# embed Python in a here-doc as a matter of course, so exempting the body
|
||||
# wholesale would disarm the check across the corpus.
|
||||
cat > "$skill/scripts/helper.sh" <<'SH'
|
||||
#!/usr/bin/env bash
|
||||
python3 - <<'PY'
|
||||
print("press enter")
|
||||
input()
|
||||
PY
|
||||
SH
|
||||
chmod +x "$skill/scripts/helper.sh"
|
||||
run bash "$SCRIPT" "$skill"
|
||||
assert_failure
|
||||
}
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# ADR-0022 — metadata.version is mandatory. FAIL tier, matching the
|
||||
# skill-frontmatter pre-commit hook: an audit that graded this lower would
|
||||
# report ready-to-ship on a file the commit gate rejects.
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
@test "ADR-0022: a SKILL.md with no metadata block at all FAILs" {
|
||||
local skill="$TMPDIR/my-skill"
|
||||
make_valid_skill "$skill"
|
||||
python3 - "$skill/SKILL.md" <<'PY'
|
||||
import sys
|
||||
p = sys.argv[1]
|
||||
s = open(p).read().replace('metadata:\n version: "1.0.0"\n', '')
|
||||
open(p, 'w').write(s)
|
||||
PY
|
||||
run bash "$SCRIPT" "$skill"
|
||||
assert_failure
|
||||
assert_output --partial "metadata.version"
|
||||
}
|
||||
|
||||
@test "ADR-0022: a metadata block with no version key FAILs" {
|
||||
local skill="$TMPDIR/my-skill"
|
||||
make_valid_skill "$skill"
|
||||
python3 - "$skill/SKILL.md" <<'PY'
|
||||
import sys
|
||||
p = sys.argv[1]
|
||||
s = open(p).read().replace(' version: "1.0.0"\n', ' category: factory\n')
|
||||
open(p, 'w').write(s)
|
||||
PY
|
||||
run bash "$SCRIPT" "$skill"
|
||||
assert_failure
|
||||
assert_output --partial "metadata.version"
|
||||
}
|
||||
|
||||
@test "ADR-0022: a two-part metadata.version FAILs as malformed, not passes as present" {
|
||||
local skill="$TMPDIR/my-skill"
|
||||
make_valid_skill "$skill"
|
||||
python3 - "$skill/SKILL.md" <<'PY'
|
||||
import sys
|
||||
p = sys.argv[1]
|
||||
s = open(p).read().replace(' version: "1.0.0"\n', ' version: 1.0\n')
|
||||
open(p, 'w').write(s)
|
||||
PY
|
||||
run bash "$SCRIPT" "$skill"
|
||||
assert_failure
|
||||
assert_output --partial "three-part semver"
|
||||
}
|
||||
|
||||
@test "ADR-0022: an unquoted three-part metadata.version passes" {
|
||||
local skill="$TMPDIR/my-skill"
|
||||
make_valid_skill "$skill"
|
||||
python3 - "$skill/SKILL.md" <<'PY'
|
||||
import sys
|
||||
p = sys.argv[1]
|
||||
s = open(p).read().replace(' version: "1.0.0"\n', ' version: 0.1.3\n')
|
||||
open(p, 'w').write(s)
|
||||
PY
|
||||
run bash "$SCRIPT" "$skill"
|
||||
assert_success
|
||||
assert_output --partial "metadata.version present: '0.1.3'"
|
||||
}
|
||||
|
||||
@test "fails when name contains consecutive hyphens" {
|
||||
local skill="$TMPDIR/my--skill"
|
||||
make_valid_skill "$skill"
|
||||
@@ -608,6 +751,8 @@ make_hand_invoked_skill() {
|
||||
echo "name: $name"
|
||||
echo "description: $desc"
|
||||
echo "disable-model-invocation: true"
|
||||
echo "metadata:"
|
||||
echo ' version: "1.0.0"'
|
||||
echo "---"
|
||||
echo ""
|
||||
python3 -c "print(' '.join(['word'] * $body_words))"
|
||||
@@ -761,6 +906,8 @@ make_hand_invoked_skill() {
|
||||
---
|
||||
name: locale-skill
|
||||
description: A valid skill description that is well within the limit.
|
||||
metadata:
|
||||
version: "1.0.0"
|
||||
---
|
||||
|
||||
## Step 1
|
||||
|
||||
Reference in New Issue
Block a user