fix(skill-audit): flag a changed provenance claim, not just its shape
validate-provenance.sh checked that a sources.md entry was internally consistent -- slugs resolve, Contributing files exist, back-references match -- but never whether the asserted contribution was true. A retrofit once turned an honest hedge into a false confident claim and every existing check passed it silently. A literal-filename cross-check (flag a description naming a .md file absent from Contributing files) was tried and rejected: 3/95 flagged against the real corpus, all three false positives, and it would not have caught the actual bug -- the bad description never named a literal filename. No bash script can verify semantic truth, so the fix uses what git can reliably detect -- a changed field -- purely as a trigger for what can verify semantics: the auditor reading the files. New check 9 flags (INFO only, never FAIL) any Description or Contributing-files text change against a base ref (default: merge-base with origin/main, overridable via --base-ref). A slug absent at the base ref is a creation, not a change, and is not flagged. skill-audit's rubric now tells the auditor a check-9 INFO means open the named files and verify by reading, not just relay it. skill-author's retrofit checklist gained a matching authoring-time guardrail: don't upgrade a hedge into a confident claim without re-reading the source first. 8 new bats tests (73 total, 0 failures). Fixes: #118 Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01EeH8SCbcrCAQrtymkNuhKP
This commit is contained in:
@@ -67,17 +67,36 @@ EOF
|
||||
EOF
|
||||
}
|
||||
|
||||
# Helper: create a fake repo (a .git marker makes find_repo_root stop there)
|
||||
# holding one skill whose single sources.md slug points at the given
|
||||
# Research doc value. Checks 7 and 8 only run for a skill inside a checkout,
|
||||
# so every upstream case needs this shape; the research doc itself is
|
||||
# written per test into "$repo/docs/research/sources.md".
|
||||
# Helper: turn dir into a real git repo with one commit of its current
|
||||
# contents, and a refs/remotes/origin/main pointing at that same commit.
|
||||
# Check 9 diffs the skill's references/sources.md against `git merge-base
|
||||
# HEAD origin/main` by default; this makes that resolve to a commit whose
|
||||
# sources.md is byte-identical to the working tree, so check 9 has
|
||||
# nothing to report there — exactly what a real repo looks like the
|
||||
# instant after a clean commit. Fixtures that go on to test something
|
||||
# else entirely (checks 3, 4, 5, 7, 8...) call this once, at the point
|
||||
# their skill's own sources.md is in its final state, so a completely
|
||||
# clean run stays completely clean.
|
||||
commit_as_base() {
|
||||
local dir="$1"
|
||||
git -C "$dir" init -q >/dev/null 2>&1
|
||||
git -C "$dir" -c user.email=test@example.com -c user.name=test add -A >/dev/null 2>&1
|
||||
git -C "$dir" -c user.email=test@example.com -c user.name=test commit -q -m base >/dev/null 2>&1
|
||||
git -C "$dir" update-ref refs/remotes/origin/main HEAD >/dev/null 2>&1
|
||||
}
|
||||
|
||||
# Helper: create a fake repo (a real git repo, one commit, makes
|
||||
# find_repo_root stop there) holding one skill whose single sources.md
|
||||
# slug points at the given Research doc value. Checks 7 and 8 only run
|
||||
# for a skill inside a checkout, so every upstream case needs this shape;
|
||||
# the research doc itself is written per test into
|
||||
# "$repo/docs/research/sources.md" — which check 9 does not examine, so
|
||||
# a test overwriting it after this helper runs does not disturb check 9.
|
||||
make_upstream_skill() {
|
||||
local repo="$1"
|
||||
local research="${2:-docs/research/sources.md}"
|
||||
local skill="$repo/my-skill"
|
||||
mkdir -p "$skill/references" "$repo/docs/research"
|
||||
touch "$repo/.git"
|
||||
cat > "$skill/SKILL.md" <<EOF
|
||||
---
|
||||
name: my-skill
|
||||
@@ -102,6 +121,7 @@ EOF
|
||||
- **Research doc:** ${research}
|
||||
- **Status:** \`extracted\`
|
||||
EOF
|
||||
commit_as_base "$repo"
|
||||
}
|
||||
}
|
||||
|
||||
@@ -790,6 +810,7 @@ source_keys: []
|
||||
|
||||
House-authored, no external source.
|
||||
EOF
|
||||
commit_as_base "$skill"
|
||||
run bash "$SCRIPT" "$skill"
|
||||
assert_success
|
||||
assert_output ""
|
||||
@@ -809,6 +830,7 @@ metadata:
|
||||
|
||||
House-authored, no external source.
|
||||
EOF
|
||||
commit_as_base "$skill"
|
||||
run bash "$SCRIPT" "$skill"
|
||||
assert_success
|
||||
assert_output ""
|
||||
@@ -1046,6 +1068,7 @@ EOF
|
||||
local skill="$TMPDIR/my-skill"
|
||||
make_skill_with_source_keys "$skill"
|
||||
make_sources_md "$skill" "my-source" "(none — not used directly)"
|
||||
commit_as_base "$skill"
|
||||
run bash "$SCRIPT" "$skill"
|
||||
assert_success
|
||||
assert_output ""
|
||||
@@ -1494,3 +1517,143 @@ EOF
|
||||
assert_failure
|
||||
assert_output --partial "not valid UTF-8"
|
||||
}
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Cycle 28 — Check 9: a changed Description or Contributing files claim is an
|
||||
# INFO, never a FAIL — the script can tell the text changed, not whether the
|
||||
# (possibly stronger) new wording is still true.
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
@test "check 9: Description text changed since base ref fires an INFO naming the slug and field" {
|
||||
local skill="$TMPDIR/my-skill"
|
||||
make_skill_with_source_keys "$skill"
|
||||
make_sources_md "$skill"
|
||||
commit_as_base "$skill"
|
||||
|
||||
# Rewritten in the working tree only, never committed — exactly the
|
||||
# shape of the bug check 9 exists to flag: a hedge upgraded to a
|
||||
# confident claim with nothing else in the entry touched.
|
||||
sed -i 's/^- \*\*Description:\*\* A test source\.$/- **Description:** A test source that Grounds the dispatch table directly./' \
|
||||
"$skill/references/sources.md"
|
||||
|
||||
run bash "$SCRIPT" "$skill"
|
||||
assert_success
|
||||
assert_output --partial "INFO"
|
||||
assert_output --partial "'Description' changed for 'my-source'"
|
||||
}
|
||||
|
||||
@test "check 9: Contributing files text changed since base ref fires an INFO naming the slug and field" {
|
||||
local skill="$TMPDIR/my-skill"
|
||||
make_skill_with_source_keys "$skill"
|
||||
make_sources_md "$skill"
|
||||
commit_as_base "$skill"
|
||||
|
||||
# Only an annotation is added. strip_note() in parse_contributing_files()
|
||||
# removes it before checks 4 and 5 compare paths, so those stay clean —
|
||||
# check 9 compares the raw field text, not the parsed path list, and
|
||||
# this is still a real wording change worth a human re-reading it.
|
||||
sed -i 's/^- \*\*Contributing files:\*\* SKILL\.md$/- **Contributing files:** SKILL.md (the dispatch table)/' \
|
||||
"$skill/references/sources.md"
|
||||
|
||||
run bash "$SCRIPT" "$skill"
|
||||
assert_success
|
||||
assert_output --partial "INFO"
|
||||
assert_output --partial "'Contributing files' changed for 'my-source'"
|
||||
refute_output --partial "does not exist"
|
||||
refute_output --partial "does not list"
|
||||
}
|
||||
|
||||
@test "check 9: an entry unchanged since base ref produces no check-9 finding" {
|
||||
local skill="$TMPDIR/my-skill"
|
||||
make_skill_with_source_keys "$skill"
|
||||
make_sources_md "$skill"
|
||||
commit_as_base "$skill"
|
||||
|
||||
run bash "$SCRIPT" "$skill"
|
||||
assert_success
|
||||
assert_output ""
|
||||
}
|
||||
|
||||
@test "check 9: a brand-new entry absent at the base ref is a creation, not a change" {
|
||||
local skill="$TMPDIR/my-skill"
|
||||
make_skill_with_source_keys "$skill"
|
||||
make_sources_md "$skill"
|
||||
commit_as_base "$skill"
|
||||
|
||||
# Added in the working tree only, after the commit above. It has no
|
||||
# earlier revision to diff against, so check 9 must stay silent about
|
||||
# it — a brand-new entry is a creation, not a rewrite of an existing
|
||||
# claim, and flagging it would be exactly the false-positive shape the
|
||||
# rejected literal-text approaches produced.
|
||||
cat >> "$skill/references/sources.md" <<EOF
|
||||
|
||||
## new-source
|
||||
|
||||
- **URL:** https://example.com/new-source
|
||||
- **Description:** A brand-new source, never committed.
|
||||
- **Contributing files:** (none)
|
||||
- **Research doc:** (none)
|
||||
- **Status:** \`extracted\`
|
||||
EOF
|
||||
|
||||
run bash "$SCRIPT" "$skill"
|
||||
assert_success
|
||||
assert_output ""
|
||||
}
|
||||
|
||||
@test "check 9: no repo root emits the one graceful INFO, not a silent skip or a crash" {
|
||||
local skill="$TMPDIR/my-skill"
|
||||
make_skill_with_source_keys "$skill"
|
||||
make_sources_md "$skill"
|
||||
# Deliberately no commit_as_base — $TMPDIR has no ancestor .git, so
|
||||
# find_repo_root() returns None.
|
||||
|
||||
run bash "$SCRIPT" "$skill"
|
||||
assert_success
|
||||
assert_output --partial "INFO"
|
||||
assert_output --partial "Check 9 skipped — no repo root above the skill directory"
|
||||
}
|
||||
|
||||
@test "check 9: an unresolvable base ref emits the one graceful INFO, not a silent skip or a crash" {
|
||||
local skill="$TMPDIR/my-skill"
|
||||
make_skill_with_source_keys "$skill"
|
||||
make_sources_md "$skill"
|
||||
commit_as_base "$skill"
|
||||
# A repo with no origin/main and no override: `git merge-base HEAD
|
||||
# origin/main` fails outright.
|
||||
git -C "$skill" update-ref -d refs/remotes/origin/main >/dev/null 2>&1
|
||||
|
||||
run bash "$SCRIPT" "$skill"
|
||||
assert_success
|
||||
assert_output --partial "INFO"
|
||||
assert_output --partial "Check 9 skipped — no base ref could be resolved"
|
||||
}
|
||||
|
||||
@test "check 9: --base-ref overrides the default origin/main resolution" {
|
||||
local skill="$TMPDIR/my-skill"
|
||||
make_skill_with_source_keys "$skill"
|
||||
make_sources_md "$skill"
|
||||
commit_as_base "$skill"
|
||||
local base_sha
|
||||
base_sha="$(git -C "$skill" rev-parse HEAD)"
|
||||
git -C "$skill" update-ref -d refs/remotes/origin/main >/dev/null 2>&1
|
||||
|
||||
sed -i 's/^- \*\*Description:\*\* A test source\.$/- **Description:** A rewritten claim./' \
|
||||
"$skill/references/sources.md"
|
||||
|
||||
run bash "$SCRIPT" "$skill" "--base-ref=$base_sha"
|
||||
assert_success
|
||||
assert_output --partial "'Description' changed for 'my-source'"
|
||||
}
|
||||
|
||||
@test "check 9: an invalid --base-ref value is reported as unresolvable, not a crash" {
|
||||
local skill="$TMPDIR/my-skill"
|
||||
make_skill_with_source_keys "$skill"
|
||||
make_sources_md "$skill"
|
||||
commit_as_base "$skill"
|
||||
|
||||
run bash "$SCRIPT" "$skill" "--base-ref=not-a-real-ref"
|
||||
assert_success
|
||||
assert_output --partial "Check 9 skipped — no base ref could be resolved"
|
||||
assert_output --partial "not-a-real-ref"
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user