#!/usr/bin/env bats setup() { REPO_ROOT="$(cd "$BATS_TEST_DIRNAME/../../../../../../" && pwd)" load "$REPO_ROOT/tests/test_helper/bats-support/load" load "$REPO_ROOT/tests/test_helper/bats-assert/load" SCRIPT="$(cd "$BATS_TEST_DIRNAME/../scripts" && pwd)/validate-provenance.sh" TMPDIR="$(mktemp -d)" # Helper: create a minimal skill directory with no sources.md and no source_keys make_clean_skill() { local dir="$1" local name name="$(basename "$dir")" mkdir -p "$dir" cat > "$dir/SKILL.md" < "$dir/SKILL.md" < "$dir/references/sources.md" < "$dir/references/sources.md" <<'EOF' # Sources ## my-source - **URL:** https://example.com/my-source - **Description:** A test source, informing the dispatch table's shape with no forge-specific content drawn directly from it beyond that. - **Contributing files:** SKILL.md - **Research doc:** (none) - **Status:** `extracted` EOF } # Helper: turn dir into a real git repo with one commit of its current # contents, and a refs/remotes/origin/main pointing at that same commit. # Check 9 diffs the skill's references/sources.md against `git merge-base # HEAD origin/main` by default; this makes that resolve to a commit whose # sources.md is byte-identical to the working tree, so check 9 has # nothing to report there — exactly what a real repo looks like the # instant after a clean commit. Fixtures that go on to test something # else entirely (checks 3, 4, 5, 7, 8...) call this once, at the point # their skill's own sources.md is in its final state, so a completely # clean run stays completely clean. commit_as_base() { local dir="$1" git -C "$dir" init -q >/dev/null 2>&1 git -C "$dir" -c user.email=test@example.com -c user.name=test add -A >/dev/null 2>&1 git -C "$dir" -c user.email=test@example.com -c user.name=test commit -q -m base >/dev/null 2>&1 git -C "$dir" update-ref refs/remotes/origin/main HEAD >/dev/null 2>&1 } # Helper: create a fake repo (a real git repo, one commit, makes # find_repo_root stop there) holding one skill whose single sources.md # slug points at the given Research doc value. Checks 7 and 8 only run # for a skill inside a checkout, so every upstream case needs this shape; # the research doc itself is written per test into # "$repo/docs/research/sources.md" — which check 9 does not examine, so # a test overwriting it after this helper runs does not disturb check 9. make_upstream_skill() { local repo="$1" local research="${2:-docs/research/sources.md}" local skill="$repo/my-skill" mkdir -p "$skill/references" "$repo/docs/research" cat > "$skill/SKILL.md" < "$skill/references/sources.md" < "$skill/references/sources.md" <> "$skill/references/sources.md" run bash "$SCRIPT" "$skill" assert_success } # --------------------------------------------------------------------------- # Cycle 5 — Check 2: source_keys slug missing from sources.md → FAIL # --------------------------------------------------------------------------- @test "FAIL: source_keys slug in SKILL.md not present as H2 in sources.md" { local skill="$TMPDIR/my-skill" make_skill_with_source_keys "$skill" mkdir -p "$skill/references" cat > "$skill/references/sources.md" < "$skill/references/sources.md" < "$skill/references/sources.md" < "$skill/SKILL.md" < "$skill/references/sources.md" <> "$skill/references/sources.md" < "$skill/references/extra.md" < "$skill/references/extra.md" < "$research_dir/sources.md" < "$skill2/SKILL.md" < "$fake_repo/docs/research/sources.md" < "$skill2/references/sources.md" < "$skill/SKILL.md" < "$fake_repo/docs/research/sources.md" < "$skill/references/sources.md" < "$skill/SKILL.md" < "$fake_repo/docs/research/sources.md" < "$skill/references/sources.md" < "$fake_repo/docs/research/sources.md" < "$fake_repo/docs/research/sources.md" < "$fake_repo/docs/research/sources.md" < "$fake_repo/docs/research/sources.md" < "$fake_repo/docs/research/sources.md" < "$skill/references/sources.md" < "$fake_repo/docs/research/sources.md" < "$fake_repo/docs/research/sources.md" < "$skill/references/extra.md" < "$skill/references/extra.md" < "$skill/references/extra.md" < "$skill/references/extra.md" < "$fake_repo/docs/research/sources.md" < "$fake_repo/docs/research/sources.md" < "$fake_repo/docs/research/sources.md" < "$fake_repo/docs/research/sources.md" < "$skill/references/sources.md" < "$skill/references/sources.md" < "$skill/references/extra.md" < "$skill/references/extra.md" < "$fake_repo/docs/research/remotes.md" < "$fake_repo/docs/research/sources.md" < "$fake_repo/docs/research/sources.md" < "$fake_repo/docs/research/sources.md" < "$fake_repo/docs/research/sources.md" < "$fake_repo/docs/research/sources.md" < "$skill/SKILL.md" cat >> "$skill/SKILL.md" < "$skill/references/sources.md" < "$skill/references/topic.md" < "$skill/references/topic.md" run bash "$SCRIPT" "$skill" assert_failure assert_output --partial "not valid UTF-8" assert_output --partial "references/topic.md" } # --------------------------------------------------------------------------- # Cycle 24 — G4: check 3 walks references/ recursively # --------------------------------------------------------------------------- @test "G4: a source_keys file in references// is validated, not skipped" { local skill="$TMPDIR/my-skill" make_skill_with_source_keys "$skill" make_sources_md "$skill" mkdir -p "$skill/references/nested" cat > "$skill/references/nested/topic.md" < "$skill/references/sources.md" < "$skill/references/sources.md" < "$skill/references/sources.md" < "$skill/references/topic.md" run bash "$SCRIPT" "$skill" assert_failure assert_output --partial "not valid UTF-8" } # --------------------------------------------------------------------------- # Cycle 28 — Check 9: a changed Description or Contributing files claim is an # INFO, never a FAIL — the script can tell the text changed, not whether the # (possibly stronger) new wording is still true. # --------------------------------------------------------------------------- @test "check 9: Description text changed since base ref fires an INFO naming the slug and field" { local skill="$TMPDIR/my-skill" make_skill_with_source_keys "$skill" make_sources_md "$skill" commit_as_base "$skill" # Rewritten in the working tree only, never committed — exactly the # shape of the bug check 9 exists to flag: a hedge upgraded to a # confident claim with nothing else in the entry touched. sed -i 's/^- \*\*Description:\*\* A test source\.$/- **Description:** A test source that Grounds the dispatch table directly./' \ "$skill/references/sources.md" run bash "$SCRIPT" "$skill" assert_success assert_output --partial "INFO" assert_output --partial "'Description' changed for 'my-source'" } @test "check 9: Contributing files text changed since base ref fires an INFO naming the slug and field" { local skill="$TMPDIR/my-skill" make_skill_with_source_keys "$skill" make_sources_md "$skill" commit_as_base "$skill" # Only an annotation is added. strip_note() in parse_contributing_files() # removes it before checks 4 and 5 compare paths, so those stay clean — # check 9 compares the raw field text, not the parsed path list, and # this is still a real wording change worth a human re-reading it. sed -i 's/^- \*\*Contributing files:\*\* SKILL\.md$/- **Contributing files:** SKILL.md (the dispatch table)/' \ "$skill/references/sources.md" run bash "$SCRIPT" "$skill" assert_success assert_output --partial "INFO" assert_output --partial "'Contributing files' changed for 'my-source'" refute_output --partial "does not exist" refute_output --partial "does not list" } @test "check 9: an entry unchanged since base ref produces no check-9 finding" { local skill="$TMPDIR/my-skill" make_skill_with_source_keys "$skill" make_sources_md "$skill" commit_as_base "$skill" run bash "$SCRIPT" "$skill" assert_success assert_output "" } @test "check 9: a brand-new entry absent at the base ref is a creation, not a change" { local skill="$TMPDIR/my-skill" make_skill_with_source_keys "$skill" make_sources_md "$skill" commit_as_base "$skill" # Added in the working tree only, after the commit above. It has no # earlier revision to diff against, so check 9 must stay silent about # it — a brand-new entry is a creation, not a rewrite of an existing # claim, and flagging it would be exactly the false-positive shape the # rejected literal-text approaches produced. cat >> "$skill/references/sources.md" </dev/null 2>&1 run bash "$SCRIPT" "$skill" assert_success assert_output --partial "INFO" assert_output --partial "Check 9 skipped — no base ref could be resolved" } @test "check 9: --base-ref overrides a default origin/main that resolves to something else" { local skill="$TMPDIR/my-skill" make_skill_with_source_keys "$skill" make_sources_md "$skill" commit_as_base "$skill" local old_sha old_sha="$(git -C "$skill" rev-parse HEAD)" # A SECOND commit carrying the rewritten claim, with origin/main moved onto # it. The default base ref therefore resolves — to a commit that matches the # working tree — so a silent default run proves there was a default here to # override. Deleting origin/main instead, as this test used to, proved only # that the flag works when nothing else does. sed -i 's/^- \*\*Description:\*\* A test source\.$/- **Description:** A rewritten claim./' \ "$skill/references/sources.md" git -C "$skill" -c user.email=test@example.com -c user.name=test commit -aqm rewrite >/dev/null 2>&1 git -C "$skill" update-ref refs/remotes/origin/main HEAD >/dev/null 2>&1 run bash "$SCRIPT" "$skill" assert_success assert_output "" run bash "$SCRIPT" "$skill" "--base-ref=$old_sha" assert_success assert_output --partial "'Description' changed for 'my-source'" } @test "check 9: VALIDATE_PROVENANCE_BASE_REF sets the base ref when no flag is given" { local skill="$TMPDIR/my-skill" make_skill_with_source_keys "$skill" make_sources_md "$skill" commit_as_base "$skill" local old_sha old_sha="$(git -C "$skill" rev-parse HEAD)" sed -i 's/^- \*\*Description:\*\* A test source\.$/- **Description:** A rewritten claim./' \ "$skill/references/sources.md" git -C "$skill" -c user.email=test@example.com -c user.name=test commit -aqm rewrite >/dev/null 2>&1 git -C "$skill" update-ref refs/remotes/origin/main HEAD >/dev/null 2>&1 run env VALIDATE_PROVENANCE_BASE_REF="$old_sha" bash "$SCRIPT" "$skill" assert_success assert_output --partial "'Description' changed for 'my-source'" } @test "check 9: the --base-ref flag wins over the environment variable" { local skill="$TMPDIR/my-skill" make_skill_with_source_keys "$skill" make_sources_md "$skill" commit_as_base "$skill" local old_sha old_sha="$(git -C "$skill" rev-parse HEAD)" sed -i 's/^- \*\*Description:\*\* A test source\.$/- **Description:** A rewritten claim./' \ "$skill/references/sources.md" git -C "$skill" -c user.email=test@example.com -c user.name=test commit -aqm rewrite >/dev/null 2>&1 local new_sha new_sha="$(git -C "$skill" rev-parse HEAD)" # The environment names the old commit (which would fire), the flag names # the new one (which would not). The usage text promises the flag wins. run env VALIDATE_PROVENANCE_BASE_REF="$old_sha" bash "$SCRIPT" "$skill" "--base-ref=$new_sha" assert_success assert_output "" } @test "check 9: an EMPTY --base-ref is still 'given' and wins over the environment variable" { local skill="$TMPDIR/my-skill" make_skill_with_source_keys "$skill" make_sources_md "$skill" commit_as_base "$skill" local old_sha old_sha="$(git -C "$skill" rev-parse HEAD)" sed -i 's/^- \*\*Description:\*\* A test source\.$/- **Description:** A rewritten claim./' \ "$skill/references/sources.md" git -C "$skill" -c user.email=test@example.com -c user.name=test commit -aqm rewrite >/dev/null 2>&1 git -C "$skill" update-ref refs/remotes/origin/main HEAD >/dev/null 2>&1 # `--base-ref=` selects the DEFAULT resolution (origin/main, which now # matches the working tree), so nothing fires. Under the old `:-` spelling # the empty value read as absent and the environment variable won, firing # the INFO and contradicting the usage text. run env VALIDATE_PROVENANCE_BASE_REF="$old_sha" bash "$SCRIPT" "$skill" "--base-ref=" assert_success assert_output "" } # --------------------------------------------------------------------------- # Cycle 29 — Check 9: a WRAPPED field value. The parser compared only the first # physical line, so a rewrite confined to a continuation line — the exact # hedge-to-confident-claim shape #118 exists to catch — produced no finding at # all, while a pure re-wrap produced a false one. # --------------------------------------------------------------------------- @test "check 9: a rewrite confined to a wrapped Description's CONTINUATION line fires an INFO" { local skill="$TMPDIR/my-skill" make_skill_with_source_keys "$skill" make_wrapped_sources_md "$skill" commit_as_base "$skill" # Hedge to confident claim, on the second physical line only. This is the # regression check 9 was written for, and the one it could not see. sed -i "s|^ no forge-specific content drawn directly from it beyond that\.\$| it grounds Step 2's dispatch table in full.|" \ "$skill/references/sources.md" run bash "$SCRIPT" "$skill" assert_success assert_output --partial "'Description' changed for 'my-source'" } @test "check 9: re-wrapping a Description with no wording change produces no finding" { local skill="$TMPDIR/my-skill" make_skill_with_source_keys "$skill" make_wrapped_sources_md "$skill" commit_as_base "$skill" # Same words, different line breaks. normalize_field_text()'s docstring # promises this is invisible; it was not, because the value was truncated # at its first newline before the whitespace collapse ever ran. python3 - "$skill/references/sources.md" <<'PY' import sys path = sys.argv[1] text = open(path).read() old = ("- **Description:** A test source, informing the dispatch table's shape with\n" " no forge-specific content drawn directly from it beyond that.") new = ("- **Description:** A test source, informing the dispatch\n" " table's shape with no forge-specific content drawn\n" " directly from it beyond that.") assert old in text open(path, 'w').write(text.replace(old, new)) PY run bash "$SCRIPT" "$skill" assert_success assert_output "" } @test "check 9: the bullet form of Contributing files is compared, not skipped" { local skill="$TMPDIR/my-skill" make_skill_with_source_keys "$skill" mkdir -p "$skill/references" cat > "$skill/references/sources.md" <<'EOF' # Sources ## my-source - **URL:** https://example.com/my-source - **Description:** A test source. **Contributing files:** - SKILL.md (the dispatch table) - references/other.md (the rubric) - **Research doc:** (none) - **Status:** `extracted` EOF printf -- '---\nsource_keys:\n - my-source\n---\n\nnotes\n' > "$skill/references/other.md" commit_as_base "$skill" # The change is in the SECOND bullet. The old loop joined bullets in # document order too, but broke on any wrapped one — and no test covered # this branch at all, both existing check-9 tests using the inline form. sed -i 's|^- references/other.md (the rubric)$|- references/other.md (the whole rubric, verbatim)|' \ "$skill/references/sources.md" run bash "$SCRIPT" "$skill" assert_success assert_output --partial "'Contributing files' changed for 'my-source'" } @test "check 9: a wrapped bullet does not silently drop the bullets after it" { local skill="$TMPDIR/my-skill" make_skill_with_source_keys "$skill" mkdir -p "$skill/references" cat > "$skill/references/sources.md" <<'EOF' # Sources ## my-source - **URL:** https://example.com/my-source - **Description:** A test source. **Contributing files:** - SKILL.md (the dispatch table, and the gates common to every branch of it) - references/other.md (the rubric) - **Research doc:** (none) - **Status:** `extracted` EOF printf -- '---\nsource_keys:\n - my-source\n---\n\nnotes\n' > "$skill/references/other.md" commit_as_base "$skill" # The old loop broke at the wrapped continuation line, so everything from # here down was never part of the compared value — a change to the last # bullet was invisible. sed -i 's|^- references/other.md (the rubric)$|- references/other.md (rewritten claim)|' \ "$skill/references/sources.md" run bash "$SCRIPT" "$skill" assert_success assert_output --partial "'Contributing files' changed for 'my-source'" } @test "check 9: a field present at the base ref and deleted since is announced" { local skill="$TMPDIR/my-skill" make_skill_with_source_keys "$skill" make_sources_md "$skill" commit_as_base "$skill" # No other check in this script requires a Description, so a deleted one # used to leave no finding anywhere: a claim could be withdrawn as # invisibly as it could be strengthened. sed -i '/^- \*\*Description:\*\*/d' "$skill/references/sources.md" run bash "$SCRIPT" "$skill" assert_success assert_output --partial "'Description' removed for 'my-source'" } @test "check 9: a sources.md untracked at the base ref is announced, not silently skipped" { local repo="$TMPDIR/repo" local skill="$repo/tracked-skill" mkdir -p "$skill" make_skill_with_source_keys "$skill" make_sources_md "$skill" commit_as_base "$repo" # A copy of the same skill at a path git does not know — the everyday case # being an installed, gitignored .claude/skills/ tree. The base ref # resolves fine; `git show :` does not. Treating that as # "creation, nothing to flag" made the whole check vanish without a word, # so the same directory reported findings at one path and silence at the # other. cp -r "$skill" "$repo/untracked-copy" run bash "$SCRIPT" "$repo/untracked-copy" assert_success assert_output --partial "INFO" assert_output --partial "is not tracked at" assert_output --partial "Check 9 did not run for any slug in this skill." } @test "check 9: an invalid --base-ref value is reported as unresolvable, not a crash" { local skill="$TMPDIR/my-skill" make_skill_with_source_keys "$skill" make_sources_md "$skill" commit_as_base "$skill" run bash "$SCRIPT" "$skill" "--base-ref=not-a-real-ref" assert_success assert_output --partial "Check 9 skipped — no base ref could be resolved" assert_output --partial "not-a-real-ref" }