feat(skill-audit): validate sources provenance chain
Add validate-provenance.sh and validate-provenance.bats to enforce the sources provenance chain introduced by skill-author. Eight checks cover slug cross-references, Contributing files existence, bidirectional source_keys linkage, Research doc: field presence, and upstream research doc alignment (forward INFO, reverse FAIL). Adds a new Provenance report dimension and INFO finding level (observational, exit-0, counted separately as · P info in the result block). Closes #8 Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
@@ -26,3 +26,4 @@ bats plugins/kyberforge/skills/skill-audit/tests/
|
||||
| File | Purpose |
|
||||
|------|---------|
|
||||
| `validate.bats` | Bats test suite for `scripts/validate.sh` |
|
||||
| `validate-provenance.bats` | Bats test suite for `scripts/validate-provenance.sh` |
|
||||
|
||||
@@ -0,0 +1,514 @@
|
||||
#!/usr/bin/env bats
|
||||
|
||||
setup() {
|
||||
REPO_ROOT="$(cd "$BATS_TEST_DIRNAME/../../../../../" && pwd)"
|
||||
load "$REPO_ROOT/tests/test_helper/bats-support/load"
|
||||
load "$REPO_ROOT/tests/test_helper/bats-assert/load"
|
||||
|
||||
SCRIPT="$(cd "$BATS_TEST_DIRNAME/../scripts" && pwd)/validate-provenance.sh"
|
||||
TMPDIR="$(mktemp -d)"
|
||||
|
||||
# Helper: create a minimal skill directory with no sources.md and no source_keys
|
||||
make_clean_skill() {
|
||||
local dir="$1"
|
||||
local name
|
||||
name="$(basename "$dir")"
|
||||
mkdir -p "$dir"
|
||||
cat > "$dir/SKILL.md" <<EOF
|
||||
---
|
||||
name: $name
|
||||
description: A valid skill description.
|
||||
---
|
||||
|
||||
## Step 1
|
||||
|
||||
Do the thing.
|
||||
EOF
|
||||
}
|
||||
|
||||
# Helper: create a skill with source_keys in SKILL.md
|
||||
make_skill_with_source_keys() {
|
||||
local dir="$1"
|
||||
local name
|
||||
name="$(basename "$dir")"
|
||||
mkdir -p "$dir"
|
||||
cat > "$dir/SKILL.md" <<EOF
|
||||
---
|
||||
name: $name
|
||||
description: A valid skill description.
|
||||
metadata:
|
||||
source_keys:
|
||||
- my-source
|
||||
---
|
||||
|
||||
## Step 1
|
||||
|
||||
Do the thing.
|
||||
EOF
|
||||
}
|
||||
|
||||
# Helper: create a valid sources.md with one entry
|
||||
make_sources_md() {
|
||||
local dir="$1"
|
||||
local slug="${2:-my-source}"
|
||||
local contrib="${3:-SKILL.md}"
|
||||
local research="${4:-(none)}"
|
||||
mkdir -p "$dir/references"
|
||||
cat > "$dir/references/sources.md" <<EOF
|
||||
# Sources
|
||||
|
||||
## ${slug}
|
||||
|
||||
- **URL:** https://example.com/${slug}
|
||||
- **Description:** A test source.
|
||||
- **Contributing files:** ${contrib}
|
||||
- **Research doc:** ${research}
|
||||
- **Status:** \`extracted\`
|
||||
EOF
|
||||
}
|
||||
}
|
||||
|
||||
teardown() {
|
||||
rm -rf "$TMPDIR"
|
||||
}
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Cycle 1 — --help
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
@test "--help exits 0" {
|
||||
run bash "$SCRIPT" --help
|
||||
assert_success
|
||||
assert_output --partial "Usage:"
|
||||
}
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Cycle 2 — Early exit: no sources.md, no source_keys → exit 0, no output
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
@test "clean pass: no sources.md and no source_keys anywhere → exit 0, no output" {
|
||||
local skill="$TMPDIR/my-skill"
|
||||
make_clean_skill "$skill"
|
||||
run bash "$SCRIPT" "$skill"
|
||||
assert_success
|
||||
assert_output ""
|
||||
}
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Cycle 3 — Check 0: source_keys present but no sources.md → FAIL
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
@test "FAIL: source_keys in SKILL.md but sources.md absent" {
|
||||
local skill="$TMPDIR/my-skill"
|
||||
make_skill_with_source_keys "$skill"
|
||||
run bash "$SCRIPT" "$skill"
|
||||
assert_failure
|
||||
assert_output --partial "FAIL"
|
||||
}
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Cycle 4 — Check 1: FILL IN: placeholder in sources.md → FAIL
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
@test "FAIL: FILL IN: placeholder in sources.md" {
|
||||
local skill="$TMPDIR/my-skill"
|
||||
make_skill_with_source_keys "$skill"
|
||||
mkdir -p "$skill/references"
|
||||
cat > "$skill/references/sources.md" <<EOF
|
||||
# Sources
|
||||
|
||||
## my-source
|
||||
|
||||
- **URL:** FILL IN: add url
|
||||
- **Description:** A test source.
|
||||
- **Contributing files:** SKILL.md
|
||||
- **Research doc:** (none)
|
||||
- **Status:** \`extracted\`
|
||||
EOF
|
||||
run bash "$SCRIPT" "$skill"
|
||||
assert_failure
|
||||
assert_output --partial "FAIL"
|
||||
}
|
||||
|
||||
@test "FILL IN: inside backticks in sources.md does not fail" {
|
||||
local skill="$TMPDIR/my-skill"
|
||||
make_skill_with_source_keys "$skill"
|
||||
make_sources_md "$skill"
|
||||
echo "Use \`FILL IN: value\` as example." >> "$skill/references/sources.md"
|
||||
run bash "$SCRIPT" "$skill"
|
||||
assert_success
|
||||
}
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Cycle 5 — Check 2: source_keys slug missing from sources.md → FAIL
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
@test "FAIL: source_keys slug in SKILL.md not present as H2 in sources.md" {
|
||||
local skill="$TMPDIR/my-skill"
|
||||
make_skill_with_source_keys "$skill"
|
||||
mkdir -p "$skill/references"
|
||||
cat > "$skill/references/sources.md" <<EOF
|
||||
# Sources
|
||||
|
||||
## different-source
|
||||
|
||||
- **URL:** https://example.com/different-source
|
||||
- **Description:** A different source.
|
||||
- **Contributing files:** SKILL.md
|
||||
- **Research doc:** (none)
|
||||
- **Status:** \`extracted\`
|
||||
EOF
|
||||
run bash "$SCRIPT" "$skill"
|
||||
assert_failure
|
||||
assert_output --partial "FAIL"
|
||||
}
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Cycle 6 — Check 4: Contributing file path doesn't exist → FAIL
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
@test "FAIL: Contributing file listed in sources.md does not exist" {
|
||||
local skill="$TMPDIR/my-skill"
|
||||
make_skill_with_source_keys "$skill"
|
||||
make_sources_md "$skill" "my-source" "references/nonexistent.md"
|
||||
run bash "$SCRIPT" "$skill"
|
||||
assert_failure
|
||||
assert_output --partial "FAIL"
|
||||
}
|
||||
|
||||
@test "pass: (none) in Contributing files is skipped" {
|
||||
local skill="$TMPDIR/my-skill"
|
||||
make_skill_with_source_keys "$skill"
|
||||
make_sources_md "$skill" "my-source" "(none — not used directly)"
|
||||
run bash "$SCRIPT" "$skill"
|
||||
assert_success
|
||||
}
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Cycle 7 — Check 6: Research doc field missing → FAIL
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
@test "FAIL: Research doc field missing from sources.md entry" {
|
||||
local skill="$TMPDIR/my-skill"
|
||||
make_skill_with_source_keys "$skill"
|
||||
mkdir -p "$skill/references"
|
||||
cat > "$skill/references/sources.md" <<EOF
|
||||
# Sources
|
||||
|
||||
## my-source
|
||||
|
||||
- **URL:** https://example.com/my-source
|
||||
- **Description:** A test source.
|
||||
- **Contributing files:** SKILL.md
|
||||
- **Status:** \`extracted\`
|
||||
EOF
|
||||
run bash "$SCRIPT" "$skill"
|
||||
assert_failure
|
||||
assert_output --partial "FAIL"
|
||||
}
|
||||
|
||||
@test "FAIL: Research doc field is FILL IN: placeholder" {
|
||||
local skill="$TMPDIR/my-skill"
|
||||
make_skill_with_source_keys "$skill"
|
||||
mkdir -p "$skill/references"
|
||||
cat > "$skill/references/sources.md" <<EOF
|
||||
# Sources
|
||||
|
||||
## my-source
|
||||
|
||||
- **URL:** https://example.com/my-source
|
||||
- **Description:** A test source.
|
||||
- **Contributing files:** SKILL.md
|
||||
- **Research doc:** FILL IN: path to research doc
|
||||
- **Status:** \`extracted\`
|
||||
EOF
|
||||
run bash "$SCRIPT" "$skill"
|
||||
assert_failure
|
||||
assert_output --partial "FAIL"
|
||||
}
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Cycle 8 — Check 5: Bidirectional mismatch → FAIL
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
@test "FAIL: Contributing file exists but does not list parent slug in source_keys" {
|
||||
local skill="$TMPDIR/my-skill"
|
||||
make_skill_with_source_keys "$skill"
|
||||
make_sources_md "$skill" "my-source" "SKILL.md"
|
||||
# SKILL.md has source_keys: my-source, but let's change it to NOT have my-source
|
||||
cat > "$skill/SKILL.md" <<EOF
|
||||
---
|
||||
name: my-skill
|
||||
description: A valid skill description.
|
||||
metadata:
|
||||
source_keys:
|
||||
- other-source
|
||||
---
|
||||
|
||||
## Step 1
|
||||
|
||||
Do the thing.
|
||||
EOF
|
||||
mkdir -p "$skill/references"
|
||||
cat > "$skill/references/sources.md" <<EOF
|
||||
# Sources
|
||||
|
||||
## other-source
|
||||
|
||||
- **URL:** https://example.com/other-source
|
||||
- **Description:** A test source.
|
||||
- **Contributing files:** SKILL.md
|
||||
- **Research doc:** (none)
|
||||
- **Status:** \`extracted\`
|
||||
EOF
|
||||
# Now add my-source that references SKILL.md but SKILL.md doesn't back-reference it
|
||||
cat >> "$skill/references/sources.md" <<EOF
|
||||
|
||||
## my-source
|
||||
|
||||
- **URL:** https://example.com/my-source
|
||||
- **Description:** Another source.
|
||||
- **Contributing files:** SKILL.md
|
||||
- **Research doc:** (none)
|
||||
- **Status:** \`extracted\`
|
||||
EOF
|
||||
run bash "$SCRIPT" "$skill"
|
||||
assert_failure
|
||||
assert_output --partial "FAIL"
|
||||
}
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Cycle 9 — Check 3: references/*.md with no source_keys → INFO (exit 0)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
@test "INFO: references doc with no source_keys frontmatter emits INFO but exits 0" {
|
||||
local skill="$TMPDIR/my-skill"
|
||||
make_skill_with_source_keys "$skill"
|
||||
make_sources_md "$skill"
|
||||
mkdir -p "$skill/references"
|
||||
cat > "$skill/references/extra.md" <<EOF
|
||||
# Extra Reference
|
||||
|
||||
No frontmatter here.
|
||||
EOF
|
||||
run bash "$SCRIPT" "$skill"
|
||||
assert_success
|
||||
assert_output --partial "INFO"
|
||||
}
|
||||
|
||||
@test "pass: references doc with source_keys all matching sources.md exits 0" {
|
||||
local skill="$TMPDIR/my-skill"
|
||||
make_skill_with_source_keys "$skill"
|
||||
make_sources_md "$skill"
|
||||
mkdir -p "$skill/references"
|
||||
cat > "$skill/references/extra.md" <<EOF
|
||||
---
|
||||
source_keys:
|
||||
- my-source
|
||||
---
|
||||
|
||||
# Extra Reference
|
||||
|
||||
Content here.
|
||||
EOF
|
||||
run bash "$SCRIPT" "$skill"
|
||||
assert_success
|
||||
}
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Cycle 10 — Clean full pass: valid sources.md, all source_keys match, files exist
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
@test "clean full pass: all checks satisfied" {
|
||||
local skill="$TMPDIR/my-skill"
|
||||
make_skill_with_source_keys "$skill"
|
||||
make_sources_md "$skill"
|
||||
run bash "$SCRIPT" "$skill"
|
||||
assert_success
|
||||
}
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Cycle 11 — Check 7: Upstream forward: slug in sources.md not in research doc → INFO
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
@test "INFO: slug in sources.md not found in research doc → INFO, exits 0" {
|
||||
local skill="$TMPDIR/my-skill"
|
||||
make_skill_with_source_keys "$skill"
|
||||
|
||||
# Create a research doc that does NOT have the slug
|
||||
local research_dir="$TMPDIR/research"
|
||||
mkdir -p "$research_dir"
|
||||
cat > "$research_dir/my-research.md" <<EOF
|
||||
# Research
|
||||
|
||||
## different-slug
|
||||
|
||||
- **Contributing files:** (none)
|
||||
- **Status:** \`extracted\`
|
||||
EOF
|
||||
|
||||
# Use a path relative to repo root — we'll place research doc inside TMPDIR
|
||||
# and reference it as absolute for test purposes.
|
||||
# The script finds repo root by walking up from skill-dir until .git is found.
|
||||
# Since TMPDIR won't have .git, we simulate a repo structure.
|
||||
local fake_repo="$TMPDIR/fakerepo"
|
||||
mkdir -p "$fake_repo"
|
||||
touch "$fake_repo/.git" # fake .git marker
|
||||
|
||||
local skill2="$fake_repo/my-skill"
|
||||
mkdir -p "$skill2"
|
||||
cat > "$skill2/SKILL.md" <<EOF
|
||||
---
|
||||
name: my-skill
|
||||
description: A valid skill description.
|
||||
metadata:
|
||||
source_keys:
|
||||
- my-source
|
||||
---
|
||||
|
||||
## Step 1
|
||||
|
||||
Do the thing.
|
||||
EOF
|
||||
|
||||
mkdir -p "$skill2/references"
|
||||
mkdir -p "$fake_repo/docs/research"
|
||||
cat > "$fake_repo/docs/research/my-research.md" <<EOF
|
||||
# Research
|
||||
|
||||
## different-slug
|
||||
|
||||
- **Contributing files:** (none)
|
||||
- **Status:** \`extracted\`
|
||||
EOF
|
||||
|
||||
cat > "$skill2/references/sources.md" <<EOF
|
||||
# Sources
|
||||
|
||||
## my-source
|
||||
|
||||
- **URL:** https://example.com/my-source
|
||||
- **Description:** A test source.
|
||||
- **Contributing files:** SKILL.md
|
||||
- **Research doc:** docs/research/my-research.md
|
||||
- **Status:** \`extracted\`
|
||||
EOF
|
||||
|
||||
run bash "$SCRIPT" "$skill2"
|
||||
assert_success
|
||||
assert_output --partial "INFO"
|
||||
}
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Cycle 12 — Check 8: Upstream reverse: extracted slug in research doc not in sources.md → FAIL
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
@test "FAIL: extracted non-(none) slug in research doc missing from skill sources.md" {
|
||||
local fake_repo="$TMPDIR/fakerepo"
|
||||
mkdir -p "$fake_repo"
|
||||
touch "$fake_repo/.git"
|
||||
|
||||
local skill="$fake_repo/my-skill"
|
||||
mkdir -p "$skill"
|
||||
cat > "$skill/SKILL.md" <<EOF
|
||||
---
|
||||
name: my-skill
|
||||
description: A valid skill description.
|
||||
metadata:
|
||||
source_keys:
|
||||
- my-source
|
||||
---
|
||||
|
||||
## Step 1
|
||||
|
||||
Do the thing.
|
||||
EOF
|
||||
|
||||
mkdir -p "$skill/references"
|
||||
mkdir -p "$fake_repo/docs/research"
|
||||
|
||||
# Research doc has my-source (extracted, with a contributing file) AND extra-source (also extracted)
|
||||
cat > "$fake_repo/docs/research/my-research.md" <<EOF
|
||||
# Research
|
||||
|
||||
## my-source
|
||||
|
||||
- **Contributing files:** some-skill/SKILL.md
|
||||
- **Status:** \`extracted\`
|
||||
|
||||
## extra-source
|
||||
|
||||
- **Contributing files:** some-skill/references/extra.md
|
||||
- **Status:** \`extracted\`
|
||||
EOF
|
||||
|
||||
cat > "$skill/references/sources.md" <<EOF
|
||||
# Sources
|
||||
|
||||
## my-source
|
||||
|
||||
- **URL:** https://example.com/my-source
|
||||
- **Description:** A test source.
|
||||
- **Contributing files:** SKILL.md
|
||||
- **Research doc:** docs/research/my-research.md
|
||||
- **Status:** \`extracted\`
|
||||
EOF
|
||||
|
||||
run bash "$SCRIPT" "$skill"
|
||||
assert_failure
|
||||
assert_output --partial "FAIL"
|
||||
}
|
||||
|
||||
@test "pass: extracted slug in research doc with (none) contributing files is not required in sources.md" {
|
||||
local fake_repo="$TMPDIR/fakerepo"
|
||||
mkdir -p "$fake_repo"
|
||||
touch "$fake_repo/.git"
|
||||
|
||||
local skill="$fake_repo/my-skill"
|
||||
mkdir -p "$skill"
|
||||
cat > "$skill/SKILL.md" <<EOF
|
||||
---
|
||||
name: my-skill
|
||||
description: A valid skill description.
|
||||
metadata:
|
||||
source_keys:
|
||||
- my-source
|
||||
---
|
||||
|
||||
## Step 1
|
||||
|
||||
Do the thing.
|
||||
EOF
|
||||
|
||||
mkdir -p "$skill/references"
|
||||
mkdir -p "$fake_repo/docs/research"
|
||||
|
||||
cat > "$fake_repo/docs/research/my-research.md" <<EOF
|
||||
# Research
|
||||
|
||||
## my-source
|
||||
|
||||
- **Contributing files:** some-skill/SKILL.md
|
||||
- **Status:** \`extracted\`
|
||||
|
||||
## extra-source
|
||||
|
||||
- **Contributing files:** (none — not relevant)
|
||||
- **Status:** \`extracted\`
|
||||
EOF
|
||||
|
||||
cat > "$skill/references/sources.md" <<EOF
|
||||
# Sources
|
||||
|
||||
## my-source
|
||||
|
||||
- **URL:** https://example.com/my-source
|
||||
- **Description:** A test source.
|
||||
- **Contributing files:** SKILL.md
|
||||
- **Research doc:** docs/research/my-research.md
|
||||
- **Status:** \`extracted\`
|
||||
EOF
|
||||
|
||||
run bash "$SCRIPT" "$skill"
|
||||
assert_success
|
||||
}
|
||||
Reference in New Issue
Block a user