Defame1297
79da149935
Add validate-provenance.sh and validate-provenance.bats to enforce the sources provenance chain introduced by skill-author. Eight checks cover slug cross-references, Contributing files existence, bidirectional source_keys linkage, Research doc: field presence, and upstream research doc alignment (forward INFO, reverse FAIL). Adds a new Provenance report dimension and INFO finding level (observational, exit-0, counted separately as · P info in the result block). Closes #8 Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
515 lines
14 KiB
Bash
515 lines
14 KiB
Bash
#!/usr/bin/env bats
|
|
|
|
setup() {
|
|
REPO_ROOT="$(cd "$BATS_TEST_DIRNAME/../../../../../" && pwd)"
|
|
load "$REPO_ROOT/tests/test_helper/bats-support/load"
|
|
load "$REPO_ROOT/tests/test_helper/bats-assert/load"
|
|
|
|
SCRIPT="$(cd "$BATS_TEST_DIRNAME/../scripts" && pwd)/validate-provenance.sh"
|
|
TMPDIR="$(mktemp -d)"
|
|
|
|
# Helper: create a minimal skill directory with no sources.md and no source_keys
|
|
make_clean_skill() {
|
|
local dir="$1"
|
|
local name
|
|
name="$(basename "$dir")"
|
|
mkdir -p "$dir"
|
|
cat > "$dir/SKILL.md" <<EOF
|
|
---
|
|
name: $name
|
|
description: A valid skill description.
|
|
---
|
|
|
|
## Step 1
|
|
|
|
Do the thing.
|
|
EOF
|
|
}
|
|
|
|
# Helper: create a skill with source_keys in SKILL.md
|
|
make_skill_with_source_keys() {
|
|
local dir="$1"
|
|
local name
|
|
name="$(basename "$dir")"
|
|
mkdir -p "$dir"
|
|
cat > "$dir/SKILL.md" <<EOF
|
|
---
|
|
name: $name
|
|
description: A valid skill description.
|
|
metadata:
|
|
source_keys:
|
|
- my-source
|
|
---
|
|
|
|
## Step 1
|
|
|
|
Do the thing.
|
|
EOF
|
|
}
|
|
|
|
# Helper: create a valid sources.md with one entry
|
|
make_sources_md() {
|
|
local dir="$1"
|
|
local slug="${2:-my-source}"
|
|
local contrib="${3:-SKILL.md}"
|
|
local research="${4:-(none)}"
|
|
mkdir -p "$dir/references"
|
|
cat > "$dir/references/sources.md" <<EOF
|
|
# Sources
|
|
|
|
## ${slug}
|
|
|
|
- **URL:** https://example.com/${slug}
|
|
- **Description:** A test source.
|
|
- **Contributing files:** ${contrib}
|
|
- **Research doc:** ${research}
|
|
- **Status:** \`extracted\`
|
|
EOF
|
|
}
|
|
}
|
|
|
|
teardown() {
|
|
rm -rf "$TMPDIR"
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Cycle 1 — --help
|
|
# ---------------------------------------------------------------------------
|
|
|
|
@test "--help exits 0" {
|
|
run bash "$SCRIPT" --help
|
|
assert_success
|
|
assert_output --partial "Usage:"
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Cycle 2 — Early exit: no sources.md, no source_keys → exit 0, no output
|
|
# ---------------------------------------------------------------------------
|
|
|
|
@test "clean pass: no sources.md and no source_keys anywhere → exit 0, no output" {
|
|
local skill="$TMPDIR/my-skill"
|
|
make_clean_skill "$skill"
|
|
run bash "$SCRIPT" "$skill"
|
|
assert_success
|
|
assert_output ""
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Cycle 3 — Check 0: source_keys present but no sources.md → FAIL
|
|
# ---------------------------------------------------------------------------
|
|
|
|
@test "FAIL: source_keys in SKILL.md but sources.md absent" {
|
|
local skill="$TMPDIR/my-skill"
|
|
make_skill_with_source_keys "$skill"
|
|
run bash "$SCRIPT" "$skill"
|
|
assert_failure
|
|
assert_output --partial "FAIL"
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Cycle 4 — Check 1: FILL IN: placeholder in sources.md → FAIL
|
|
# ---------------------------------------------------------------------------
|
|
|
|
@test "FAIL: FILL IN: placeholder in sources.md" {
|
|
local skill="$TMPDIR/my-skill"
|
|
make_skill_with_source_keys "$skill"
|
|
mkdir -p "$skill/references"
|
|
cat > "$skill/references/sources.md" <<EOF
|
|
# Sources
|
|
|
|
## my-source
|
|
|
|
- **URL:** FILL IN: add url
|
|
- **Description:** A test source.
|
|
- **Contributing files:** SKILL.md
|
|
- **Research doc:** (none)
|
|
- **Status:** \`extracted\`
|
|
EOF
|
|
run bash "$SCRIPT" "$skill"
|
|
assert_failure
|
|
assert_output --partial "FAIL"
|
|
}
|
|
|
|
@test "FILL IN: inside backticks in sources.md does not fail" {
|
|
local skill="$TMPDIR/my-skill"
|
|
make_skill_with_source_keys "$skill"
|
|
make_sources_md "$skill"
|
|
echo "Use \`FILL IN: value\` as example." >> "$skill/references/sources.md"
|
|
run bash "$SCRIPT" "$skill"
|
|
assert_success
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Cycle 5 — Check 2: source_keys slug missing from sources.md → FAIL
|
|
# ---------------------------------------------------------------------------
|
|
|
|
@test "FAIL: source_keys slug in SKILL.md not present as H2 in sources.md" {
|
|
local skill="$TMPDIR/my-skill"
|
|
make_skill_with_source_keys "$skill"
|
|
mkdir -p "$skill/references"
|
|
cat > "$skill/references/sources.md" <<EOF
|
|
# Sources
|
|
|
|
## different-source
|
|
|
|
- **URL:** https://example.com/different-source
|
|
- **Description:** A different source.
|
|
- **Contributing files:** SKILL.md
|
|
- **Research doc:** (none)
|
|
- **Status:** \`extracted\`
|
|
EOF
|
|
run bash "$SCRIPT" "$skill"
|
|
assert_failure
|
|
assert_output --partial "FAIL"
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Cycle 6 — Check 4: Contributing file path doesn't exist → FAIL
|
|
# ---------------------------------------------------------------------------
|
|
|
|
@test "FAIL: Contributing file listed in sources.md does not exist" {
|
|
local skill="$TMPDIR/my-skill"
|
|
make_skill_with_source_keys "$skill"
|
|
make_sources_md "$skill" "my-source" "references/nonexistent.md"
|
|
run bash "$SCRIPT" "$skill"
|
|
assert_failure
|
|
assert_output --partial "FAIL"
|
|
}
|
|
|
|
@test "pass: (none) in Contributing files is skipped" {
|
|
local skill="$TMPDIR/my-skill"
|
|
make_skill_with_source_keys "$skill"
|
|
make_sources_md "$skill" "my-source" "(none — not used directly)"
|
|
run bash "$SCRIPT" "$skill"
|
|
assert_success
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Cycle 7 — Check 6: Research doc field missing → FAIL
|
|
# ---------------------------------------------------------------------------
|
|
|
|
@test "FAIL: Research doc field missing from sources.md entry" {
|
|
local skill="$TMPDIR/my-skill"
|
|
make_skill_with_source_keys "$skill"
|
|
mkdir -p "$skill/references"
|
|
cat > "$skill/references/sources.md" <<EOF
|
|
# Sources
|
|
|
|
## my-source
|
|
|
|
- **URL:** https://example.com/my-source
|
|
- **Description:** A test source.
|
|
- **Contributing files:** SKILL.md
|
|
- **Status:** \`extracted\`
|
|
EOF
|
|
run bash "$SCRIPT" "$skill"
|
|
assert_failure
|
|
assert_output --partial "FAIL"
|
|
}
|
|
|
|
@test "FAIL: Research doc field is FILL IN: placeholder" {
|
|
local skill="$TMPDIR/my-skill"
|
|
make_skill_with_source_keys "$skill"
|
|
mkdir -p "$skill/references"
|
|
cat > "$skill/references/sources.md" <<EOF
|
|
# Sources
|
|
|
|
## my-source
|
|
|
|
- **URL:** https://example.com/my-source
|
|
- **Description:** A test source.
|
|
- **Contributing files:** SKILL.md
|
|
- **Research doc:** FILL IN: path to research doc
|
|
- **Status:** \`extracted\`
|
|
EOF
|
|
run bash "$SCRIPT" "$skill"
|
|
assert_failure
|
|
assert_output --partial "FAIL"
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Cycle 8 — Check 5: Bidirectional mismatch → FAIL
|
|
# ---------------------------------------------------------------------------
|
|
|
|
@test "FAIL: Contributing file exists but does not list parent slug in source_keys" {
|
|
local skill="$TMPDIR/my-skill"
|
|
make_skill_with_source_keys "$skill"
|
|
make_sources_md "$skill" "my-source" "SKILL.md"
|
|
# SKILL.md has source_keys: my-source, but let's change it to NOT have my-source
|
|
cat > "$skill/SKILL.md" <<EOF
|
|
---
|
|
name: my-skill
|
|
description: A valid skill description.
|
|
metadata:
|
|
source_keys:
|
|
- other-source
|
|
---
|
|
|
|
## Step 1
|
|
|
|
Do the thing.
|
|
EOF
|
|
mkdir -p "$skill/references"
|
|
cat > "$skill/references/sources.md" <<EOF
|
|
# Sources
|
|
|
|
## other-source
|
|
|
|
- **URL:** https://example.com/other-source
|
|
- **Description:** A test source.
|
|
- **Contributing files:** SKILL.md
|
|
- **Research doc:** (none)
|
|
- **Status:** \`extracted\`
|
|
EOF
|
|
# Now add my-source that references SKILL.md but SKILL.md doesn't back-reference it
|
|
cat >> "$skill/references/sources.md" <<EOF
|
|
|
|
## my-source
|
|
|
|
- **URL:** https://example.com/my-source
|
|
- **Description:** Another source.
|
|
- **Contributing files:** SKILL.md
|
|
- **Research doc:** (none)
|
|
- **Status:** \`extracted\`
|
|
EOF
|
|
run bash "$SCRIPT" "$skill"
|
|
assert_failure
|
|
assert_output --partial "FAIL"
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Cycle 9 — Check 3: references/*.md with no source_keys → INFO (exit 0)
|
|
# ---------------------------------------------------------------------------
|
|
|
|
@test "INFO: references doc with no source_keys frontmatter emits INFO but exits 0" {
|
|
local skill="$TMPDIR/my-skill"
|
|
make_skill_with_source_keys "$skill"
|
|
make_sources_md "$skill"
|
|
mkdir -p "$skill/references"
|
|
cat > "$skill/references/extra.md" <<EOF
|
|
# Extra Reference
|
|
|
|
No frontmatter here.
|
|
EOF
|
|
run bash "$SCRIPT" "$skill"
|
|
assert_success
|
|
assert_output --partial "INFO"
|
|
}
|
|
|
|
@test "pass: references doc with source_keys all matching sources.md exits 0" {
|
|
local skill="$TMPDIR/my-skill"
|
|
make_skill_with_source_keys "$skill"
|
|
make_sources_md "$skill"
|
|
mkdir -p "$skill/references"
|
|
cat > "$skill/references/extra.md" <<EOF
|
|
---
|
|
source_keys:
|
|
- my-source
|
|
---
|
|
|
|
# Extra Reference
|
|
|
|
Content here.
|
|
EOF
|
|
run bash "$SCRIPT" "$skill"
|
|
assert_success
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Cycle 10 — Clean full pass: valid sources.md, all source_keys match, files exist
|
|
# ---------------------------------------------------------------------------
|
|
|
|
@test "clean full pass: all checks satisfied" {
|
|
local skill="$TMPDIR/my-skill"
|
|
make_skill_with_source_keys "$skill"
|
|
make_sources_md "$skill"
|
|
run bash "$SCRIPT" "$skill"
|
|
assert_success
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Cycle 11 — Check 7: Upstream forward: slug in sources.md not in research doc → INFO
|
|
# ---------------------------------------------------------------------------
|
|
|
|
@test "INFO: slug in sources.md not found in research doc → INFO, exits 0" {
|
|
local skill="$TMPDIR/my-skill"
|
|
make_skill_with_source_keys "$skill"
|
|
|
|
# Create a research doc that does NOT have the slug
|
|
local research_dir="$TMPDIR/research"
|
|
mkdir -p "$research_dir"
|
|
cat > "$research_dir/my-research.md" <<EOF
|
|
# Research
|
|
|
|
## different-slug
|
|
|
|
- **Contributing files:** (none)
|
|
- **Status:** \`extracted\`
|
|
EOF
|
|
|
|
# Use a path relative to repo root — we'll place research doc inside TMPDIR
|
|
# and reference it as absolute for test purposes.
|
|
# The script finds repo root by walking up from skill-dir until .git is found.
|
|
# Since TMPDIR won't have .git, we simulate a repo structure.
|
|
local fake_repo="$TMPDIR/fakerepo"
|
|
mkdir -p "$fake_repo"
|
|
touch "$fake_repo/.git" # fake .git marker
|
|
|
|
local skill2="$fake_repo/my-skill"
|
|
mkdir -p "$skill2"
|
|
cat > "$skill2/SKILL.md" <<EOF
|
|
---
|
|
name: my-skill
|
|
description: A valid skill description.
|
|
metadata:
|
|
source_keys:
|
|
- my-source
|
|
---
|
|
|
|
## Step 1
|
|
|
|
Do the thing.
|
|
EOF
|
|
|
|
mkdir -p "$skill2/references"
|
|
mkdir -p "$fake_repo/docs/research"
|
|
cat > "$fake_repo/docs/research/my-research.md" <<EOF
|
|
# Research
|
|
|
|
## different-slug
|
|
|
|
- **Contributing files:** (none)
|
|
- **Status:** \`extracted\`
|
|
EOF
|
|
|
|
cat > "$skill2/references/sources.md" <<EOF
|
|
# Sources
|
|
|
|
## my-source
|
|
|
|
- **URL:** https://example.com/my-source
|
|
- **Description:** A test source.
|
|
- **Contributing files:** SKILL.md
|
|
- **Research doc:** docs/research/my-research.md
|
|
- **Status:** \`extracted\`
|
|
EOF
|
|
|
|
run bash "$SCRIPT" "$skill2"
|
|
assert_success
|
|
assert_output --partial "INFO"
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Cycle 12 — Check 8: Upstream reverse: extracted slug in research doc not in sources.md → FAIL
|
|
# ---------------------------------------------------------------------------
|
|
|
|
@test "FAIL: extracted non-(none) slug in research doc missing from skill sources.md" {
|
|
local fake_repo="$TMPDIR/fakerepo"
|
|
mkdir -p "$fake_repo"
|
|
touch "$fake_repo/.git"
|
|
|
|
local skill="$fake_repo/my-skill"
|
|
mkdir -p "$skill"
|
|
cat > "$skill/SKILL.md" <<EOF
|
|
---
|
|
name: my-skill
|
|
description: A valid skill description.
|
|
metadata:
|
|
source_keys:
|
|
- my-source
|
|
---
|
|
|
|
## Step 1
|
|
|
|
Do the thing.
|
|
EOF
|
|
|
|
mkdir -p "$skill/references"
|
|
mkdir -p "$fake_repo/docs/research"
|
|
|
|
# Research doc has my-source (extracted, with a contributing file) AND extra-source (also extracted)
|
|
cat > "$fake_repo/docs/research/my-research.md" <<EOF
|
|
# Research
|
|
|
|
## my-source
|
|
|
|
- **Contributing files:** some-skill/SKILL.md
|
|
- **Status:** \`extracted\`
|
|
|
|
## extra-source
|
|
|
|
- **Contributing files:** some-skill/references/extra.md
|
|
- **Status:** \`extracted\`
|
|
EOF
|
|
|
|
cat > "$skill/references/sources.md" <<EOF
|
|
# Sources
|
|
|
|
## my-source
|
|
|
|
- **URL:** https://example.com/my-source
|
|
- **Description:** A test source.
|
|
- **Contributing files:** SKILL.md
|
|
- **Research doc:** docs/research/my-research.md
|
|
- **Status:** \`extracted\`
|
|
EOF
|
|
|
|
run bash "$SCRIPT" "$skill"
|
|
assert_failure
|
|
assert_output --partial "FAIL"
|
|
}
|
|
|
|
@test "pass: extracted slug in research doc with (none) contributing files is not required in sources.md" {
|
|
local fake_repo="$TMPDIR/fakerepo"
|
|
mkdir -p "$fake_repo"
|
|
touch "$fake_repo/.git"
|
|
|
|
local skill="$fake_repo/my-skill"
|
|
mkdir -p "$skill"
|
|
cat > "$skill/SKILL.md" <<EOF
|
|
---
|
|
name: my-skill
|
|
description: A valid skill description.
|
|
metadata:
|
|
source_keys:
|
|
- my-source
|
|
---
|
|
|
|
## Step 1
|
|
|
|
Do the thing.
|
|
EOF
|
|
|
|
mkdir -p "$skill/references"
|
|
mkdir -p "$fake_repo/docs/research"
|
|
|
|
cat > "$fake_repo/docs/research/my-research.md" <<EOF
|
|
# Research
|
|
|
|
## my-source
|
|
|
|
- **Contributing files:** some-skill/SKILL.md
|
|
- **Status:** \`extracted\`
|
|
|
|
## extra-source
|
|
|
|
- **Contributing files:** (none — not relevant)
|
|
- **Status:** \`extracted\`
|
|
EOF
|
|
|
|
cat > "$skill/references/sources.md" <<EOF
|
|
# Sources
|
|
|
|
## my-source
|
|
|
|
- **URL:** https://example.com/my-source
|
|
- **Description:** A test source.
|
|
- **Contributing files:** SKILL.md
|
|
- **Research doc:** docs/research/my-research.md
|
|
- **Status:** \`extracted\`
|
|
EOF
|
|
|
|
run bash "$SCRIPT" "$skill"
|
|
assert_success
|
|
}
|