test(kyberforge): pin the provenance checker's three new behaviours

27a7669 changed validate-provenance.sh in three ways and tested none of them: the bullet-form parser returning None rather than [] on unparsable input, the source_keys: [] house-authored declaration, and the stripping of section annotations off a Research doc path. All three are checks that previously failed by staying silent, which is the failure mode this repo keeps rediscovering. Shipping them untested left nothing to catch a regression back to silence. The 22 cases assert consequences rather than return values -- that check 8 runs on an unparsable block, that an unresolvable path emits an INFO naming its slug -- and each was verified against a deliberate mutation of the behaviour it covers. Addresses #111.
2026-08-31 08:28:25 +00:00
parent 8599b4a058
commit 5b80f305e9
1 changed files with 473 additions and 0 deletions
--- a/plugins/kyberforge/.apm/skills/skill-audit/tests/validate-provenance.bats
+++ b/plugins/kyberforge/.apm/skills/skill-audit/tests/validate-provenance.bats
@@ -64,6 +64,43 @@ EOF
 - **Contributing files:** ${contrib}
 - **Research doc:** ${research}
 - **Status:** \`extracted\`
+EOF
+    }
+
+    # Helper: create a fake repo (a .git marker makes find_repo_root stop there)
+    # holding one skill whose single sources.md slug points at the given
+    # Research doc value. Checks 7 and 8 only run for a skill inside a checkout,
+    # so every upstream case needs this shape; the research doc itself is
+    # written per test into "$repo/docs/research/my-research.md".
+    make_upstream_skill() {
+        local repo="$1"
+        local research="${2:-docs/research/my-research.md}"
+        local skill="$repo/my-skill"
+        mkdir -p "$skill/references" "$repo/docs/research"
+        touch "$repo/.git"
+        cat > "$skill/SKILL.md" <<EOF
+---
+name: my-skill
+description: A valid skill description.
+metadata:
+  source_keys:
+    - my-source
+---
+
+## Step 1
+
+Do the thing.
+EOF
+        cat > "$skill/references/sources.md" <<EOF
+# Sources
+
+## my-source
+
+- **URL:** https://example.com/my-source
+- **Description:** A test source.
+- **Contributing files:** SKILL.md
+- **Research doc:** ${research}
+- **Status:** \`extracted\`
 EOF
    }
 }
@@ -512,3 +549,439 @@ EOF
    run bash "$SCRIPT" "$skill"
    assert_success
 }
+
+# ---------------------------------------------------------------------------
+# Cycle 13 — parse_contributing_files: None ("could not parse") is NOT []
+#            ("explicitly (none)")
+#
+# Check 8 reads [] as "the research doc deliberately records no contributing
+# files" and SKIPS the slug on that basis. A block the parser cannot read must
+# therefore return None, so the slug stays exposed to check 8. Each case below
+# asserts that CONSEQUENCE — check 8 firing on the unreadable entry — not the
+# parser's return value, because returning [] is exactly the shape that makes
+# the check silently do nothing while still exiting 0.
+# ---------------------------------------------------------------------------
+
+@test "check 8 runs: bullet form with '*' asterisk bullets is unparsable, not '(none)'" {
+    local fake_repo="$TMPDIR/fakerepo"
+    make_upstream_skill "$fake_repo"
+
+    cat > "$fake_repo/docs/research/my-research.md" <<EOF
+# Research
+
+## my-source
+
+- **Contributing files:** some-skill/SKILL.md
+- **Status:** \`extracted\`
+
+## extra-source
+
+**Contributing files:**
+* some-skill/references/extra.md
+
+- **Status:** \`extracted\`
+EOF
+
+    run bash "$SCRIPT" "$fake_repo/my-skill"
+    assert_failure
+    assert_output --partial "FAIL"
+    assert_output --partial "extra-source"
+}
+
+@test "check 8 runs: bullet form with a numbered list is unparsable, not '(none)'" {
+    local fake_repo="$TMPDIR/fakerepo"
+    make_upstream_skill "$fake_repo"
+
+    cat > "$fake_repo/docs/research/my-research.md" <<EOF
+# Research
+
+## my-source
+
+- **Contributing files:** some-skill/SKILL.md
+- **Status:** \`extracted\`
+
+## extra-source
+
+**Contributing files:**
+1. some-skill/references/extra.md
+
+- **Status:** \`extracted\`
+EOF
+
+    run bash "$SCRIPT" "$fake_repo/my-skill"
+    assert_failure
+    assert_output --partial "FAIL"
+    assert_output --partial "extra-source"
+}
+
+@test "check 8 runs: bullet form followed by prose is unparsable, not '(none)'" {
+    local fake_repo="$TMPDIR/fakerepo"
+    make_upstream_skill "$fake_repo"
+
+    cat > "$fake_repo/docs/research/my-research.md" <<EOF
+# Research
+
+## my-source
+
+- **Contributing files:** some-skill/SKILL.md
+- **Status:** \`extracted\`
+
+## extra-source
+
+**Contributing files:**
+See the table below for the file list.
+
+- **Status:** \`extracted\`
+EOF
+
+    run bash "$SCRIPT" "$fake_repo/my-skill"
+    assert_failure
+    assert_output --partial "FAIL"
+    assert_output --partial "extra-source"
+}
+
+@test "check 8 runs: bullet form heading with a blank line and nothing after is unparsable, not '(none)'" {
+    local fake_repo="$TMPDIR/fakerepo"
+    make_upstream_skill "$fake_repo"
+
+    cat > "$fake_repo/docs/research/my-research.md" <<EOF
+# Research
+
+## my-source
+
+- **Contributing files:** some-skill/SKILL.md
+- **Status:** \`extracted\`
+
+## extra-source
+
+- **Status:** \`extracted\`
+
+**Contributing files:**
+
+EOF
+
+    run bash "$SCRIPT" "$fake_repo/my-skill"
+    assert_failure
+    assert_output --partial "FAIL"
+    assert_output --partial "extra-source"
+}
+
+@test "check 8 runs: inline form whose whole value is a parenthetical is unparsable, not '(none)'" {
+    local fake_repo="$TMPDIR/fakerepo"
+    make_upstream_skill "$fake_repo"
+
+    cat > "$fake_repo/docs/research/my-research.md" <<EOF
+# Research
+
+## my-source
+
+- **Contributing files:** some-skill/SKILL.md
+- **Status:** \`extracted\`
+
+## extra-source
+
+- **Contributing files:** (see notes below)
+- **Status:** \`extracted\`
+EOF
+
+    run bash "$SCRIPT" "$fake_repo/my-skill"
+    assert_failure
+    assert_output --partial "FAIL"
+    assert_output --partial "extra-source"
+}
+
+@test "check 4 runs: bullet form with '-' hyphen bullets still parses each path" {
+    local skill="$TMPDIR/my-skill"
+    make_skill_with_source_keys "$skill"
+    mkdir -p "$skill/references"
+    cat > "$skill/references/sources.md" <<EOF
+# Sources
+
+## my-source
+
+- **URL:** https://example.com/my-source
+- **Description:** A test source.
+
+**Contributing files:**
+- SKILL.md (the dispatch table, with a comma in the note)
+- references/nonexistent.md (why this one matters)
+
+- **Research doc:** (none)
+- **Status:** \`extracted\`
+EOF
+    run bash "$SCRIPT" "$skill"
+    assert_failure
+    assert_output --partial "Contributing file 'references/nonexistent.md' does not exist"
+}
+
+@test "check 4 runs: inline comma-separated form still parses each path" {
+    local skill="$TMPDIR/my-skill"
+    make_skill_with_source_keys "$skill"
+    make_sources_md "$skill" "my-source" "SKILL.md, references/nonexistent.md"
+    run bash "$SCRIPT" "$skill"
+    assert_failure
+    assert_output --partial "Contributing file 'references/nonexistent.md' does not exist"
+}
+
+@test "check 8 skips: bullet form '- (none)' is an explicit declaration" {
+    local fake_repo="$TMPDIR/fakerepo"
+    make_upstream_skill "$fake_repo"
+
+    cat > "$fake_repo/docs/research/my-research.md" <<EOF
+# Research
+
+## my-source
+
+- **Contributing files:** some-skill/SKILL.md
+- **Status:** \`extracted\`
+
+## extra-source
+
+**Contributing files:**
+- (none — nothing was extracted from this section)
+
+- **Status:** \`extracted\`
+EOF
+
+    run bash "$SCRIPT" "$fake_repo/my-skill"
+    assert_success
+}
+
+@test "check 8 skips: inline bare '(none)' is an explicit declaration" {
+    local fake_repo="$TMPDIR/fakerepo"
+    make_upstream_skill "$fake_repo"
+
+    cat > "$fake_repo/docs/research/my-research.md" <<EOF
+# Research
+
+## my-source
+
+- **Contributing files:** some-skill/SKILL.md
+- **Status:** \`extracted\`
+
+## extra-source
+
+- **Contributing files:** (none)
+- **Status:** \`extracted\`
+EOF
+
+    run bash "$SCRIPT" "$fake_repo/my-skill"
+    assert_success
+}
+
+# ---------------------------------------------------------------------------
+# Cycle 14 — Check 3 (#111): an explicit 'source_keys: []' is a house-authored
+#            declaration, a bare 'source_keys:' is truncation
+#
+# Three states, three outcomes. Only the middle one — declared empty — is
+# silent; collapsing any pair of them is the defect #111 filed.
+# ---------------------------------------------------------------------------
+
+@test "check 3 silent: references doc with top-level 'source_keys: []' emits nothing" {
+    local skill="$TMPDIR/my-skill"
+    make_skill_with_source_keys "$skill"
+    make_sources_md "$skill"
+    cat > "$skill/references/extra.md" <<EOF
+---
+source_keys: []
+---
+
+# Extra Reference
+
+House-authored, no external source.
+EOF
+    run bash "$SCRIPT" "$skill"
+    assert_success
+    assert_output ""
+}
+
+@test "check 3 silent: references doc with 'source_keys: []' nested under metadata: emits nothing" {
+    local skill="$TMPDIR/my-skill"
+    make_skill_with_source_keys "$skill"
+    make_sources_md "$skill"
+    cat > "$skill/references/extra.md" <<EOF
+---
+metadata:
+  source_keys: []
+---
+
+# Extra Reference
+
+House-authored, no external source.
+EOF
+    run bash "$SCRIPT" "$skill"
+    assert_success
+    assert_output ""
+}
+
+@test "INFO: references doc with a bare 'source_keys:' and no value still emits INFO" {
+    local skill="$TMPDIR/my-skill"
+    make_skill_with_source_keys "$skill"
+    make_sources_md "$skill"
+    cat > "$skill/references/extra.md" <<EOF
+---
+source_keys:
+---
+
+# Extra Reference
+
+Truncated frontmatter — this is not a decision.
+EOF
+    run bash "$SCRIPT" "$skill"
+    assert_success
+    assert_output --partial "INFO"
+    assert_output --partial "No source_keys frontmatter"
+}
+
+@test "INFO: references doc with frontmatter but no source_keys key still emits INFO" {
+    local skill="$TMPDIR/my-skill"
+    make_skill_with_source_keys "$skill"
+    make_sources_md "$skill"
+    cat > "$skill/references/extra.md" <<EOF
+---
+title: Extra Reference
+---
+
+# Extra Reference
+
+Never said either way.
+EOF
+    run bash "$SCRIPT" "$skill"
+    assert_success
+    assert_output --partial "INFO"
+    assert_output --partial "No source_keys frontmatter"
+}
+
+# ---------------------------------------------------------------------------
+# Cycle 15 — Checks 7 and 8: Research doc annotation stripping, and the INFO
+#            that replaced the silent skip
+#
+# A Research doc value is very often a path PLUS a section annotation, and
+# os.path.isfile() is false for every such string. Before the strip, checks 7
+# and 8 skipped SILENTLY on those — so the negative assertions here (an INFO
+# that must appear) are what distinguishes a running check from a dead one.
+# ---------------------------------------------------------------------------
+
+@test "check 7 runs: '§' section annotation is stripped before the path is resolved" {
+    local fake_repo="$TMPDIR/fakerepo"
+    make_upstream_skill "$fake_repo" 'docs/research/my-research.md § "Some Section"'
+
+    cat > "$fake_repo/docs/research/my-research.md" <<EOF
+# Research
+
+## different-slug
+
+- **Contributing files:** (none)
+- **Status:** \`extracted\`
+EOF
+
+    run bash "$SCRIPT" "$fake_repo/my-skill"
+    assert_success
+    assert_output --partial "Slug 'my-source' not found as H2 in research doc 'docs/research/my-research.md'"
+    refute_output --partial "§"
+}
+
+@test "check 7 runs: '→' section annotation is stripped before the path is resolved" {
+    local fake_repo="$TMPDIR/fakerepo"
+    make_upstream_skill "$fake_repo" 'docs/research/my-research.md → `## Pushing`'
+
+    cat > "$fake_repo/docs/research/my-research.md" <<EOF
+# Research
+
+## different-slug
+
+- **Contributing files:** (none)
+- **Status:** \`extracted\`
+EOF
+
+    run bash "$SCRIPT" "$fake_repo/my-skill"
+    assert_success
+    assert_output --partial "Slug 'my-source' not found as H2 in research doc 'docs/research/my-research.md'"
+    refute_output --partial "→"
+}
+
+@test "check 7 runs: parenthetical annotation is stripped before the path is resolved" {
+    local fake_repo="$TMPDIR/fakerepo"
+    make_upstream_skill "$fake_repo" "docs/research/my-research.md (whole-document reference)"
+
+    cat > "$fake_repo/docs/research/my-research.md" <<EOF
+# Research
+
+## different-slug
+
+- **Contributing files:** (none)
+- **Status:** \`extracted\`
+EOF
+
+    run bash "$SCRIPT" "$fake_repo/my-skill"
+    assert_success
+    assert_output --partial "Slug 'my-source' not found as H2 in research doc 'docs/research/my-research.md'"
+    refute_output --partial "whole-document reference"
+}
+
+@test "check 7 runs: a bare path with no annotation survives the strip intact" {
+    local fake_repo="$TMPDIR/fakerepo"
+    make_upstream_skill "$fake_repo" "docs/research/my-research.md"
+
+    cat > "$fake_repo/docs/research/my-research.md" <<EOF
+# Research
+
+## my-source
+
+- **Contributing files:** (none)
+- **Status:** \`extracted\`
+EOF
+
+    run bash "$SCRIPT" "$fake_repo/my-skill"
+    assert_success
+    assert_output ""
+}
+
+@test "checks 7 and 8 skipped silently: Research doc '(none)' is recognised before the strip" {
+    local fake_repo="$TMPDIR/fakerepo"
+    make_upstream_skill "$fake_repo" "(none)"
+
+    run bash "$SCRIPT" "$fake_repo/my-skill"
+    assert_success
+    assert_output ""
+}
+
+@test "checks 7 and 8 skipped silently: bare 'none — reason' is recognised as a declaration" {
+    local fake_repo="$TMPDIR/fakerepo"
+    make_upstream_skill "$fake_repo" "none — org convention, no upstream research doc"
+
+    run bash "$SCRIPT" "$fake_repo/my-skill"
+    assert_success
+    assert_output ""
+}
+
+@test "INFO: a stripped path that does not resolve names the slug instead of skipping silently" {
+    local fake_repo="$TMPDIR/fakerepo"
+    make_upstream_skill "$fake_repo" 'docs/research/missing.md § "Some Section"'
+
+    run bash "$SCRIPT" "$fake_repo/my-skill"
+    assert_success
+    assert_output --partial "INFO"
+    assert_output --partial "Upstream checks skipped for 'my-source' — research doc 'docs/research/missing.md' does not exist"
+}
+
+@test "INFO: a Research doc value that is entirely annotation names the slug instead of skipping silently" {
+    local fake_repo="$TMPDIR/fakerepo"
+    make_upstream_skill "$fake_repo" '§ "Some Section"'
+
+    run bash "$SCRIPT" "$fake_repo/my-skill"
+    assert_success
+    assert_output --partial "INFO"
+    assert_output --partial "Upstream checks skipped for 'my-source' — Research doc value names no path"
+}
+
+@test "INFO: no repo root above the skill directory names the slug instead of skipping silently" {
+    local skill="$TMPDIR/my-skill"
+    make_skill_with_source_keys "$skill"
+    make_sources_md "$skill" "my-source" "SKILL.md" "docs/research/my-research.md"
+
+    run bash "$SCRIPT" "$skill"
+    assert_success
+    assert_output --partial "INFO"
+    assert_output --partial "Upstream checks skipped for 'my-source' — no repo root above the skill directory"
+}