feat(kyberforge): add agent-audit provenance chain validation (closes #60)

## Why agent-author produces agents/sources.md at plugin scope to record which research sources informed which agent files. agent-audit had no way to validate this chain, leaving stale or missing provenance undetected. ## Implementation Notes Validation is per-pair (the given agent file + its counterpart) rather than plugin-wide, keeping the scope consistent with validate.sh. The script exits 0 silently for non-plugin-scope agents. source_keys is top-level in both CC .md and Copilot .agent.md files (not under metadata:) to avoid conflict with Copilot's own metadata field semantics. Checks 0, 1, 2, 4, 5, 6 mirror the skill provenance set; upstream research-doc cross-reference checks (7, 8) are deferred. agent-author Steps 2, 3, and 4 updated to formally specify the agents/sources.md format and instruct authors to add source_keys to both files when research sources are in context. Refs: #60 Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_0147vXtL5sP6vorDdqXGJJU9
2026-07-04 10:37:44 +00:00
parent a4235d197d
commit 1333d2c1b1
7 changed files with 703 additions and 6 deletions
--- a/plugins/kyberforge/skills/agent-audit/tests/validate-provenance.bats
+++ b/plugins/kyberforge/skills/agent-audit/tests/validate-provenance.bats
@@ -0,0 +1,383 @@
+#!/usr/bin/env bats
+
+setup() {
+    REPO_ROOT="$(cd "$BATS_TEST_DIRNAME/../../../../../" && pwd)"
+    load "$REPO_ROOT/tests/test_helper/bats-support/load"
+    load "$REPO_ROOT/tests/test_helper/bats-assert/load"
+
+    SCRIPT="$(cd "$BATS_TEST_DIRNAME/../scripts" && pwd)/validate-provenance.sh"
+    TMPDIR="$(mktemp -d)"
+
+    # Helper: create a plugin root with plugin.json and an agents/ directory
+    make_plugin() {
+        local root="$1"
+        mkdir -p "$root/agents"
+        echo '{"name":"test-plugin","version":"0.1.0"}' > "$root/plugin.json"
+    }
+
+    # Helper: create a clean agent pair (no source_keys)
+    make_clean_pair() {
+        local root="$1"
+        local name="${2:-my-agent}"
+        cat > "$root/agents/${name}.md" <<EOF
+---
+name: ${name}
+description: A valid agent description.
+---
+
+You are a test agent.
+EOF
+        cat > "$root/agents/${name}.agent.md" <<EOF
+---
+name: ${name}
+description: A valid agent description.
+---
+
+You are a test agent.
+EOF
+    }
+
+    # Helper: create a CC agent file with source_keys
+    make_cc_with_source_keys() {
+        local root="$1"
+        local name="${2:-my-agent}"
+        local slug="${3:-my-source}"
+        cat > "$root/agents/${name}.md" <<EOF
+---
+name: ${name}
+description: A valid agent description.
+source_keys:
+  - ${slug}
+---
+
+You are a test agent.
+EOF
+    }
+
+    # Helper: create a Copilot agent file with source_keys
+    make_copilot_with_source_keys() {
+        local root="$1"
+        local name="${2:-my-agent}"
+        local slug="${3:-my-source}"
+        cat > "$root/agents/${name}.agent.md" <<EOF
+---
+name: ${name}
+description: A valid agent description.
+source_keys:
+  - ${slug}
+---
+
+You are a test agent.
+EOF
+    }
+
+    # Helper: create a minimal Copilot file without source_keys
+    make_copilot_clean() {
+        local root="$1"
+        local name="${2:-my-agent}"
+        cat > "$root/agents/${name}.agent.md" <<EOF
+---
+name: ${name}
+description: A valid agent description.
+---
+
+You are a test agent.
+EOF
+    }
+
+    # Helper: create a valid agents/sources.md with one entry
+    make_sources_md() {
+        local root="$1"
+        local slug="${2:-my-source}"
+        local contrib="${3:-agents/my-agent.md, agents/my-agent.agent.md}"
+        local research="${4:-(none)}"
+        cat > "$root/agents/sources.md" <<EOF
+# Sources
+
+## ${slug}
+
+- **URL:** https://example.com/${slug}
+- **Description:** A test source.
+- **Contributing files:** ${contrib}
+- **Research doc:** ${research}
+- **Status:** \`extracted\`
+EOF
+    }
+}
+
+teardown() {
+    rm -rf "$TMPDIR"
+}
+
+# ---------------------------------------------------------------------------
+# --help
+# ---------------------------------------------------------------------------
+
+@test "--help exits 0" {
+    run bash "$SCRIPT" --help
+    assert_success
+    assert_output --partial "Usage:"
+}
+
+# ---------------------------------------------------------------------------
+# Non-plugin scope → exit 0 silently
+# ---------------------------------------------------------------------------
+
+@test "non-plugin scope: no plugin.json in tree → exit 0, no output" {
+    local dir="$TMPDIR/no-plugin"
+    mkdir -p "$dir/agents"
+    cat > "$dir/agents/my-agent.md" <<EOF
+---
+name: my-agent
+description: A valid agent description.
+source_keys:
+  - my-source
+---
+
+You are a test agent.
+EOF
+    run bash "$SCRIPT" "$dir/agents/my-agent.md"
+    assert_success
+    assert_output ""
+}
+
+# ---------------------------------------------------------------------------
+# Early exit: no sources.md, no source_keys → exit 0, no output
+# ---------------------------------------------------------------------------
+
+@test "clean pass: no sources.md and no source_keys → exit 0, no output" {
+    local root="$TMPDIR/plugin"
+    make_plugin "$root"
+    make_clean_pair "$root"
+    run bash "$SCRIPT" "$root/agents/my-agent.md"
+    assert_success
+    assert_output ""
+}
+
+# ---------------------------------------------------------------------------
+# Check 0: source_keys present but agents/sources.md absent → FAIL
+# ---------------------------------------------------------------------------
+
+@test "FAIL: source_keys in CC file but agents/sources.md absent" {
+    local root="$TMPDIR/plugin"
+    make_plugin "$root"
+    make_cc_with_source_keys "$root"
+    make_copilot_clean "$root"
+    run bash "$SCRIPT" "$root/agents/my-agent.md"
+    assert_failure
+    assert_output --partial "FAIL"
+}
+
+@test "FAIL: source_keys in Copilot file but agents/sources.md absent" {
+    local root="$TMPDIR/plugin"
+    make_plugin "$root"
+    make_clean_pair "$root"
+    make_copilot_with_source_keys "$root"
+    run bash "$SCRIPT" "$root/agents/my-agent.agent.md"
+    assert_failure
+    assert_output --partial "FAIL"
+}
+
+# ---------------------------------------------------------------------------
+# Check 1: FILL IN: placeholder in agents/sources.md → FAIL
+# ---------------------------------------------------------------------------
+
+@test "FAIL: FILL IN: placeholder in agents/sources.md" {
+    local root="$TMPDIR/plugin"
+    make_plugin "$root"
+    make_cc_with_source_keys "$root"
+    make_copilot_with_source_keys "$root"
+    cat > "$root/agents/sources.md" <<EOF
+# Sources
+
+## my-source
+
+- **URL:** FILL IN: add url
+- **Description:** A test source.
+- **Contributing files:** agents/my-agent.md, agents/my-agent.agent.md
+- **Research doc:** (none)
+- **Status:** \`extracted\`
+EOF
+    run bash "$SCRIPT" "$root/agents/my-agent.md"
+    assert_failure
+    assert_output --partial "FAIL"
+}
+
+@test "FILL IN: inside backticks in agents/sources.md does not fail" {
+    local root="$TMPDIR/plugin"
+    make_plugin "$root"
+    make_cc_with_source_keys "$root"
+    make_copilot_with_source_keys "$root"
+    make_sources_md "$root"
+    echo "Use \`FILL IN: value\` as example." >> "$root/agents/sources.md"
+    run bash "$SCRIPT" "$root/agents/my-agent.md"
+    assert_success
+}
+
+# ---------------------------------------------------------------------------
+# Check 2: source_keys slug missing from agents/sources.md → FAIL
+# ---------------------------------------------------------------------------
+
+@test "FAIL: source_keys slug in CC file not present as H2 in agents/sources.md" {
+    local root="$TMPDIR/plugin"
+    make_plugin "$root"
+    make_cc_with_source_keys "$root" "my-agent" "my-source"
+    make_copilot_clean "$root"
+    make_sources_md "$root" "different-source" "(none)" "(none)"
+    run bash "$SCRIPT" "$root/agents/my-agent.md"
+    assert_failure
+    assert_output --partial "FAIL"
+}
+
+@test "FAIL: source_keys slug in Copilot file not present as H2 in agents/sources.md" {
+    local root="$TMPDIR/plugin"
+    make_plugin "$root"
+    make_clean_pair "$root"
+    make_copilot_with_source_keys "$root" "my-agent" "my-source"
+    make_sources_md "$root" "different-source" "(none)" "(none)"
+    run bash "$SCRIPT" "$root/agents/my-agent.agent.md"
+    assert_failure
+    assert_output --partial "FAIL"
+}
+
+# ---------------------------------------------------------------------------
+# Check 4: Contributing file path doesn't exist → FAIL
+# ---------------------------------------------------------------------------
+
+@test "FAIL: Contributing file listed in agents/sources.md does not exist" {
+    local root="$TMPDIR/plugin"
+    make_plugin "$root"
+    make_cc_with_source_keys "$root"
+    make_copilot_with_source_keys "$root"
+    make_sources_md "$root" "my-source" "agents/nonexistent.md"
+    run bash "$SCRIPT" "$root/agents/my-agent.md"
+    assert_failure
+    assert_output --partial "FAIL"
+}
+
+@test "pass: (none) in Contributing files is skipped" {
+    local root="$TMPDIR/plugin"
+    make_plugin "$root"
+    make_cc_with_source_keys "$root"
+    make_copilot_with_source_keys "$root"
+    make_sources_md "$root" "my-source" "(none — not used directly)"
+    run bash "$SCRIPT" "$root/agents/my-agent.md"
+    assert_success
+}
+
+# ---------------------------------------------------------------------------
+# Check 6: Research doc field missing or placeholder → FAIL
+# ---------------------------------------------------------------------------
+
+@test "FAIL: Research doc field missing from agents/sources.md entry" {
+    local root="$TMPDIR/plugin"
+    make_plugin "$root"
+    make_cc_with_source_keys "$root"
+    make_copilot_with_source_keys "$root"
+    cat > "$root/agents/sources.md" <<EOF
+# Sources
+
+## my-source
+
+- **URL:** https://example.com/my-source
+- **Description:** A test source.
+- **Contributing files:** agents/my-agent.md, agents/my-agent.agent.md
+- **Status:** \`extracted\`
+EOF
+    run bash "$SCRIPT" "$root/agents/my-agent.md"
+    assert_failure
+    assert_output --partial "FAIL"
+}
+
+@test "FAIL: Research doc field is FILL IN: placeholder" {
+    local root="$TMPDIR/plugin"
+    make_plugin "$root"
+    make_cc_with_source_keys "$root"
+    make_copilot_with_source_keys "$root"
+    cat > "$root/agents/sources.md" <<EOF
+# Sources
+
+## my-source
+
+- **URL:** https://example.com/my-source
+- **Description:** A test source.
+- **Contributing files:** agents/my-agent.md, agents/my-agent.agent.md
+- **Research doc:** FILL IN: path to research doc
+- **Status:** \`extracted\`
+EOF
+    run bash "$SCRIPT" "$root/agents/my-agent.md"
+    assert_failure
+    assert_output --partial "FAIL"
+}
+
+# ---------------------------------------------------------------------------
+# Check 5: Bidirectional — contributing file missing slug in source_keys → FAIL
+# ---------------------------------------------------------------------------
+
+@test "FAIL: Contributing file exists but does not list parent slug in source_keys" {
+    local root="$TMPDIR/plugin"
+    make_plugin "$root"
+    # CC file has source_keys: other-source (not my-source)
+    cat > "$root/agents/my-agent.md" <<EOF
+---
+name: my-agent
+description: A valid agent description.
+source_keys:
+  - other-source
+---
+
+You are a test agent.
+EOF
+    make_copilot_clean "$root"
+    # sources.md says my-agent.md contributed to my-source, but my-agent.md doesn't list my-source
+    cat > "$root/agents/sources.md" <<EOF
+# Sources
+
+## other-source
+
+- **URL:** https://example.com/other-source
+- **Description:** A test source.
+- **Contributing files:** agents/my-agent.md
+- **Research doc:** (none)
+- **Status:** \`extracted\`
+
+## my-source
+
+- **URL:** https://example.com/my-source
+- **Description:** Another source.
+- **Contributing files:** agents/my-agent.md
+- **Research doc:** (none)
+- **Status:** \`extracted\`
+EOF
+    run bash "$SCRIPT" "$root/agents/my-agent.md"
+    assert_failure
+    assert_output --partial "FAIL"
+}
+
+# ---------------------------------------------------------------------------
+# Entry via Copilot file path
+# ---------------------------------------------------------------------------
+
+@test "accepts Copilot file path as entry point" {
+    local root="$TMPDIR/plugin"
+    make_plugin "$root"
+    make_cc_with_source_keys "$root"
+    make_copilot_with_source_keys "$root"
+    make_sources_md "$root"
+    run bash "$SCRIPT" "$root/agents/my-agent.agent.md"
+    assert_success
+}
+
+# ---------------------------------------------------------------------------
+# Clean full pass
+# ---------------------------------------------------------------------------
+
+@test "clean full pass: all checks satisfied via CC file" {
+    local root="$TMPDIR/plugin"
+    make_plugin "$root"
+    make_cc_with_source_keys "$root"
+    make_copilot_with_source_keys "$root"
+    make_sources_md "$root"
+    run bash "$SCRIPT" "$root/agents/my-agent.md"
+    assert_success
+}