holocron/plugins/kyberforge/skills/agent-audit/tests/validate-provenance.bats

#!/usr/bin/env bats

setup() {
    REPO_ROOT="$(cd "$BATS_TEST_DIRNAME/../../../../../" && pwd)"
    load "$REPO_ROOT/tests/test_helper/bats-support/load"
    load "$REPO_ROOT/tests/test_helper/bats-assert/load"

    SCRIPT="$(cd "$BATS_TEST_DIRNAME/../scripts" && pwd)/validate-provenance.sh"
    TMPDIR="$(mktemp -d)"

    # Helper: create an APM package root at <root> (apm.yml with a top-level
    # type: line — a real package manifest, not marketplace-only) plus a
    # single vendor-neutral agent file at <root>/.apm/agents/<name>.agent.md.
    make_package() {
        local root="$1"
        mkdir -p "$root/.apm/agents"
        cat > "$root/apm.yml" <<EOF
name: test-package
version: 0.1.0
type: skill
EOF
    }

    # Helper: create a clean agent file (no source_keys)
    make_clean_agent() {
        local root="$1"
        local name="${2:-my-agent}"
        cat > "$root/.apm/agents/${name}.agent.md" <<EOF
---
name: ${name}
description: A valid agent description.
---

You are a test agent.
EOF
    }

    # Helper: create an agent file with source_keys
    make_agent_with_source_keys() {
        local root="$1"
        local name="${2:-my-agent}"
        local slug="${3:-my-source}"
        cat > "$root/.apm/agents/${name}.agent.md" <<EOF
---
name: ${name}
description: A valid agent description.
source_keys:
  - ${slug}
---

You are a test agent.
EOF
    }

    # Helper: create a valid sources.md with one entry
    make_sources_md() {
        local root="$1"
        local slug="${2:-my-source}"
        local contrib="${3:-.apm/agents/my-agent.agent.md}"
        local research="${4:-(none)}"
        cat > "$root/sources.md" <<EOF
# Sources

## ${slug}

- **URL:** https://example.com/${slug}
- **Description:** A test source.
- **Contributing files:** ${contrib}
- **Research doc:** ${research}
- **Status:** \`extracted\`
EOF
    }
}

teardown() {
    rm -rf "$TMPDIR"
}

# ---------------------------------------------------------------------------
# --help
# ---------------------------------------------------------------------------

@test "--help exits 0" {
    run bash "$SCRIPT" --help
    assert_success
    assert_output --partial "Usage:"
}

# ---------------------------------------------------------------------------
# Non-plugin/APM scope → exit 0 silently
# ---------------------------------------------------------------------------

@test "non-plugin scope: no apm.yml in tree → exit 0, no output" {
    local dir="$TMPDIR/no-package"
    mkdir -p "$dir/.apm/agents"
    cat > "$dir/.apm/agents/my-agent.agent.md" <<EOF
---
name: my-agent
description: A valid agent description.
source_keys:
  - my-source
---

You are a test agent.
EOF
    run bash "$SCRIPT" "$dir/.apm/agents/my-agent.agent.md"
    assert_success
    assert_output ""
}

@test "non-plugin scope: apm.yml present but type:-less (marketplace-only) → exit 0, no output" {
    local dir="$TMPDIR/marketplace-only"
    mkdir -p "$dir/.apm/agents"
    cat > "$dir/apm.yml" <<EOF
name: root-marketplace
marketplace:
  owner: someone
  packages:
    - ./packages/plugin-a
EOF
    cat > "$dir/.apm/agents/my-agent.agent.md" <<EOF
---
name: my-agent
description: A valid agent description.
source_keys:
  - my-source
---

You are a test agent.
EOF
    run bash "$SCRIPT" "$dir/.apm/agents/my-agent.agent.md"
    assert_success
    assert_output ""
}

@test "non-plugin scope: bare plugin.json (no apm.yml) is no longer a scope signal → exit 0, no output" {
    local dir="$TMPDIR/old-plugin-json-only"
    mkdir -p "$dir/agents"
    echo '{"name":"test-plugin","version":"0.1.0"}' > "$dir/plugin.json"
    cat > "$dir/agents/my-agent.md" <<EOF
---
name: my-agent
description: A valid agent description.
source_keys:
  - my-source
---

You are a test agent.
EOF
    run bash "$SCRIPT" "$dir/agents/my-agent.md"
    assert_success
    assert_output ""
}

@test "non-plugin scope: walk-up stops at .git boundary before reaching an ancestor apm.yml" {
    local dir="$TMPDIR/repo"
    mkdir -p "$dir/.git" "$dir/.apm/agents"
    cat > "$dir/apm.yml" <<EOF
name: test-package
version: 0.1.0
type: skill
EOF
    mkdir -p "$dir/sub/.apm/agents"
    cat > "$dir/sub/.apm/agents/my-agent.agent.md" <<EOF
---
name: my-agent
description: A valid agent description.
source_keys:
  - my-source
---

You are a test agent.
EOF
    # sub/ has no .git and no apm.yml of its own; the real package apm.yml
    # lives at $dir, but $dir/.git means the walk from sub/ should stop at
    # sub/ itself only if sub/ had a .git — here .git is at $dir, ABOVE
    # sub/, so the walk from sub/ reaches $dir/apm.yml before any .git.
    # This test instead verifies the walk finds that package root correctly
    # (a positive case) — see the dedicated .git-stops-first test below for
    # the negative case.
    run bash "$SCRIPT" "$dir/sub/.apm/agents/my-agent.agent.md"
    assert_failure
    assert_output --partial "FAIL"
}

@test "non-plugin scope: .git between the agent file and an ancestor apm.yml stops the walk first" {
    local dir="$TMPDIR/repo2"
    mkdir -p "$dir/.apm/agents"
    cat > "$dir/apm.yml" <<EOF
name: test-package
version: 0.1.0
type: skill
EOF
    mkdir -p "$dir/sub/.git" "$dir/sub/.apm/agents"
    cat > "$dir/sub/.apm/agents/my-agent.agent.md" <<EOF
---
name: my-agent
description: A valid agent description.
source_keys:
  - my-source
---

You are a test agent.
EOF
    run bash "$SCRIPT" "$dir/sub/.apm/agents/my-agent.agent.md"
    assert_success
    assert_output ""
}

# ---------------------------------------------------------------------------
# Early exit: no sources.md, no source_keys → exit 0, no output
# ---------------------------------------------------------------------------

@test "clean pass: no sources.md and no source_keys → exit 0, no output" {
    local root="$TMPDIR/package"
    make_package "$root"
    make_clean_agent "$root"
    run bash "$SCRIPT" "$root/.apm/agents/my-agent.agent.md"
    assert_success
    assert_output ""
}

# ---------------------------------------------------------------------------
# Check 0: source_keys present but sources.md absent → FAIL
# ---------------------------------------------------------------------------

@test "FAIL: source_keys in agent file but sources.md absent" {
    local root="$TMPDIR/package"
    make_package "$root"
    make_agent_with_source_keys "$root"
    run bash "$SCRIPT" "$root/.apm/agents/my-agent.agent.md"
    assert_failure
    assert_output --partial "FAIL"
}

# ---------------------------------------------------------------------------
# Check 1: FILL IN: placeholder in sources.md → FAIL
# ---------------------------------------------------------------------------

@test "FAIL: FILL IN: placeholder in sources.md" {
    local root="$TMPDIR/package"
    make_package "$root"
    make_agent_with_source_keys "$root"
    cat > "$root/sources.md" <<EOF
# Sources

## my-source

- **URL:** FILL IN: add url
- **Description:** A test source.
- **Contributing files:** .apm/agents/my-agent.agent.md
- **Research doc:** (none)
- **Status:** \`extracted\`
EOF
    run bash "$SCRIPT" "$root/.apm/agents/my-agent.agent.md"
    assert_failure
    assert_output --partial "FAIL"
}

@test "FILL IN: inside backticks in sources.md does not fail" {
    local root="$TMPDIR/package"
    make_package "$root"
    make_agent_with_source_keys "$root"
    make_sources_md "$root"
    echo "Use \`FILL IN: value\` as example." >> "$root/sources.md"
    run bash "$SCRIPT" "$root/.apm/agents/my-agent.agent.md"
    assert_success
}

# ---------------------------------------------------------------------------
# Check 2: source_keys slug missing from sources.md → FAIL
# ---------------------------------------------------------------------------

@test "FAIL: source_keys slug in agent file not present as H2 in sources.md" {
    local root="$TMPDIR/package"
    make_package "$root"
    make_agent_with_source_keys "$root" "my-agent" "my-source"
    make_sources_md "$root" "different-source" "(none)" "(none)"
    run bash "$SCRIPT" "$root/.apm/agents/my-agent.agent.md"
    assert_failure
    assert_output --partial "FAIL"
}

# ---------------------------------------------------------------------------
# Check 3: Contributing file path doesn't exist → FAIL
# ---------------------------------------------------------------------------

@test "FAIL: Contributing file listed in sources.md does not exist" {
    local root="$TMPDIR/package"
    make_package "$root"
    make_agent_with_source_keys "$root"
    make_sources_md "$root" "my-source" ".apm/agents/nonexistent.agent.md"
    run bash "$SCRIPT" "$root/.apm/agents/my-agent.agent.md"
    assert_failure
    assert_output --partial "FAIL"
}

@test "pass: (none) in Contributing files is skipped" {
    local root="$TMPDIR/package"
    make_package "$root"
    make_agent_with_source_keys "$root"
    make_sources_md "$root" "my-source" "(none — not used directly)"
    run bash "$SCRIPT" "$root/.apm/agents/my-agent.agent.md"
    assert_success
}

# ---------------------------------------------------------------------------
# Check 5: Research doc field missing or placeholder → FAIL
# ---------------------------------------------------------------------------

@test "FAIL: Research doc field missing from sources.md entry" {
    local root="$TMPDIR/package"
    make_package "$root"
    make_agent_with_source_keys "$root"
    cat > "$root/sources.md" <<EOF
# Sources

## my-source

- **URL:** https://example.com/my-source
- **Description:** A test source.
- **Contributing files:** .apm/agents/my-agent.agent.md
- **Status:** \`extracted\`
EOF
    run bash "$SCRIPT" "$root/.apm/agents/my-agent.agent.md"
    assert_failure
    assert_output --partial "FAIL"
}

@test "FAIL: Research doc field is FILL IN: placeholder" {
    local root="$TMPDIR/package"
    make_package "$root"
    make_agent_with_source_keys "$root"
    cat > "$root/sources.md" <<EOF
# Sources

## my-source

- **URL:** https://example.com/my-source
- **Description:** A test source.
- **Contributing files:** .apm/agents/my-agent.agent.md
- **Research doc:** FILL IN: path to research doc
- **Status:** \`extracted\`
EOF
    run bash "$SCRIPT" "$root/.apm/agents/my-agent.agent.md"
    assert_failure
    assert_output --partial "FAIL"
}

# ---------------------------------------------------------------------------
# Check 4: Bidirectional — contributing file missing slug in source_keys → FAIL
# ---------------------------------------------------------------------------

@test "FAIL: Contributing file exists but does not list parent slug in source_keys" {
    local root="$TMPDIR/package"
    make_package "$root"
    # agent file has source_keys: other-source (not my-source)
    cat > "$root/.apm/agents/my-agent.agent.md" <<EOF
---
name: my-agent
description: A valid agent description.
source_keys:
  - other-source
---

You are a test agent.
EOF
    # sources.md says my-agent.agent.md contributed to my-source, but
    # my-agent.agent.md doesn't list my-source
    cat > "$root/sources.md" <<EOF
# Sources

## other-source

- **URL:** https://example.com/other-source
- **Description:** A test source.
- **Contributing files:** .apm/agents/my-agent.agent.md
- **Research doc:** (none)
- **Status:** \`extracted\`

## my-source

- **URL:** https://example.com/my-source
- **Description:** Another source.
- **Contributing files:** .apm/agents/my-agent.agent.md
- **Research doc:** (none)
- **Status:** \`extracted\`
EOF
    run bash "$SCRIPT" "$root/.apm/agents/my-agent.agent.md"
    assert_failure
    assert_output --partial "FAIL"
}

# ---------------------------------------------------------------------------
# Clean full pass
# ---------------------------------------------------------------------------

@test "clean full pass: all checks satisfied" {
    local root="$TMPDIR/package"
    make_package "$root"
    make_agent_with_source_keys "$root"
    make_sources_md "$root"
    run bash "$SCRIPT" "$root/.apm/agents/my-agent.agent.md"
    assert_success
}