holocron/plugins/kyberforge/.apm/skills/skill-audit/tests/validate-provenance.bats

#!/usr/bin/env bats

setup() {
    REPO_ROOT="$(cd "$BATS_TEST_DIRNAME/../../../../../../" && pwd)"
    load "$REPO_ROOT/tests/test_helper/bats-support/load"
    load "$REPO_ROOT/tests/test_helper/bats-assert/load"

    SCRIPT="$(cd "$BATS_TEST_DIRNAME/../scripts" && pwd)/validate-provenance.sh"
    TMPDIR="$(mktemp -d)"

    # Helper: create a minimal skill directory with no sources.md and no source_keys
    make_clean_skill() {
        local dir="$1"
        local name
        name="$(basename "$dir")"
        mkdir -p "$dir"
        cat > "$dir/SKILL.md" <<EOF
---
name: $name
description: A valid skill description.
---

## Step 1

Do the thing.
EOF
    }

    # Helper: create a skill with source_keys in SKILL.md
    make_skill_with_source_keys() {
        local dir="$1"
        local name
        name="$(basename "$dir")"
        mkdir -p "$dir"
        cat > "$dir/SKILL.md" <<EOF
---
name: $name
description: A valid skill description.
metadata:
  source_keys:
    - my-source
---

## Step 1

Do the thing.
EOF
    }

    # Helper: create a valid sources.md with one entry
    make_sources_md() {
        local dir="$1"
        local slug="${2:-my-source}"
        local contrib="${3:-SKILL.md}"
        local research="${4:-(none)}"
        mkdir -p "$dir/references"
        cat > "$dir/references/sources.md" <<EOF
# Sources

## ${slug}

- **URL:** https://example.com/${slug}
- **Description:** A test source.
- **Contributing files:** ${contrib}
- **Research doc:** ${research}
- **Status:** \`extracted\`
EOF
    }
}

teardown() {
    rm -rf "$TMPDIR"
}

# ---------------------------------------------------------------------------
# Cycle 1 — --help
# ---------------------------------------------------------------------------

@test "--help exits 0" {
    run bash "$SCRIPT" --help
    assert_success
    assert_output --partial "Usage:"
}

# ---------------------------------------------------------------------------
# Cycle 2 — Early exit: no sources.md, no source_keys → exit 0, no output
# ---------------------------------------------------------------------------

@test "clean pass: no sources.md and no source_keys anywhere → exit 0, no output" {
    local skill="$TMPDIR/my-skill"
    make_clean_skill "$skill"
    run bash "$SCRIPT" "$skill"
    assert_success
    assert_output ""
}

# ---------------------------------------------------------------------------
# Cycle 3 — Check 0: source_keys present but no sources.md → FAIL
# ---------------------------------------------------------------------------

@test "FAIL: source_keys in SKILL.md but sources.md absent" {
    local skill="$TMPDIR/my-skill"
    make_skill_with_source_keys "$skill"
    run bash "$SCRIPT" "$skill"
    assert_failure
    assert_output --partial "FAIL"
}

# ---------------------------------------------------------------------------
# Cycle 4 — Check 1: FILL IN: placeholder in sources.md → FAIL
# ---------------------------------------------------------------------------

@test "FAIL: FILL IN: placeholder in sources.md" {
    local skill="$TMPDIR/my-skill"
    make_skill_with_source_keys "$skill"
    mkdir -p "$skill/references"
    cat > "$skill/references/sources.md" <<EOF
# Sources

## my-source

- **URL:** FILL IN: add url
- **Description:** A test source.
- **Contributing files:** SKILL.md
- **Research doc:** (none)
- **Status:** \`extracted\`
EOF
    run bash "$SCRIPT" "$skill"
    assert_failure
    assert_output --partial "FAIL"
}

@test "FILL IN: inside backticks in sources.md does not fail" {
    local skill="$TMPDIR/my-skill"
    make_skill_with_source_keys "$skill"
    make_sources_md "$skill"
    echo "Use \`FILL IN: value\` as example." >> "$skill/references/sources.md"
    run bash "$SCRIPT" "$skill"
    assert_success
}

# ---------------------------------------------------------------------------
# Cycle 5 — Check 2: source_keys slug missing from sources.md → FAIL
# ---------------------------------------------------------------------------

@test "FAIL: source_keys slug in SKILL.md not present as H2 in sources.md" {
    local skill="$TMPDIR/my-skill"
    make_skill_with_source_keys "$skill"
    mkdir -p "$skill/references"
    cat > "$skill/references/sources.md" <<EOF
# Sources

## different-source

- **URL:** https://example.com/different-source
- **Description:** A different source.
- **Contributing files:** SKILL.md
- **Research doc:** (none)
- **Status:** \`extracted\`
EOF
    run bash "$SCRIPT" "$skill"
    assert_failure
    assert_output --partial "FAIL"
}

# ---------------------------------------------------------------------------
# Cycle 6 — Check 4: Contributing file path doesn't exist → FAIL
# ---------------------------------------------------------------------------

@test "FAIL: Contributing file listed in sources.md does not exist" {
    local skill="$TMPDIR/my-skill"
    make_skill_with_source_keys "$skill"
    make_sources_md "$skill" "my-source" "references/nonexistent.md"
    run bash "$SCRIPT" "$skill"
    assert_failure
    assert_output --partial "FAIL"
}

@test "pass: (none) in Contributing files is skipped" {
    local skill="$TMPDIR/my-skill"
    make_skill_with_source_keys "$skill"
    make_sources_md "$skill" "my-source" "(none — not used directly)"
    run bash "$SCRIPT" "$skill"
    assert_success
}

# ---------------------------------------------------------------------------
# Cycle 7 — Check 6: Research doc field missing → FAIL
# ---------------------------------------------------------------------------

@test "FAIL: Research doc field missing from sources.md entry" {
    local skill="$TMPDIR/my-skill"
    make_skill_with_source_keys "$skill"
    mkdir -p "$skill/references"
    cat > "$skill/references/sources.md" <<EOF
# Sources

## my-source

- **URL:** https://example.com/my-source
- **Description:** A test source.
- **Contributing files:** SKILL.md
- **Status:** \`extracted\`
EOF
    run bash "$SCRIPT" "$skill"
    assert_failure
    assert_output --partial "FAIL"
}

@test "FAIL: Research doc field is FILL IN: placeholder" {
    local skill="$TMPDIR/my-skill"
    make_skill_with_source_keys "$skill"
    mkdir -p "$skill/references"
    cat > "$skill/references/sources.md" <<EOF
# Sources

## my-source

- **URL:** https://example.com/my-source
- **Description:** A test source.
- **Contributing files:** SKILL.md
- **Research doc:** FILL IN: path to research doc
- **Status:** \`extracted\`
EOF
    run bash "$SCRIPT" "$skill"
    assert_failure
    assert_output --partial "FAIL"
}

# ---------------------------------------------------------------------------
# Cycle 8 — Check 5: Bidirectional mismatch → FAIL
# ---------------------------------------------------------------------------

@test "FAIL: Contributing file exists but does not list parent slug in source_keys" {
    local skill="$TMPDIR/my-skill"
    make_skill_with_source_keys "$skill"
    make_sources_md "$skill" "my-source" "SKILL.md"
    # SKILL.md has source_keys: my-source, but let's change it to NOT have my-source
    cat > "$skill/SKILL.md" <<EOF
---
name: my-skill
description: A valid skill description.
metadata:
  source_keys:
    - other-source
---

## Step 1

Do the thing.
EOF
    mkdir -p "$skill/references"
    cat > "$skill/references/sources.md" <<EOF
# Sources

## other-source

- **URL:** https://example.com/other-source
- **Description:** A test source.
- **Contributing files:** SKILL.md
- **Research doc:** (none)
- **Status:** \`extracted\`
EOF
    # Now add my-source that references SKILL.md but SKILL.md doesn't back-reference it
    cat >> "$skill/references/sources.md" <<EOF

## my-source

- **URL:** https://example.com/my-source
- **Description:** Another source.
- **Contributing files:** SKILL.md
- **Research doc:** (none)
- **Status:** \`extracted\`
EOF
    run bash "$SCRIPT" "$skill"
    assert_failure
    assert_output --partial "FAIL"
}

# ---------------------------------------------------------------------------
# Cycle 9 — Check 3: references/*.md with no source_keys → INFO (exit 0)
# ---------------------------------------------------------------------------

@test "INFO: references doc with no source_keys frontmatter emits INFO but exits 0" {
    local skill="$TMPDIR/my-skill"
    make_skill_with_source_keys "$skill"
    make_sources_md "$skill"
    mkdir -p "$skill/references"
    cat > "$skill/references/extra.md" <<EOF
# Extra Reference

No frontmatter here.
EOF
    run bash "$SCRIPT" "$skill"
    assert_success
    assert_output --partial "INFO"
}

@test "pass: references doc with source_keys all matching sources.md exits 0" {
    local skill="$TMPDIR/my-skill"
    make_skill_with_source_keys "$skill"
    make_sources_md "$skill"
    mkdir -p "$skill/references"
    cat > "$skill/references/extra.md" <<EOF
---
source_keys:
  - my-source
---

# Extra Reference

Content here.
EOF
    run bash "$SCRIPT" "$skill"
    assert_success
}

# ---------------------------------------------------------------------------
# Cycle 10 — Clean full pass: valid sources.md, all source_keys match, files exist
# ---------------------------------------------------------------------------

@test "clean full pass: all checks satisfied" {
    local skill="$TMPDIR/my-skill"
    make_skill_with_source_keys "$skill"
    make_sources_md "$skill"
    run bash "$SCRIPT" "$skill"
    assert_success
}

# ---------------------------------------------------------------------------
# Cycle 11 — Check 7: Upstream forward: slug in sources.md not in research doc → INFO
# ---------------------------------------------------------------------------

@test "INFO: slug in sources.md not found in research doc → INFO, exits 0" {
    local skill="$TMPDIR/my-skill"
    make_skill_with_source_keys "$skill"

    # Create a research doc that does NOT have the slug
    local research_dir="$TMPDIR/research"
    mkdir -p "$research_dir"
    cat > "$research_dir/my-research.md" <<EOF
# Research

## different-slug

- **Contributing files:** (none)
- **Status:** \`extracted\`
EOF

    # Use a path relative to repo root — we'll place research doc inside TMPDIR
    # and reference it as absolute for test purposes.
    # The script finds repo root by walking up from skill-dir until .git is found.
    # Since TMPDIR won't have .git, we simulate a repo structure.
    local fake_repo="$TMPDIR/fakerepo"
    mkdir -p "$fake_repo"
    touch "$fake_repo/.git"  # fake .git marker

    local skill2="$fake_repo/my-skill"
    mkdir -p "$skill2"
    cat > "$skill2/SKILL.md" <<EOF
---
name: my-skill
description: A valid skill description.
metadata:
  source_keys:
    - my-source
---

## Step 1

Do the thing.
EOF

    mkdir -p "$skill2/references"
    mkdir -p "$fake_repo/docs/research"
    cat > "$fake_repo/docs/research/my-research.md" <<EOF
# Research

## different-slug

- **Contributing files:** (none)
- **Status:** \`extracted\`
EOF

    cat > "$skill2/references/sources.md" <<EOF
# Sources

## my-source

- **URL:** https://example.com/my-source
- **Description:** A test source.
- **Contributing files:** SKILL.md
- **Research doc:** docs/research/my-research.md
- **Status:** \`extracted\`
EOF

    run bash "$SCRIPT" "$skill2"
    assert_success
    assert_output --partial "INFO"
}

# ---------------------------------------------------------------------------
# Cycle 12 — Check 8: Upstream reverse: extracted slug in research doc not in sources.md → FAIL
# ---------------------------------------------------------------------------

@test "FAIL: extracted non-(none) slug in research doc missing from skill sources.md" {
    local fake_repo="$TMPDIR/fakerepo"
    mkdir -p "$fake_repo"
    touch "$fake_repo/.git"

    local skill="$fake_repo/my-skill"
    mkdir -p "$skill"
    cat > "$skill/SKILL.md" <<EOF
---
name: my-skill
description: A valid skill description.
metadata:
  source_keys:
    - my-source
---

## Step 1

Do the thing.
EOF

    mkdir -p "$skill/references"
    mkdir -p "$fake_repo/docs/research"

    # Research doc has my-source (extracted, with a contributing file) AND extra-source (also extracted)
    cat > "$fake_repo/docs/research/my-research.md" <<EOF
# Research

## my-source

- **Contributing files:** some-skill/SKILL.md
- **Status:** \`extracted\`

## extra-source

- **Contributing files:** some-skill/references/extra.md
- **Status:** \`extracted\`
EOF

    cat > "$skill/references/sources.md" <<EOF
# Sources

## my-source

- **URL:** https://example.com/my-source
- **Description:** A test source.
- **Contributing files:** SKILL.md
- **Research doc:** docs/research/my-research.md
- **Status:** \`extracted\`
EOF

    run bash "$SCRIPT" "$skill"
    assert_failure
    assert_output --partial "FAIL"
}

@test "pass: extracted slug in research doc with (none) contributing files is not required in sources.md" {
    local fake_repo="$TMPDIR/fakerepo"
    mkdir -p "$fake_repo"
    touch "$fake_repo/.git"

    local skill="$fake_repo/my-skill"
    mkdir -p "$skill"
    cat > "$skill/SKILL.md" <<EOF
---
name: my-skill
description: A valid skill description.
metadata:
  source_keys:
    - my-source
---

## Step 1

Do the thing.
EOF

    mkdir -p "$skill/references"
    mkdir -p "$fake_repo/docs/research"

    cat > "$fake_repo/docs/research/my-research.md" <<EOF
# Research

## my-source

- **Contributing files:** some-skill/SKILL.md
- **Status:** \`extracted\`

## extra-source

- **Contributing files:** (none — not relevant)
- **Status:** \`extracted\`
EOF

    cat > "$skill/references/sources.md" <<EOF
# Sources

## my-source

- **URL:** https://example.com/my-source
- **Description:** A test source.
- **Contributing files:** SKILL.md
- **Research doc:** docs/research/my-research.md
- **Status:** \`extracted\`
EOF

    run bash "$SCRIPT" "$skill"
    assert_success
}