holocron/plugins/kyberforge/skills/agent-audit/scripts/validate-provenance.sh

#!/usr/bin/env bash
set -euo pipefail

usage() {
  cat <<EOF
Usage: validate-provenance.sh <agent-file>

Validate that an agent's sources provenance chain is complete and internally consistent.
Operates at plugin/APM scope only (a single vendor-neutral .apm/agents/<name>.agent.md
inside a package with a type:-bearing apm.yml) — exits 0 silently for project and user
scope agents.

Arguments:
  agent-file  Path to either the Claude Code .md or Copilot .agent.md agent file.

Exit codes:
  0  All checks passed (or nothing to validate, or not plugin scope)
  1  One or more checks failed
  2  Script error (unrecognized file extension — expected .md or .agent.md)

Checks performed:
  0  source_keys present in agent pair but sources.md absent
  1  FILL IN: placeholders in sources.md
  2  source_keys in agent files → slug exists in sources.md
  3  Contributing files listed in sources.md exist on disk (plugin-root relative)
  4  Contributing files back-reference the parent slug in their source_keys
  5  Research doc field present and not placeholder
EOF
}

if [[ "${1:-}" == "--help" || "${1:-}" == "-h" ]]; then
  usage
  exit 0
fi

if [[ $# -lt 1 ]]; then
  echo "Error: agent-file is required." >&2
  echo "" >&2
  usage >&2
  exit 1
fi

python3 -u - "$1" <<'PYTHON'
import sys
import os
import re

agent_file = os.path.abspath(sys.argv[1])
fname = os.path.basename(agent_file)
agent_dir = os.path.dirname(agent_file)

# --- Sanity-check extension (single vendor-neutral .agent.md file at plugin/APM scope) ---
if not (fname.endswith('.agent.md') or fname.endswith('.md')):
    print(f"Error: unrecognized extension '{fname}' — expected .md or .agent.md", file=sys.stderr)
    sys.exit(2)

# Matches a top-level `type:` line whose value is exactly one of the four
# package content types — identical to validate.sh's APM_TYPE_RE. Group 1's
# optional quote must be closed by \1 (or nothing), and the value must be
# followed by whitespace/end-of-line so a malformed value like `prompts-only`
# doesn't false-match on the `prompts` prefix.
TYPE_RE = re.compile(r"^type:\s*(['\"]?)(instructions|skill|hybrid|prompts)\1(?:\s|$)")

# --- Find package root: walk up for the nearest ancestor apm.yml that
# declares a top-level type: field. An apm.yml with no type: field is a
# marketplace-only manifest (see monorepo-and-repo-shapes.md) — skip it and
# keep walking. Stop at a $HOME boundary, a .git boundary, or the filesystem
# root: none of these is plugin/APM scope, so this script has nothing to
# check there.
def find_plugin_root(start_dir):
    home = os.path.expanduser('~')
    current = os.path.abspath(start_dir)
    while True:
        apm_yml = os.path.join(current, 'apm.yml')
        if os.path.isfile(apm_yml):
            with open(apm_yml) as f:
                if any(TYPE_RE.match(line) for line in f):
                    return current
        # $HOME is a non-plugin-scope boundary — checked before the .git test
        # below (mirrors validate.sh's detect_scope ordering), so a
        # dotfiles-managed $HOME (yadm, chezmoi bare-repo, etc.) can't shadow
        # this check by being its own .git repo. Without this, the walk could
        # continue past $HOME toward the filesystem root looking for a
        # type-bearing apm.yml, misclassifying a user/project-scope file as
        # plugin scope in rare ancestor layouts.
        if current == home:
            return None
        # .git is a directory in a normal checkout but a file (`gitdir: ...`)
        # in a git worktree — exists() covers both.
        if os.path.exists(os.path.join(current, '.git')):
            return None
        parent = os.path.dirname(current)
        if parent == current:
            return None
        current = parent

plugin_root = find_plugin_root(agent_dir)
if plugin_root is None:
    sys.exit(0)

sources_md_path = os.path.join(plugin_root, 'sources.md')

# --- Helpers ---
PLACEHOLDER_RE = re.compile(r'(?<!`)FILL IN:[^`\n]')

def parse_frontmatter(content):
    m = re.match(r'^---\n(.*?)\n---', content, re.DOTALL)
    if not m:
        return None, content
    return m.group(1), content[m.end():]

def parse_source_keys(fm):
    """Extract top-level source_keys list from frontmatter string."""
    if fm is None:
        return []
    keys = []
    in_source_keys = False
    for line in fm.splitlines():
        if re.match(r'^source_keys:', line):
            in_source_keys = True
            continue
        if in_source_keys:
            m = re.match(r'^[ \t]+-\s+(\S+)', line)
            if m:
                keys.append(m.group(1).strip())
            elif line and not line[0].isspace():
                in_source_keys = False
    return keys

def parse_h2_slugs(content):
    return re.findall(r'^## (.+)$', content, re.MULTILINE)

def parse_contributing_files(content, slug):
    """Find the Contributing files for a given slug H2 in content.

    Accepts the inline form and the bullet form; recognising only the
    inline one silently skips the contributing-file checks on every
    sources.md written the other way. Returns a list of paths with any
    trailing parenthetical note stripped; "(none)" returns an empty list
    and a slug with no entry returns None. Kept behaviourally identical to
    skill-audit's copy, which is where the bug was found.
    """
    pattern = re.compile(
        r'^## ' + re.escape(slug) + r'\s*\n(.*?)(?=^## |\Z)',
        re.MULTILINE | re.DOTALL
    )
    m = pattern.search(content)
    if not m:
        return None
    block = m.group(1)

    def strip_note(entry):
        return re.sub(r'\s*\(.*$', '', entry).strip()

    cf_m = re.search(r'^\- \*\*Contributing files:\*\* (.+)$', block, re.MULTILINE)
    if cf_m:
        value = cf_m.group(1).strip()
        if value.startswith("(none"):
            return []
        return [p for p in (strip_note(x) for x in value.split(",")) if p]

    cf_m = re.search(r'^\*\*Contributing files:\*\*\s*$', block, re.MULTILINE)
    if not cf_m:
        return None
    files = []
    for line in block[cf_m.end():].splitlines():
        line = line.strip()
        if not line:
            if files:
                break
            continue
        if not line.startswith("- "):
            break
        entry = line[2:].strip()
        if entry.startswith("(none"):
            return []
        entry = strip_note(entry)
        if entry:
            files.append(entry)
    return files

def parse_research_doc(content, slug):
    pattern = re.compile(
        r'^## ' + re.escape(slug) + r'\s*\n(.*?)(?=^## |\Z)',
        re.MULTILINE | re.DOTALL
    )
    m = pattern.search(content)
    if not m:
        return None
    block = m.group(1)
    rd_m = re.search(r'^\- \*\*Research doc:\*\* (.+)$', block, re.MULTILINE)
    if not rd_m:
        return None
    return rd_m.group(1).strip()

findings = []
has_fail = False

def emit_fail(desc, fpath, why, fix):
    global has_fail
    has_fail = True
    findings.append(("FAIL", desc, fpath, why, fix))

def print_findings():
    for kind, desc, fpath, why, fix in findings:
        print(f"FAIL  {desc} — {fpath}")
        print(f"      Why: {why}")
        print(f"      Fix: {fix}")
        print()

# --- Collect source_keys from agent pair ---
def get_source_keys_from_file(fpath):
    if not os.path.isfile(fpath):
        return []
    try:
        with open(fpath) as f:
            content = f.read()
    except Exception:
        return []
    fm, _ = parse_frontmatter(content)
    return parse_source_keys(fm)

# Plugin/APM scope is a single vendor-neutral file — no counterpart to merge.
given_keys = get_source_keys_from_file(agent_file)
all_source_keys = given_keys

sources_md_exists = os.path.isfile(sources_md_path)

# Early exit: nothing to validate
if not all_source_keys and not sources_md_exists:
    sys.exit(0)

sources_content = None
sources_slugs = set()
if sources_md_exists:
    with open(sources_md_path) as f:
        sources_content = f.read()
    sources_slugs = set(parse_h2_slugs(sources_content))

# --- Check 0: source_keys present but sources.md absent ---
if not sources_md_exists and all_source_keys:
    rel_given = os.path.relpath(agent_file, plugin_root)
    emit_fail(
        "source_keys declared but sources.md is absent",
        rel_given,
        "source_keys references research provenance that has no sources index to validate against.",
        "Create sources.md with an H2 entry for each slug referenced by source_keys."
    )
    print_findings()
    sys.exit(1)

# --- Check 1: FILL IN: placeholders in sources.md ---
for line in sources_content.splitlines():
    if PLACEHOLDER_RE.search(line):
        emit_fail(
            "Unfilled FILL IN: placeholder",
            "sources.md",
            "sources.md contains an unfilled placeholder, meaning provenance is incomplete.",
            "Replace all 'FILL IN:' values in sources.md with real content."
        )
        break

# --- Check 2: source_keys in the agent file → slug exists in sources.md ---
for fpath, keys in [(agent_file, given_keys)]:
    if not keys:
        continue
    rel = os.path.relpath(fpath, plugin_root)
    for slug in keys:
        if slug not in sources_slugs:
            emit_fail(
                f"source_keys slug '{slug}' not found in sources.md",
                rel,
                f"'{rel}' declares '{slug}' as a source but there is no '## {slug}' heading in sources.md.",
                f"Add '## {slug}' entry to sources.md or remove '{slug}' from {rel} source_keys."
            )

# --- Checks 3, 4, 5: Per-slug checks in sources.md ---
for slug in parse_h2_slugs(sources_content):
    # Check 3: Contributing files exist (paths relative to plugin root)
    cf_files = parse_contributing_files(sources_content, slug)
    if cf_files:
        for cf_rel in cf_files:
            cf_abs = os.path.join(plugin_root, cf_rel)
            if not os.path.isfile(cf_abs):
                emit_fail(
                    f"Contributing file '{cf_rel}' does not exist",
                    f"sources.md (## {slug})",
                    f"sources.md claims '{cf_rel}' was contributed to by slug '{slug}' but the file does not exist.",
                    f"Create '{cf_rel}' relative to the plugin root, or correct the path in sources.md."
                )
            else:
                # Check 4: Bidirectional — file should list slug in its source_keys
                with open(cf_abs) as f:
                    cf_content = f.read()
                cf_fm, _ = parse_frontmatter(cf_content)
                cf_keys = parse_source_keys(cf_fm)
                if slug not in cf_keys:
                    emit_fail(
                        f"Contributing file '{cf_rel}' does not list '{slug}' in its source_keys",
                        f"sources.md (## {slug})",
                        f"sources.md says '{cf_rel}' was informed by '{slug}', but '{cf_rel}' does not declare '{slug}' in its top-level source_keys.",
                        f"Add '{slug}' to the top-level source_keys frontmatter in '{cf_rel}'."
                    )

    # Check 5: Research doc field required
    rd_value = parse_research_doc(sources_content, slug)
    if rd_value is None:
        emit_fail(
            "Research doc field missing",
            f"sources.md (## {slug})",
            f"The '## {slug}' entry in sources.md has no '- **Research doc:**' line.",
            f"Add '- **Research doc:** <path-or-(none)>' to the '## {slug}' entry in sources.md."
        )
    elif rd_value == "" or PLACEHOLDER_RE.search(rd_value):
        emit_fail(
            "Research doc field is empty or placeholder",
            f"sources.md (## {slug})",
            f"The '## {slug}' entry has an unfilled Research doc value.",
            "Set '- **Research doc:**' to a real path relative to repo root, or '(none)' if not applicable."
        )

print_findings()
sys.exit(1 if has_fail else 0)
PYTHON