holocron/plugins/kyberforge/skills/skill-audit/scripts/validate-provenance.sh

#!/usr/bin/env bash
set -euo pipefail

usage() {
  cat <<EOF
Usage: validate-provenance.sh <skill-dir>

Validate that a skill's sources provenance chain is complete and internally consistent.

Arguments:
  skill-dir  Path to the skill directory to validate.

Exit codes:
  0  All checks passed (or nothing to validate)
  1  One or more checks failed

Checks performed:
  0  source_keys present but references/sources.md absent
  1  FILL IN: placeholders in sources.md
  2  source_keys in SKILL.md → slug exists in sources.md
  3  source_keys in references/*.md → slug exists in sources.md (INFO if no
     source_keys; an explicit 'source_keys: []' declares the file house-authored
     and passes silently)
  4  Contributing files listed in sources.md exist on disk. An explicit
     '(none)' skips silently; a Contributing files block this parser cannot
     read is reported as an INFO saying checks 4 and 5 did not run, never
     skipped silently.
  5  Contributing files back-reference the parent slug in their source_keys
  6  Research doc field present and not placeholder
  7  Slug in sources.md present in upstream research doc (INFO only). A section
     annotation ('§ ...', '→ ...', '(...)') is stripped before the path is
     resolved; a path that still does not resolve is reported as an INFO saying
     checks 7 and 8 did not run, never skipped silently.
  8  Extracted non-(none) slug in research doc present in sources.md
EOF
}

if [[ "${1:-}" == "--help" || "${1:-}" == "-h" ]]; then
  usage
  exit 0
fi

if [[ $# -lt 1 ]]; then
  echo "Error: skill-dir is required." >&2
  echo "" >&2
  usage >&2
  exit 1
fi

python3 -u - "$1" <<'PYTHON'
import sys
import os
import re

skill_dir = os.path.abspath(sys.argv[1])
sources_md_path = os.path.join(skill_dir, "references", "sources.md")
refs_dir = os.path.join(skill_dir, "references")

# --- Helpers ---

PLACEHOLDER_RE = re.compile(r'(?<!`)FILL IN:[^`\n]')

def parse_frontmatter(content):
    """Return (frontmatter_str, body_str) or (None, content) if no frontmatter."""
    m = re.match(r'^---\n(.*?)\n---', content, re.DOTALL)
    if not m:
        return None, content
    return m.group(1), content[m.end():]

def parse_source_keys(fm):
    """Extract list of source_keys from frontmatter string (handles metadata.source_keys and top-level)."""
    if fm is None:
        return []
    keys = []
    # Match either:
    #   metadata:\n  source_keys:\n    - slug
    # or:
    #   source_keys:\n  - slug
    in_source_keys = False
    in_metadata = False
    for line in fm.splitlines():
        if re.match(r'^metadata:', line):
            in_metadata = True
            continue
        if in_metadata and re.match(r'^  source_keys:', line):
            in_source_keys = True
            continue
        if not in_metadata and re.match(r'^source_keys:', line):
            in_source_keys = True
            continue
        if in_source_keys:
            m = re.match(r'^[ \t]+-\s+(\S+)', line)
            if m:
                keys.append(m.group(1).strip())
            elif line and not line[0].isspace():
                in_source_keys = False
                in_metadata = False
    return keys

# An explicit `source_keys: []` — top-level or under metadata: — is a
# DECLARATION that the file is house-authored and has no external source.
# parse_source_keys() returns [] both for that and for a file with no
# source_keys key at all, so the two are indistinguishable downstream and
# check 3 emitted the same INFO for each (#111). That left no honest way to
# record "this file has no external source": the only ways to silence the INFO
# were to invent a slug or borrow an unrelated one, both false provenance
# claims that then have to be maintained in sources.md as well. A bare
# `source_keys:` with nothing after it is NOT accepted here — that reads as a
# truncated or half-written entry, not a decision.
#
# The indent is pinned to the two positions parse_source_keys() actually reads
# — column 0, or two spaces under `metadata:`. A permissive `^\s*` matched a
# `source_keys: []` nested at ANY depth under an unrelated key, which
# parse_source_keys() never reads, so a stray nested key silenced the check-3
# INFO for a file that had declared nothing.
EMPTY_SOURCE_KEYS_RE = re.compile(r'^(?:  )?source_keys:\s*\[\s*\]\s*$')

def declares_empty_source_keys(fm):
    """True when frontmatter carries an explicit, empty `source_keys: []`."""
    if fm is None:
        return False
    return any(EMPTY_SOURCE_KEYS_RE.match(line) for line in fm.splitlines())

def parse_h2_slugs(content):
    """Return list of H2 heading values from a markdown file."""
    return re.findall(r'^## (.+)$', content, re.MULTILINE)

# ===== BEGIN SHARED CONTRIBUTING-FILES PARSER =====
# ONE parser, embedded VERBATIM in two scripts:
#   plugins/kyberforge/.apm/skills/skill-audit/scripts/validate-provenance.sh
#   plugins/kyberforge/.apm/skills/agent-audit/scripts/validate-provenance.sh
# The block between these markers must stay byte-identical in both. It is
# copied rather than imported because a cache-installed plugin's scripts cannot
# read files outside their own plugin directory, so there is no single file both
# can share — the same constraint that forces the ADR-0020 boundary resolver to
# be duplicated across three scripts. Edit one copy, then paste it over the
# other.
#
# tests/test-adr0020-contract.sh hashes both copies and fails on drift. Before
# it did, the agent-audit copy's docstring merely ASSERTED the two were
# "behaviourally identical" and nothing checked it — which is how the two
# already-diverged spellings of the bullet loop went unnoticed.
#
# Requires: re (imported by the host script).


def parse_contributing_files(content, slug):
    """Find the Contributing files for a given slug H2 in content.

    Both authored forms are accepted, because both are in use across the
    corpus and only recognising the first silently skipped the contributing-
    file checks on every sources.md written the other way:

        - **Contributing files:** SKILL.md, references/a.md

        **Contributing files:**
        - SKILL.md (what this source contributed)
        - references/a.md (what this source contributed)

    Returns a list of paths with any trailing parenthetical note stripped.
    Note the bullet form's notes may themselves contain commas, so the list
    is built per bullet rather than by splitting the joined value.

    The three return values are NOT interchangeable, and callers depend on
    the distinction:

        [path, ...]  the entry names contributing files
        []           the entry EXPLICITLY records "(none)"
        None         the entry says nothing this parser can read

    Only an explicit "(none)" yields []. A "Contributing files:" heading
    followed by a numbered list, by `*` bullets, or by prose parses nothing
    and returns None, never [] — a caller reads [] as a deliberate "no
    contributing files" record and SKIPS its check on that basis, so a parse
    failure returning [] would silently disable the check instead of leaving
    the unreadable entry exposed to it.
    """
    pattern = re.compile(
        r'^## ' + re.escape(slug) + r'\s*\n(.*?)(?=^## |\Z)',
        re.MULTILINE | re.DOTALL
    )
    m = pattern.search(content)
    if not m:
        return None
    block = m.group(1)

    def strip_note(entry):
        # "references/a.md (why)" -> "references/a.md"
        return re.sub(r'\s*\(.*$', '', entry).strip()

    # Inline form: value on the same line, comma-separated, no notes.
    cf_m = re.search(r'^\- \*\*Contributing files:\*\* (.+)$', block, re.MULTILINE)
    if cf_m:
        value = cf_m.group(1).strip()
        if value.startswith("(none"):
            return []
        return [p for p in (strip_note(x) for x in value.split(","))
                if p] or None

    # Bullet form: heading on its own line, one file per following bullet.
    cf_m = re.search(r'^\*\*Contributing files:\*\*\s*$', block, re.MULTILINE)
    if not cf_m:
        return None
    files = []
    for line in block[cf_m.end():].splitlines():
        line = line.strip()
        if not line:
            if files:
                break
            continue
        if not line.startswith("- "):
            break
        entry = line[2:].strip()
        if entry.startswith("(none"):
            return []
        entry = strip_note(entry)
        if entry:
            files.append(entry)
    return files or None
# ===== END SHARED CONTRIBUTING-FILES PARSER =====

def parse_research_doc(content, slug):
    """Find the Research doc value for a given slug H2 in content."""
    pattern = re.compile(
        r'^## ' + re.escape(slug) + r'\s*\n(.*?)(?=^## |\Z)',
        re.MULTILINE | re.DOTALL
    )
    m = pattern.search(content)
    if not m:
        return None
    block = m.group(1)
    rd_m = re.search(r'^\- \*\*Research doc:\*\* (.+)$', block, re.MULTILINE)
    if not rd_m:
        return None
    return rd_m.group(1).strip()

# A Research doc value is a path, and very often a path PLUS an annotation
# naming the section the slug came from:
#
#   plugins/git/docs/research/docs/git/gitflow.md (whole-document reference)
#   plugins/git/docs/research/docs/git/remotes.md → `## Pushing (`git push`)`
#   .../pre-commit/hooks-reference.md § "pre-commit-hooks (official collection)"
#
# os.path.isfile() is false for every one of those strings, and checks 7 and 8
# used to skip SILENTLY whenever the path did not resolve. The effect was that
# both checks were dead on eight of the nine git skills — git-history, the one
# skill writing a bare path, was the only place they ran, which is why it was
# the only skill ever reporting a check-7 INFO. Strip the annotation before
# resolving, and report when the result still does not resolve: a check that
# quietly does not run is worse than one that fails.
RESEARCH_DOC_ANNOTATION_RE = re.compile(r'[§→(]')

def strip_research_doc_annotation(value):
    """Path part of a Research doc value, with any section annotation removed."""
    return RESEARCH_DOC_ANNOTATION_RE.split(value, maxsplit=1)[0].strip()

def research_doc_is_none(value):
    """True when a Research doc value declares that no research doc backs the slug.

    Both '(none)' and the bare 'none — org convention, ...' spelling are in
    use; recognising only the parenthesised one would report the other as an
    unresolvable path. Checked BEFORE the annotation strip, because '(none)'
    is itself a parenthesis and would strip to the empty string.
    """
    return re.match(r'\(?none\b', value.strip(), re.IGNORECASE) is not None

def parse_status(content, slug):
    """Find the Status value for a given slug H2 in content."""
    pattern = re.compile(
        r'^## ' + re.escape(slug) + r'\s*\n(.*?)(?=^## |\Z)',
        re.MULTILINE | re.DOTALL
    )
    m = pattern.search(content)
    if not m:
        return None
    block = m.group(1)
    st_m = re.search(r'^\- \*\*Status:\*\* (.+)$', block, re.MULTILINE)
    if not st_m:
        return None
    return st_m.group(1).strip()

def find_repo_root(start_dir):
    """Walk up from start_dir until we find a directory containing .git."""
    current = start_dir
    while True:
        if os.path.exists(os.path.join(current, ".git")):
            return current
        parent = os.path.dirname(current)
        if parent == current:
            return None
        current = parent

findings = []
has_fail = False

def emit_fail(desc, fpath, why, fix):
    global has_fail
    has_fail = True
    findings.append(("FAIL", desc, fpath, why, fix, None))

def emit_info(desc, fpath, note):
    findings.append(("INFO", desc, fpath, None, None, note))

def print_findings():
    for entry in findings:
        kind = entry[0]
        desc = entry[1]
        fpath = entry[2]
        why = entry[3]
        fix = entry[4]
        note = entry[5]
        if kind == "FAIL":
            print(f"FAIL  {desc} — {fpath}")
            print(f"      Why: {why}")
            print(f"      Fix: {fix}")
            print()
        else:
            print(f"INFO  {desc} — {fpath}")
            print(f"      Note: {note}")
            print()

# --- Scan for any file with source_keys ---

def file_has_source_keys(fpath):
    try:
        with open(fpath) as f:
            content = f.read()
    except Exception:
        return False
    fm, _ = parse_frontmatter(content)
    if fm is None:
        return False
    return bool(parse_source_keys(fm))

def find_files_with_source_keys():
    """Return list of (relative_path, abs_path) for all skill files with source_keys."""
    results = []
    for root, dirs, files in os.walk(skill_dir):
        # Skip hidden dirs
        dirs[:] = [d for d in dirs if not d.startswith('.')]
        for fname in files:
            if fname.endswith('.md'):
                abs_path = os.path.join(root, fname)
                if file_has_source_keys(abs_path):
                    rel = os.path.relpath(abs_path, skill_dir)
                    results.append((rel, abs_path))
    return results

sources_md_exists = os.path.isfile(sources_md_path)
files_with_source_keys = find_files_with_source_keys()

# Early exit: nothing to validate
if not sources_md_exists and not files_with_source_keys:
    sys.exit(0)

# Load sources.md if it exists
sources_content = None
if sources_md_exists:
    with open(sources_md_path) as f:
        sources_content = f.read()
    sources_slugs = set(parse_h2_slugs(sources_content))
else:
    sources_slugs = set()

# --- Check 0: source_keys without sources.md ---
if not sources_md_exists:
    for rel, abs_path in files_with_source_keys:
        emit_fail(
            f"source_keys declared but references/sources.md is absent",
            rel,
            "source_keys references research provenance that has no sources index to validate against.",
            "Create references/sources.md with an H2 entry for each slug referenced by source_keys."
        )
    print_findings()
    sys.exit(1)

# --- Check 1: FILL IN: placeholders in sources.md ---
for line in sources_content.splitlines():
    if PLACEHOLDER_RE.search(line):
        emit_fail(
            "Unfilled FILL IN: placeholder",
            "references/sources.md",
            "sources.md contains an unfilled placeholder, meaning provenance is incomplete.",
            "Replace all 'FILL IN:' values in references/sources.md with real content."
        )
        break

# --- Check 2: source_keys in SKILL.md → slug exists in sources.md ---
skill_md_path = os.path.join(skill_dir, "SKILL.md")
if os.path.isfile(skill_md_path):
    with open(skill_md_path) as f:
        skill_content = f.read()
    skill_fm, _ = parse_frontmatter(skill_content)
    skill_source_keys = parse_source_keys(skill_fm)
    for slug in skill_source_keys:
        if slug not in sources_slugs:
            emit_fail(
                f"source_keys slug '{slug}' not found in sources.md",
                "SKILL.md",
                f"SKILL.md declares '{slug}' as a source but there is no '## {slug}' heading in references/sources.md.",
                f"Add '## {slug}' entry to references/sources.md or remove '{slug}' from SKILL.md source_keys."
            )

# --- Check 3: source_keys in references/*.md → slug exists in sources.md (INFO if no source_keys) ---
if os.path.isdir(refs_dir):
    for fname in sorted(os.listdir(refs_dir)):
        if not fname.endswith('.md'):
            continue
        if fname == "sources.md":
            continue
        fpath = os.path.join(refs_dir, fname)
        rel = os.path.relpath(fpath, skill_dir)
        with open(fpath) as f:
            ref_content = f.read()
        ref_fm, _ = parse_frontmatter(ref_content)
        ref_keys = parse_source_keys(ref_fm)
        if not ref_keys:
            # An explicit `source_keys: []` is a deliberate declaration that
            # the file is house-authored, and passes silently. The INFO is for
            # files that never said either way.
            if declares_empty_source_keys(ref_fm):
                continue
            emit_info(
                f"No source_keys frontmatter",
                rel,
                "This references file has no source_keys — provenance cannot be verified. "
                "Add source_keys frontmatter listing the slugs from references/sources.md that informed this file, "
                "or declare an explicit 'source_keys: []' if the file is house-authored and has no external source."
            )
        else:
            for slug in ref_keys:
                if slug not in sources_slugs:
                    emit_fail(
                        f"source_keys slug '{slug}' not found in sources.md",
                        rel,
                        f"'{rel}' declares '{slug}' as a source but there is no '## {slug}' heading in references/sources.md.",
                        f"Add '## {slug}' entry to references/sources.md or remove '{slug}' from {rel} source_keys."
                    )

# --- Checks 4, 5, 6, 7, 8: Per-slug checks in sources.md ---

repo_root = find_repo_root(skill_dir)

# Collect all research doc paths we'll check (for Check 8)
research_docs_seen = {}  # abs_path → set of slugs in sources.md that reference it

for slug in parse_h2_slugs(sources_content):
    # Checks 4 and 5: Contributing files exist, and back-reference the slug.
    # `[]` and None are NOT the same answer here. `[]` is the author writing
    # "(none)" — there is nothing to check and the skip is correct. None is a
    # Contributing-files block this parser cannot read, and skipping THAT
    # silently disables both checks on the one entry least likely to be right,
    # which is the failure mode parse_contributing_files' own docstring warns
    # about. Say so out loud instead, the same way an unresolvable Research doc
    # value does.
    cf_files = parse_contributing_files(sources_content, slug)
    if cf_files is None:
        emit_info(
            f"Contributing-file checks skipped for '{slug}' — the Contributing files block could not be parsed",
            f"references/sources.md (## {slug})",
            f"The '## {slug}' entry has no Contributing files list this parser can read — a missing field, a bare heading, '*' bullets, a numbered list, or prose all read as unparsable rather than as an empty declaration. "
            f"Checks 4 and 5 did not run for this slug, so nothing verified that its contributing files exist or name it back. "
            f"Write the value as '- **Contributing files:** <comma-separated paths>', or as a '**Contributing files:**' heading followed by '- ' bullets — "
            f"or record '(none)' if this source contributed no files."
        )
    elif cf_files:
        for cf_rel in cf_files:
            cf_abs = os.path.join(skill_dir, cf_rel)
            if not os.path.isfile(cf_abs):
                emit_fail(
                    f"Contributing file '{cf_rel}' does not exist",
                    f"references/sources.md (## {slug})",
                    f"sources.md claims '{cf_rel}' was contributed to by slug '{slug}' but the file does not exist.",
                    f"Create '{cf_rel}' relative to the skill directory, or correct the path in sources.md."
                )
            else:
                # Check 5: Bidirectional — file should list slug in its source_keys
                # Skip sources.md itself
                if cf_rel == "references/sources.md":
                    continue
                with open(cf_abs) as f:
                    cf_content = f.read()
                cf_fm, _ = parse_frontmatter(cf_content)
                cf_keys = parse_source_keys(cf_fm)
                if slug not in cf_keys:
                    emit_fail(
                        f"Contributing file '{cf_rel}' does not list '{slug}' in its source_keys",
                        f"references/sources.md (## {slug})",
                        f"sources.md says '{cf_rel}' was informed by '{slug}', but '{cf_rel}' does not declare '{slug}' in its source_keys frontmatter.",
                        f"Add '{slug}' to the source_keys frontmatter of '{cf_rel}'."
                    )

    # Check 6: Research doc field required
    rd_value = parse_research_doc(sources_content, slug)
    if rd_value is None:
        emit_fail(
            f"Research doc field missing",
            f"references/sources.md (## {slug})",
            f"The '## {slug}' entry in sources.md has no '- **Research doc:**' line.",
            f"Add '- **Research doc:** <path-or-(none)>' to the '## {slug}' entry in references/sources.md."
        )
    elif rd_value == "" or PLACEHOLDER_RE.search(rd_value):
        emit_fail(
            f"Research doc field is empty or placeholder",
            f"references/sources.md (## {slug})",
            f"The '## {slug}' entry has an unfilled Research doc value.",
            f"Set '- **Research doc:**' to a real path relative to repo root, or '(none)' if not applicable."
        )
    elif not research_doc_is_none(rd_value):
        # Check 7: Upstream forward — slug should appear in research doc.
        # Every path out of here that does NOT run the check says so out loud.
        rd_path = strip_research_doc_annotation(rd_value)
        if not repo_root:
            emit_info(
                f"Upstream checks skipped for '{slug}' — no repo root above the skill directory",
                f"references/sources.md (## {slug})",
                f"'{rd_value}' is a path relative to the repo root, but no ancestor of the skill directory contains a .git entry, "
                f"so it cannot be resolved. Checks 7 and 8 did not run for this slug. "
                f"Run this script against a skill inside a checkout."
            )
        elif not rd_path:
            emit_info(
                f"Upstream checks skipped for '{slug}' — Research doc value names no path",
                f"references/sources.md (## {slug})",
                f"The Research doc value '{rd_value}' is entirely annotation — stripping the section marker leaves no path. "
                f"Checks 7 and 8 did not run for this slug. "
                f"Give the value a file path relative to the repo root, or record '(none)' if no research doc backs this entry."
            )
        else:
            rd_abs = os.path.join(repo_root, rd_path)
            if not os.path.isfile(rd_abs):
                emit_info(
                    f"Upstream checks skipped for '{slug}' — research doc '{rd_path}' does not exist",
                    f"references/sources.md (## {slug})",
                    f"'{rd_value}' resolves to '{rd_path}' relative to the repo root and no file is there. "
                    f"Checks 7 and 8 did not run for this slug, so nothing verified that the research doc still backs it. "
                    f"Point the value at one existing file — a brace expansion, a comma-separated list of paths, or a bare section title does not resolve — "
                    f"or record '(none)' if no research doc backs this entry."
                )
            else:
                with open(rd_abs) as f:
                    rd_content = f.read()
                rd_slugs = set(parse_h2_slugs(rd_content))
                if slug not in rd_slugs:
                    emit_info(
                        f"Slug '{slug}' not found as H2 in research doc '{rd_path}'",
                        f"references/sources.md (## {slug})",
                        f"The research doc '{rd_path}' does not have a '## {slug}' heading. "
                        f"The provenance link may be imprecise — the slug name in sources.md may differ from the research doc's heading."
                    )
                # Track for Check 8
                if rd_abs not in research_docs_seen:
                    research_docs_seen[rd_abs] = (rd_path, set())
                research_docs_seen[rd_abs][1].add(slug)

# --- Check 8: Upstream reverse ---
for rd_abs, (rd_rel, known_slugs) in research_docs_seen.items():
    with open(rd_abs) as f:
        rd_content = f.read()
    for rd_slug in parse_h2_slugs(rd_content):
        # Parse this slug's Contributing files and Status in the research doc
        rd_cf = parse_contributing_files(rd_content, rd_slug)
        rd_status = parse_status(rd_content, rd_slug)
        # Skip if the research doc explicitly records no contributing files
        if rd_cf == []:
            continue
        # Skip if status is not `extracted`
        if rd_status != "`extracted`":
            continue
        # This slug should be in sources.md
        if rd_slug not in sources_slugs:
            emit_fail(
                f"Research doc slug '{rd_slug}' missing from skill sources.md",
                f"references/sources.md",
                f"The research doc '{rd_rel}' has '## {rd_slug}' with status `extracted` and contributing files, "
                f"but this skill's sources.md has no '## {rd_slug}' entry.",
                f"Add '## {rd_slug}' to references/sources.md or mark it as '(none)' in the research doc's Contributing files."
            )

print_findings()
sys.exit(1 if has_fail else 0)
PYTHON