#!/usr/bin/env bash
set -euo pipefail

usage() {
  cat <<EOF
Usage: validate-provenance.sh <skill-dir>

Validate that a skill's sources provenance chain is complete and internally consistent.

Arguments:
  skill-dir  Path to the skill directory to validate.

Exit codes:
  0  All checks passed (or nothing to validate)
  1  One or more checks failed

Checks performed:
  0  source_keys present but references/sources.md absent
  1  FILL IN: placeholders in sources.md
  2  source_keys in SKILL.md → slug exists in sources.md
  3  source_keys in references/*.md → slug exists in sources.md (INFO if no source_keys)
  4  Contributing files listed in sources.md exist on disk
  5  Contributing files back-reference the parent slug in their source_keys
  6  Research doc field present and not placeholder
  7  Slug in sources.md present in upstream research doc (INFO only)
  8  Extracted non-(none) slug in research doc present in sources.md
EOF
}

if [[ "${1:-}" == "--help" || "${1:-}" == "-h" ]]; then
  usage
  exit 0
fi

if [[ $# -lt 1 ]]; then
  echo "Error: skill-dir is required." >&2
  echo "" >&2
  usage >&2
  exit 1
fi

python3 -u - "$1" <<'PYTHON'
import sys
import os
import re

skill_dir = os.path.abspath(sys.argv[1])
sources_md_path = os.path.join(skill_dir, "references", "sources.md")
refs_dir = os.path.join(skill_dir, "references")

# --- Helpers ---

PLACEHOLDER_RE = re.compile(r'(?<!`)FILL IN:[^`\n]')

def parse_frontmatter(content):
    """Return (frontmatter_str, body_str) or (None, content) if no frontmatter."""
    m = re.match(r'^---\n(.*?)\n---', content, re.DOTALL)
    if not m:
        return None, content
    return m.group(1), content[m.end():]

def parse_source_keys(fm):
    """Extract list of source_keys from frontmatter string (handles metadata.source_keys and top-level)."""
    if fm is None:
        return []
    keys = []
    # Match either:
    #   metadata:\n  source_keys:\n    - slug
    # or:
    #   source_keys:\n  - slug
    in_source_keys = False
    in_metadata = False
    for line in fm.splitlines():
        if re.match(r'^metadata:', line):
            in_metadata = True
            continue
        if in_metadata and re.match(r'^  source_keys:', line):
            in_source_keys = True
            continue
        if not in_metadata and re.match(r'^source_keys:', line):
            in_source_keys = True
            continue
        if in_source_keys:
            m = re.match(r'^[ \t]+-\s+(\S+)', line)
            if m:
                keys.append(m.group(1).strip())
            elif line and not line[0].isspace():
                in_source_keys = False
                in_metadata = False
    return keys

def parse_h2_slugs(content):
    """Return list of H2 heading values from a markdown file."""
    return re.findall(r'^## (.+)$', content, re.MULTILINE)

def parse_contributing_files(content, slug):
    """Find the Contributing files for a given slug H2 in content.

    Both authored forms are accepted, because both are in use across the
    corpus and only recognising the first silently skipped checks 4 and 5
    on every skill using the second:

        - **Contributing files:** SKILL.md, references/a.md

        **Contributing files:**
        - SKILL.md (what this source contributed)
        - references/a.md (what this source contributed)

    Returns a list of paths with any trailing parenthetical note stripped.
    A "(none)" value returns an empty list; a slug with no Contributing
    files entry at all returns None. Note the bullet form's notes may
    themselves contain commas, so the list is built per bullet rather than
    by splitting the joined value.
    """
    pattern = re.compile(
        r'^## ' + re.escape(slug) + r'\s*\n(.*?)(?=^## |\Z)',
        re.MULTILINE | re.DOTALL
    )
    m = pattern.search(content)
    if not m:
        return None
    block = m.group(1)

    def strip_note(entry):
        # "references/a.md (why)" -> "references/a.md"
        return re.sub(r'\s*\(.*$', '', entry).strip()

    # Inline form: value on the same line, comma-separated, no notes.
    cf_m = re.search(r'^\- \*\*Contributing files:\*\* (.+)$', block, re.MULTILINE)
    if cf_m:
        value = cf_m.group(1).strip()
        if value.startswith("(none"):
            return []
        return [p for p in (strip_note(x) for x in value.split(",")) if p]

    # Bullet form: heading on its own line, one file per following bullet.
    cf_m = re.search(r'^\*\*Contributing files:\*\*\s*$', block, re.MULTILINE)
    if not cf_m:
        return None
    rest = block[cf_m.end():]
    files = []
    for line in rest.splitlines():
        line = line.strip()
        if not line:
            if files:
                break
            continue
        if not line.startswith("- "):
            break
        entry = line[2:].strip()
        if entry.startswith("(none"):
            return []
        entry = strip_note(entry)
        if entry:
            files.append(entry)
    return files

def parse_research_doc(content, slug):
    """Find the Research doc value for a given slug H2 in content."""
    pattern = re.compile(
        r'^## ' + re.escape(slug) + r'\s*\n(.*?)(?=^## |\Z)',
        re.MULTILINE | re.DOTALL
    )
    m = pattern.search(content)
    if not m:
        return None
    block = m.group(1)
    rd_m = re.search(r'^\- \*\*Research doc:\*\* (.+)$', block, re.MULTILINE)
    if not rd_m:
        return None
    return rd_m.group(1).strip()

def parse_status(content, slug):
    """Find the Status value for a given slug H2 in content."""
    pattern = re.compile(
        r'^## ' + re.escape(slug) + r'\s*\n(.*?)(?=^## |\Z)',
        re.MULTILINE | re.DOTALL
    )
    m = pattern.search(content)
    if not m:
        return None
    block = m.group(1)
    st_m = re.search(r'^\- \*\*Status:\*\* (.+)$', block, re.MULTILINE)
    if not st_m:
        return None
    return st_m.group(1).strip()

def find_repo_root(start_dir):
    """Walk up from start_dir until we find a directory containing .git."""
    current = start_dir
    while True:
        if os.path.exists(os.path.join(current, ".git")):
            return current
        parent = os.path.dirname(current)
        if parent == current:
            return None
        current = parent

findings = []
has_fail = False

def emit_fail(desc, fpath, why, fix):
    global has_fail
    has_fail = True
    findings.append(("FAIL", desc, fpath, why, fix, None))

def emit_info(desc, fpath, note):
    findings.append(("INFO", desc, fpath, None, None, note))

def print_findings():
    for entry in findings:
        kind = entry[0]
        desc = entry[1]
        fpath = entry[2]
        why = entry[3]
        fix = entry[4]
        note = entry[5]
        if kind == "FAIL":
            print(f"FAIL  {desc} — {fpath}")
            print(f"      Why: {why}")
            print(f"      Fix: {fix}")
            print()
        else:
            print(f"INFO  {desc} — {fpath}")
            print(f"      Note: {note}")
            print()

# --- Scan for any file with source_keys ---

def file_has_source_keys(fpath):
    try:
        with open(fpath) as f:
            content = f.read()
    except Exception:
        return False
    fm, _ = parse_frontmatter(content)
    if fm is None:
        return False
    return bool(parse_source_keys(fm))

def find_files_with_source_keys():
    """Return list of (relative_path, abs_path) for all skill files with source_keys."""
    results = []
    for root, dirs, files in os.walk(skill_dir):
        # Skip hidden dirs
        dirs[:] = [d for d in dirs if not d.startswith('.')]
        for fname in files:
            if fname.endswith('.md'):
                abs_path = os.path.join(root, fname)
                if file_has_source_keys(abs_path):
                    rel = os.path.relpath(abs_path, skill_dir)
                    results.append((rel, abs_path))
    return results

sources_md_exists = os.path.isfile(sources_md_path)
files_with_source_keys = find_files_with_source_keys()

# Early exit: nothing to validate
if not sources_md_exists and not files_with_source_keys:
    sys.exit(0)

# Load sources.md if it exists
sources_content = None
if sources_md_exists:
    with open(sources_md_path) as f:
        sources_content = f.read()
    sources_slugs = set(parse_h2_slugs(sources_content))
else:
    sources_slugs = set()

# --- Check 0: source_keys without sources.md ---
if not sources_md_exists:
    for rel, abs_path in files_with_source_keys:
        emit_fail(
            f"source_keys declared but references/sources.md is absent",
            rel,
            "source_keys references research provenance that has no sources index to validate against.",
            "Create references/sources.md with an H2 entry for each slug referenced by source_keys."
        )
    print_findings()
    sys.exit(1)

# --- Check 1: FILL IN: placeholders in sources.md ---
for line in sources_content.splitlines():
    if PLACEHOLDER_RE.search(line):
        emit_fail(
            "Unfilled FILL IN: placeholder",
            "references/sources.md",
            "sources.md contains an unfilled placeholder, meaning provenance is incomplete.",
            "Replace all 'FILL IN:' values in references/sources.md with real content."
        )
        break

# --- Check 2: source_keys in SKILL.md → slug exists in sources.md ---
skill_md_path = os.path.join(skill_dir, "SKILL.md")
if os.path.isfile(skill_md_path):
    with open(skill_md_path) as f:
        skill_content = f.read()
    skill_fm, _ = parse_frontmatter(skill_content)
    skill_source_keys = parse_source_keys(skill_fm)
    for slug in skill_source_keys:
        if slug not in sources_slugs:
            emit_fail(
                f"source_keys slug '{slug}' not found in sources.md",
                "SKILL.md",
                f"SKILL.md declares '{slug}' as a source but there is no '## {slug}' heading in references/sources.md.",
                f"Add '## {slug}' entry to references/sources.md or remove '{slug}' from SKILL.md source_keys."
            )

# --- Check 3: source_keys in references/*.md → slug exists in sources.md (INFO if no source_keys) ---
if os.path.isdir(refs_dir):
    for fname in sorted(os.listdir(refs_dir)):
        if not fname.endswith('.md'):
            continue
        if fname == "sources.md":
            continue
        fpath = os.path.join(refs_dir, fname)
        rel = os.path.relpath(fpath, skill_dir)
        with open(fpath) as f:
            ref_content = f.read()
        ref_fm, _ = parse_frontmatter(ref_content)
        ref_keys = parse_source_keys(ref_fm)
        if not ref_keys:
            emit_info(
                f"No source_keys frontmatter",
                rel,
                "This references file has no source_keys — provenance cannot be verified. "
                "Add source_keys frontmatter listing the slugs from references/sources.md that informed this file."
            )
        else:
            for slug in ref_keys:
                if slug not in sources_slugs:
                    emit_fail(
                        f"source_keys slug '{slug}' not found in sources.md",
                        rel,
                        f"'{rel}' declares '{slug}' as a source but there is no '## {slug}' heading in references/sources.md.",
                        f"Add '## {slug}' entry to references/sources.md or remove '{slug}' from {rel} source_keys."
                    )

# --- Checks 4, 5, 6, 7, 8: Per-slug checks in sources.md ---

repo_root = find_repo_root(skill_dir)

# Collect all research doc paths we'll check (for Check 8)
research_docs_seen = {}  # abs_path → set of slugs in sources.md that reference it

for slug in parse_h2_slugs(sources_content):
    # Check 4: Contributing files exist
    cf_files = parse_contributing_files(sources_content, slug)
    if cf_files:
        for cf_rel in cf_files:
            cf_abs = os.path.join(skill_dir, cf_rel)
            if not os.path.isfile(cf_abs):
                emit_fail(
                    f"Contributing file '{cf_rel}' does not exist",
                    f"references/sources.md (## {slug})",
                    f"sources.md claims '{cf_rel}' was contributed to by slug '{slug}' but the file does not exist.",
                    f"Create '{cf_rel}' relative to the skill directory, or correct the path in sources.md."
                )
            else:
                # Check 5: Bidirectional — file should list slug in its source_keys
                # Skip sources.md itself
                if cf_rel == "references/sources.md":
                    continue
                with open(cf_abs) as f:
                    cf_content = f.read()
                cf_fm, _ = parse_frontmatter(cf_content)
                cf_keys = parse_source_keys(cf_fm)
                if slug not in cf_keys:
                    emit_fail(
                        f"Contributing file '{cf_rel}' does not list '{slug}' in its source_keys",
                        f"references/sources.md (## {slug})",
                        f"sources.md says '{cf_rel}' was informed by '{slug}', but '{cf_rel}' does not declare '{slug}' in its source_keys frontmatter.",
                        f"Add '{slug}' to the source_keys frontmatter of '{cf_rel}'."
                    )

    # Check 6: Research doc field required
    rd_value = parse_research_doc(sources_content, slug)
    if rd_value is None:
        emit_fail(
            f"Research doc field missing",
            f"references/sources.md (## {slug})",
            f"The '## {slug}' entry in sources.md has no '- **Research doc:**' line.",
            f"Add '- **Research doc:** <path-or-(none)>' to the '## {slug}' entry in references/sources.md."
        )
    elif rd_value == "" or PLACEHOLDER_RE.search(rd_value):
        emit_fail(
            f"Research doc field is empty or placeholder",
            f"references/sources.md (## {slug})",
            f"The '## {slug}' entry has an unfilled Research doc value.",
            f"Set '- **Research doc:**' to a real path relative to repo root, or '(none)' if not applicable."
        )
    else:
        # Check 7: Upstream forward — slug should appear in research doc
        if repo_root and not rd_value.startswith("(none"):
            rd_abs = os.path.join(repo_root, rd_value)
            if os.path.isfile(rd_abs):
                with open(rd_abs) as f:
                    rd_content = f.read()
                rd_slugs = set(parse_h2_slugs(rd_content))
                if slug not in rd_slugs:
                    emit_info(
                        f"Slug '{slug}' not found as H2 in research doc '{rd_value}'",
                        f"references/sources.md (## {slug})",
                        f"The research doc '{rd_value}' does not have a '## {slug}' heading. "
                        f"The provenance link may be imprecise — the slug name in sources.md may differ from the research doc's heading."
                    )
                # Track for Check 8
                if rd_abs not in research_docs_seen:
                    research_docs_seen[rd_abs] = (rd_value, set())
                research_docs_seen[rd_abs][1].add(slug)

# --- Check 8: Upstream reverse ---
for rd_abs, (rd_rel, known_slugs) in research_docs_seen.items():
    with open(rd_abs) as f:
        rd_content = f.read()
    for rd_slug in parse_h2_slugs(rd_content):
        # Parse this slug's Contributing files and Status in the research doc
        rd_cf = parse_contributing_files(rd_content, rd_slug)
        rd_status = parse_status(rd_content, rd_slug)
        # Skip if the research doc explicitly records no contributing files
        if rd_cf == []:
            continue
        # Skip if status is not `extracted`
        if rd_status != "`extracted`":
            continue
        # This slug should be in sources.md
        if rd_slug not in sources_slugs:
            emit_fail(
                f"Research doc slug '{rd_slug}' missing from skill sources.md",
                f"references/sources.md",
                f"The research doc '{rd_rel}' has '## {rd_slug}' with status `extracted` and contributing files, "
                f"but this skill's sources.md has no '## {rd_slug}' entry.",
                f"Add '## {rd_slug}' to references/sources.md or mark it as '(none)' in the research doc's Contributing files."
            )

print_findings()
sys.exit(1 if has_fail else 0)
PYTHON