Defame1297
484357a3b9
validate-provenance.sh matched Contributing files only as a single inline line beginning "- **Contributing files:**". Seven skills write it as a bare "**Contributing files:**" heading above a bullet list, so parse_contributing_files returned None and checks 4 (contributing file exists) and 5 (bidirectional source_keys) silently verified nothing on git-branches, git-remotes, git-submodules, git-workflow, git-worktrees, gitea-files and gitea-releases. Those are among the skills this branch changed most — git-branches alone gained five reference files — and the retrofit's mandatory sources.md collateral went in unchecked. Demonstrated rather than argued: planting a nonexistent contributing path in git-remotes yields 0 findings under the old parser and 1 FAIL under the new one. Both forms are now accepted. The bullet form is parsed per bullet rather than by splitting a joined value, because its per-file notes contain commas that would otherwise be read as path separators. The return type becomes a list of note-stripped paths, with "(none)" as an empty list and an absent entry as None, so the two callers no longer re-split a string. Applied to agent-audit's copy as well. No agent ships a sources.md today, so it is latent there, but it is the same defect. This is a third gate blind spot alongside #117 and #118, and was unfiled. One real defect surfaced immediately and is fixed separately. Refs #99
440 lines
16 KiB
Bash
Executable File
440 lines
16 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
set -euo pipefail
|
|
|
|
usage() {
|
|
cat <<EOF
|
|
Usage: validate-provenance.sh <skill-dir>
|
|
|
|
Validate that a skill's sources provenance chain is complete and internally consistent.
|
|
|
|
Arguments:
|
|
skill-dir Path to the skill directory to validate.
|
|
|
|
Exit codes:
|
|
0 All checks passed (or nothing to validate)
|
|
1 One or more checks failed
|
|
|
|
Checks performed:
|
|
0 source_keys present but references/sources.md absent
|
|
1 FILL IN: placeholders in sources.md
|
|
2 source_keys in SKILL.md → slug exists in sources.md
|
|
3 source_keys in references/*.md → slug exists in sources.md (INFO if no source_keys)
|
|
4 Contributing files listed in sources.md exist on disk
|
|
5 Contributing files back-reference the parent slug in their source_keys
|
|
6 Research doc field present and not placeholder
|
|
7 Slug in sources.md present in upstream research doc (INFO only)
|
|
8 Extracted non-(none) slug in research doc present in sources.md
|
|
EOF
|
|
}
|
|
|
|
if [[ "${1:-}" == "--help" || "${1:-}" == "-h" ]]; then
|
|
usage
|
|
exit 0
|
|
fi
|
|
|
|
if [[ $# -lt 1 ]]; then
|
|
echo "Error: skill-dir is required." >&2
|
|
echo "" >&2
|
|
usage >&2
|
|
exit 1
|
|
fi
|
|
|
|
python3 -u - "$1" <<'PYTHON'
|
|
import sys
|
|
import os
|
|
import re
|
|
|
|
skill_dir = os.path.abspath(sys.argv[1])
|
|
sources_md_path = os.path.join(skill_dir, "references", "sources.md")
|
|
refs_dir = os.path.join(skill_dir, "references")
|
|
|
|
# --- Helpers ---
|
|
|
|
PLACEHOLDER_RE = re.compile(r'(?<!`)FILL IN:[^`\n]')
|
|
|
|
def parse_frontmatter(content):
|
|
"""Return (frontmatter_str, body_str) or (None, content) if no frontmatter."""
|
|
m = re.match(r'^---\n(.*?)\n---', content, re.DOTALL)
|
|
if not m:
|
|
return None, content
|
|
return m.group(1), content[m.end():]
|
|
|
|
def parse_source_keys(fm):
|
|
"""Extract list of source_keys from frontmatter string (handles metadata.source_keys and top-level)."""
|
|
if fm is None:
|
|
return []
|
|
keys = []
|
|
# Match either:
|
|
# metadata:\n source_keys:\n - slug
|
|
# or:
|
|
# source_keys:\n - slug
|
|
in_source_keys = False
|
|
in_metadata = False
|
|
for line in fm.splitlines():
|
|
if re.match(r'^metadata:', line):
|
|
in_metadata = True
|
|
continue
|
|
if in_metadata and re.match(r'^ source_keys:', line):
|
|
in_source_keys = True
|
|
continue
|
|
if not in_metadata and re.match(r'^source_keys:', line):
|
|
in_source_keys = True
|
|
continue
|
|
if in_source_keys:
|
|
m = re.match(r'^[ \t]+-\s+(\S+)', line)
|
|
if m:
|
|
keys.append(m.group(1).strip())
|
|
elif line and not line[0].isspace():
|
|
in_source_keys = False
|
|
in_metadata = False
|
|
return keys
|
|
|
|
def parse_h2_slugs(content):
|
|
"""Return list of H2 heading values from a markdown file."""
|
|
return re.findall(r'^## (.+)$', content, re.MULTILINE)
|
|
|
|
def parse_contributing_files(content, slug):
|
|
"""Find the Contributing files for a given slug H2 in content.
|
|
|
|
Both authored forms are accepted, because both are in use across the
|
|
corpus and only recognising the first silently skipped checks 4 and 5
|
|
on every skill using the second:
|
|
|
|
- **Contributing files:** SKILL.md, references/a.md
|
|
|
|
**Contributing files:**
|
|
- SKILL.md (what this source contributed)
|
|
- references/a.md (what this source contributed)
|
|
|
|
Returns a list of paths with any trailing parenthetical note stripped.
|
|
A "(none)" value returns an empty list; a slug with no Contributing
|
|
files entry at all returns None. Note the bullet form's notes may
|
|
themselves contain commas, so the list is built per bullet rather than
|
|
by splitting the joined value.
|
|
"""
|
|
pattern = re.compile(
|
|
r'^## ' + re.escape(slug) + r'\s*\n(.*?)(?=^## |\Z)',
|
|
re.MULTILINE | re.DOTALL
|
|
)
|
|
m = pattern.search(content)
|
|
if not m:
|
|
return None
|
|
block = m.group(1)
|
|
|
|
def strip_note(entry):
|
|
# "references/a.md (why)" -> "references/a.md"
|
|
return re.sub(r'\s*\(.*$', '', entry).strip()
|
|
|
|
# Inline form: value on the same line, comma-separated, no notes.
|
|
cf_m = re.search(r'^\- \*\*Contributing files:\*\* (.+)$', block, re.MULTILINE)
|
|
if cf_m:
|
|
value = cf_m.group(1).strip()
|
|
if value.startswith("(none"):
|
|
return []
|
|
return [p for p in (strip_note(x) for x in value.split(",")) if p]
|
|
|
|
# Bullet form: heading on its own line, one file per following bullet.
|
|
cf_m = re.search(r'^\*\*Contributing files:\*\*\s*$', block, re.MULTILINE)
|
|
if not cf_m:
|
|
return None
|
|
rest = block[cf_m.end():]
|
|
files = []
|
|
for line in rest.splitlines():
|
|
line = line.strip()
|
|
if not line:
|
|
if files:
|
|
break
|
|
continue
|
|
if not line.startswith("- "):
|
|
break
|
|
entry = line[2:].strip()
|
|
if entry.startswith("(none"):
|
|
return []
|
|
entry = strip_note(entry)
|
|
if entry:
|
|
files.append(entry)
|
|
return files
|
|
|
|
def parse_research_doc(content, slug):
|
|
"""Find the Research doc value for a given slug H2 in content."""
|
|
pattern = re.compile(
|
|
r'^## ' + re.escape(slug) + r'\s*\n(.*?)(?=^## |\Z)',
|
|
re.MULTILINE | re.DOTALL
|
|
)
|
|
m = pattern.search(content)
|
|
if not m:
|
|
return None
|
|
block = m.group(1)
|
|
rd_m = re.search(r'^\- \*\*Research doc:\*\* (.+)$', block, re.MULTILINE)
|
|
if not rd_m:
|
|
return None
|
|
return rd_m.group(1).strip()
|
|
|
|
def parse_status(content, slug):
|
|
"""Find the Status value for a given slug H2 in content."""
|
|
pattern = re.compile(
|
|
r'^## ' + re.escape(slug) + r'\s*\n(.*?)(?=^## |\Z)',
|
|
re.MULTILINE | re.DOTALL
|
|
)
|
|
m = pattern.search(content)
|
|
if not m:
|
|
return None
|
|
block = m.group(1)
|
|
st_m = re.search(r'^\- \*\*Status:\*\* (.+)$', block, re.MULTILINE)
|
|
if not st_m:
|
|
return None
|
|
return st_m.group(1).strip()
|
|
|
|
def find_repo_root(start_dir):
|
|
"""Walk up from start_dir until we find a directory containing .git."""
|
|
current = start_dir
|
|
while True:
|
|
if os.path.exists(os.path.join(current, ".git")):
|
|
return current
|
|
parent = os.path.dirname(current)
|
|
if parent == current:
|
|
return None
|
|
current = parent
|
|
|
|
findings = []
|
|
has_fail = False
|
|
|
|
def emit_fail(desc, fpath, why, fix):
|
|
global has_fail
|
|
has_fail = True
|
|
findings.append(("FAIL", desc, fpath, why, fix, None))
|
|
|
|
def emit_info(desc, fpath, note):
|
|
findings.append(("INFO", desc, fpath, None, None, note))
|
|
|
|
def print_findings():
|
|
for entry in findings:
|
|
kind = entry[0]
|
|
desc = entry[1]
|
|
fpath = entry[2]
|
|
why = entry[3]
|
|
fix = entry[4]
|
|
note = entry[5]
|
|
if kind == "FAIL":
|
|
print(f"FAIL {desc} — {fpath}")
|
|
print(f" Why: {why}")
|
|
print(f" Fix: {fix}")
|
|
print()
|
|
else:
|
|
print(f"INFO {desc} — {fpath}")
|
|
print(f" Note: {note}")
|
|
print()
|
|
|
|
# --- Scan for any file with source_keys ---
|
|
|
|
def file_has_source_keys(fpath):
|
|
try:
|
|
with open(fpath) as f:
|
|
content = f.read()
|
|
except Exception:
|
|
return False
|
|
fm, _ = parse_frontmatter(content)
|
|
if fm is None:
|
|
return False
|
|
return bool(parse_source_keys(fm))
|
|
|
|
def find_files_with_source_keys():
|
|
"""Return list of (relative_path, abs_path) for all skill files with source_keys."""
|
|
results = []
|
|
for root, dirs, files in os.walk(skill_dir):
|
|
# Skip hidden dirs
|
|
dirs[:] = [d for d in dirs if not d.startswith('.')]
|
|
for fname in files:
|
|
if fname.endswith('.md'):
|
|
abs_path = os.path.join(root, fname)
|
|
if file_has_source_keys(abs_path):
|
|
rel = os.path.relpath(abs_path, skill_dir)
|
|
results.append((rel, abs_path))
|
|
return results
|
|
|
|
sources_md_exists = os.path.isfile(sources_md_path)
|
|
files_with_source_keys = find_files_with_source_keys()
|
|
|
|
# Early exit: nothing to validate
|
|
if not sources_md_exists and not files_with_source_keys:
|
|
sys.exit(0)
|
|
|
|
# Load sources.md if it exists
|
|
sources_content = None
|
|
if sources_md_exists:
|
|
with open(sources_md_path) as f:
|
|
sources_content = f.read()
|
|
sources_slugs = set(parse_h2_slugs(sources_content))
|
|
else:
|
|
sources_slugs = set()
|
|
|
|
# --- Check 0: source_keys without sources.md ---
|
|
if not sources_md_exists:
|
|
for rel, abs_path in files_with_source_keys:
|
|
emit_fail(
|
|
f"source_keys declared but references/sources.md is absent",
|
|
rel,
|
|
"source_keys references research provenance that has no sources index to validate against.",
|
|
"Create references/sources.md with an H2 entry for each slug referenced by source_keys."
|
|
)
|
|
print_findings()
|
|
sys.exit(1)
|
|
|
|
# --- Check 1: FILL IN: placeholders in sources.md ---
|
|
for line in sources_content.splitlines():
|
|
if PLACEHOLDER_RE.search(line):
|
|
emit_fail(
|
|
"Unfilled FILL IN: placeholder",
|
|
"references/sources.md",
|
|
"sources.md contains an unfilled placeholder, meaning provenance is incomplete.",
|
|
"Replace all 'FILL IN:' values in references/sources.md with real content."
|
|
)
|
|
break
|
|
|
|
# --- Check 2: source_keys in SKILL.md → slug exists in sources.md ---
|
|
skill_md_path = os.path.join(skill_dir, "SKILL.md")
|
|
if os.path.isfile(skill_md_path):
|
|
with open(skill_md_path) as f:
|
|
skill_content = f.read()
|
|
skill_fm, _ = parse_frontmatter(skill_content)
|
|
skill_source_keys = parse_source_keys(skill_fm)
|
|
for slug in skill_source_keys:
|
|
if slug not in sources_slugs:
|
|
emit_fail(
|
|
f"source_keys slug '{slug}' not found in sources.md",
|
|
"SKILL.md",
|
|
f"SKILL.md declares '{slug}' as a source but there is no '## {slug}' heading in references/sources.md.",
|
|
f"Add '## {slug}' entry to references/sources.md or remove '{slug}' from SKILL.md source_keys."
|
|
)
|
|
|
|
# --- Check 3: source_keys in references/*.md → slug exists in sources.md (INFO if no source_keys) ---
|
|
if os.path.isdir(refs_dir):
|
|
for fname in sorted(os.listdir(refs_dir)):
|
|
if not fname.endswith('.md'):
|
|
continue
|
|
if fname == "sources.md":
|
|
continue
|
|
fpath = os.path.join(refs_dir, fname)
|
|
rel = os.path.relpath(fpath, skill_dir)
|
|
with open(fpath) as f:
|
|
ref_content = f.read()
|
|
ref_fm, _ = parse_frontmatter(ref_content)
|
|
ref_keys = parse_source_keys(ref_fm)
|
|
if not ref_keys:
|
|
emit_info(
|
|
f"No source_keys frontmatter",
|
|
rel,
|
|
"This references file has no source_keys — provenance cannot be verified. "
|
|
"Add source_keys frontmatter listing the slugs from references/sources.md that informed this file."
|
|
)
|
|
else:
|
|
for slug in ref_keys:
|
|
if slug not in sources_slugs:
|
|
emit_fail(
|
|
f"source_keys slug '{slug}' not found in sources.md",
|
|
rel,
|
|
f"'{rel}' declares '{slug}' as a source but there is no '## {slug}' heading in references/sources.md.",
|
|
f"Add '## {slug}' entry to references/sources.md or remove '{slug}' from {rel} source_keys."
|
|
)
|
|
|
|
# --- Checks 4, 5, 6, 7, 8: Per-slug checks in sources.md ---
|
|
|
|
repo_root = find_repo_root(skill_dir)
|
|
|
|
# Collect all research doc paths we'll check (for Check 8)
|
|
research_docs_seen = {} # abs_path → set of slugs in sources.md that reference it
|
|
|
|
for slug in parse_h2_slugs(sources_content):
|
|
# Check 4: Contributing files exist
|
|
cf_files = parse_contributing_files(sources_content, slug)
|
|
if cf_files:
|
|
for cf_rel in cf_files:
|
|
cf_abs = os.path.join(skill_dir, cf_rel)
|
|
if not os.path.isfile(cf_abs):
|
|
emit_fail(
|
|
f"Contributing file '{cf_rel}' does not exist",
|
|
f"references/sources.md (## {slug})",
|
|
f"sources.md claims '{cf_rel}' was contributed to by slug '{slug}' but the file does not exist.",
|
|
f"Create '{cf_rel}' relative to the skill directory, or correct the path in sources.md."
|
|
)
|
|
else:
|
|
# Check 5: Bidirectional — file should list slug in its source_keys
|
|
# Skip sources.md itself
|
|
if cf_rel == "references/sources.md":
|
|
continue
|
|
with open(cf_abs) as f:
|
|
cf_content = f.read()
|
|
cf_fm, _ = parse_frontmatter(cf_content)
|
|
cf_keys = parse_source_keys(cf_fm)
|
|
if slug not in cf_keys:
|
|
emit_fail(
|
|
f"Contributing file '{cf_rel}' does not list '{slug}' in its source_keys",
|
|
f"references/sources.md (## {slug})",
|
|
f"sources.md says '{cf_rel}' was informed by '{slug}', but '{cf_rel}' does not declare '{slug}' in its source_keys frontmatter.",
|
|
f"Add '{slug}' to the source_keys frontmatter of '{cf_rel}'."
|
|
)
|
|
|
|
# Check 6: Research doc field required
|
|
rd_value = parse_research_doc(sources_content, slug)
|
|
if rd_value is None:
|
|
emit_fail(
|
|
f"Research doc field missing",
|
|
f"references/sources.md (## {slug})",
|
|
f"The '## {slug}' entry in sources.md has no '- **Research doc:**' line.",
|
|
f"Add '- **Research doc:** <path-or-(none)>' to the '## {slug}' entry in references/sources.md."
|
|
)
|
|
elif rd_value == "" or PLACEHOLDER_RE.search(rd_value):
|
|
emit_fail(
|
|
f"Research doc field is empty or placeholder",
|
|
f"references/sources.md (## {slug})",
|
|
f"The '## {slug}' entry has an unfilled Research doc value.",
|
|
f"Set '- **Research doc:**' to a real path relative to repo root, or '(none)' if not applicable."
|
|
)
|
|
else:
|
|
# Check 7: Upstream forward — slug should appear in research doc
|
|
if repo_root and not rd_value.startswith("(none"):
|
|
rd_abs = os.path.join(repo_root, rd_value)
|
|
if os.path.isfile(rd_abs):
|
|
with open(rd_abs) as f:
|
|
rd_content = f.read()
|
|
rd_slugs = set(parse_h2_slugs(rd_content))
|
|
if slug not in rd_slugs:
|
|
emit_info(
|
|
f"Slug '{slug}' not found as H2 in research doc '{rd_value}'",
|
|
f"references/sources.md (## {slug})",
|
|
f"The research doc '{rd_value}' does not have a '## {slug}' heading. "
|
|
f"The provenance link may be imprecise — the slug name in sources.md may differ from the research doc's heading."
|
|
)
|
|
# Track for Check 8
|
|
if rd_abs not in research_docs_seen:
|
|
research_docs_seen[rd_abs] = (rd_value, set())
|
|
research_docs_seen[rd_abs][1].add(slug)
|
|
|
|
# --- Check 8: Upstream reverse ---
|
|
for rd_abs, (rd_rel, known_slugs) in research_docs_seen.items():
|
|
with open(rd_abs) as f:
|
|
rd_content = f.read()
|
|
for rd_slug in parse_h2_slugs(rd_content):
|
|
# Parse this slug's Contributing files and Status in the research doc
|
|
rd_cf = parse_contributing_files(rd_content, rd_slug)
|
|
rd_status = parse_status(rd_content, rd_slug)
|
|
# Skip if the research doc explicitly records no contributing files
|
|
if rd_cf == []:
|
|
continue
|
|
# Skip if status is not `extracted`
|
|
if rd_status != "`extracted`":
|
|
continue
|
|
# This slug should be in sources.md
|
|
if rd_slug not in sources_slugs:
|
|
emit_fail(
|
|
f"Research doc slug '{rd_slug}' missing from skill sources.md",
|
|
f"references/sources.md",
|
|
f"The research doc '{rd_rel}' has '## {rd_slug}' with status `extracted` and contributing files, "
|
|
f"but this skill's sources.md has no '## {rd_slug}' entry.",
|
|
f"Add '## {rd_slug}' to references/sources.md or mark it as '(none)' in the research doc's Contributing files."
|
|
)
|
|
|
|
print_findings()
|
|
sys.exit(1 if has_fail else 0)
|
|
PYTHON
|