#!/usr/bin/env bash set -euo pipefail usage() { cat < Validate that an agent's sources provenance chain is complete and internally consistent. Operates at plugin/APM scope only (a single vendor-neutral .apm/agents/.agent.md inside a package with a type:-bearing apm.yml) — exits 0 silently for project and user scope agents. Arguments: agent-file Path to either the Claude Code .md or Copilot .agent.md agent file. Exit codes: 0 All checks passed (or nothing to validate, or not plugin scope) 1 One or more checks failed 2 Usage error, or the argument is not an agent file this script can read An exit code of 2 is NOT a finding. SKILL.md tells the auditor to surface a non-zero exit as findings, so a usage error leaving exit 1 with nothing on stdout was indistinguishable from a clean-but-failing run. Environment and argument problems exit 2; only real findings exit 1. Exit 2 and the silent exit 0 answer two DIFFERENT questions, and neither may be spelled with the other's code: exit 2 the argument is not something this script can audit at all — it is missing, doubled, not a file, or not named .md / .agent.md. Decided before the scope walk-up runs, from the argument alone. exit 0 the argument IS a readable agent file, and the scope walk-up found no type:-bearing apm.yml above it before hitting the \$HOME, .git or filesystem-root boundary. That is a real verdict about a real file — "this agent is user or project scope, so plugin-scope provenance does not apply to it" — not a rejected input. scripts/check-scope-walkup-sync.sh's fixture 6 pins the second: a real agent file under a \$HOME with a type-bearing apm.yml ABOVE it must exit 0 with empty output. Widening exit 2 to cover "the walk-up found no package" would break that fixture AND would be wrong on its own terms, because new-agent.sh happily scaffolds exactly that layout. Checks performed: 0 source_keys present in agent pair but sources.md absent 1 FILL IN: placeholders in sources.md 2 source_keys in agent files → slug exists in sources.md 3 Contributing files listed in sources.md exist on disk (plugin-root relative). An explicit '(none)' skips silently; a Contributing files block this parser cannot read is reported as an INFO saying checks 3 and 4 did not run, never skipped silently. 4 Contributing files back-reference the parent slug in their source_keys 5 Research doc field present and not placeholder This script has no counterpart to skill-audit's checks 6, 7 and 8 (Research doc field / upstream forward / upstream reverse are numbered 6, 7, 8 there and 5 here): an agent at plugin scope is a single file with a plugin-root sources.md, so there is no references/ tree to walk and no upstream research source index to cross-check. parse_status() and the sources.md-basename gate that those checks need exist only in the skill-audit copy. EOF } if [[ "${1:-}" == "--help" || "${1:-}" == "-h" ]]; then usage exit 0 fi # Usage and environment problems exit 2, findings exit 1. See the usage text # above for why the two must not share a code, and for why "not plugin scope" # is neither of them. This is a deliberate divergence from validate.sh, which # has no 2 tier for content: validate.sh always prints PASS lines, so a usage # error there is visibly not a findings report. This script prints NOTHING on a # clean run, so exit 1 plus empty stdout was the only signal a caller got # either way. if [[ $# -lt 1 ]]; then echo "Error: agent-file is required." >&2 echo "" >&2 usage >&2 exit 2 fi # Extra positional arguments were silently dropped, so a typo'd flag or a second # path looked like it had been honoured. if [[ $# -gt 1 ]]; then echo "Error: expected exactly one argument, got $#: $*" >&2 echo "" >&2 usage >&2 exit 2 fi # python3 is a HARD dependency. Without this preflight a missing interpreter # produced 'line NN: python3: command not found' and exit 127 — an exit code no # caller maps to anything, from a message that names this script's line number # rather than the missing dependency. if ! command -v python3 > /dev/null 2>&1; then echo "Error: python3 is required but was not found on PATH." >&2 echo " Why: skipping the provenance checks entirely would be a vacuous pass." >&2 echo " Fix: install python3 (pre-commit itself is a Python application, so it is almost certainly already present)." >&2 exit 2 fi # A path that does not exist, or exists but is not a regular file, used to reach # the Python body, get os.path.dirname()'d into some ancestor directory and then # either report a silent exit 0 (no package above it) or — worse — audit a # DIFFERENT agent's package while naming the typo'd path. A typo'd target was # indistinguishable from a clean agent. vale-wrap.sh hard-errors on a # nonexistent path for exactly this reason. # # This is decided from the argument alone, before any walk-up runs, so it cannot # collide with the not-plugin-scope exit 0: that verdict is only ever reached by # a file that got past here. if [[ ! -e "$1" ]]; then echo "Error: no such file: $1" >&2 echo " Why: a nonexistent target would otherwise report a silent pass." >&2 echo " Fix: pass the path of the agent file to validate." >&2 exit 2 fi if [[ ! -f "$1" ]]; then echo "Error: not a regular file: $1" >&2 echo " Why: this script audits one agent file, not a directory of them, and reporting a directory as a pass hides the wrong-target mistake." >&2 echo " Fix: pass the agent file itself — .apm/agents/.agent.md — not its parent directory." >&2 exit 2 fi # The extension check used to live inside the Python body. It stays exit 2 and # keeps its wording; it moves up here so that every "this argument is not # auditable" verdict is reached in one place, before the interpreter starts and # before the scope walk-up can turn a bad argument into a silent exit 0. case "$1" in *.agent.md | *.md) ;; *) echo "Error: unrecognized extension '$(basename "$1")' — expected .md or .agent.md" >&2 exit 2 ;; esac python3 -u - "$1" <<'PYTHON' import sys import os import re # Output is UTF-8 for the same reason input is: under LC_ALL=C the streams # default to ASCII, and every finding this script prints contains an em dash. # Pinning only the reads moved the crash from the read to the write — a # UnicodeEncodeError inside print_findings(), which loses the whole report # after all the checks have already run. for _stream in (sys.stdout, sys.stderr): try: _stream.reconfigure(encoding='utf-8') except AttributeError: # pragma: no cover — Python < 3.7 pass agent_file = os.path.abspath(sys.argv[1]) agent_dir = os.path.dirname(agent_file) # --- Input ---------------------------------------------------------------- # Ported from the skill-audit copy, where the same two problems were already # fixed. # # read_text() pins UTF-8 explicitly instead of inheriting # locale.getpreferredencoding(), which is ASCII under LC_ALL=C — an ordinary em # dash in an agent file or in sources.md then aborted the run with a bare # UnicodeDecodeError traceback, or, at the one call site that wrapped its read # in `except Exception: return []`, reported the unreadable file as having no # source_keys and therefore as clean. A file that genuinely is not UTF-8 still # fails; it just says which file and why. # # strip_bom() runs on every read because a leading BOM defeats # parse_frontmatter()'s `^---` anchor, which silently disabled check 2 on a # BOM-prefixed agent file: no frontmatter parsed means no source_keys parsed # means nothing to validate. class EncodingError(Exception): pass def strip_bom(text): return text[1:] if text.startswith(u'\ufeff') else text def read_text(path): """File contents as text, UTF-8 and BOM-free, with a diagnostic instead of a traceback.""" try: with open(path, encoding='utf-8') as fh: return strip_bom(fh.read()) except UnicodeDecodeError as exc: raise EncodingError( "not valid UTF-8 (%s at byte %d) — re-save the file as UTF-8; " "this gate does not guess at other encodings" % (exc.reason, exc.start)) # Matches a top-level `type:` line whose value is exactly one of the four # package content types — identical to validate.sh's APM_TYPE_RE. Group 1's # optional quote must be closed by \1 (or nothing), and the value must be # followed by whitespace/end-of-line so a malformed value like `prompts-only` # doesn't false-match on the `prompts` prefix. TYPE_RE = re.compile(r"^type:\s*(['\"]?)(instructions|skill|hybrid|prompts)\1(?:\s|$)") # --- Find package root: walk up for the nearest ancestor apm.yml that # declares a top-level type: field. An apm.yml with no type: field is a # marketplace-only manifest (see monorepo-and-repo-shapes.md) — skip it and # keep walking. Stop at a $HOME boundary, a .git boundary, or the filesystem # root: none of these is plugin/APM scope, so this script has nothing to # check there. # # Returning None here means NOT PLUGIN SCOPE, which is a verdict, not an error: # the caller exits 0 silently, and scripts/check-scope-walkup-sync.sh fixture 6 # pins that. It is deliberately NOT folded into the exit-2 tier above. def find_plugin_root(start_dir): home = os.path.expanduser('~') current = os.path.abspath(start_dir) while True: apm_yml = os.path.join(current, 'apm.yml') if os.path.isfile(apm_yml): # An apm.yml is a manifest this script must be able to READ to # classify scope at all. Under LC_ALL=C the old bare open() decoded # as ASCII, so a manifest with an accented author name raised # UnicodeDecodeError mid-walk and killed the run with a traceback. # It is an environment problem, not a finding, so it exits 2 rather # than being swallowed into a silent "no package here". try: content = read_text(apm_yml) except EncodingError as exc: print( "Error: %s is %s" % (apm_yml, exc), file=sys.stderr) sys.exit(2) if any(TYPE_RE.match(line) for line in content.splitlines()): return current # $HOME is a non-plugin-scope boundary — checked before the .git test # below (mirrors validate.sh's detect_scope ordering), so a # dotfiles-managed $HOME (yadm, chezmoi bare-repo, etc.) can't shadow # this check by being its own .git repo. Without this, the walk could # continue past $HOME toward the filesystem root looking for a # type-bearing apm.yml, misclassifying a user/project-scope file as # plugin scope in rare ancestor layouts. if current == home: return None # .git is a directory in a normal checkout but a file (`gitdir: ...`) # in a git worktree — exists() covers both. if os.path.exists(os.path.join(current, '.git')): return None parent = os.path.dirname(current) if parent == current: return None current = parent plugin_root = find_plugin_root(agent_dir) if plugin_root is None: sys.exit(0) sources_md_path = os.path.join(plugin_root, 'sources.md') # --- Helpers --- # The trailing character class used to be CONSUMING — `[^`\n]` — so a # `FILL IN:` at end of line matched nothing and escaped checks 1 and 5 # entirely. `- **Description:** FILL IN:` is the most likely spelling of a # half-written entry, and it was the one spelling the placeholder gate could # not see. The exclusion it was really expressing is "not inside backticks", # which a lookahead states without eating a character. PLACEHOLDER_RE = re.compile(r'(? "references/a.md" return re.sub(r'\s*\(.*$', '', entry).strip() # Inline form: value on the same line, comma-separated, no notes. cf_m = re.search(r'^\- \*\*Contributing files:\*\* (.+)$', block, re.MULTILINE) if cf_m: value = cf_m.group(1).strip() if value.startswith("(none"): return [] return [p for p in (strip_note(x) for x in value.split(",")) if p] or None # Bullet form: heading on its own line, one file per following bullet. cf_m = re.search(r'^\*\*Contributing files:\*\*\s*$', block, re.MULTILINE) if not cf_m: return None files = [] for line in block[cf_m.end():].splitlines(): line = line.strip() if not line: if files: break continue if not line.startswith("- "): break entry = line[2:].strip() if entry.startswith("(none"): return [] entry = strip_note(entry) if entry: files.append(entry) return files or None # ===== END SHARED CONTRIBUTING-FILES PARSER ===== def parse_research_docs(content, slug): """Every Research doc value under a given slug H2, in document order. The caller uses the first and reports the rest. Returning only the first — what this did before — meant a second '- **Research doc:**' line in one entry was silently ignored, so an author who added a doc rather than replacing one got check 5 run against the old value and no hint that the new one was never looked at. """ pattern = re.compile( r'^## ' + re.escape(slug) + r'\s*\n(.*?)(?=^## |\Z)', re.MULTILINE | re.DOTALL ) m = pattern.search(content) if not m: return [] block = m.group(1) return [v.strip() for v in re.findall(r'^\- \*\*Research doc:\*\* (.+)$', block, re.MULTILINE)] findings = [] has_fail = False # A finding identical in every field is the same finding, and the same file is # now reached by more than one check — the agent file is read once for its own # source_keys and again as a contributing file, so an unreadable one would # otherwise be reported twice with the same words. Distinct findings about the # same file still both appear. def _record(entry): if entry not in findings: findings.append(entry) def emit_fail(desc, fpath, why, fix): global has_fail has_fail = True _record(("FAIL", desc, fpath, why, fix, None)) # INFO does not set has_fail and does not change the exit code. It is for a # check that could not RUN — an unverified entry, not a broken one — and it # exists so that "did not run" is never spelled the same way as "passed". def emit_info(desc, fpath, note): _record(("INFO", desc, fpath, None, None, note)) def print_findings(): for entry in findings: kind = entry[0] desc = entry[1] fpath = entry[2] why = entry[3] fix = entry[4] note = entry[5] if kind == "FAIL": print(f"FAIL {desc} — {fpath}") print(f" Why: {why}") print(f" Fix: {fix}") print() else: print(f"INFO {desc} — {fpath}") print(f" Note: {note}") print() def emit_unreadable(rel, exc): """Report a file this script cannot decode. Never a silent skip.""" emit_fail( f"File is {exc}", rel, f"'{rel}' cannot be decoded, so its frontmatter — and any source_keys in it — " f"cannot be read. This used to be swallowed by a bare 'except Exception: return []', " f"which reported the unreadable file as having no source_keys and therefore as clean.", f"Re-save '{rel}' as UTF-8." ) # --- Collect source_keys from agent pair --- def get_source_keys_from_file(fpath, rel): if not os.path.isfile(fpath): return [] try: content = read_text(fpath) except EncodingError as exc: emit_unreadable(rel, exc) return [] fm, _ = parse_frontmatter(content) return parse_source_keys(fm) # Plugin/APM scope is a single vendor-neutral file — no counterpart to merge. rel_given = os.path.relpath(agent_file, plugin_root) given_keys = get_source_keys_from_file(agent_file, rel_given) all_source_keys = given_keys sources_md_exists = os.path.isfile(sources_md_path) # Early exit: nothing to validate. The read above can itself raise a finding — # an unreadable agent file — so print before leaving; the clean case still # prints nothing and exits 0. if not all_source_keys and not sources_md_exists: print_findings() sys.exit(1 if has_fail else 0) sources_content = None sources_slugs = set() if sources_md_exists: try: sources_content = read_text(sources_md_path) except EncodingError as exc: emit_unreadable("sources.md", exc) print_findings() sys.exit(1) sources_slugs = set(parse_h2_slugs(sources_content)) # --- Check 0: source_keys present but sources.md absent --- if not sources_md_exists and all_source_keys: emit_fail( "source_keys declared but sources.md is absent", rel_given, "source_keys references research provenance that has no sources index to validate against.", "Create sources.md with an H2 entry for each slug referenced by source_keys." ) print_findings() sys.exit(1) # --- Check 1: FILL IN: placeholders in sources.md --- for line in sources_content.splitlines(): if PLACEHOLDER_RE.search(line): emit_fail( "Unfilled FILL IN: placeholder", "sources.md", "sources.md contains an unfilled placeholder, meaning provenance is incomplete.", "Replace all 'FILL IN:' values in sources.md with real content." ) break # --- Check 2: source_keys in the agent file → slug exists in sources.md --- for fpath, keys in [(agent_file, given_keys)]: if not keys: continue rel = os.path.relpath(fpath, plugin_root) for slug in keys: if slug not in sources_slugs: emit_fail( f"source_keys slug '{slug}' not found in sources.md", rel, f"'{rel}' declares '{slug}' as a source but there is no '## {slug}' heading in sources.md.", f"Add '## {slug}' entry to sources.md or remove '{slug}' from {rel} source_keys." ) # --- Checks 3, 4, 5: Per-slug checks in sources.md --- # Every per-slug parser below — parse_contributing_files, parse_research_docs — # locates its block with pattern.search(), so a slug written twice resolves to # the FIRST block every time. Iterating the raw heading list therefore checked # the first block's fields twice and the second block's never: a duplicated slug # is half-validated, and looked fully validated. The duplicate is announced and # the repeat visit dropped. all_slugs = parse_h2_slugs(sources_content) unique_slugs = [] for _slug in all_slugs: if _slug in unique_slugs: continue unique_slugs.append(_slug) _count = all_slugs.count(_slug) if _count > 1: emit_info( f"Duplicate '## {_slug}' entry in sources.md — only the first block is checked", f"sources.md (## {_slug})", f"'## {_slug}' appears {_count} times. Every field parser here takes the first match, so the " f"second and later blocks' Contributing files and Research doc are never validated — " f"checks 3, 4 and 5 did not run for them. " f"Merge the blocks into one entry, or give each a distinct slug and reference it from source_keys." ) for slug in unique_slugs: # Checks 3 and 4: Contributing files exist (paths relative to plugin root), # and back-reference the slug. `[]` and None are NOT the same answer here. # `[]` is the author writing "(none)" — there is nothing to check and the # skip is correct. None is a Contributing-files block this parser cannot # read, and skipping THAT silently disables both checks on the one entry # least likely to be right, which is the failure mode # parse_contributing_files' own docstring warns about. Say so out loud. cf_files = parse_contributing_files(sources_content, slug) if cf_files is None: emit_info( f"Contributing-file checks skipped for '{slug}' — the Contributing files block could not be parsed", f"sources.md (## {slug})", f"The '## {slug}' entry has no Contributing files list this parser can read — a missing field, a bare heading, '*' bullets, a numbered list, or prose all read as unparsable rather than as an empty declaration. " f"Checks 3 and 4 did not run for this slug, so nothing verified that its contributing files exist or name it back. " f"Write the value as '- **Contributing files:** ', or as a '**Contributing files:**' heading followed by '- ' bullets — " f"or record '(none)' if this source contributed no files." ) elif cf_files: for cf_rel in cf_files: cf_abs = os.path.join(plugin_root, cf_rel) if not os.path.isfile(cf_abs): emit_fail( f"Contributing file '{cf_rel}' does not exist", f"sources.md (## {slug})", f"sources.md claims '{cf_rel}' was contributed to by slug '{slug}' but the file does not exist.", f"Create '{cf_rel}' relative to the plugin root, or correct the path in sources.md." ) else: # Check 4: Bidirectional — file should list slug in its source_keys try: cf_content = read_text(cf_abs) except EncodingError as exc: emit_unreadable(cf_rel, exc) continue cf_fm, _ = parse_frontmatter(cf_content) cf_keys = parse_source_keys(cf_fm) if slug not in cf_keys: emit_fail( f"Contributing file '{cf_rel}' does not list '{slug}' in its source_keys", f"sources.md (## {slug})", f"sources.md says '{cf_rel}' was informed by '{slug}', but '{cf_rel}' does not declare '{slug}' in its top-level source_keys.", f"Add '{slug}' to the top-level source_keys frontmatter in '{cf_rel}'." ) # Check 5: Research doc field required rd_values = parse_research_docs(sources_content, slug) if len(rd_values) > 1: emit_info( f"Multiple '- **Research doc:**' lines for '{slug}' — only the first is used", f"sources.md (## {slug})", f"The '## {slug}' entry has {len(rd_values)} Research doc lines; check 5 ran against the first " f"('{rd_values[0]}') and never looked at the rest. " f"Keep one Research doc line per entry — if a slug genuinely came from two documents, split it into two slugs, " f"or name the extra document inside the first value's annotation where it is at least visible." ) rd_value = rd_values[0] if rd_values else None if rd_value is None: emit_fail( "Research doc field missing", f"sources.md (## {slug})", f"The '## {slug}' entry in sources.md has no '- **Research doc:**' line.", f"Add '- **Research doc:** ' to the '## {slug}' entry in sources.md." ) elif rd_value == "" or PLACEHOLDER_RE.search(rd_value): emit_fail( "Research doc field is empty or placeholder", f"sources.md (## {slug})", f"The '## {slug}' entry has an unfilled Research doc value.", "Set '- **Research doc:**' to a real path relative to repo root, or '(none)' if not applicable." ) print_findings() sys.exit(1 if has_fail else 0) PYTHON