feat(kyberforge): restructure agent-audit for plugin-scope apm agents
Validates the new single-file .apm/agents/<name>.agent.md shape agent-author now produces at plugin/APM scope: frontmatter allowlist (name/description/ model only, from a new apm-agent-allowlist entry in field-inventory.md), no counterpart derivation, and Pair Consistency dropped from that scope's report entirely (nothing to pair by design). Adds a plugin/APM-scope-only SUGGESTION when an agent's description/body implies a tool restriction or Claude-only behavior the vendor-neutral frontmatter can no longer express (ADR-0016). Scope detection in both validate.sh and validate-provenance.sh switches from a flat plugin.json/.claude-plugin/plugin.json check to a walk-up for the nearest ancestor apm.yml with a top-level type: field, skipping type:-less marketplace-only manifests — full switch, no dual-mode fallback to the old plugin.json signal. validate-provenance.sh's walk-up was fixed to match validate.sh's (it still used the old plugin.json check, and its counterpart-merge logic was rewritten to read a single file's source_keys instead of merging a CC+Copilot pair, since plugin/APM scope has no counterpart). Project/user scope validation is unchanged in both scripts. Refs: #89
This commit is contained in:
@@ -5,8 +5,10 @@ usage() {
|
||||
cat <<EOF
|
||||
Usage: validate-provenance.sh <agent-file>
|
||||
|
||||
Validate that an agent pair's sources provenance chain is complete and internally consistent.
|
||||
Operates at plugin scope only — exits 0 silently for project and user scope agents.
|
||||
Validate that an agent's sources provenance chain is complete and internally consistent.
|
||||
Operates at plugin/APM scope only (a single vendor-neutral .apm/agents/<name>.agent.md
|
||||
inside a package with a type:-bearing apm.yml) — exits 0 silently for project and user
|
||||
scope agents.
|
||||
|
||||
Arguments:
|
||||
agent-file Path to either the Claude Code .md or Copilot .agent.md agent file.
|
||||
@@ -47,23 +49,28 @@ agent_file = os.path.abspath(sys.argv[1])
|
||||
fname = os.path.basename(agent_file)
|
||||
agent_dir = os.path.dirname(agent_file)
|
||||
|
||||
# --- Detect provider ---
|
||||
if fname.endswith('.agent.md'):
|
||||
provider = 'copilot'
|
||||
name_stem = fname[:-len('.agent.md')]
|
||||
elif fname.endswith('.md'):
|
||||
provider = 'claude-code'
|
||||
name_stem = fname[:-len('.md')]
|
||||
else:
|
||||
# --- Sanity-check extension (single vendor-neutral .agent.md file at plugin/APM scope) ---
|
||||
if not (fname.endswith('.agent.md') or fname.endswith('.md')):
|
||||
print(f"Error: unrecognized extension '{fname}' — expected .md or .agent.md", file=sys.stderr)
|
||||
sys.exit(2)
|
||||
|
||||
# --- Find plugin root ---
|
||||
TYPE_RE = re.compile(r'^type:\s*(instructions|skill|hybrid|prompts)\b')
|
||||
|
||||
# --- Find package root: walk up for the nearest ancestor apm.yml that
|
||||
# declares a top-level type: field. An apm.yml with no type: field is a
|
||||
# marketplace-only manifest (see monorepo-and-repo-shapes.md) — skip it and
|
||||
# keep walking. Stop at a .git boundary or the filesystem root: neither is
|
||||
# plugin/APM scope, so this script has nothing to check there.
|
||||
def find_plugin_root(start_dir):
|
||||
current = os.path.abspath(start_dir)
|
||||
while True:
|
||||
if (os.path.isfile(os.path.join(current, 'plugin.json')) or os.path.isfile(os.path.join(current, '.claude-plugin', 'plugin.json'))):
|
||||
return current
|
||||
apm_yml = os.path.join(current, 'apm.yml')
|
||||
if os.path.isfile(apm_yml):
|
||||
with open(apm_yml) as f:
|
||||
if any(TYPE_RE.match(line) for line in f):
|
||||
return current
|
||||
if os.path.isdir(os.path.join(current, '.git')):
|
||||
return None
|
||||
parent = os.path.dirname(current)
|
||||
if parent == current:
|
||||
return None
|
||||
@@ -73,12 +80,6 @@ plugin_root = find_plugin_root(agent_dir)
|
||||
if plugin_root is None:
|
||||
sys.exit(0)
|
||||
|
||||
# --- Derive counterpart ---
|
||||
if provider == 'copilot':
|
||||
counterpart = os.path.join(agent_dir, name_stem + '.md')
|
||||
else:
|
||||
counterpart = os.path.join(agent_dir, name_stem + '.agent.md')
|
||||
|
||||
sources_md_path = os.path.join(plugin_root, 'sources.md')
|
||||
|
||||
# --- Helpers ---
|
||||
@@ -166,15 +167,9 @@ def get_source_keys_from_file(fpath):
|
||||
fm, _ = parse_frontmatter(content)
|
||||
return parse_source_keys(fm)
|
||||
|
||||
# Plugin/APM scope is a single vendor-neutral file — no counterpart to merge.
|
||||
given_keys = get_source_keys_from_file(agent_file)
|
||||
counterpart_keys = get_source_keys_from_file(counterpart)
|
||||
# Deduplicated union, preserving order
|
||||
seen = set()
|
||||
all_source_keys = []
|
||||
for k in given_keys + counterpart_keys:
|
||||
if k not in seen:
|
||||
seen.add(k)
|
||||
all_source_keys.append(k)
|
||||
all_source_keys = given_keys
|
||||
|
||||
sources_md_exists = os.path.isfile(sources_md_path)
|
||||
|
||||
@@ -212,8 +207,8 @@ for line in sources_content.splitlines():
|
||||
)
|
||||
break
|
||||
|
||||
# --- Check 2: source_keys in agent files → slug exists in sources.md ---
|
||||
for fpath, keys in [(agent_file, given_keys), (counterpart, counterpart_keys)]:
|
||||
# --- Check 2: source_keys in the agent file → slug exists in sources.md ---
|
||||
for fpath, keys in [(agent_file, given_keys)]:
|
||||
if not keys:
|
||||
continue
|
||||
rel = os.path.relpath(fpath, plugin_root)
|
||||
|
||||
@@ -5,10 +5,15 @@ usage() {
|
||||
cat <<EOF
|
||||
Usage: validate.sh <agent-file>
|
||||
|
||||
Validate a Claude Code or Copilot agent file pair against the agent definition spec.
|
||||
Validate an agent definition file against the agent definition spec.
|
||||
|
||||
At plugin/APM scope, <agent-file> is a single vendor-neutral
|
||||
.apm/agents/<name>.agent.md file (frontmatter allowlist: name, description,
|
||||
model — no counterpart file). At project or user scope, <agent-file> is
|
||||
either half of a Claude Code .md / Copilot .agent.md pair.
|
||||
|
||||
Arguments:
|
||||
agent-file Path to either the Claude Code .md or Copilot .agent.md agent file.
|
||||
agent-file Path to the agent file (or either half of a project/user-scope pair).
|
||||
|
||||
Exit codes:
|
||||
0 All checks passed (may include SUGGESTIONs)
|
||||
@@ -74,6 +79,7 @@ def parse_section_tokens(content, section_name):
|
||||
cc_only_fields = parse_section_tokens(inv_content, 'claude-code-only-fields')
|
||||
copilot_only_fields = parse_section_tokens(inv_content, 'copilot-only-fields')
|
||||
plugin_ignored_fields = parse_section_tokens(inv_content, 'plugin-silently-ignored-fields')
|
||||
apm_agent_allowlist = parse_section_tokens(inv_content, 'apm-agent-allowlist')
|
||||
|
||||
# Tools the runtime withholds from subagents regardless of the tools field
|
||||
SUBAGENT_UNAVAILABLE_TOOLS = {
|
||||
@@ -83,48 +89,7 @@ SUBAGENT_UNAVAILABLE_TOOLS = {
|
||||
# Copilot body length limit (chars) — content beyond this is silently truncated
|
||||
COPILOT_BODY_LIMIT = 30000
|
||||
|
||||
# --- Detect scope ---
|
||||
def detect_scope(start_dir):
|
||||
current = os.path.abspath(start_dir)
|
||||
while True:
|
||||
if (os.path.isfile(os.path.join(current, 'plugin.json')) or
|
||||
os.path.isfile(os.path.join(current, '.claude-plugin', 'plugin.json'))):
|
||||
return 'plugin', current
|
||||
if os.path.isdir(os.path.join(current, '.git')):
|
||||
return 'project', current
|
||||
parent = os.path.dirname(current)
|
||||
if parent == current:
|
||||
return 'user', os.path.expanduser('~')
|
||||
current = parent
|
||||
|
||||
agent_dir = os.path.dirname(agent_file)
|
||||
scope, scope_root = detect_scope(agent_dir)
|
||||
|
||||
# --- Derive counterpart path ---
|
||||
if scope == 'plugin':
|
||||
if provider == 'copilot':
|
||||
counterpart = os.path.join(agent_dir, name_stem + '.md')
|
||||
counterpart_provider = 'claude-code'
|
||||
else:
|
||||
counterpart = os.path.join(agent_dir, name_stem + '.agent.md')
|
||||
counterpart_provider = 'copilot'
|
||||
elif scope == 'project':
|
||||
if provider == 'claude-code':
|
||||
counterpart = os.path.join(scope_root, '.github', 'agents', name_stem + '.agent.md')
|
||||
counterpart_provider = 'copilot'
|
||||
else:
|
||||
counterpart = os.path.join(scope_root, '.claude', 'agents', name_stem + '.md')
|
||||
counterpart_provider = 'claude-code'
|
||||
else: # user
|
||||
home = os.path.expanduser('~')
|
||||
if provider == 'claude-code':
|
||||
counterpart = os.path.join(home, '.copilot', 'agents', name_stem + '.agent.md')
|
||||
counterpart_provider = 'copilot'
|
||||
else:
|
||||
counterpart = os.path.join(home, '.claude', 'agents', name_stem + '.md')
|
||||
counterpart_provider = 'claude-code'
|
||||
|
||||
# --- Helpers ---
|
||||
# --- Helpers (shared by every scope) ---
|
||||
failed = False
|
||||
suggestions = []
|
||||
|
||||
@@ -167,6 +132,113 @@ def is_copilot_cloud_ide(fpath):
|
||||
"""True if the file is a cloud/IDE Copilot agent (name is optional for these)."""
|
||||
return '.github/copilot/agents' in os.path.abspath(fpath).replace(os.sep, '/')
|
||||
|
||||
# --- Detect scope ---
|
||||
# APM_TYPE_RE matches a top-level (column-0) `type:` line in apm.yml whose value is
|
||||
# one of the four package content types. `[\'"]?` tolerates a quoted value; the
|
||||
# pattern doesn't anchor the line end, so trailing whitespace/comments don't matter.
|
||||
APM_TYPE_RE = re.compile(r"^type:\s*['\"]?(instructions|skill|hybrid|prompts)\b")
|
||||
|
||||
def find_apm_package_root(apm_yml_path):
|
||||
"""Return True if apm_yml_path has a top-level type: line (i.e. is a package
|
||||
manifest, not a type:-less marketplace-only apm.yml)."""
|
||||
with open(apm_yml_path) as f:
|
||||
for line in f:
|
||||
if APM_TYPE_RE.match(line):
|
||||
return True
|
||||
return False
|
||||
|
||||
def detect_scope(start_dir):
|
||||
current = os.path.abspath(start_dir)
|
||||
while True:
|
||||
apm_yml = os.path.join(current, 'apm.yml')
|
||||
if os.path.isfile(apm_yml) and find_apm_package_root(apm_yml):
|
||||
return 'plugin', current
|
||||
if os.path.isdir(os.path.join(current, '.git')):
|
||||
return 'project', current
|
||||
parent = os.path.dirname(current)
|
||||
if parent == current:
|
||||
return 'user', os.path.expanduser('~')
|
||||
current = parent
|
||||
|
||||
agent_dir = os.path.dirname(agent_file)
|
||||
scope, scope_root = detect_scope(agent_dir)
|
||||
|
||||
# --- Plugin/APM scope: single vendor-neutral file, no counterpart ---
|
||||
def check_apm_agent_file(fpath, allowlist, stem):
|
||||
local_fname = os.path.basename(fpath)
|
||||
with open(fpath) as f:
|
||||
content = f.read()
|
||||
|
||||
fm, body = parse_frontmatter(content)
|
||||
if fm is None:
|
||||
fail(f"no valid YAML frontmatter (---...---) — {local_fname}")
|
||||
return
|
||||
|
||||
# Allowlist: only name/description/model may appear — no tools, no
|
||||
# Claude-only or Copilot-only fields. apm compile verbatim-copies
|
||||
# frontmatter to every target, so anything else is unsafe on at least
|
||||
# one harness (ADR-0016).
|
||||
fm_keys = get_frontmatter_keys(fm)
|
||||
for key in sorted(fm_keys):
|
||||
if key not in allowlist:
|
||||
fail(f"field '{key}' is not in the vendor-neutral APM agent allowlist "
|
||||
f"({', '.join(sorted(allowlist))}) — {local_fname}")
|
||||
|
||||
# name — required, kebab-case, must match filename stem (file is <name>.agent.md)
|
||||
name_val = extract_field(fm, 'name')
|
||||
if not name_val:
|
||||
fail(f"name field is missing or empty — {local_fname}")
|
||||
else:
|
||||
if not re.match(r'^[a-z0-9]+(-[a-z0-9]+)*$', name_val):
|
||||
fail(f"name '{name_val}' is not kebab-case — {local_fname}")
|
||||
if name_val != stem:
|
||||
fail(f"name '{name_val}' does not match filename stem '{stem}' — {local_fname}")
|
||||
|
||||
# description — required, non-empty, no placeholder
|
||||
desc_val = extract_field(fm, 'description')
|
||||
if not desc_val:
|
||||
fail(f"description field is missing or empty — {local_fname}")
|
||||
else:
|
||||
if PLACEHOLDER_RE.search(desc_val):
|
||||
fail(f"description contains unfilled FILL IN: placeholder — {local_fname}")
|
||||
|
||||
# body — required, non-empty, no placeholder; same Copilot truncation risk
|
||||
# applies since this file compiles verbatim into a real Copilot file downstream.
|
||||
if not body.strip():
|
||||
fail(f"system prompt body is empty — {local_fname}")
|
||||
else:
|
||||
if PLACEHOLDER_RE.search(body):
|
||||
fail(f"body contains unfilled FILL IN: placeholder — {local_fname}")
|
||||
if len(body) > COPILOT_BODY_LIMIT:
|
||||
suggest(f"body exceeds {COPILOT_BODY_LIMIT:,} characters ({len(body):,} chars) — "
|
||||
f"content beyond the limit is silently truncated by the Copilot runtime "
|
||||
f"once apm compile emits it downstream — {local_fname}")
|
||||
|
||||
if scope == 'plugin':
|
||||
check_apm_agent_file(agent_file, apm_agent_allowlist, name_stem)
|
||||
for s in suggestions:
|
||||
print(f"SUGGESTION {s}")
|
||||
sys.exit(1 if failed else 0)
|
||||
|
||||
# --- Project/user scope: unchanged CC/Copilot pair validation ---
|
||||
|
||||
# --- Derive counterpart path ---
|
||||
if scope == 'project':
|
||||
if provider == 'claude-code':
|
||||
counterpart = os.path.join(scope_root, '.github', 'agents', name_stem + '.agent.md')
|
||||
counterpart_provider = 'copilot'
|
||||
else:
|
||||
counterpart = os.path.join(scope_root, '.claude', 'agents', name_stem + '.md')
|
||||
counterpart_provider = 'claude-code'
|
||||
else: # user
|
||||
home = os.path.expanduser('~')
|
||||
if provider == 'claude-code':
|
||||
counterpart = os.path.join(home, '.copilot', 'agents', name_stem + '.agent.md')
|
||||
counterpart_provider = 'copilot'
|
||||
else:
|
||||
counterpart = os.path.join(home, '.claude', 'agents', name_stem + '.md')
|
||||
counterpart_provider = 'claude-code'
|
||||
|
||||
def check_file(fpath, file_provider, is_plugin_scope):
|
||||
local_fname = os.path.basename(fpath)
|
||||
with open(fpath) as f:
|
||||
|
||||
Reference in New Issue
Block a user