Files
holocron/plugins/kyberforge/.apm/skills/agent-audit/scripts/validate.sh
Defame1297 4a5c3c0cff feat(kyberforge): enforce the ADR-0020 context contract for skills and agents
Skill name+description pairs are preloaded into every session, costing
~6,200 tokens across 39 skills before any skill is invoked. The authoring
rules mandated that growth: skill-author:104 and description-quality.md:21
both required padding, while skill-author:102 (the deflating rule) had no
FAIL condition behind it.

Gates (blocking, no baseline file):
- description 250 chars SUGGESTION / 400 FAIL, measured on the folded
  YAML value
- body-only 600 words SUGGESTION / 900 FAIL, independent of the unchanged
  whole-file 2770-word / 500-line spec backstop
- every boundary-clause routing target must resolve to a real skill or
  agent; catches skill-improve, neuledge-context and gitea-labels
- agents take the description gates but deliberately no body gate; a test
  pins that absence

Vale: DescriptionOpener widened to ^This\b, new CompositionNote rule
banning architecture notes from descriptions. 10 hits, 0 false positives.

Kyberforge's own four skills retrofitted: descriptions 3,364 -> 938 chars
(-72%), bodies 8,306 -> 2,487 words (-70%), all via the apm-workflow
dispatch pattern. Fixes the skill-improve dangling route and the
agent-author misroute to manual review.

Also fixes a pre-existing false positive where any line-initial 'read '
was flagged as interactive input, which had already caused two scripts to
be rewritten around it.

Refs: ADR-0020
2026-08-14 21:13:13 +00:00

487 lines
21 KiB
Bash
Executable File

#!/usr/bin/env bash
set -euo pipefail
usage() {
cat <<EOF
Usage: validate.sh <agent-file>
Validate an agent definition file against the agent definition spec.
At plugin/APM scope, <agent-file> is a single vendor-neutral
.apm/agents/<name>.agent.md file with no counterpart. Its frontmatter allowlist
is not restated here: it is read at load time from the apm-agent-allowlist
section of references/field-inventory.md, which is the authoritative list.
At project or user scope, <agent-file> is either half of a Claude Code .md /
Copilot .agent.md pair.
Arguments:
agent-file Path to the agent file (or either half of a project/user-scope pair).
Exit codes:
0 All checks passed (may include SUGGESTIONs)
1 One or more checks failed
2 Script error (unrecognized file extension or missing field-inventory.md)
EOF
}
if [[ "${1:-}" == "--help" || "${1:-}" == "-h" ]]; then
usage
exit 0
fi
if [[ $# -lt 1 ]]; then
echo "Error: agent-file is required." >&2
echo "" >&2
usage >&2
exit 1
fi
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
python3 -u - "$1" "$SCRIPT_DIR" <<'PYTHON'
import sys
import os
import re
agent_file = os.path.abspath(sys.argv[1])
script_dir = sys.argv[2]
fname = os.path.basename(agent_file)
# --- Detect provider (check .agent.md before .md) ---
if fname.endswith('.agent.md'):
provider = 'copilot'
name_stem = fname[:-len('.agent.md')]
elif fname.endswith('.md'):
provider = 'claude-code'
name_stem = fname[:-len('.md')]
else:
print(f"Error: unrecognized extension '{fname}' — expected .md or .agent.md", file=sys.stderr)
sys.exit(2)
# --- Load field-inventory.md ---
inv_path = os.path.normpath(os.path.join(script_dir, '..', 'references', 'field-inventory.md'))
if not os.path.isfile(inv_path):
print(f"Error: field-inventory.md not found at {inv_path}", file=sys.stderr)
sys.exit(2)
with open(inv_path) as f:
inv_content = f.read()
def parse_section_tokens(content, section_name):
lines = content.splitlines()
for i, line in enumerate(lines):
if line.strip() == f'## {section_name}':
for j in range(i + 1, len(lines)):
stripped = lines[j].strip()
if stripped and not stripped.startswith('#') and not stripped.startswith('---'):
return set(stripped.split())
return set()
cc_only_fields = parse_section_tokens(inv_content, 'claude-code-only-fields')
copilot_only_fields = parse_section_tokens(inv_content, 'copilot-only-fields')
apm_agent_allowlist = parse_section_tokens(inv_content, 'apm-agent-allowlist')
# Tools the runtime withholds from subagents regardless of the tools field
SUBAGENT_UNAVAILABLE_TOOLS = {
'AskUserQuestion', 'EnterPlanMode', 'ExitPlanMode', 'ScheduleWakeup', 'WaitForMcpServers',
}
# Copilot body length limit (chars) — content beyond this is silently truncated
COPILOT_BODY_LIMIT = 30000
# ADR-0020 description budget. An agent's name + description is preloaded into
# every session exactly like a skill's, so agents take the SAME description
# gates. These two constants are DUPLICATED from scripts/skill-size-check.sh
# and skill-audit/scripts/validate.sh rather than shared from one file: a
# cache-installed plugin's scripts cannot read files outside their own plugin
# directory, so there is no single source to share (same rationale as
# vale-wrap.sh's per-plugin duplication). tests/test-skill-size-check.sh
# asserts all copies agree, so drift fails CI rather than silently diverging.
#
# Agents deliberately take NO body word gate, and adding one here would
# contradict ADR-0020: a skill body is loaded into the caller's context and
# competes with the live conversation, while an agent body becomes the system
# prompt of a fresh context. The rationale for the 900-word skill ceiling does
# not transfer. Agent body length falls out of the delegation rule instead.
DESC_SUGGEST_CHARS = 250
DESC_MAX_CHARS = 400
# --- Helpers (shared by every scope) ---
failed = False
suggestions = []
def fail(msg):
global failed
failed = True
print(f"FAIL {msg}")
def suggest(msg):
suggestions.append(msg)
PLACEHOLDER_RE = re.compile(r'(?<!`)FILL IN:[^`\n]')
def parse_frontmatter(content):
m = re.match(r'^---\n(.*?)\n---', content, re.DOTALL)
if not m:
return None, content
return m.group(1), content[m.end():]
def extract_field(fm, field):
m = re.search(rf'^{re.escape(field)}:\s*(.+)', fm, re.MULTILINE)
return m.group(1).strip() if m else None
def get_frontmatter_keys(fm):
keys = set()
for line in fm.splitlines():
m = re.match(r'^([a-zA-Z][a-zA-Z0-9_-]*):', line)
if m:
keys.add(m.group(1))
return keys
def normalize_scalar(value):
return re.sub(r'\s+', ' ', value).strip()
def fold_description_fallback(fm_text):
"""Resolve `description:` without PyYAML.
Not a YAML parser — it recognises exactly the shapes agent frontmatter
uses: an inline scalar (optionally quoted, optionally continued on
following indented lines) and a `>`/`|` block scalar with optional
indentation and chomping indicators.
"""
lines = fm_text.splitlines()
for i, line in enumerate(lines):
m = re.match(r'^description:[ \t]*(.*)$', line)
if not m:
continue
head = m.group(1).strip()
block = bool(re.match(r'^[>|][0-9]*[-+]?$|^[>|][-+]?[0-9]*$', head))
parts = [] if block else [head]
for nxt in lines[i + 1:]:
if not nxt.strip():
parts.append('')
continue
if not re.match(r'^[ \t]', nxt):
break
parts.append(nxt.strip())
value = ' '.join(parts)
if not block:
value = value.strip()
if len(value) >= 2 and value[0] == value[-1] and value[0] in '"\'':
value = value[1:-1]
return value
return ''
def description_value(fm):
"""The description VALUE with YAML folding resolved.
extract_field() reads one raw line, which is the right shape for the
presence and placeholder checks but the wrong one for a length gate: a
`>`-folded description measured off its first raw line is not the value the
host preloads. Parse instead of regexing the raw text.
"""
try:
import yaml
data = yaml.safe_load(fm)
if isinstance(data, dict):
value = data.get('description')
if isinstance(value, str):
return normalize_scalar(value)
if value is not None:
return normalize_scalar(str(value))
return ''
except Exception:
pass
return normalize_scalar(fold_description_fallback(fm))
def check_description_budget(fm, local_fname):
"""ADR-0020 description gates — identical for every scope."""
value = description_value(fm)
if not value:
return
dlen = len(value)
if dlen > DESC_MAX_CHARS:
fail(f"description is {dlen} chars — exceeds the {DESC_MAX_CHARS}-character "
f"ADR-0020 ceiling. It is preloaded into every session whether or not the "
f"agent is invoked. Keep a trigger clause, at most one capability clause, "
f"and a boundary clause; move capability enumeration, output-format detail, "
f"composition notes and implementation detail to the body — {local_fname}")
elif dlen > DESC_SUGGEST_CHARS:
suggest(f"description is {dlen} chars — over the {DESC_SUGGEST_CHARS}-character "
f"ADR-0020 target (hard fail at {DESC_MAX_CHARS}). The SUGGESTION tier is "
f"what moves the corpus average; the FAIL tier only stops outliers "
f"— {local_fname}")
def extract_tools_list(fm):
"""Extract tool names from the tools frontmatter field (space or comma separated)."""
val = extract_field(fm, 'tools')
if not val:
return set()
return set(re.split(r'[\s,]+', val.strip()))
def is_copilot_cloud_ide(fpath):
"""True if the file is a cloud/IDE Copilot agent (name is optional for these)."""
return '.github/copilot/agents' in os.path.abspath(fpath).replace(os.sep, '/')
# --- Detect scope ---
# APM_TYPE_RE matches a top-level (column-0) `type:` line in apm.yml whose value is
# exactly one of the four package content types. Group 1 captures an optional
# opening quote; \1 requires the same character (or nothing) to close it, so
# "skill" and '"skill"' both match but a mismatched quote doesn't. The value
# must then be followed by whitespace or end-of-line — not just a non-word
# character — so a malformed value like `prompts-only` is correctly rejected
# instead of false-matching on the `prompts` prefix.
APM_TYPE_RE = re.compile(r"^type:\s*(['\"]?)(instructions|skill|hybrid|prompts)\1(?:\s|$)")
def find_apm_package_root(apm_yml_path):
"""Return True if apm_yml_path has a top-level type: line (i.e. is a package
manifest, not a type:-less marketplace-only apm.yml)."""
with open(apm_yml_path) as f:
for line in f:
if APM_TYPE_RE.match(line):
return True
return False
def detect_scope(start_dir):
home = os.path.expanduser('~')
original_start = os.path.abspath(start_dir)
# Agent files conventionally live exactly two path segments below their
# scope root — <root>/.claude/agents, <root>/.github/agents,
# <root>/.copilot/agents, or <root>/.apm/agents (see new-agent.sh's
# CC_DIR/CP_DIR and user-scope dirs). Stripping those two segments
# recovers the same root new-agent.sh would have been invoked with to
# produce this exact file, independent of how far the walk below has to
# travel to find (or fail to find) a marker — mirrors new-agent.sh's
# `root` vs `current` distinction even though validate.sh is handed a
# file's directory, not the scope root itself.
#
# That arithmetic is only trustworthy when the path actually has this
# shape: parent directory literally named "agents", grandparent one of
# the four known scope-dir names. A hand-placed or otherwise
# non-conventional agent file (never produced by new-agent.sh) has no
# such guarantee — blindly trusting two-segments-up there could point at
# an unrelated ancestor. conventional_shape gates every use of
# conventional_root below; when it's false, the walked-to `current`
# directory is used instead, the same fallback this function used before
# conventional_root existed.
scope_dir_name = os.path.basename(os.path.dirname(original_start))
conventional_shape = (
os.path.basename(original_start) == 'agents'
and scope_dir_name in ('.claude', '.github', '.copilot', '.apm')
)
conventional_root = os.path.dirname(os.path.dirname(original_start))
current = original_start
while True:
apm_yml = os.path.join(current, 'apm.yml')
if os.path.isfile(apm_yml) and find_apm_package_root(apm_yml):
return 'plugin', current
# $HOME is the user-scope boundary — checked before the .git test
# below, so a dotfiles-managed $HOME (yadm, chezmoi bare-repo, etc.)
# can't shadow user scope by being its own .git repo. 'user' scope
# requires EITHER start_dir to BE $HOME itself (no walk-up — the
# new-agent.sh "root exactly $HOME" case) OR start_dir to sit at the
# conventional two-segments-below-root depth (i.e. $HOME IS that
# root, matching the real ~/.claude/agents or ~/.copilot/agents
# shape). Any other walk-up into $HOME — a marker-less directory
# nested deeper than that convention — resolves to project scope
# instead: a stray directory under $HOME can't be silently
# redirected into the shared global ~/.claude or ~/.copilot agent
# directories.
if current == home:
if original_start == home or (conventional_shape and conventional_root == home):
return 'user', home
return 'project', conventional_root if conventional_shape else current
# .git is a directory in a normal checkout but a file (`gitdir: ...`)
# in a git worktree — exists() covers both. Returns conventional_root,
# not current: new-agent.sh's project-scope file placement always
# uses its `$ROOT` argument directly, never the walked-up `.git`
# location, so a <root> one or more levels below the repo's .git
# (a subdirectory of a larger git-tracked tree — explicitly a
# supported case per new-agent.sh's usage text) must resolve to the
# same root new-agent.sh actually wrote to, not to the .git dir —
# unless the path lacks the conventional shape, in which case that
# arithmetic isn't trustworthy and current is used instead.
if os.path.exists(os.path.join(current, '.git')):
return 'project', conventional_root if conventional_shape else current
parent = os.path.dirname(current)
if parent == current:
return 'project', conventional_root if conventional_shape else current
current = parent
agent_dir = os.path.dirname(agent_file)
scope, scope_root = detect_scope(agent_dir)
# --- Plugin/APM scope: single vendor-neutral file, no counterpart ---
def check_apm_agent_file(fpath, allowlist, stem):
local_fname = os.path.basename(fpath)
with open(fpath) as f:
content = f.read()
fm, body = parse_frontmatter(content)
if fm is None:
fail(f"no valid YAML frontmatter (---...---) — {local_fname}")
return
# The apm-agent.md template embeds its authoring guidance as HTML
# comments inside the frontmatter block (so they render invisible in a
# Markdown preview but stay visible in the raw file). get_frontmatter_keys
# silently ignores any line that isn't a `key:` match, so a comment left
# behind at ship time would otherwise pass unnoticed — yet apm compile
# copies this frontmatter verbatim to both harnesses, and `<!-- -->` is
# not valid YAML, so yaml.safe_load breaks on both downstream (ADR-0016).
if re.search(r'<!--|-->', fm):
fail(f"frontmatter still contains template HTML comments (<!-- ... -->) "
f"— delete them before shipping — {local_fname}")
# Allowlist: the permitted keys are data, read at load time from
# references/field-inventory.md's `## apm-agent-allowlist` section — do not
# restate them here, or this comment goes stale the next time that line
# changes. apm compile verbatim-copies frontmatter to every target, so a key
# outside the list is unsafe on at least one harness (ADR-0016). Note the
# list admits denylist-shaped restrictions (disallowedTools) but never
# allowlist-shaped ones (tools), whose value shape differs per harness.
fm_keys = get_frontmatter_keys(fm)
for key in sorted(fm_keys):
if key not in allowlist:
fail(f"field '{key}' is not in the vendor-neutral APM agent allowlist "
f"({', '.join(sorted(allowlist))}) — {local_fname}")
# name — required, kebab-case, must match filename stem (file is <name>.agent.md)
name_val = extract_field(fm, 'name')
if not name_val:
fail(f"name field is missing or empty — {local_fname}")
else:
if not re.match(r'^[a-z0-9]+(-[a-z0-9]+)*$', name_val):
fail(f"name '{name_val}' is not kebab-case — {local_fname}")
if name_val != stem:
fail(f"name '{name_val}' does not match filename stem '{stem}' — {local_fname}")
# description — required, non-empty, no placeholder
desc_val = extract_field(fm, 'description')
if not desc_val:
fail(f"description field is missing or empty — {local_fname}")
else:
if PLACEHOLDER_RE.search(desc_val):
fail(f"description contains unfilled FILL IN: placeholder — {local_fname}")
check_description_budget(fm, local_fname)
# body — required, non-empty, no placeholder; same Copilot truncation risk
# applies since this file compiles verbatim into a real Copilot file downstream.
if not body.strip():
fail(f"system prompt body is empty — {local_fname}")
else:
if PLACEHOLDER_RE.search(body):
fail(f"body contains unfilled FILL IN: placeholder — {local_fname}")
if len(body) > COPILOT_BODY_LIMIT:
suggest(f"body exceeds {COPILOT_BODY_LIMIT:,} characters ({len(body):,} chars) — "
f"content beyond the limit is silently truncated by the Copilot runtime "
f"once apm compile emits it downstream — {local_fname}")
if scope == 'plugin':
check_apm_agent_file(agent_file, apm_agent_allowlist, name_stem)
for s in suggestions:
print(f"SUGGESTION {s}")
sys.exit(1 if failed else 0)
# --- Project/user scope: unchanged CC/Copilot pair validation ---
# --- Derive counterpart path ---
if scope == 'project':
if provider == 'claude-code':
counterpart = os.path.join(scope_root, '.github', 'agents', name_stem + '.agent.md')
counterpart_provider = 'copilot'
else:
counterpart = os.path.join(scope_root, '.claude', 'agents', name_stem + '.md')
counterpart_provider = 'claude-code'
else: # user
home = os.path.expanduser('~')
if provider == 'claude-code':
counterpart = os.path.join(home, '.copilot', 'agents', name_stem + '.agent.md')
counterpart_provider = 'copilot'
else:
counterpart = os.path.join(home, '.claude', 'agents', name_stem + '.md')
counterpart_provider = 'claude-code'
def check_file(fpath, file_provider):
local_fname = os.path.basename(fpath)
with open(fpath) as f:
content = f.read()
fm, body = parse_frontmatter(content)
if fm is None:
fail(f"no valid YAML frontmatter (---...---) — {local_fname}")
return
# name — required for CC and Copilot CLI; optional for Copilot cloud/IDE agents
cloud_ide = (file_provider == 'copilot' and is_copilot_cloud_ide(fpath))
name_val = extract_field(fm, 'name')
if not cloud_ide:
if not name_val:
fail(f"name field is missing or empty — {local_fname}")
else:
if not re.match(r'^[a-z0-9]+(-[a-z0-9]+)*$', name_val):
fail(f"name '{name_val}' is not kebab-case — {local_fname}")
# Stem check applies to Copilot CLI only; CC docs say filename need not match name
if file_provider == 'copilot':
stem = local_fname[:-len('.agent.md')]
if name_val != stem:
fail(f"name '{name_val}' does not match filename stem '{stem}' — {local_fname}")
elif name_val and not re.match(r'^[a-z0-9]+(-[a-z0-9]+)*$', name_val):
# cloud/IDE: name is optional, but if present it must be valid
fail(f"name '{name_val}' is not kebab-case — {local_fname}")
# description
desc_val = extract_field(fm, 'description')
if not desc_val:
fail(f"description field is missing or empty — {local_fname}")
else:
if PLACEHOLDER_RE.search(desc_val):
fail(f"description contains unfilled FILL IN: placeholder — {local_fname}")
check_description_budget(fm, local_fname)
# body
if not body.strip():
fail(f"system prompt body is empty — {local_fname}")
else:
if PLACEHOLDER_RE.search(body):
fail(f"body contains unfilled FILL IN: placeholder — {local_fname}")
# Copilot body length limit
if file_provider == 'copilot' and len(body) > COPILOT_BODY_LIMIT:
suggest(f"body exceeds {COPILOT_BODY_LIMIT:,} characters ({len(body):,} chars) — content beyond the limit is silently truncated by the Copilot runtime — {local_fname}")
# CC-only fields in Copilot file
if file_provider == 'copilot':
fm_keys = get_frontmatter_keys(fm)
for key in sorted(fm_keys):
if key in cc_only_fields:
fail(f"CC-only field '{key}' present in Copilot file — {local_fname}")
# Copilot-only fields in CC file
if file_provider == 'claude-code':
fm_keys = get_frontmatter_keys(fm)
for key in sorted(fm_keys):
if key in copilot_only_fields:
fail(f"Copilot-only field '{key}' present in CC file — {local_fname}")
# Subagent-unavailable tools listed in tools field
tools = extract_tools_list(fm)
unavailable = tools & SUBAGENT_UNAVAILABLE_TOOLS
for tool in sorted(unavailable):
suggest(f"'{tool}' is listed in tools but is never available to subagents — the runtime withholds it regardless — {local_fname}")
# --- Check counterpart exists ---
if not os.path.isfile(counterpart):
fail(f"counterpart file not found: {counterpart}")
sys.exit(1)
# --- Check both files ---
check_file(agent_file, provider)
check_file(counterpart, counterpart_provider)
for s in suggestions:
print(f"SUGGESTION {s}")
sys.exit(1 if failed else 0)
PYTHON