feat(kyberforge): enforce the ADR-0020 context contract for skills and agents
Skill name+description pairs are preloaded into every session, costing ~6,200 tokens across 39 skills before any skill is invoked. The authoring rules mandated that growth: skill-author:104 and description-quality.md:21 both required padding, while skill-author:102 (the deflating rule) had no FAIL condition behind it. Gates (blocking, no baseline file): - description 250 chars SUGGESTION / 400 FAIL, measured on the folded YAML value - body-only 600 words SUGGESTION / 900 FAIL, independent of the unchanged whole-file 2770-word / 500-line spec backstop - every boundary-clause routing target must resolve to a real skill or agent; catches skill-improve, neuledge-context and gitea-labels - agents take the description gates but deliberately no body gate; a test pins that absence Vale: DescriptionOpener widened to ^This\b, new CompositionNote rule banning architecture notes from descriptions. 10 hits, 0 false positives. Kyberforge's own four skills retrofitted: descriptions 3,364 -> 938 chars (-72%), bodies 8,306 -> 2,487 words (-70%), all via the apm-workflow dispatch pattern. Fixes the skill-improve dangling route and the agent-author misroute to manual review. Also fixes a pre-existing false positive where any line-initial 'read ' was flagged as interactive input, which had already caused two scripts to be rewritten around it. Refs: ADR-0020
This commit is contained in:
@@ -11,7 +11,7 @@ Arguments:
|
||||
skill-dir Path to the skill directory containing SKILL.md.
|
||||
|
||||
Exit codes:
|
||||
0 All checks passed
|
||||
0 All checks passed (may include SUGGESTIONs)
|
||||
1 One or more checks failed
|
||||
EOF
|
||||
}
|
||||
@@ -32,6 +32,7 @@ python3 -u - "$1" <<'PYTHON'
|
||||
import sys
|
||||
import os
|
||||
import re
|
||||
import glob
|
||||
|
||||
skill_dir = os.path.abspath(sys.argv[1])
|
||||
skill_md = os.path.join(skill_dir, "SKILL.md")
|
||||
@@ -44,6 +45,7 @@ with open(skill_md) as f:
|
||||
content = f.read()
|
||||
|
||||
failed = False
|
||||
suggestions = []
|
||||
|
||||
def ok(msg):
|
||||
print(f"PASS {msg}")
|
||||
@@ -53,6 +55,13 @@ def fail(msg):
|
||||
print(f"FAIL {msg}")
|
||||
failed = True
|
||||
|
||||
def suggest(msg):
|
||||
# SUGGESTIONs are printed after every check and NEVER touch the exit code.
|
||||
# skill-audit's Step 4 report counts them into its `PASS (N suggestions)`
|
||||
# result line, which is what makes the ADR-0020 SUGGESTION tier visible
|
||||
# rather than another silently-ignored warning (ADR-0013).
|
||||
suggestions.append(msg)
|
||||
|
||||
# --- Parse frontmatter ---
|
||||
fm_match = re.match(r'^---\n(.*?)\n---', content, re.DOTALL)
|
||||
if not fm_match:
|
||||
@@ -66,16 +75,58 @@ body_start = fm_match.end()
|
||||
name_m = re.search(r'^name:\s*(\S+)', fm, re.MULTILINE)
|
||||
name = name_m.group(1).strip('"\'') if name_m else ""
|
||||
|
||||
# Extract description — inline or block scalar (> or |)
|
||||
desc = ""
|
||||
desc_m = re.search(r'^description:\s*([>|])\n((?:[ \t]+.+\n?)+)', fm, re.MULTILINE)
|
||||
if desc_m:
|
||||
raw = desc_m.group(2)
|
||||
desc = re.sub(r'\s+', ' ', raw).strip()
|
||||
else:
|
||||
desc_inline = re.search(r'^description:\s*(.+)', fm, re.MULTILINE)
|
||||
if desc_inline:
|
||||
desc = desc_inline.group(1).strip()
|
||||
# Extract description — the VALUE, with YAML folding resolved. Most of this
|
||||
# corpus writes descriptions as `>`-folded block scalars, so the raw lines
|
||||
# carry indentation and newlines that are not part of the value: every length
|
||||
# measurement below is wrong unless the scalar is folded first. PyYAML is used
|
||||
# when importable (it is a real parser); the fallback recognises exactly the
|
||||
# shapes this corpus uses — an inline scalar, optionally quoted and optionally
|
||||
# continued on following indented lines, and a `>`/`|` block scalar with
|
||||
# optional indentation and chomping indicators.
|
||||
|
||||
def normalize(value):
|
||||
return re.sub(r'\s+', ' ', value).strip()
|
||||
|
||||
def fold_description_fallback(fm_text):
|
||||
lines = fm_text.splitlines()
|
||||
for i, line in enumerate(lines):
|
||||
m = re.match(r'^description:[ \t]*(.*)$', line)
|
||||
if not m:
|
||||
continue
|
||||
head = m.group(1).strip()
|
||||
block = bool(re.match(r'^[>|][0-9]*[-+]?$|^[>|][-+]?[0-9]*$', head))
|
||||
parts = [] if block else [head]
|
||||
for nxt in lines[i + 1:]:
|
||||
if not nxt.strip():
|
||||
parts.append('')
|
||||
continue
|
||||
if not re.match(r'^[ \t]', nxt):
|
||||
break
|
||||
parts.append(nxt.strip())
|
||||
value = ' '.join(parts)
|
||||
if not block:
|
||||
value = value.strip()
|
||||
if len(value) >= 2 and value[0] == value[-1] and value[0] in '"\'':
|
||||
value = value[1:-1]
|
||||
return value
|
||||
return ''
|
||||
|
||||
def extract_description(fm_text):
|
||||
try:
|
||||
import yaml
|
||||
data = yaml.safe_load(fm_text)
|
||||
if isinstance(data, dict):
|
||||
value = data.get('description')
|
||||
if isinstance(value, str):
|
||||
return normalize(value)
|
||||
if value is not None:
|
||||
return normalize(str(value))
|
||||
return ''
|
||||
except Exception:
|
||||
pass
|
||||
return normalize(fold_description_fallback(fm_text))
|
||||
|
||||
desc = extract_description(fm)
|
||||
|
||||
dir_name = os.path.basename(skill_dir)
|
||||
|
||||
@@ -114,11 +165,13 @@ if desc:
|
||||
else:
|
||||
fail("description field is missing or empty")
|
||||
|
||||
# description length
|
||||
# description length — agentskills.io spec backstop. UNCHANGED by ADR-0020:
|
||||
# 1024 is the specification's hard limit, and the ADR-0020 budget gate below
|
||||
# sits underneath it rather than replacing it.
|
||||
if desc:
|
||||
dlen = len(desc)
|
||||
if dlen <= 1024:
|
||||
ok(f"description length {dlen} chars (limit: 1024)")
|
||||
ok(f"description length {dlen} chars (agentskills.io spec limit: 1024)")
|
||||
else:
|
||||
fail(f"description length {dlen} chars — exceeds 1024-character limit")
|
||||
|
||||
@@ -146,6 +199,26 @@ MAX_LINES = 500
|
||||
# for the full measurement.
|
||||
MAX_WORDS = 2770
|
||||
|
||||
# ADR-0020 context-budget gates. DUPLICATED from scripts/skill-size-check.sh
|
||||
# for exactly the same cache-isolation reason as MAX_LINES/MAX_WORDS above, and
|
||||
# carrying the same warning — tests/test-skill-size-check.sh asserts the copies
|
||||
# agree, so drift fails CI instead of shipping an audit that disagrees with the
|
||||
# commit hook. agent-audit/scripts/validate.sh holds a third copy of the two
|
||||
# description constants; per ADR-0020 agents take the description gates and
|
||||
# deliberately take NO body word gate, because an agent body becomes the system
|
||||
# prompt of a fresh context rather than competing with a live conversation.
|
||||
#
|
||||
# These are NOT the same measurements as MAX_LINES/MAX_WORDS and must not be
|
||||
# unified with them: MAX_WORDS counts the WHOLE FILE including frontmatter and
|
||||
# is a spec-conformance backstop; BODY_MAX_WORDS counts the body ONLY and is a
|
||||
# quality gate. Likewise the 1024-character description limit above is the
|
||||
# agentskills.io spec ceiling and stays exactly as it is — DESC_MAX_CHARS sits
|
||||
# underneath it.
|
||||
DESC_SUGGEST_CHARS = 250
|
||||
DESC_MAX_CHARS = 400
|
||||
BODY_SUGGEST_WORDS = 600
|
||||
BODY_MAX_WORDS = 900
|
||||
|
||||
line_count = len(content.splitlines())
|
||||
if line_count <= MAX_LINES:
|
||||
ok(f"SKILL.md line count {line_count} (limit: {MAX_LINES})")
|
||||
@@ -160,14 +233,220 @@ if word_count <= MAX_WORDS:
|
||||
else:
|
||||
fail(f"SKILL.md word count {word_count} — exceeds {MAX_WORDS}-word limit (proxy for ~5,000 tokens)")
|
||||
|
||||
# Body unfilled placeholders
|
||||
body = content[body_start:]
|
||||
|
||||
# --- ADR-0020: description budget -----------------------------------------
|
||||
if desc:
|
||||
dlen = len(desc)
|
||||
if dlen > DESC_MAX_CHARS:
|
||||
fail(f"description is {dlen} chars — exceeds the {DESC_MAX_CHARS}-character "
|
||||
f"ADR-0020 ceiling. It is preloaded into every session whether or not the "
|
||||
f"skill is invoked. Keep a trigger clause, at most one capability clause, "
|
||||
f"and a boundary clause; move capability enumeration, output-format detail, "
|
||||
f"composition notes and implementation detail to the body or README.md")
|
||||
elif dlen > DESC_SUGGEST_CHARS:
|
||||
suggest(f"description is {dlen} chars — over the {DESC_SUGGEST_CHARS}-character "
|
||||
f"ADR-0020 target (hard fail at {DESC_MAX_CHARS}). The SUGGESTION tier is "
|
||||
f"what moves the corpus average; the FAIL tier only stops outliers")
|
||||
else:
|
||||
ok(f"description length {dlen} chars (ADR-0020 target: {DESC_SUGGEST_CHARS})")
|
||||
|
||||
# --- ADR-0020: body budget -------------------------------------------------
|
||||
# Counts the BODY ONLY — everything after the closing --- of the frontmatter.
|
||||
# This is a different measurement from MAX_WORDS above, which counts the whole
|
||||
# file including frontmatter as a spec-conformance backstop. Both are reported.
|
||||
body_word_count = len(body.split())
|
||||
if body_word_count > BODY_MAX_WORDS:
|
||||
fail(f"SKILL.md body is {body_word_count} words — exceeds the {BODY_MAX_WORDS}-word "
|
||||
f"ADR-0020 ceiling (body only; separate from the {MAX_WORDS}-word whole-file "
|
||||
f"limit above). Move lookup tables, spec restatements, output schemas, templates "
|
||||
f"and rationale prose to references/ behind an explicit "
|
||||
f"\"If X, read `references/file.md`\" trigger. At two or more mutually exclusive "
|
||||
f"flows, dispatch is mandatory: the body carries the dispatch table and the gates "
|
||||
f"common to every branch, each flow gets its own self-contained references/ file")
|
||||
elif body_word_count > BODY_SUGGEST_WORDS:
|
||||
suggest(f"SKILL.md body is {body_word_count} words — over the {BODY_SUGGEST_WORDS}-word "
|
||||
f"ADR-0020 target (hard fail at {BODY_MAX_WORDS})")
|
||||
else:
|
||||
ok(f"SKILL.md body word count {body_word_count} (ADR-0020 target: {BODY_SUGGEST_WORDS})")
|
||||
|
||||
# --- ADR-0020: resolvable boundary targets ---------------------------------
|
||||
# A boundary clause names another skill — or an agent, which is an equally
|
||||
# valid routing target (git-workflow routes to the git-orchestrate agent). Every
|
||||
# named target is resolved against the AUTHORING SOURCE, plugins/*/.apm/skills/
|
||||
# and plugins/*/.apm/agents/, so the check works offline and before an
|
||||
# `apm install` has deployed anything into .claude/skills/.
|
||||
#
|
||||
# False positives are the design constraint here, not recall. Two rules do the
|
||||
# work:
|
||||
# * A BARE hyphenated word is read as a routing target only inside a boundary
|
||||
# sentence (one carrying "do not"/"instead"/"rather than"/"not for").
|
||||
# Without that, pc-run's "run pre-commit hooks" reads as a route to a
|
||||
# non-existent `pre-commit` skill.
|
||||
# * A BARE arrow target counts only in ADR-0020's compressed boundary form,
|
||||
# `Not <thing> -> <skill-name>`. Without that, diagnose's process chain
|
||||
# "fix -> regression-test" reads as a route to `regression-test`.
|
||||
# Backticked and /slash-command targets are unambiguous and always count. Tool
|
||||
# names (Read, Write, Edit) are excluded by the lowercase-only name pattern;
|
||||
# MCP tool names (issue_write, pull_request_write) by its rejection of
|
||||
# underscores; file names by its rejection of dots and slashes.
|
||||
|
||||
NAME_ANY = r"[a-z0-9]+(?:-[a-z0-9]+)*"
|
||||
NAME_HYPH = r"[a-z0-9]+(?:-[a-z0-9]+)+"
|
||||
ROUTE_VERB = (r"(?:use|uses|using|run|runs|invoke|invokes|invoking|try|see"
|
||||
r"|that'?s|compose|composes|call|calls)")
|
||||
MARKED_TARGET = r"(?:`/?(%s)`|(?<![\w./*-])/(%s)\b)" % (NAME_ANY, NAME_ANY)
|
||||
ANY_TARGET = r"(?:%s|(%s)\b)" % (MARKED_TARGET, NAME_HYPH)
|
||||
ROUTE_MARKED = re.compile(r"\b%s\s+(?:the\s+|an?\s+)?%s" % (ROUTE_VERB, MARKED_TARGET), re.I)
|
||||
ROUTE_ANY = re.compile(r"\b%s\s+(?:the\s+|an?\s+)?%s" % (ROUTE_VERB, ANY_TARGET), re.I)
|
||||
CONT_MARKED = re.compile(r"\s*(?:or|and|/|,)\s*%s" % MARKED_TARGET)
|
||||
CONT_ANY = re.compile(r"\s*(?:or|and|/|,)\s*%s" % ANY_TARGET)
|
||||
ARROW_MARKED = re.compile(r"(?:->|→)\s*%s" % MARKED_TARGET)
|
||||
ARROW_BOUNDARY = re.compile(r"\bnot\b[^.;]*?(?:->|→)\s*(%s)\b" % NAME_HYPH, re.I)
|
||||
BACKTICK = re.compile(r"`(%s)`" % NAME_HYPH)
|
||||
BOUNDARY_MARKER = re.compile(r"\b(?:do\s+not|instead|rather\s+than|not\s+for)\b", re.I)
|
||||
SENTENCE_SPLIT = re.compile('(?<=[.!?])\\s+(?=[A-Z"“(])')
|
||||
|
||||
def _first_group(groups):
|
||||
for g in groups:
|
||||
if g:
|
||||
return g
|
||||
return None
|
||||
|
||||
def _scan_routes(text, route_re, cont_re, out):
|
||||
for m in route_re.finditer(text):
|
||||
target = _first_group(m.groups())
|
||||
if not target:
|
||||
continue
|
||||
out.append(target)
|
||||
# Conjoined targets: "use git-history or git-branches instead",
|
||||
# "use gitea-issues / gitea-prs".
|
||||
pos = m.end()
|
||||
while True:
|
||||
cm = cont_re.match(text, pos)
|
||||
if not cm:
|
||||
break
|
||||
nxt = _first_group(cm.groups())
|
||||
if nxt:
|
||||
out.append(nxt)
|
||||
pos = cm.end()
|
||||
|
||||
def boundary_targets(description):
|
||||
out = []
|
||||
for sentence in SENTENCE_SPLIT.split(description):
|
||||
boundary = bool(BOUNDARY_MARKER.search(sentence))
|
||||
_scan_routes(sentence,
|
||||
ROUTE_ANY if boundary else ROUTE_MARKED,
|
||||
CONT_ANY if boundary else CONT_MARKED,
|
||||
out)
|
||||
for m in ARROW_MARKED.finditer(sentence):
|
||||
target = _first_group(m.groups())
|
||||
if target:
|
||||
out.append(target)
|
||||
for m in ARROW_BOUNDARY.finditer(sentence):
|
||||
out.append(m.group(1))
|
||||
out.extend(BACKTICK.findall(sentence))
|
||||
return sorted(set(out))
|
||||
|
||||
def known_targets(start_dir):
|
||||
names = set()
|
||||
# Sibling skills/agents. This is the branch that works in a cache-installed
|
||||
# plugin and in a deployed .claude/skills/ tree, neither of which has a
|
||||
# plugins/ directory above it.
|
||||
parent = os.path.dirname(os.path.abspath(start_dir))
|
||||
if os.path.basename(parent) == 'skills' and os.path.isdir(parent):
|
||||
for entry in os.listdir(parent):
|
||||
if os.path.isdir(os.path.join(parent, entry)):
|
||||
names.add(entry)
|
||||
agents_dir = os.path.join(os.path.dirname(parent), 'agents')
|
||||
if os.path.isdir(agents_dir):
|
||||
for entry in os.listdir(agents_dir):
|
||||
if entry.endswith('.agent.md'):
|
||||
names.add(entry[:-len('.agent.md')])
|
||||
elif entry.endswith('.md'):
|
||||
names.add(entry[:-len('.md')])
|
||||
# Walk up for a monorepo root (plugins/*/.apm/) or a plugin root (.apm/).
|
||||
# Capped at ten levels so a pathological path can't become a filesystem
|
||||
# crawl; that covers every real layout by a wide margin.
|
||||
current = os.path.abspath(start_dir)
|
||||
for _ in range(10):
|
||||
# Never glob the filesystem root: a stray /.apm/skills/ (a scaffolding
|
||||
# test's leftover, say) would otherwise become part of every skill's
|
||||
# resolution universe on that machine.
|
||||
if os.path.dirname(current) == current:
|
||||
break
|
||||
for pattern in ('plugins/*/.apm/skills/*/', '.apm/skills/*/'):
|
||||
for path in glob.glob(os.path.join(current, pattern)):
|
||||
names.add(os.path.basename(path.rstrip('/')))
|
||||
for pattern in ('plugins/*/.apm/agents/*.agent.md', '.apm/agents/*.agent.md'):
|
||||
for path in glob.glob(os.path.join(current, pattern)):
|
||||
names.add(os.path.basename(path)[:-len('.agent.md')])
|
||||
current = os.path.dirname(current)
|
||||
return names
|
||||
|
||||
if desc:
|
||||
routing_targets = boundary_targets(desc)
|
||||
known = known_targets(skill_dir) if routing_targets else set()
|
||||
# An empty universe means no authoring source was found anywhere above this
|
||||
# skill — reporting every target as dangling there would be noise, not a
|
||||
# finding, so the check declines to run rather than guessing.
|
||||
if routing_targets and known:
|
||||
unresolved = [t for t in routing_targets if t not in known]
|
||||
for target in unresolved:
|
||||
fail(f"description routes to '{target}', which resolves to no skill under "
|
||||
f"plugins/*/.apm/skills/ and no agent under plugins/*/.apm/agents/ — "
|
||||
f"a boundary clause naming a non-existent target sends the router nowhere")
|
||||
if not unresolved:
|
||||
ok(f"all {len(routing_targets)} boundary target(s) resolve: "
|
||||
f"{', '.join(routing_targets)}")
|
||||
|
||||
# Body unfilled placeholders
|
||||
fill_matches = PLACEHOLDER_RE.findall(body)
|
||||
if fill_matches:
|
||||
fail(f"SKILL.md body contains {len(fill_matches)} unfilled 'FILL IN:' placeholder(s)")
|
||||
else:
|
||||
ok("SKILL.md body has no unfilled placeholders")
|
||||
|
||||
# Interactive prompt heuristic.
|
||||
#
|
||||
# A line-initial `read` only blocks an agent when its stdin is the terminal.
|
||||
# These forms never touch a TTY and are ordinary data plumbing, so flagging
|
||||
# them is a false positive — one that has already cost two authors a
|
||||
# contorted rewrite of working source:
|
||||
#
|
||||
# read -r MODE ROOT <<< "$WALK_OUTPUT" here-string
|
||||
# read -r X <<EOF here-doc
|
||||
# read -r line < "$file" redirect from a file
|
||||
# printf '%s' "$v" | piped stdin — the pipe ends the
|
||||
# read -r X PREVIOUS line, not this one
|
||||
#
|
||||
# So a `read` is reported only when it has neither a stdin redirection on its
|
||||
# own line nor a pipe terminating the previous logical line. `read -r ANSWER`,
|
||||
# `read -p "..." X` and a bare `read` still fail, which is the case the check
|
||||
# exists for.
|
||||
def stdin_redirected(line, prev_line):
|
||||
# Quoted spans are stripped first so a `<` inside a prompt string is not
|
||||
# mistaken for a redirect: `read -p "enter <name>: " X` is interactive and
|
||||
# must still fail.
|
||||
unquoted = re.sub(r'"[^"]*"|\'[^\']*\'', '', line)
|
||||
return '<' in unquoted or prev_line.rstrip().endswith('|')
|
||||
|
||||
def interactive_reads(source):
|
||||
hits = []
|
||||
prev_line = ''
|
||||
for line in source.splitlines():
|
||||
stripped = line.strip()
|
||||
if re.match(r'read(\s|$)', stripped):
|
||||
if not stdin_redirected(line, prev_line):
|
||||
hits.append(stripped)
|
||||
elif re.match(r'input\(', stripped):
|
||||
hits.append(stripped)
|
||||
# Blank lines and comments cannot carry the pipe that feeds a
|
||||
# following `read`, so they never displace the previous line.
|
||||
if stripped and not stripped.startswith('#'):
|
||||
prev_line = line
|
||||
return hits
|
||||
|
||||
# Scripts checks
|
||||
scripts_dir = os.path.join(skill_dir, "scripts")
|
||||
if os.path.isdir(scripts_dir):
|
||||
@@ -177,9 +456,10 @@ if os.path.isdir(scripts_dir):
|
||||
fpath = os.path.join(scripts_dir, fname)
|
||||
with open(fpath) as f:
|
||||
sc = f.read()
|
||||
# Interactive prompt heuristic
|
||||
if re.search(r'^\s*(read\s|input\()', sc, re.MULTILINE):
|
||||
fail(f"scripts/{fname}: may use interactive input (read/input detected)")
|
||||
interactive = interactive_reads(sc)
|
||||
if interactive:
|
||||
fail(f"scripts/{fname}: may use interactive input "
|
||||
f"(read/input from a terminal detected): {interactive[0]}")
|
||||
else:
|
||||
ok(f"scripts/{fname}: no interactive prompts detected")
|
||||
# Executable bit
|
||||
@@ -190,8 +470,17 @@ if os.path.isdir(scripts_dir):
|
||||
|
||||
# Summary
|
||||
print()
|
||||
for s in suggestions:
|
||||
print(f"SUGGESTION {s}")
|
||||
if suggestions:
|
||||
print()
|
||||
if not failed:
|
||||
print("All checks passed.")
|
||||
if suggestions:
|
||||
# Feeds skill-audit's Step 4 `PASS (N suggestions)` result line. A
|
||||
# SUGGESTION never changes the exit code — only a FAIL does.
|
||||
print(f"All checks passed ({len(suggestions)} suggestion(s)).")
|
||||
else:
|
||||
print("All checks passed.")
|
||||
sys.exit(0)
|
||||
else:
|
||||
print("One or more checks failed.")
|
||||
|
||||
Reference in New Issue
Block a user