feat(kyberforge): improve agent-audit skill with platform-accurate checks and description quality reference
- Fix validate.sh: remove incorrect name==stem check for CC files (CC docs say filename need not match name field); keep check for Copilot CLI only - Fix validate.sh: plugin scope detection now checks both plugin.json and .claude-plugin/plugin.json - Fix validate.sh: Copilot cloud/IDE agents (.github/copilot/agents/) have name as optional; path-based guard added - Add validate.sh checks: Copilot body length >30,000 chars (SUGGESTION), Copilot-only fields in CC files (FAIL), subagent-unavailable tools in tools field (SUGGESTION) - Add references/description-quality.md as conditional escape hatch for borderline description findings - SKILL.md: name five audit dimensions in description; sharpen indirect-trigger phrasing - SKILL.md: label pair-mandate as kyberforge project convention, not platform requirement - SKILL.md: scope redundant name-match/body-empty checks to manual fallback only - SKILL.md: add conditional reference to description-quality.md; update provider-safety description for new check categories; fix plugin scope gotcha to mention .claude-plugin/plugin.json - SKILL.md: add INFO tier to result block template - Add source_keys frontmatter to references/README.md; update sources.md to add description-quality.md to contributing files Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_0147vXtL5sP6vorDdqXGJJU9
This commit is contained in:
@@ -11,7 +11,7 @@ Arguments:
|
||||
agent-file Path to either the Claude Code .md or Copilot .agent.md agent file.
|
||||
|
||||
Exit codes:
|
||||
0 All checks passed
|
||||
0 All checks passed (may include SUGGESTIONs)
|
||||
1 One or more checks failed
|
||||
2 Script error (unrecognized file extension or missing field-inventory.md)
|
||||
EOF
|
||||
@@ -72,13 +72,23 @@ def parse_section_tokens(content, section_name):
|
||||
return set()
|
||||
|
||||
cc_only_fields = parse_section_tokens(inv_content, 'claude-code-only-fields')
|
||||
copilot_only_fields = parse_section_tokens(inv_content, 'copilot-only-fields')
|
||||
plugin_ignored_fields = parse_section_tokens(inv_content, 'plugin-silently-ignored-fields')
|
||||
|
||||
# Tools the runtime withholds from subagents regardless of the tools field
|
||||
SUBAGENT_UNAVAILABLE_TOOLS = {
|
||||
'AskUserQuestion', 'EnterPlanMode', 'ExitPlanMode', 'ScheduleWakeup', 'WaitForMcpServers',
|
||||
}
|
||||
|
||||
# Copilot body length limit (chars) — content beyond this is silently truncated
|
||||
COPILOT_BODY_LIMIT = 30000
|
||||
|
||||
# --- Detect scope ---
|
||||
def detect_scope(start_dir):
|
||||
current = os.path.abspath(start_dir)
|
||||
while True:
|
||||
if os.path.isfile(os.path.join(current, 'plugin.json')):
|
||||
if (os.path.isfile(os.path.join(current, 'plugin.json')) or
|
||||
os.path.isfile(os.path.join(current, '.claude-plugin', 'plugin.json'))):
|
||||
return 'plugin', current
|
||||
if os.path.isdir(os.path.join(current, '.git')):
|
||||
return 'project', current
|
||||
@@ -116,12 +126,16 @@ else: # user
|
||||
|
||||
# --- Helpers ---
|
||||
failed = False
|
||||
suggestions = []
|
||||
|
||||
def fail(msg):
|
||||
global failed
|
||||
failed = True
|
||||
print(f"FAIL {msg}")
|
||||
|
||||
def suggest(msg):
|
||||
suggestions.append(msg)
|
||||
|
||||
PLACEHOLDER_RE = re.compile(r'(?<!`)FILL IN:[^`\n]')
|
||||
|
||||
def parse_frontmatter(content):
|
||||
@@ -142,6 +156,17 @@ def get_frontmatter_keys(fm):
|
||||
keys.add(m.group(1))
|
||||
return keys
|
||||
|
||||
def extract_tools_list(fm):
|
||||
"""Extract tool names from the tools frontmatter field (space or comma separated)."""
|
||||
val = extract_field(fm, 'tools')
|
||||
if not val:
|
||||
return set()
|
||||
return set(re.split(r'[\s,]+', val.strip()))
|
||||
|
||||
def is_copilot_cloud_ide(fpath):
|
||||
"""True if the file is a cloud/IDE Copilot agent (name is optional for these)."""
|
||||
return '.github/copilot/agents' in os.path.abspath(fpath).replace(os.sep, '/')
|
||||
|
||||
def check_file(fpath, file_provider, is_plugin_scope):
|
||||
local_fname = os.path.basename(fpath)
|
||||
with open(fpath) as f:
|
||||
@@ -152,20 +177,23 @@ def check_file(fpath, file_provider, is_plugin_scope):
|
||||
fail(f"no valid YAML frontmatter (---...---) — {local_fname}")
|
||||
return
|
||||
|
||||
# name
|
||||
# name — required for CC and Copilot CLI; optional for Copilot cloud/IDE agents
|
||||
cloud_ide = (file_provider == 'copilot' and is_copilot_cloud_ide(fpath))
|
||||
name_val = extract_field(fm, 'name')
|
||||
if not name_val:
|
||||
fail(f"name field is missing or empty — {local_fname}")
|
||||
else:
|
||||
if not re.match(r'^[a-z0-9]+(-[a-z0-9]+)*$', name_val):
|
||||
fail(f"name '{name_val}' is not kebab-case — {local_fname}")
|
||||
# stem check
|
||||
if file_provider == 'copilot':
|
||||
stem = local_fname[:-len('.agent.md')]
|
||||
if not cloud_ide:
|
||||
if not name_val:
|
||||
fail(f"name field is missing or empty — {local_fname}")
|
||||
else:
|
||||
stem = local_fname[:-len('.md')]
|
||||
if name_val != stem:
|
||||
fail(f"name '{name_val}' does not match filename stem '{stem}' — {local_fname}")
|
||||
if not re.match(r'^[a-z0-9]+(-[a-z0-9]+)*$', name_val):
|
||||
fail(f"name '{name_val}' is not kebab-case — {local_fname}")
|
||||
# Stem check applies to Copilot CLI only; CC docs say filename need not match name
|
||||
if file_provider == 'copilot':
|
||||
stem = local_fname[:-len('.agent.md')]
|
||||
if name_val != stem:
|
||||
fail(f"name '{name_val}' does not match filename stem '{stem}' — {local_fname}")
|
||||
elif name_val and not re.match(r'^[a-z0-9]+(-[a-z0-9]+)*$', name_val):
|
||||
# cloud/IDE: name is optional, but if present it must be valid
|
||||
fail(f"name '{name_val}' is not kebab-case — {local_fname}")
|
||||
|
||||
# description
|
||||
desc_val = extract_field(fm, 'description')
|
||||
@@ -181,6 +209,9 @@ def check_file(fpath, file_provider, is_plugin_scope):
|
||||
else:
|
||||
if PLACEHOLDER_RE.search(body):
|
||||
fail(f"body contains unfilled FILL IN: placeholder — {local_fname}")
|
||||
# Copilot body length limit
|
||||
if file_provider == 'copilot' and len(body) > COPILOT_BODY_LIMIT:
|
||||
suggest(f"body exceeds {COPILOT_BODY_LIMIT:,} characters ({len(body):,} chars) — content beyond the limit is silently truncated by the Copilot runtime — {local_fname}")
|
||||
|
||||
# CC-only fields in Copilot file
|
||||
if file_provider == 'copilot':
|
||||
@@ -189,6 +220,13 @@ def check_file(fpath, file_provider, is_plugin_scope):
|
||||
if key in cc_only_fields:
|
||||
fail(f"CC-only field '{key}' present in Copilot file — {local_fname}")
|
||||
|
||||
# Copilot-only fields in CC file
|
||||
if file_provider == 'claude-code':
|
||||
fm_keys = get_frontmatter_keys(fm)
|
||||
for key in sorted(fm_keys):
|
||||
if key in copilot_only_fields:
|
||||
fail(f"Copilot-only field '{key}' present in CC file — {local_fname}")
|
||||
|
||||
# Silently-ignored fields in plugin-scope CC file
|
||||
if file_provider == 'claude-code' and is_plugin_scope:
|
||||
fm_keys = get_frontmatter_keys(fm)
|
||||
@@ -196,6 +234,12 @@ def check_file(fpath, file_provider, is_plugin_scope):
|
||||
if key in plugin_ignored_fields:
|
||||
fail(f"plugin-silently-ignored field '{key}' present in plugin-scope CC file — {local_fname}")
|
||||
|
||||
# Subagent-unavailable tools listed in tools field
|
||||
tools = extract_tools_list(fm)
|
||||
unavailable = tools & SUBAGENT_UNAVAILABLE_TOOLS
|
||||
for tool in sorted(unavailable):
|
||||
suggest(f"'{tool}' is listed in tools but is never available to subagents — the runtime withholds it regardless — {local_fname}")
|
||||
|
||||
# --- Check counterpart exists ---
|
||||
if not os.path.isfile(counterpart):
|
||||
fail(f"counterpart file not found: {counterpart}")
|
||||
@@ -206,5 +250,8 @@ is_plugin = (scope == 'plugin')
|
||||
check_file(agent_file, provider, is_plugin)
|
||||
check_file(counterpart, counterpart_provider, is_plugin)
|
||||
|
||||
for s in suggestions:
|
||||
print(f"SUGGESTION {s}")
|
||||
|
||||
sys.exit(1 if failed else 0)
|
||||
PYTHON
|
||||
|
||||
Reference in New Issue
Block a user