holocron/plugins/kyberforge/skills/agent-audit/scripts/validate.sh

#!/usr/bin/env bash
set -euo pipefail

usage() {
  cat <<EOF
Usage: validate.sh <agent-file>

Validate an agent definition file against the agent definition spec.

At plugin/APM scope, <agent-file> is a single vendor-neutral
.apm/agents/<name>.agent.md file (frontmatter allowlist: name, description,
model — no counterpart file). At project or user scope, <agent-file> is
either half of a Claude Code .md / Copilot .agent.md pair.

Arguments:
  agent-file  Path to the agent file (or either half of a project/user-scope pair).

Exit codes:
  0  All checks passed (may include SUGGESTIONs)
  1  One or more checks failed
  2  Script error (unrecognized file extension or missing field-inventory.md)
EOF
}

if [[ "${1:-}" == "--help" || "${1:-}" == "-h" ]]; then
  usage
  exit 0
fi

if [[ $# -lt 1 ]]; then
  echo "Error: agent-file is required." >&2
  echo "" >&2
  usage >&2
  exit 1
fi

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"

python3 -u - "$1" "$SCRIPT_DIR" <<'PYTHON'
import sys
import os
import re

agent_file = os.path.abspath(sys.argv[1])
script_dir = sys.argv[2]

fname = os.path.basename(agent_file)

# --- Detect provider (check .agent.md before .md) ---
if fname.endswith('.agent.md'):
    provider = 'copilot'
    name_stem = fname[:-len('.agent.md')]
elif fname.endswith('.md'):
    provider = 'claude-code'
    name_stem = fname[:-len('.md')]
else:
    print(f"Error: unrecognized extension '{fname}' — expected .md or .agent.md", file=sys.stderr)
    sys.exit(2)

# --- Load field-inventory.md ---
inv_path = os.path.normpath(os.path.join(script_dir, '..', 'references', 'field-inventory.md'))
if not os.path.isfile(inv_path):
    print(f"Error: field-inventory.md not found at {inv_path}", file=sys.stderr)
    sys.exit(2)

with open(inv_path) as f:
    inv_content = f.read()

def parse_section_tokens(content, section_name):
    lines = content.splitlines()
    for i, line in enumerate(lines):
        if line.strip() == f'## {section_name}':
            for j in range(i + 1, len(lines)):
                stripped = lines[j].strip()
                if stripped and not stripped.startswith('#') and not stripped.startswith('---'):
                    return set(stripped.split())
    return set()

cc_only_fields = parse_section_tokens(inv_content, 'claude-code-only-fields')
copilot_only_fields = parse_section_tokens(inv_content, 'copilot-only-fields')
apm_agent_allowlist = parse_section_tokens(inv_content, 'apm-agent-allowlist')

# Tools the runtime withholds from subagents regardless of the tools field
SUBAGENT_UNAVAILABLE_TOOLS = {
    'AskUserQuestion', 'EnterPlanMode', 'ExitPlanMode', 'ScheduleWakeup', 'WaitForMcpServers',
}

# Copilot body length limit (chars) — content beyond this is silently truncated
COPILOT_BODY_LIMIT = 30000

# --- Helpers (shared by every scope) ---
failed = False
suggestions = []

def fail(msg):
    global failed
    failed = True
    print(f"FAIL  {msg}")

def suggest(msg):
    suggestions.append(msg)

PLACEHOLDER_RE = re.compile(r'(?<!`)FILL IN:[^`\n]')

def parse_frontmatter(content):
    m = re.match(r'^---\n(.*?)\n---', content, re.DOTALL)
    if not m:
        return None, content
    return m.group(1), content[m.end():]

def extract_field(fm, field):
    m = re.search(rf'^{re.escape(field)}:\s*(.+)', fm, re.MULTILINE)
    return m.group(1).strip() if m else None

def get_frontmatter_keys(fm):
    keys = set()
    for line in fm.splitlines():
        m = re.match(r'^([a-zA-Z][a-zA-Z0-9_-]*):', line)
        if m:
            keys.add(m.group(1))
    return keys

def extract_tools_list(fm):
    """Extract tool names from the tools frontmatter field (space or comma separated)."""
    val = extract_field(fm, 'tools')
    if not val:
        return set()
    return set(re.split(r'[\s,]+', val.strip()))

def is_copilot_cloud_ide(fpath):
    """True if the file is a cloud/IDE Copilot agent (name is optional for these)."""
    return '.github/copilot/agents' in os.path.abspath(fpath).replace(os.sep, '/')

# --- Detect scope ---
# APM_TYPE_RE matches a top-level (column-0) `type:` line in apm.yml whose value is
# exactly one of the four package content types. Group 1 captures an optional
# opening quote; \1 requires the same character (or nothing) to close it, so
# "skill" and '"skill"' both match but a mismatched quote doesn't. The value
# must then be followed by whitespace or end-of-line — not just a non-word
# character — so a malformed value like `prompts-only` is correctly rejected
# instead of false-matching on the `prompts` prefix.
APM_TYPE_RE = re.compile(r"^type:\s*(['\"]?)(instructions|skill|hybrid|prompts)\1(?:\s|$)")

def find_apm_package_root(apm_yml_path):
    """Return True if apm_yml_path has a top-level type: line (i.e. is a package
    manifest, not a type:-less marketplace-only apm.yml)."""
    with open(apm_yml_path) as f:
        for line in f:
            if APM_TYPE_RE.match(line):
                return True
    return False

def detect_scope(start_dir):
    home = os.path.expanduser('~')
    original_start = os.path.abspath(start_dir)
    # Agent files conventionally live exactly two path segments below their
    # scope root — <root>/.claude/agents, <root>/.github/agents,
    # <root>/.copilot/agents, or <root>/.apm/agents (see new-agent.sh's
    # CC_DIR/CP_DIR and user-scope dirs). Stripping those two segments
    # recovers the same root new-agent.sh would have been invoked with to
    # produce this exact file, independent of how far the walk below has to
    # travel to find (or fail to find) a marker — mirrors new-agent.sh's
    # `root` vs `current` distinction even though validate.sh is handed a
    # file's directory, not the scope root itself.
    #
    # That arithmetic is only trustworthy when the path actually has this
    # shape: parent directory literally named "agents", grandparent one of
    # the four known scope-dir names. A hand-placed or otherwise
    # non-conventional agent file (never produced by new-agent.sh) has no
    # such guarantee — blindly trusting two-segments-up there could point at
    # an unrelated ancestor. conventional_shape gates every use of
    # conventional_root below; when it's false, the walked-to `current`
    # directory is used instead, the same fallback this function used before
    # conventional_root existed.
    scope_dir_name = os.path.basename(os.path.dirname(original_start))
    conventional_shape = (
        os.path.basename(original_start) == 'agents'
        and scope_dir_name in ('.claude', '.github', '.copilot', '.apm')
    )
    conventional_root = os.path.dirname(os.path.dirname(original_start))
    current = original_start
    while True:
        apm_yml = os.path.join(current, 'apm.yml')
        if os.path.isfile(apm_yml) and find_apm_package_root(apm_yml):
            return 'plugin', current
        # $HOME is the user-scope boundary — checked before the .git test
        # below, so a dotfiles-managed $HOME (yadm, chezmoi bare-repo, etc.)
        # can't shadow user scope by being its own .git repo. 'user' scope
        # requires EITHER start_dir to BE $HOME itself (no walk-up — the
        # new-agent.sh "root exactly $HOME" case) OR start_dir to sit at the
        # conventional two-segments-below-root depth (i.e. $HOME IS that
        # root, matching the real ~/.claude/agents or ~/.copilot/agents
        # shape). Any other walk-up into $HOME — a marker-less directory
        # nested deeper than that convention — resolves to project scope
        # instead: a stray directory under $HOME can't be silently
        # redirected into the shared global ~/.claude or ~/.copilot agent
        # directories.
        if current == home:
            if original_start == home or (conventional_shape and conventional_root == home):
                return 'user', home
            return 'project', conventional_root if conventional_shape else current
        # .git is a directory in a normal checkout but a file (`gitdir: ...`)
        # in a git worktree — exists() covers both. Returns conventional_root,
        # not current: new-agent.sh's project-scope file placement always
        # uses its `$ROOT` argument directly, never the walked-up `.git`
        # location, so a <root> one or more levels below the repo's .git
        # (a subdirectory of a larger git-tracked tree — explicitly a
        # supported case per new-agent.sh's usage text) must resolve to the
        # same root new-agent.sh actually wrote to, not to the .git dir —
        # unless the path lacks the conventional shape, in which case that
        # arithmetic isn't trustworthy and current is used instead.
        if os.path.exists(os.path.join(current, '.git')):
            return 'project', conventional_root if conventional_shape else current
        parent = os.path.dirname(current)
        if parent == current:
            return 'project', conventional_root if conventional_shape else current
        current = parent

agent_dir = os.path.dirname(agent_file)
scope, scope_root = detect_scope(agent_dir)

# --- Plugin/APM scope: single vendor-neutral file, no counterpart ---
def check_apm_agent_file(fpath, allowlist, stem):
    local_fname = os.path.basename(fpath)
    with open(fpath) as f:
        content = f.read()

    fm, body = parse_frontmatter(content)
    if fm is None:
        fail(f"no valid YAML frontmatter (---...---) — {local_fname}")
        return

    # The apm-agent.md template embeds its authoring guidance as HTML
    # comments inside the frontmatter block (so they render invisible in a
    # Markdown preview but stay visible in the raw file). get_frontmatter_keys
    # silently ignores any line that isn't a `key:` match, so a comment left
    # behind at ship time would otherwise pass unnoticed — yet apm compile
    # copies this frontmatter verbatim to both harnesses, and `<!-- -->` is
    # not valid YAML, so yaml.safe_load breaks on both downstream (ADR-0016).
    if re.search(r'<!--|-->', fm):
        fail(f"frontmatter still contains template HTML comments (<!-- ... -->) "
             f"— delete them before shipping — {local_fname}")

    # Allowlist: the permitted keys are data, read at load time from
    # references/field-inventory.md's `## apm-agent-allowlist` section — do not
    # restate them here, or this comment goes stale the next time that line
    # changes. apm compile verbatim-copies frontmatter to every target, so a key
    # outside the list is unsafe on at least one harness (ADR-0016). Note the
    # list admits denylist-shaped restrictions (disallowedTools) but never
    # allowlist-shaped ones (tools), whose value shape differs per harness.
    fm_keys = get_frontmatter_keys(fm)
    for key in sorted(fm_keys):
        if key not in allowlist:
            fail(f"field '{key}' is not in the vendor-neutral APM agent allowlist "
                 f"({', '.join(sorted(allowlist))}) — {local_fname}")

    # name — required, kebab-case, must match filename stem (file is <name>.agent.md)
    name_val = extract_field(fm, 'name')
    if not name_val:
        fail(f"name field is missing or empty — {local_fname}")
    else:
        if not re.match(r'^[a-z0-9]+(-[a-z0-9]+)*$', name_val):
            fail(f"name '{name_val}' is not kebab-case — {local_fname}")
        if name_val != stem:
            fail(f"name '{name_val}' does not match filename stem '{stem}' — {local_fname}")

    # description — required, non-empty, no placeholder
    desc_val = extract_field(fm, 'description')
    if not desc_val:
        fail(f"description field is missing or empty — {local_fname}")
    else:
        if PLACEHOLDER_RE.search(desc_val):
            fail(f"description contains unfilled FILL IN: placeholder — {local_fname}")

    # body — required, non-empty, no placeholder; same Copilot truncation risk
    # applies since this file compiles verbatim into a real Copilot file downstream.
    if not body.strip():
        fail(f"system prompt body is empty — {local_fname}")
    else:
        if PLACEHOLDER_RE.search(body):
            fail(f"body contains unfilled FILL IN: placeholder — {local_fname}")
        if len(body) > COPILOT_BODY_LIMIT:
            suggest(f"body exceeds {COPILOT_BODY_LIMIT:,} characters ({len(body):,} chars) — "
                    f"content beyond the limit is silently truncated by the Copilot runtime "
                    f"once apm compile emits it downstream — {local_fname}")

if scope == 'plugin':
    check_apm_agent_file(agent_file, apm_agent_allowlist, name_stem)
    for s in suggestions:
        print(f"SUGGESTION  {s}")
    sys.exit(1 if failed else 0)

# --- Project/user scope: unchanged CC/Copilot pair validation ---

# --- Derive counterpart path ---
if scope == 'project':
    if provider == 'claude-code':
        counterpart = os.path.join(scope_root, '.github', 'agents', name_stem + '.agent.md')
        counterpart_provider = 'copilot'
    else:
        counterpart = os.path.join(scope_root, '.claude', 'agents', name_stem + '.md')
        counterpart_provider = 'claude-code'
else:  # user
    home = os.path.expanduser('~')
    if provider == 'claude-code':
        counterpart = os.path.join(home, '.copilot', 'agents', name_stem + '.agent.md')
        counterpart_provider = 'copilot'
    else:
        counterpart = os.path.join(home, '.claude', 'agents', name_stem + '.md')
        counterpart_provider = 'claude-code'

def check_file(fpath, file_provider):
    local_fname = os.path.basename(fpath)
    with open(fpath) as f:
        content = f.read()

    fm, body = parse_frontmatter(content)
    if fm is None:
        fail(f"no valid YAML frontmatter (---...---) — {local_fname}")
        return

    # name — required for CC and Copilot CLI; optional for Copilot cloud/IDE agents
    cloud_ide = (file_provider == 'copilot' and is_copilot_cloud_ide(fpath))
    name_val = extract_field(fm, 'name')
    if not cloud_ide:
        if not name_val:
            fail(f"name field is missing or empty — {local_fname}")
        else:
            if not re.match(r'^[a-z0-9]+(-[a-z0-9]+)*$', name_val):
                fail(f"name '{name_val}' is not kebab-case — {local_fname}")
            # Stem check applies to Copilot CLI only; CC docs say filename need not match name
            if file_provider == 'copilot':
                stem = local_fname[:-len('.agent.md')]
                if name_val != stem:
                    fail(f"name '{name_val}' does not match filename stem '{stem}' — {local_fname}")
    elif name_val and not re.match(r'^[a-z0-9]+(-[a-z0-9]+)*$', name_val):
        # cloud/IDE: name is optional, but if present it must be valid
        fail(f"name '{name_val}' is not kebab-case — {local_fname}")

    # description
    desc_val = extract_field(fm, 'description')
    if not desc_val:
        fail(f"description field is missing or empty — {local_fname}")
    else:
        if PLACEHOLDER_RE.search(desc_val):
            fail(f"description contains unfilled FILL IN: placeholder — {local_fname}")

    # body
    if not body.strip():
        fail(f"system prompt body is empty — {local_fname}")
    else:
        if PLACEHOLDER_RE.search(body):
            fail(f"body contains unfilled FILL IN: placeholder — {local_fname}")
        # Copilot body length limit
        if file_provider == 'copilot' and len(body) > COPILOT_BODY_LIMIT:
            suggest(f"body exceeds {COPILOT_BODY_LIMIT:,} characters ({len(body):,} chars) — content beyond the limit is silently truncated by the Copilot runtime — {local_fname}")

    # CC-only fields in Copilot file
    if file_provider == 'copilot':
        fm_keys = get_frontmatter_keys(fm)
        for key in sorted(fm_keys):
            if key in cc_only_fields:
                fail(f"CC-only field '{key}' present in Copilot file — {local_fname}")

    # Copilot-only fields in CC file
    if file_provider == 'claude-code':
        fm_keys = get_frontmatter_keys(fm)
        for key in sorted(fm_keys):
            if key in copilot_only_fields:
                fail(f"Copilot-only field '{key}' present in CC file — {local_fname}")

    # Subagent-unavailable tools listed in tools field
    tools = extract_tools_list(fm)
    unavailable = tools & SUBAGENT_UNAVAILABLE_TOOLS
    for tool in sorted(unavailable):
        suggest(f"'{tool}' is listed in tools but is never available to subagents — the runtime withholds it regardless — {local_fname}")

# --- Check counterpart exists ---
if not os.path.isfile(counterpart):
    fail(f"counterpart file not found: {counterpart}")
    sys.exit(1)

# --- Check both files ---
check_file(agent_file, provider)
check_file(counterpart, counterpart_provider)

for s in suggestions:
    print(f"SUGGESTION  {s}")

sys.exit(1 if failed else 0)
PYTHON