holocron/tests/test-adr0020-differential.sh

#!/usr/bin/env bash
# Differential test: scripts/skill-size-check.sh (the pre-commit hook) and
# factory-audit/scripts/validate.sh in SKILL mode (the in-skill auditor) must
# reach the SAME ADR-0020 verdict on the same file. ADR-0025 merged skill-audit
# and agent-audit behind one auto-detecting entry point; every fixture here is a
# skill directory, so every invocation below runs the skill flow. The agent flow
# has no counterpart hook to differ from — there is no agent-file size gate in
# .pre-commit-hooks.yaml — so it is out of this suite's scope, not dropped from it.
#
# Why this exists as a separate suite. tests/test-skill-size-check.sh already
# asserts the two agree on their CONSTANTS, and that assertion is necessary but
# demonstrably not sufficient: a previous review found the two scripts disagreeing
# on real files while every constant matched perfectly. Constants are one of the
# ways two hand-duplicated implementations diverge; comparison operators, message
# wording, which value gets measured, and which branch runs first are the others,
# and none of them is visible to a constant check.
#
# The consequence of divergence is specific and bad: the auditor reports a skill
# ready to ship and the commit hook then rejects it, or worse, the reverse. So the
# comparison here is over VERDICTS on files, not over source text.
#
# Scope: every axis the two scripts share. The ADR-0020 ones — description
# length and tier, body word count and tier, dangling routing targets, missing
# references/ pointers, the two Gotchas suggestions, the missing-boundary-clause
# suggestion, a declined resolution, an empty description — plus the two
# agentskills.io spec ceilings, MAX_LINES and MAX_WORDS.
#
# Those last two were EXCLUDED from this comparison until a real divergence
# shipped behind the exclusion. The header used to say "the hook checks
# whole-file lines and words" as if the auditor did not; it does, from its own
# copy of the same two constants, and the two implementations disagreed on
# Unicode whitespace for as long as nobody compared them. An axis both scripts
# measure is in scope by definition — the only lines still ignored are the ones
# a single script owns outright (validate.sh's name/directory agreement, script
# executability and 1024-char description backstop).
#
# Run over the real 39-skill corpus AND over purpose-built fixtures that sit ON
# each boundary. The corpus alone is not enough — it happens not to contain a
# file at exactly 900 body words, which is precisely where an inclusive/exclusive
# comparison mismatch would hide.
set -euo pipefail

REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
HOOK="$REPO_ROOT/scripts/skill-size-check.sh"
SKILL_VALIDATE="$REPO_ROOT/plugins/kyberforge/.apm/skills/factory-audit/scripts/validate.sh"

TMPDIR_T="$(mktemp -d)"
trap 'rm -rf "$TMPDIR_T"' EXIT

# ---------------------------------------------------------------------------
# Fixtures: one per ADR-0020 axis, placed ON the boundary wherever there is one.
# ---------------------------------------------------------------------------
# Built inside a synthetic plugin monorepo so boundary-target resolution actually
# runs for both scripts (in a bare temp dir both would decline, and "both
# declined" is agreement about nothing).
FIXTURE_ROOT="$TMPDIR_T/fixtures"
FX="$FIXTURE_ROOT/plugins/fixture-plugin/.apm/skills"
mkdir -p "$FX/sibling-skill" "$FIXTURE_ROOT/plugins/fixture-plugin/.apm/agents"

make_fx() {
  local name="$1" desc="$2" body_words="$3"
  mkdir -p "$FX/$name"
  {
    echo "---"
    echo "name: $name"
    echo "description: $desc"
    echo "metadata:"
    echo "  version: \"1.0.0\""
    echo "---"
    echo ""
    python3 -c "print(' '.join(['word'] * $body_words))"
  } > "$FX/$name/SKILL.md"
}
desc_of_length() {
  python3 - "$1" <<'PY'
import sys
n = int(sys.argv[1])
prefix = 'Use when doing the thing. Do not use for anything else. '
print(prefix + 'x' * (n - len(prefix)))
PY
}
CLEAN="Use when doing the thing. Do not use for anything else."

# Description tier boundaries, both sides of both thresholds.
make_fx desc-249 "$(desc_of_length 249)" 10
make_fx desc-250 "$(desc_of_length 250)" 10
make_fx desc-251 "$(desc_of_length 251)" 10
make_fx desc-400 "$(desc_of_length 400)" 10
make_fx desc-401 "$(desc_of_length 401)" 10
# Body tier boundaries, both sides of both thresholds.
make_fx body-599 "$CLEAN" 599
make_fx body-600 "$CLEAN" 600
make_fx body-601 "$CLEAN" 601
make_fx body-900 "$CLEAN" 900
make_fx body-901 "$CLEAN" 901
# Folding: the value has to be measured after YAML folding in both scripts.
mkdir -p "$FX/folded-desc"
{
  echo "---"
  echo "name: folded-desc"
  echo "description: >"
  python3 -c "print('\n'.join(['  ' + 'x' * 40] * 11))"
  echo "metadata:"
  echo "  version: \"1.0.0\""
  echo "---"
  echo ""
  echo "Do the thing."
} > "$FX/folded-desc/SKILL.md"
# Routing targets, one per tier the resolver can produce: resolves, dangles with
# in-sentence corroboration (ERROR/FAIL), dangles alone (SUGGESTION on both
# sides), route notation (ERROR/FAIL without corroboration), attributive
# (silent). Each tier is here because the two scripts have to agree on the TIER,
# not merely on the finding — a copy that promoted or demoted one of them would
# otherwise pass this comparison.
make_fx target-resolves "Use when doing the thing. Do not use for the other thing — use sibling-skill instead." 10
make_fx target-dangles "Use when doing the thing. Do not use for the other thing — use sibling-skill or no-such-skill instead." 10
make_fx target-dangles-lone "Use when doing the thing. Do not use for the other thing — use no-such-lone-skill instead." 10
make_fx target-dangles-notation "Use when doing the thing. Do not use for the other thing — use /no-such-notation-skill instead." 10
make_fx target-attributive "Use when doing the thing. Use pre-commit hooks instead of ad-hoc scripts." 10
# No boundary clause at all.
make_fx no-boundary "Use when the user wants the thing done." 10
# A missing references/ pointer, and a present one.
make_fx ref-missing "$CLEAN" 10
printf '\nIf the caller needs detail, read references/absent.md first.\n' >> "$FX/ref-missing/SKILL.md"
make_fx ref-present "$CLEAN" 10
printf '\nIf the caller needs detail, read references/there.md first.\n' >> "$FX/ref-present/SKILL.md"
mkdir -p "$FX/ref-present/references"
echo "detail" > "$FX/ref-present/references/there.md"
# Gotchas, over each guideline.
make_fx gotchas-many "$CLEAN" 0
cat >> "$FX/gotchas-many/SKILL.md" <<'EOF'

## Gotchas

- one trap here
- two trap here
- three trap here
- four trap here
- five trap here
- six trap here
- seven trap here

## Notes

EOF
python3 -c "print(' '.join(['word'] * 200))" >> "$FX/gotchas-many/SKILL.md"
# Gotchas over the 25% body-fraction guideline. Prose, not list items, so the
# entry guideline cannot fire and the two suggestions stay separable: 26 section
# words in a 100-word body is one word past the threshold.
make_fx gotchas-fraction "$CLEAN" 0
{
  echo ""
  echo "## Gotchas"
  echo ""
  python3 -c "print(' '.join(['word'] * 26))"
  echo ""
  echo "## Notes"
  echo ""
  python3 -c "print(' '.join(['word'] * 70))"
} >> "$FX/gotchas-fraction/SKILL.md"

# The agentskills.io spec ceilings, measured over Unicode whitespace.
#
# These two are in the comparison at all because they used to be excluded from
# it — `_non_adr_hook_error()` waved a spec-ceiling exit through as "not a
# disagreement", and that exclusion is exactly why the divergence below stayed
# invisible. The hook counted lines and words in a single awk pass (NR / NF)
# while skill-audit counted them with Python's splitlines() / split(). The two
# primitives do not agree: splitlines() also breaks on U+2028, U+2029, \x0b,
# \x0c, \x1c-\x1e and \x85, and split() breaks on every Unicode space. Same
# constants, same file, different verdict — hook green, audit FAIL, which is the
# precise failure mode ("passes its own audit, blocked by the commit hook",
# inverted) this whole suite exists to catch.
#
# One fixture per primitive, each sitting just past its ceiling on the Python
# measurement and nowhere near it on the awk one.
python3 - "$FX" <<'PY'
import os
import sys

fx = sys.argv[1]
# Spelled as escapes, never as literals. An invisible separator pasted into a
# source file is unreviewable and one editor round-trip away from becoming an
# ordinary space, which would silently turn both fixtures into nothing.
SEP_LINE = '\u2028'   # LINE SEPARATOR: splitlines() breaks on it, awk's NR does not
SEP_WORD = '\u00a0'   # NO-BREAK SPACE: split() breaks on it, awk's NF does not
head = ('---\nname: %s\n'
        'description: Use when doing the thing. Do not use for anything else.\n'
        '---\n\n')
# 600 U+2028-separated segments: 605 lines to splitlines(), 6 to awk's NR.
# Word count stays far below the 2,770 ceiling, so this fixture isolates lines.
cases = {
    'spec-lines-u2028': SEP_LINE.join(['word'] * 600),
    # 2,800 U+00A0-separated words: 2,816 words to split(), 17 to awk's NF.
    'spec-words-u00a0': SEP_WORD.join(['word'] * 2800),
}
for name, body in cases.items():
    d = os.path.join(fx, name)
    os.makedirs(d, exist_ok=True)
    with open(os.path.join(d, 'SKILL.md'), 'w', encoding='utf-8') as fh:
        fh.write(head % name + body + '\n')
PY

# Empty description — the shape that used to exit 0 in silence.
mkdir -p "$FX/empty-desc"
printf -- '---\nname: empty-desc\ndescription:\nmodel: sonnet\n---\n\nDo the thing.\n' \
  > "$FX/empty-desc/SKILL.md"
# Every boundary shape at once, so a divergence that only appears when several
# findings fire together is not missed.
make_fx combined "$(desc_of_length 401)" 901
printf '\nIf the caller needs detail, read references/absent.md first.\n' >> "$FX/combined/SKILL.md"

# A skill with routing targets and NO authoring root above it — deliberately
# OUTSIDE the fixture plugin tree. Both scripts must decline out loud, and both
# must decline identically; "both declined" is only meaningful as agreement if
# the declining path is exercised on purpose somewhere.
ORPHAN_ROOT="$TMPDIR_T/orphan"
mkdir -p "$ORPHAN_ROOT/no-universe"
{
  echo "---"
  echo "name: no-universe"
  echo "description: Use when doing the thing. Do not use for the other thing — use some-other-skill instead."
  echo "metadata:"
  echo "  version: \"1.0.0\""
  echo "---"
  echo ""
  echo "Do the thing."
} > "$ORPHAN_ROOT/no-universe/SKILL.md"

# ---------------------------------------------------------------------------
# The comparison
# ---------------------------------------------------------------------------
python3 - "$HOOK" "$SKILL_VALIDATE" "$REPO_ROOT" "$FX" "$ORPHAN_ROOT" <<'PYTHON'
import glob
import os
import re
import subprocess
import sys

hook, validate, repo_root, fixture_dir, orphan_dir = sys.argv[1:6]

passes = 0
failures = 0


def ok(msg):
    global passes
    passes += 1
    print("  PASS: %s" % msg)


def bad(msg):
    global failures
    failures += 1
    print("  FAIL: %s" % msg)


# Tier prefixes. The hook writes `ERROR: ` / `SUGGESTION: ` / `INFO: `; the
# auditor writes `FAIL  ` / `SUGGESTION  ` / `INFO  ` and additionally `PASS  `
# lines, which carry no finding and are dropped.
TIERS = (
    ('ERROR', ('ERROR:', 'FAIL ')),
    ('SUGGESTION', ('SUGGESTION:', 'SUGGESTION ')),
    ('INFO', ('INFO:', 'INFO ')),
)

# Each rule turns a finding line into a canonical token. Wording differs between
# the two scripts by design (one addresses a committer, the other an auditor), so
# the tokens deliberately capture the MEASUREMENT and not the sentence.
RULES = (
    ('DESC_CHARS', re.compile(r'description is (\d+) char')),
    ('BODY_WORDS', re.compile(r'body is (\d+) words')),
    ('ROUTE', re.compile(r"routes to '([^']+)'")),
    ('MISSING_REF', re.compile(r'points at (references/[^\s,]+)')),
    ('GOTCHA_ENTRIES', re.compile(r'Gotchas section has (\d+) entries')),
    ('GOTCHA_FRACTION', re.compile(r'Gotchas section is (\d+) of (\d+) body words')),
    ('NO_BOUNDARY_CLAUSE', re.compile(r'(description has no boundary clause)')),
    ('RESOLUTION_DECLINED', re.compile(r'(boundary-target resolution DID NOT RUN)')),
    ('DESC_EMPTY', re.compile(r'(description field is missing or empty)')),
    # The agentskills.io spec ceilings. These were EXCLUDED from the comparison
    # until the awk/Python divergence shipped, on the reasoning that "the hook
    # checks whole-file lines and words" and the auditor did not. It does — with
    # the same two constants — so the exclusion was never a scope decision, only
    # an untested assumption, and it hid a real disagreement. Both scripts spell
    # the finding differently, so the patterns match either wording and capture
    # only the MEASUREMENT:
    #   hook:  "... has 605 lines, exceeding the 500-line ceiling ..."
    #   audit: "SKILL.md line count 605 — exceeds 500-line limit"
    ('SPEC_LINES', re.compile(r'(?:has|line count) (\d+)(?: lines,)? (?:exceeding|—)')),
    ('SPEC_WORDS', re.compile(
        r'(?:has|word count) (\d+)(?: words \(proxy for tokens\),)? (?:exceeding|—)')),
)


def verdict(output):
    """The set of ADR-0020 findings in a script's output, tier included.

    Lines that match no rule are dropped rather than compared: the two scripts
    legitimately check different things outside ADR-0020 (name/directory
    agreement, script executability, the 1024-char spec backstop), and forcing
    those into the comparison would report a difference that is not a
    disagreement.

    The whole-file line and word ceilings are NOT in that list. They were
    excluded once, on the untested assumption that awk and splitlines() agree;
    they do not, and the divergence was invisible for exactly as long as the
    exclusion stood. SPEC_LINES/SPEC_WORDS are compared like any other rule —
    see the file header. Do not re-add an exclusion for them.
    """
    found = set()
    for raw in output.splitlines():
        line = raw.strip()
        tier = None
        for name, prefixes in TIERS:
            if any(line.startswith(p) for p in prefixes):
                tier = name
                break
        if tier is None:
            continue
        for token, pattern in RULES:
            match = pattern.search(line)
            if match:
                found.add((tier, token) + tuple(match.groups()))
    return found


def run(cmd):
    proc = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
    return proc.returncode, proc.stdout.decode('utf-8', 'replace')


def compare(label, skill_dir):
    skill_md = os.path.join(skill_dir, 'SKILL.md')
    hook_rc, hook_out = run(['bash', hook, skill_md])
    audit_rc, audit_out = run(['bash', validate, skill_dir])
    hook_v = verdict(hook_out)
    audit_v = verdict(audit_out)

    problems = []
    only_hook = sorted(hook_v - audit_v)
    only_audit = sorted(audit_v - hook_v)
    if only_hook:
        problems.append('only the hook reported %s' % (only_hook,))
    if only_audit:
        problems.append('only the auditor reported %s' % (only_audit,))

    # Exit codes are compared on the ADR-0020 axis only: an ERROR-tier ADR-0020
    # finding must make BOTH scripts non-zero, and neither may be turned
    # non-zero by a SUGGESTION. The raw codes cannot be compared directly —
    # validate.sh also fails on checks the hook does not run at all.
    hook_err = any(t == 'ERROR' for t, *_ in hook_v)
    audit_err = any(t == 'ERROR' for t, *_ in audit_v)
    if hook_err and hook_rc == 0:
        problems.append('the hook reported an ADR-0020 ERROR but exited 0')
    if audit_err and audit_rc == 0:
        problems.append('the auditor reported an ADR-0020 FAIL but exited 0')
    # No escape hatch here any more. There used to be one — a
    # `_non_adr_hook_error()` helper that waved through a non-zero hook exit
    # explained by MAX_LINES / MAX_WORDS, on the grounds that those two were
    # outside the comparison. They are inside it now (see SPEC_LINES /
    # SPEC_WORDS in RULES), so every ERROR the hook can raise is a token this
    # comparison holds both scripts to.
    if not hook_err and hook_rc != 0:
        problems.append('the hook exited %d with no compared ERROR at all — it has an '
                        'ERROR source this comparison does not know about' % hook_rc)

    if problems:
        bad('%s: %s' % (label, '; '.join(problems)))
    else:
        return True
    return False


# --- The real corpus -------------------------------------------------------
corpus = sorted(glob.glob(os.path.join(repo_root, 'plugins', '*', '.apm', 'skills', '*')))
corpus = [d for d in corpus if os.path.isfile(os.path.join(d, 'SKILL.md'))]
print("")
print("--- the two scripts agree on every skill in the live corpus (%d files) ---" % len(corpus))
if len(corpus) < 30:
    bad('only %d corpus skills were discovered — the glob is wrong, so this leg '
        'proves nothing' % len(corpus))
else:
    ok('discovered %d corpus skills to compare' % len(corpus))
agreed = 0
for skill_dir in corpus:
    rel = os.path.relpath(skill_dir, repo_root)
    if compare(rel, skill_dir):
        agreed += 1
if agreed == len(corpus):
    ok('all %d corpus skills produce identical ADR-0020 verdicts from both scripts' % agreed)

# --- Boundary fixtures -----------------------------------------------------
fixtures = sorted(d for d in (glob.glob(os.path.join(fixture_dir, '*'))
                                 + glob.glob(os.path.join(orphan_dir, '*')))
                  if os.path.isfile(os.path.join(d, 'SKILL.md')))
print("")
print("--- the two scripts agree on every boundary fixture (%d files) ---" % len(fixtures))
if len(fixtures) < 15:
    bad('only %d fixtures were built — the fixture set is incomplete, so the '
        'boundaries the corpus does not cover are untested' % len(fixtures))
else:
    ok('built %d boundary fixtures to compare' % len(fixtures))
fx_agreed = 0
for skill_dir in fixtures:
    if compare(os.path.basename(skill_dir), skill_dir):
        fx_agreed += 1
if fx_agreed == len(fixtures):
    ok('all %d boundary fixtures produce identical ADR-0020 verdicts from both scripts' % fx_agreed)

# --- The comparison must not be vacuous ------------------------------------
# Everything above would also pass if verdict() extracted nothing at all. So the
# fixtures are required to have produced findings across every axis this suite
# claims to compare — if a rule stops matching (a reworded message, say), that is
# a silent loss of coverage and it fails here instead.
print("")
print("--- the comparison actually extracted findings on every axis it claims to cover ---")
seen_tokens = set()
for skill_dir in fixtures:
    _, out = run(['bash', hook, os.path.join(skill_dir, 'SKILL.md')])
    for entry in verdict(out):
        seen_tokens.add(entry[1])
    _, out = run(['bash', validate, skill_dir])
    for entry in verdict(out):
        seen_tokens.add(entry[1])
expected_tokens = {t for t, _ in RULES}
missing = sorted(expected_tokens - seen_tokens)
if missing:
    bad('no fixture produced a finding for %s — verdict() may no longer match '
        'those messages, and any disagreement on them would go unseen' % missing)
else:
    ok('every one of the %d compared axes was exercised by at least one fixture'
       % len(expected_tokens))

# --- The Unicode-whitespace fixtures, named and asserted directly -----------
# The two comparisons above would catch this divergence, but only as "fixture
# spec-lines-u2028 disagreed" — one line among 65. Spelled out here so the
# failure names the primitive, and so the ceiling is asserted to FIRE in both
# scripts rather than merely to be reported the same way by both.
print("")
print("--- both scripts break the spec ceilings on the same Unicode whitespace ---")
for name, token, expected in (('spec-lines-u2028', 'SPEC_LINES', '605'),
                              ('spec-words-u00a0', 'SPEC_WORDS', '2816')):
    skill_dir = os.path.join(fixture_dir, name)
    _, h_out = run(['bash', hook, os.path.join(skill_dir, 'SKILL.md')])
    _, a_out = run(['bash', validate, skill_dir])
    want = ('ERROR', token, expected)
    missing = [who for who, v in (('the hook', verdict(h_out)),
                                  ('the auditor', verdict(a_out)))
               if want not in v]
    if missing:
        bad('%s: %s did not report %s=%s. The two scripts must count with the '
            'same primitive — Python splitlines()/split(), not awk NR/NF, which '
            'does not break on this character' % (name, ' and '.join(missing),
                                                  token, expected))
    else:
        ok('%s: both scripts measure %s=%s and raise the ceiling ERROR'
           % (name, token, expected))

print("")
print("Results: %d passed, %d failed" % (passes, failures))
sys.exit(1 if failures else 0)
PYTHON