holocron/tests/test-adr0020-frontmatter.sh

#!/usr/bin/env bash
# Regression test for the two ways an ADR-0020 gate can be made to check NOTHING
# while still exiting 0. Both were live defects, both were silent, and both sit
# in the shared resolver block that all three scripts embed verbatim — so every
# case below runs against all three.
#
#   1. THE FRONTMATTER BLOCKER. The frontmatter matcher used to be `^---\n`. A
#      UTF-8 BOM, a leading blank line, a trailing space after either marker, or
#      CRLF line endings all defeated it, and the miss was not reported: every
#      ADR-0020 check was skipped and the file passed. Measured at the time: a
#      550-character description with a 1,000-word body exited 0 behind a BOM.
#      So this file asserts two complementary things — that each of those four
#      shapes is now TOLERATED (the findings actually fire), and that
#      frontmatter which genuinely cannot be parsed is a hard ERROR rather than
#      a quiet skip. A file that cannot be measured must never report green.
#
#   2. THE VALUELESS DESCRIPTION. `description:` with no value, followed by
#      another key, let a line regex's `\s*` cross the newline and capture the
#      NEXT key. The value then looked present (so "missing or empty" never
#      fired) and was empty once folded (so every ADR-0020 gate early-returned).
#      An agent file with one exited 0 with zero output through a BLOCKING
#      pre-push gate. All five spellings of "no value" are pinned here, plus the
#      three shapes where the value is present but is not TEXT — a list, a
#      mapping, a bool. Those used to be `str()`-coerced and then measured as a
#      Python repr, so `description: true` was the four-character "True" and
#      passed the 400-character gate.
#
#   3. THE INDENTED CLOSING MARKER. The mirror image of (1): content the pattern
#      was too LOOSE to reject. `\r?\n[ \t]*---` matched an indented `---` inside
#      a `>`-folded description, truncating the frontmatter mid-value — the
#      description gate then measured a fragment and the body gate measured the
#      discarded description text.
#
# Every needle names the specific branch or measurement the case is about. A
# needle loose enough to match two branches is how the yaml-none fixture spent
# its life asserting the wrong one: it emitted `---\n---\n`, which never matched
# the frontmatter pattern at all, and passed on the bare word "frontmatter".
#
# Both fixtures carry an over-ceiling description AND an over-ceiling body on
# purpose: asserting a non-zero exit alone would be satisfied by the "cannot
# parse" error itself, so the tolerated shapes are asserted on the CONTENT of
# the findings, not on the exit code.
set -euo pipefail

REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
HOOK="$REPO_ROOT/scripts/skill-size-check.sh"
SKILL_VALIDATE="$REPO_ROOT/plugins/kyberforge/.apm/skills/skill-audit/scripts/validate.sh"
AGENT_VALIDATE="$REPO_ROOT/plugins/kyberforge/.apm/skills/agent-audit/scripts/validate.sh"
PASS=0
FAIL=0

pass() { echo "  PASS: $1"; PASS=$((PASS + 1)); }
fail() { echo "  FAIL: $1"; FAIL=$((FAIL + 1)); }

TMPDIR_T="$(mktemp -d)"
trap 'rm -rf "$TMPDIR_T"' EXIT

DESC_CHARS=450
BODY_WORDS=1000

# write_fixture <kind> <path> <name> — one generator for both file shapes.
#
# Byte-level control is the point: BOM placement, line endings and trailing
# whitespace are exactly what is under test, so the file is emitted in binary
# mode rather than through a shell heredoc that would normalise them.
write_fixture() {
  python3 - "$1" "$2" "$3" "$DESC_CHARS" "$BODY_WORDS" <<'PY'
import sys

kind, path, name, desc_chars, body_words = sys.argv[1:6]
desc = 'x' * int(desc_chars)
body = ' '.join(['word'] * int(body_words))

# The default, well-formed shape. Variants below mutate it.
open_marker = '---'
close_marker = '---'
prefix = ''
newline = '\n'
fm_lines = ['name: ' + name, 'description: ' + desc]

if kind == 'plain':
    pass
elif kind == 'bom':
    prefix = ''
elif kind == 'leading-blanks':
    prefix = '\n\n   \n'
elif kind == 'trailing-ws':
    open_marker = '---   '
    close_marker = '---\t '
elif kind == 'crlf':
    newline = '\r\n'
elif kind == 'no-close':
    close_marker = None
elif kind == 'yaml-list':
    fm_lines = ['- one', '- two']
elif kind == 'yaml-string':
    fm_lines = ['just a bare scalar, not a mapping']
elif kind == 'yaml-none':
    # A comment-only block, NOT an empty one. `---\n---\n` does not match
    # FRONTMATTER_RE at all (the pattern needs a `\n` between the markers), so
    # it lands on the "no parseable frontmatter" branch and never reaches the
    # `data is None` -> "not a YAML mapping" branch this fixture is named for.
    # It passed anyway because the needle used to be the bare word
    # "frontmatter", which both messages contain. A comment is real frontmatter
    # text that yaml.safe_load() returns None for, which is the branch.
    fm_lines = ['# nothing but a comment']
elif kind == 'yaml-empty-block':
    # The shape the fixture above USED to have, kept as its own case so the
    # "no parseable frontmatter block" branch is covered on purpose rather than
    # by accident.
    fm_lines = []
elif kind == 'desc-folded-indented':
    # A `>`-folded description whose CONTENT contains an indented `---` line.
    # YAML block-scalar content must be indented deeper than its key, so this is
    # a value, not a document marker — but the closing pattern used to be
    # `\r?\n[ \t]*---`, which matched it, truncated the frontmatter mid-value
    # and silently reclassified the rest of the description as body. Both halves
    # of that are vacuous greens: the description gate measured a fragment, and
    # the body gate measured description text.
    #
    # The value is padded to exactly desc_chars AFTER folding, and the boundary
    # clause naming a target sits in the part the truncation used to discard.
    head = 'Use when doing the thing. '
    tail = ' Do not use for improvements — use no-such-folded-target instead.'
    span = int(desc_chars) - len(head) - len(tail) - len(' --- ')
    if span < 2:
        raise SystemExit('desc_chars too small for the folded fixture')
    fm_lines = [
        'name: ' + name,
        'description: >',
        '  ' + head + 'x' * (span // 2),
        '  ---',
        '  ' + 'x' * (span - span // 2) + tail,
    ]
elif kind == 'desc-list':
    fm_lines = ['name: ' + name, 'description:', '  - one', '  - two']
elif kind == 'desc-mapping':
    fm_lines = ['name: ' + name, 'description:', '  text: a description']
elif kind == 'desc-bool':
    fm_lines = ['name: ' + name, 'description: true']
elif kind == 'yaml-malformed':
    fm_lines = ['name: ' + name, 'description: "unterminated', 'tabs:\t- a']
elif kind == 'desc-no-value':
    # The exact shape that exited 0 with zero output: a line regex's `\s*`
    # crosses the newline and captures `model: sonnet` as the description.
    fm_lines = ['name: ' + name, 'description:', 'model: sonnet']
elif kind == 'desc-null':
    fm_lines = ['name: ' + name, 'description: null']
elif kind == 'desc-single-quoted-empty':
    fm_lines = ['name: ' + name, "description: ''"]
elif kind == 'desc-double-quoted-empty':
    fm_lines = ['name: ' + name, 'description: ""']
elif kind == 'desc-empty-fold':
    fm_lines = ['name: ' + name, 'description: >']
else:
    raise SystemExit('unknown fixture kind: %s' % kind)

parts = [prefix, open_marker, newline]
for line in fm_lines:
    parts.append(line)
    parts.append(newline)
if close_marker is not None:
    parts.append(close_marker)
    parts.append(newline)
parts.append(newline)
parts.append(body)
parts.append(newline)

with open(path, 'wb') as fh:
    fh.write(''.join(parts).encode('utf-8'))
PY
}

# Builds all three subjects for one fixture kind and echoes nothing; the paths
# are fixed by convention so the probes below can find them.
#
# skill-audit takes a DIRECTORY (SKILL.md inside it, name matching the dir);
# agent-audit takes a FILE inside an apm package. The hook takes the SKILL.md
# directly, so it and skill-audit share one file.
build_subjects() {
  local kind="$1" base="$TMPDIR_T/$1"
  rm -rf "$base"
  mkdir -p "$base/skill/my-skill" "$base/agent/.apm/agents"
  cat > "$base/agent/apm.yml" <<'EOF'
name: test-package
version: 0.1.0
type: skill
EOF
  write_fixture "$kind" "$base/skill/my-skill/SKILL.md" my-skill
  write_fixture "$kind" "$base/agent/.apm/agents/my-agent.agent.md" my-agent
}

# probe_all <label> <kind> <needle>... — runs all three scripts over the fixture
# and requires every one of them to exit non-zero AND report every needle. One
# assertion per script would let two of them drift apart while the suite stayed
# green; the whole point of the shared resolver block is that they cannot.
#
# A needle written `@skills:<text>` is asserted for the hook and skill-audit but
# NOT for agent-audit. There is exactly one such needle in this file — the body
# word ceiling — and the exemption is the ADR, not a workaround: ADR-0020 gives
# agents the description gates and deliberately NO body word gate, because an
# agent body becomes the system prompt of a fresh context rather than competing
# with the caller's live conversation. Demanding a body finding from agent-audit
# would be demanding the ADR be contradicted.
probe_all() {
  local label="$1" kind="$2"
  shift 2
  local base="$TMPDIR_T/$kind"
  local -a targets=(
    "hook|$HOOK|$base/skill/my-skill/SKILL.md"
    "skill-audit|$SKILL_VALIDATE|$base/skill/my-skill"
    "agent-audit|$AGENT_VALIDATE|$base/agent/.apm/agents/my-agent.agent.md"
  )
  local problems=""
  for target in "${targets[@]}"; do
    local who="${target%%|*}" rest="${target#*|}"
    local script="${rest%%|*}" arg="${rest#*|}"
    local out status=0
    set +e
    out="$(bash "$script" "$arg" 2>&1)"
    status=$?
    set -e
    if [[ $status -eq 0 ]]; then
      problems="$problems [$who exited 0: ${out:-<no output>}]"
      continue
    fi
    for needle in "$@"; do
      if [[ "$needle" == @skills:* ]]; then
        # Spelled as a full `if`, not `[[ ... ]] && continue`. Under `set -e` the
        # short form's exit status is the test's when it is false, and relying on
        # the &&-list exemption to keep that from aborting the run is a footgun
        # one edit away from biting.
        if [[ "$who" == agent-audit ]]; then
          continue
        fi
        needle="${needle#@skills:}"
      fi
      if [[ "$out" != *"$needle"* ]]; then
        problems="$problems [$who never said '$needle': $out]"
      fi
    done
  done
  if [[ -z "$problems" ]]; then
    pass "$label"
  else
    fail "$label —$problems"
  fi
}

# ---------------------------------------------------------------------------
# 1a. Tolerated frontmatter shapes — the gates must RUN, not merely not-pass
# ---------------------------------------------------------------------------
# The needles are the FINDINGS, not the exit code. A script that rejected the BOM
# outright would exit non-zero too, and would still be skipping every ADR-0020
# measurement — which is the defect, one error message later.
echo ""
echo "--- a BOM, leading blanks, trailing marker whitespace and CRLF are all tolerated, and the gates still fire ---"
for kind in plain bom leading-blanks trailing-ws crlf; do
  build_subjects "$kind"
done
probe_all "control: a well-formed over-ceiling file fails on BOTH the description and the body" \
  plain "description is $DESC_CHARS char" "@skills:body is $BODY_WORDS words"
probe_all "a UTF-8 BOM does not hide an over-ceiling description or body" \
  bom "description is $DESC_CHARS char" "@skills:body is $BODY_WORDS words"
probe_all "leading blank lines before the opening --- do not hide the findings" \
  leading-blanks "description is $DESC_CHARS char" "@skills:body is $BODY_WORDS words"
probe_all "trailing whitespace after either --- marker does not hide the findings" \
  trailing-ws "description is $DESC_CHARS char" "@skills:body is $BODY_WORDS words"
# Note on what this last one can and cannot detect. read_text() opens the file in
# TEXT mode, so Python's universal-newline translation turns \r\n into \n before
# the frontmatter matcher ever sees it — verified by mutation: reverting
# FRONTMATTER_RE to the old `^---\n(.*?)\n---` breaks the leading-blanks and
# trailing-whitespace cases above but NOT this one. So this case pins the
# end-to-end behaviour (a CRLF file is measured, not skipped) rather than the
# `\r?\n` alternations in the regex, and it would catch a future switch to binary
# reads or to a newline='' open. Kept for that reason, and labelled so nobody
# reads it as covering more than it does.
probe_all "CRLF line endings do not hide the findings" \
  crlf "description is $DESC_CHARS char" "@skills:body is $BODY_WORDS words"

# ---------------------------------------------------------------------------
# 1a-bis. An indented `---` inside a block scalar is CONTENT, not a marker
# ---------------------------------------------------------------------------
# The mirror image of the four shapes above. Those were markers the pattern was
# too strict to accept; this is content the pattern was too loose to reject. The
# closing marker used to be `\r?\n[ \t]*---`, so an indented `---` inside a
# `>`-folded description ended the frontmatter early: the description gate then
# measured a truncated fragment (under every ceiling, so silent) and the body
# gate measured the discarded description text as body. Measured on the fixture
# below, the old code exited 0 with nothing but a spurious "no boundary clause"
# SUGGESTION — the clause is in the half it threw away.
#
# The needle is the full-value length, so a script that merely rejected the file
# would not satisfy it.
echo ""
echo "--- an indented --- inside a >-folded description is content, not the end of the frontmatter ---"
build_subjects desc-folded-indented
probe_all "a folded description containing an indented '---' is measured whole" \
  desc-folded-indented "description is $DESC_CHARS char"

# ---------------------------------------------------------------------------
# 1b. Unparseable frontmatter is a hard ERROR, never a quiet skip
# ---------------------------------------------------------------------------
echo ""
echo "--- genuinely unparseable frontmatter exits non-zero with a message, rather than passing quietly ---"
# Each needle names the BRANCH the fixture is supposed to reach, not the word
# "frontmatter" — which every one of these messages contains, and which is why
# the yaml-none fixture below passed for years while landing on the wrong branch
# entirely.
for kind in no-close yaml-list yaml-string yaml-none yaml-empty-block yaml-malformed; do
  build_subjects "$kind"
done
probe_all "frontmatter with no closing --- is reported, not skipped" \
  no-close "parseable YAML frontmatter block"
probe_all "frontmatter that parses to a LIST is reported, not skipped" \
  yaml-list "frontmatter is not a YAML mapping"
probe_all "frontmatter that parses to a STRING is reported, not skipped" \
  yaml-string "frontmatter is not a YAML mapping"
probe_all "frontmatter that parses to None (a comment-only block) is reported, not skipped" \
  yaml-none "frontmatter is not a YAML mapping"
probe_all "a completely empty '---/---' block is reported, not skipped" \
  yaml-empty-block "parseable YAML frontmatter block"
# Two needles, both naming the SYNTAX branch specifically. "frontmatter is not
# valid YAML" is now exclusive to it — the wrong-typed-description failures reach
# the same wrapper and no longer borrow that phrase (see 2c below) — and the
# scanner context proves the parser's own diagnostic survives the wrapper rather
# than being replaced by a generic one. Do not needle the tail of PyYAML's
# message: an earlier attempt used "could not find expected", which PyYAML 6.0.3
# does not emit for this fixture at all, so the case failed on the assertion
# rather than on the behaviour.
probe_all "malformed YAML in the frontmatter is reported, not skipped" \
  yaml-malformed "frontmatter is not valid YAML" "while scanning a quoted scalar"

# ---------------------------------------------------------------------------
# 2. A valueless description is a hard FAIL in all three scripts
# ---------------------------------------------------------------------------
# All five spellings mean the same thing to a YAML parser — an empty value — and
# all five have to be decided on the FOLDED value rather than on a line regex.
# `description:` followed by `model: sonnet` is the one that shipped: it made the
# value look present, skipped the "missing or empty" failure, and then
# early-returned out of every ADR-0020 gate on the genuinely empty folded value.
echo ""
echo "--- every spelling of a valueless description hard-FAILs in all three scripts ---"
for kind in desc-no-value desc-null desc-single-quoted-empty desc-double-quoted-empty desc-empty-fold; do
  build_subjects "$kind"
done
probe_all "'description:' with no value (next key not captured as the value) FAILs" \
  desc-no-value "description field is missing or empty"
probe_all "'description: null' FAILs" \
  desc-null "description field is missing or empty"
probe_all "\"description: ''\" FAILs" \
  desc-single-quoted-empty "description field is missing or empty"
probe_all "'description: \"\"' FAILs" \
  desc-double-quoted-empty "description field is missing or empty"
probe_all "'description: >' with nothing folded under it FAILs" \
  desc-empty-fold "description field is missing or empty"

# ---------------------------------------------------------------------------
# 2b. A description that is not a STRING is a parse failure, not a measurement
# ---------------------------------------------------------------------------
# The other half of the same family, and the reason it belongs beside the five
# above: all eight shapes are "the description is not a description", and seven
# of them used to be handled while this one was silently coerced. A non-string
# value went through `str()` and was then measured as a Python repr —
# `description: true` became the four-character "True" and sailed through the
# 400-character gate, a list became "['one', 'two']", a mapping its dict repr.
# None of those is text a host can preload, so measuring one is a green verdict
# on a file that was never measured.
echo ""
echo "--- a description that is a list, a mapping or a bool hard-FAILs in all three scripts ---"
for kind in desc-list desc-mapping desc-bool; do
  build_subjects "$kind"
done
probe_all "a LIST description FAILs rather than being measured as its repr" \
  desc-list "description is a list, not a string"
probe_all "a MAPPING description FAILs rather than being measured as its repr" \
  desc-mapping "description is a dict, not a string"
probe_all "a BOOL description FAILs rather than being measured as the 4-char 'True'" \
  desc-bool "description is a bool, not a string"

# ---------------------------------------------------------------------------
# 2c. The FAILURE CLASS reported has to be the one that happened
# ---------------------------------------------------------------------------
# The three fixtures above reach the same wrapper as a genuine YAML syntax
# error, and that wrapper used to prefix a hard-coded "frontmatter is not valid
# YAML (...)" onto all of them. For a non-string description that is false: the
# block parses, only the field's TYPE is wrong. On a blocking gate with no
# baseline it sent the author hunting for a syntax error that is not there. The
# assertion runs in both directions, because fixing it by dropping the phrase
# everywhere would trade one wrong diagnosis for another.
echo ""
echo "--- 'not valid YAML' is said for a syntax error and NOT for a wrong-typed description ---"
YAML_CLASS_PROBLEMS=""
for spec in "yaml-malformed|yes" "desc-list|no" "desc-mapping|no" "desc-bool|no"; do
  kind="${spec%%|*}"
  want="${spec#*|}"
  build_subjects "$kind"
  for target in \
    "hook|$HOOK|$TMPDIR_T/$kind/skill/my-skill/SKILL.md" \
    "skill-audit|$SKILL_VALIDATE|$TMPDIR_T/$kind/skill/my-skill" \
    "agent-audit|$AGENT_VALIDATE|$TMPDIR_T/$kind/agent/.apm/agents/my-agent.agent.md"
  do
    who="${target%%|*}"; rest="${target#*|}"
    script="${rest%%|*}"; arg="${rest#*|}"
    set +e
    out="$(bash "$script" "$arg" 2>&1)"
    set -e
    if [[ "$want" == yes && "$out" != *"frontmatter is not valid YAML"* ]]; then
      YAML_CLASS_PROBLEMS="$YAML_CLASS_PROBLEMS [$who did not call $kind a YAML syntax error: $out]"
    fi
    if [[ "$want" == no && "$out" == *"not valid YAML"* ]]; then
      YAML_CLASS_PROBLEMS="$YAML_CLASS_PROBLEMS [$who called $kind invalid YAML, but the frontmatter parsed: $out]"
    fi
  done
done
if [[ -z "$YAML_CLASS_PROBLEMS" ]]; then
  pass "a type error is reported as a type error and a syntax error as a syntax error"
else
  fail "wrong failure class reported —$YAML_CLASS_PROBLEMS"
fi

# The specific regression, spelled out: the valueless-description agent file must
# not merely fail — it must not be SILENT. Zero output on a blocking gate is what
# made this un-diagnosable, so the output is asserted non-empty independently.
echo ""
echo "--- the valueless-description agent file produces output, not silence ---"
build_subjects desc-no-value
set +e
SILENT_OUT="$(bash "$AGENT_VALIDATE" "$TMPDIR_T/desc-no-value/agent/.apm/agents/my-agent.agent.md" 2>&1)"
SILENT_RC=$?
set -e
if [[ $SILENT_RC -ne 0 && -n "$SILENT_OUT" ]]; then
  pass "agent-audit reports a valueless description rather than exiting 0 with zero output"
else
  fail "agent-audit exited $SILENT_RC with output '${SILENT_OUT:-<empty>}' — the original defect was exit 0 and total silence on a blocking pre-push gate"
fi

echo ""
echo "Results: $PASS passed, $FAIL failed"
[[ $FAIL -eq 0 ]]