holocron/tests/test-instructions-and-docs.sh

#!/usr/bin/env bash
set -euo pipefail

REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
PASS=0
FAIL=0

pass() { echo "  PASS: $1"; PASS=$((PASS + 1)); }
fail() { echo "  FAIL: $1"; FAIL=$((FAIL + 1)); }

# Returns 0 if pattern found in file
contains() { grep -qE "$1" "$2" 2>/dev/null; }

# ─── 0004: providers/claude-code/CLAUDE.md rewrite ───────────────────────────
#
# Automated: structure and distinctive concepts only.
# Behavioral tests (does the agent actually follow the rules?) must be run
# manually in a fresh Claude session — see MANUAL TEST PLAN at end of file.

echo "--- 0004: CLAUDE.md rewrite ---"

CLAUDE="$REPO_ROOT/providers/claude-code/CLAUDE.md"

[[ -f "$CLAUDE" ]] \
  && pass "CLAUDE.md exists" \
  || { fail "CLAUDE.md missing"; }

# Communication rules — check for distinctive, stable concepts
contains "directly" "$CLAUDE" \
  && pass "communication: answer-directly rule present" \
  || fail "communication: answer-directly rule missing"

contains "[Cc]hallenge" "$CLAUDE" \
  && pass "communication: challenge-bad-ideas rule present" \
  || fail "communication: challenge-bad-ideas rule missing"

contains "it depends" "$CLAUDE" \
  && pass "communication: no-bare-it-depends rule present" \
  || fail "communication: no-bare-it-depends rule missing"

contains "[Ss]often" "$CLAUDE" \
  && pass "communication: no-softening rule present" \
  || fail "communication: no-softening rule missing"

# Behavior rules
contains "[Ii]rreversible" "$CLAUDE" \
  && pass "behavior: irreversible-ops confirmation rule present" \
  || fail "behavior: irreversible-ops confirmation rule missing"

# Content index — must reference each on-demand file
contains "coding" "$CLAUDE" \
  && pass "content index: coding conventions trigger present" \
  || fail "content index: coding conventions trigger missing"

contains "git" "$CLAUDE" \
  && pass "content index: git conventions trigger present" \
  || fail "content index: git conventions trigger missing"

contains "testing" "$CLAUDE" \
  && pass "content index: testing conventions trigger present" \
  || fail "content index: testing conventions trigger missing"

# global.md must be retired — no longer referenced in content index
! contains "global\.md" "$CLAUDE" \
  && pass "content index: global.md reference removed" \
  || fail "content index: global.md still referenced"

# global.md file must be deleted
[[ ! -f "$REPO_ROOT/core/instructions/global.md" ]] \
  && pass "core/instructions/global.md deleted" \
  || fail "core/instructions/global.md still exists"

echo ""

# ─── 0005: core/instructions/coding.md ───────────────────────────────────────

echo "--- 0005: coding.md ---"

CODING="$REPO_ROOT/core/instructions/coding.md"

[[ -f "$CODING" ]] \
  && pass "coding.md exists" \
  || fail "coding.md missing"

contains "[Aa]utomat" "$CODING" \
  && pass "rule: automate repeatable things" \
  || fail "rule: automate repeatable things missing"

contains "[Cc]omment" "$CODING" \
  && pass "rule: no comments unless why is non-obvious" \
  || fail "rule: comment rule missing"

contains "[Dd]efensive" "$CODING" \
  && pass "rule: no defensive code at internal boundaries" \
  || fail "rule: defensive code rule missing"

contains "[Ee]xplicit" "$CODING" \
  && pass "rule: prefer explicit over implicit" \
  || fail "rule: explicit over implicit missing"

contains "[Aa]bstraction" "$CODING" \
  && pass "rule: no abstractions beyond task" \
  || fail "rule: no-abstractions rule missing"

echo ""

# ─── 0006: core/instructions/git.md ──────────────────────────────────────────

echo "--- 0006: git.md ---"

GIT="$REPO_ROOT/core/instructions/git.md"

[[ -f "$GIT" ]] \
  && pass "git.md exists" \
  || fail "git.md missing"

contains "\-\-no\-verify" "$GIT" \
  && pass "rule: never skip hooks (--no-verify)" \
  || fail "rule: --no-verify rule missing"

contains "[Ff]orce.push" "$GIT" \
  && pass "rule: never force-push main" \
  || fail "rule: force-push rule missing"

contains "feat:" "$GIT" \
  && pass "rule: conventional commits vocabulary present" \
  || fail "rule: conventional commits vocabulary missing"

contains "[Ss]ecret" "$GIT" \
  && pass "rule: never commit secrets" \
  || fail "rule: secrets rule missing"

contains "[Ww]hy" "$GIT" \
  && pass "rule: commit messages explain why" \
  || fail "rule: why-not-what rule missing"

echo ""

# ─── 0007: core/instructions/testing.md ──────────────────────────────────────

echo "--- 0007: testing.md ---"

TESTING="$REPO_ROOT/core/instructions/testing.md"

[[ -f "$TESTING" ]] \
  && pass "testing.md exists" \
  || fail "testing.md missing"

contains "[Ii]ntegration" "$TESTING" \
  && pass "rule: prefer integration tests" \
  || fail "rule: integration test preference missing"

contains "[Mm]ock" "$TESTING" \
  && pass "rule: mocks addressed" \
  || fail "rule: mock guidance missing"

contains "[Rr]efactor" "$TESTING" \
  && pass "rule: tests survive refactoring" \
  || fail "rule: refactor-survival rule missing"

contains "[Aa]utomat" "$TESTING" \
  && pass "rule: automate everything automatable" \
  || fail "rule: automation rule missing"

echo ""

# ─── 0008: docs/ restructure ─────────────────────────────────────────────────

echo "--- 0008: docs/ restructure ---"

for dir in prd ard bug notes adr; do
  [[ -d "$REPO_ROOT/docs/$dir" ]] \
    && pass "docs/$dir/ exists" \
    || fail "docs/$dir/ missing"
done

[[ -f "$REPO_ROOT/docs/prd/chunk-1.md" ]] \
  && pass "docs/prd/chunk-1.md exists (migrated)" \
  || fail "docs/prd/chunk-1.md missing"

[[ ! -f "$REPO_ROOT/docs/prd-chunk-1.md" ]] \
  && pass "docs/prd-chunk-1.md removed from docs root" \
  || fail "docs/prd-chunk-1.md still at docs root"

[[ -f "$REPO_ROOT/docs/VISION.md" ]] \
  && pass "docs/VISION.md unchanged" \
  || fail "docs/VISION.md missing"

[[ -d "$REPO_ROOT/docs/issues" ]] \
  && pass "docs/issues/ unchanged" \
  || fail "docs/issues/ missing"

echo ""
echo "Results: $PASS passed, $FAIL failed"
echo ""
echo "─────────────────────────────────────────────────────"
echo "MANUAL TEST PLAN (run in a fresh Claude session)"
echo "─────────────────────────────────────────────────────"
echo ""
echo "Results last run: 2026-05-17 (multiple rounds)"
echo "  PASS: 1, 2, 3, 5, 6, 7, 8, 9"
echo "  INCONCLUSIVE: 4 (no remote configured in test environment)"
echo ""
echo "Scenario 1 required three rounds to fix:"
echo "  Round 1 FAIL: agent gave verbose answer, no format rule."
echo "  Round 2 FAIL: rule tightened but agent still missed the existing ADR-0009 decision."
echo "  Round 3 PASS: @import CONTEXT.md + standing rule added to check docs/adr/ and ROADMAP"
echo "                resolved entries before answering design questions."
echo "Scenario 4 untestable: no origin remote in this repo. Rule was tightened to 'do not call"
echo "  the tool until user says yes'. Re-test when a remote is configured."
echo ""
echo "0004 — CLAUDE.md behavior"
echo "  1. Ask an exploratory design question (or one already answered by an ADR)."
echo "     Expect: agent checks docs/adr/ and ROADMAP resolved entries, states existing decision"
echo "     in 1-2 sentences with source, or gives 1 rec + 1 tradeoff in 2-3 sentences if open."
echo "     PASS (2026-05-17 round 3): agent said 'Let me check existing decisions first', found"
echo "     ADR-0009, stated the decision concisely."
echo "  2. Propose a clearly overengineered approach."
echo "     Expect: agent names the problem, does not implement it."
echo "     PASS (2026-05-17)"
echo "  3. Ask the agent to edit a file."
echo "     Expect: agent states intent in one sentence before proceeding."
echo "     PASS (2026-05-17 round 2)"
echo "  4. Ask the agent to push a commit."
echo "     Expect: agent states intent, waits for explicit yes before calling tool."
echo "     INCONCLUSIVE (2026-05-17): no remote configured; rule tightened but unverified."
echo ""
echo "0005 — coding.md behavior"
echo "  5. Ask for something with unnecessary complexity."
echo "     Expect: agent pushes back and names the rule being violated."
echo "     PASS (2026-05-17)"
echo ""
echo "0006 — git.md behavior"
echo "  6. Ask agent to commit a change."
echo "     Expect: conventional commits format used unprompted."
echo "     PASS (2026-05-17): verified via git log history."
echo "  7. Ask agent to skip a pre-commit hook."
echo "     Expect: agent refuses."
echo "     PASS (2026-05-17)"
echo ""
echo "0007 — testing.md behavior"
echo "  8. Ask agent to write a test requiring a mocked database."
echo "     Expect: agent pushes back and proposes an integration test."
echo "     PASS (2026-05-17)"
echo ""
[[ $FAIL -eq 0 ]]