holocron/tests/test-check-provenance-corpus.sh

#!/usr/bin/env bash
set -euo pipefail

REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
SCRIPT="$REPO_ROOT/scripts/check-provenance-corpus.sh"
VALIDATOR_DIR="plugins/kyberforge/.apm/skills/factory-audit/scripts"
PASS=0
FAIL=0

pass() { echo "  PASS: $1"; PASS=$((PASS + 1)); }
fail() { echo "  FAIL: $1"; FAIL=$((FAIL + 1)); }

FIXTURES=()
cleanup() { [[ ${#FIXTURES[@]} -eq 0 ]] || rm -rf "${FIXTURES[@]}"; }
trap cleanup EXIT

# Per-run scratch for captured output, for the reason check-scope-walkup-sync's
# test gives: tests/run-tests.sh fans test scripts out concurrently.
RUN_TMP="$(mktemp -d)"
FIXTURES+=("$RUN_TMP")

# A minimal REPO_ROOT: a .git entry (the validator's find_repo_root stops at
# it), a copy of the real validator at its real relative path, and one plugin
# holding a Research registry. Copying the real validator means the fixtures
# exercise the actual FAIL/INFO/exit contract rather than a stub of it.
make_repo() {
  local dir
  dir="$(mktemp -d)"
  FIXTURES+=("$dir")
  mkdir -p "$dir/.git" "$dir/$VALIDATOR_DIR" "$dir/plugins/p/docs/research/docs/t"
  cp -R "$REPO_ROOT/$VALIDATOR_DIR/." "$dir/$VALIDATOR_DIR/"
  cat > "$dir/plugins/p/docs/research/docs/t/sources.md" <<'EOF'
# Sources

## known-slug

**Status:** `extracted`
EOF
  echo "$dir"
}

# make_skill <repo> <name> <slug> <research-doc-value>
make_skill() {
  local repo="$1" name="$2" slug="$3" research="$4"
  local skill="$repo/plugins/p/.apm/skills/$name"
  mkdir -p "$skill/references"
  cat > "$skill/SKILL.md" <<EOF
---
name: $name
description: A valid skill description.
metadata:
  source_keys:
    - $slug
---

## Step 1

Do the thing.
EOF
  cat > "$skill/references/sources.md" <<EOF
# Sources

## $slug

- **URL:** https://example.com/$slug
- **Description:** A test source.
- **Contributing files:** SKILL.md
- **Research doc:** $research
- **Status:** \`extracted\`
EOF
}

REGISTRY="plugins/p/docs/research/docs/t/sources.md"

# --- 1. A skill whose slug resolves in the registry passes, quietly ---
echo ""
echo "--- passing skill ---"
R="$(make_repo)"
make_skill "$R" good known-slug "$REGISTRY"
if bash "$SCRIPT" "$R" > "$RUN_TMP/good.out" 2>&1; then
  pass "exits 0 when every skill validates"
else
  fail "exited non-zero on a clean corpus: $(cat "$RUN_TMP/good.out")"
fi

# --- 2. A slug missing from the registry is a FAIL and is named ---
echo ""
echo "--- failing skill ---"
R="$(make_repo)"
make_skill "$R" good known-slug "$REGISTRY"
make_skill "$R" bad missing-slug "$REGISTRY"
set +e
bash "$SCRIPT" "$R" > "$RUN_TMP/bad.out" 2>&1
rc=$?
set -e
if [[ $rc -eq 1 ]]; then
  pass "exits 1 when one skill has a slug missing from its registry"
else
  fail "expected exit 1, got $rc: $(cat "$RUN_TMP/bad.out")"
fi
if grep -q "bad" "$RUN_TMP/bad.out" && ! grep -qE "Failing skills:.*good" "$RUN_TMP/bad.out"; then
  pass "summary line names the failing skill and not the passing one"
else
  fail "summary did not name only the failing skill: $(cat "$RUN_TMP/bad.out")"
fi

# --- 3. INFO-only passes but the INFO is printed, not swallowed ---
echo ""
echo "--- INFO-only skill ---"
R="$(make_repo)"
make_skill "$R" info-only known-slug "plugins/p/docs/research/docs/gone/sources.md"
if bash "$SCRIPT" "$R" > "$RUN_TMP/info.out" 2>&1; then
  pass "exits 0 when the only findings are INFO"
else
  fail "INFO-only corpus failed the gate: $(cat "$RUN_TMP/info.out")"
fi
if grep -q "INFO" "$RUN_TMP/info.out"; then
  pass "INFO findings are printed"
else
  fail "INFO finding was swallowed: $(cat "$RUN_TMP/info.out")"
fi

# --- 4. Zero skills discovered is an error, not a pass ---
echo ""
echo "--- zero skills ---"
R="$(make_repo)"
set +e
bash "$SCRIPT" "$R" > "$RUN_TMP/zero.out" 2>&1
rc=$?
set -e
if [[ $rc -eq 2 ]]; then
  pass "exits 2 when no skill with references/sources.md is found"
else
  fail "expected exit 2 for an empty corpus, got $rc: $(cat "$RUN_TMP/zero.out")"
fi

# --- 5. A missing validator is a gate error (exit 2), never a pass ---
echo ""
echo "--- missing validator ---"
R="$(make_repo)"
make_skill "$R" good known-slug "$REGISTRY"
rm -rf "${R:?}/$VALIDATOR_DIR"
set +e
bash "$SCRIPT" "$R" > "$RUN_TMP/novalidator.out" 2>&1
rc=$?
set -e
if [[ $rc -eq 2 ]]; then
  pass "exits 2 when the validator is missing"
else
  fail "expected exit 2 for a missing validator, got $rc: $(cat "$RUN_TMP/novalidator.out")"
fi

# --- 6. A validator exit 2 (unauditable input) is a gate error, not a FAIL ---
echo ""
echo "--- validator exit 2 ---"
R="$(make_repo)"
make_skill "$R" good known-slug "$REGISTRY"
# Replace the entry point with a stub that reports "not auditable".
printf '#!/usr/bin/env bash\necho "stub: not auditable" >&2\nexit 2\n' \
  > "$R/$VALIDATOR_DIR/validate-provenance.sh"
set +e
bash "$SCRIPT" "$R" > "$RUN_TMP/exit2.out" 2>&1
rc=$?
set -e
if [[ $rc -eq 2 ]]; then
  pass "a validator exit 2 surfaces as gate exit 2, not as a skill FAIL"
else
  fail "expected exit 2 to propagate, got $rc: $(cat "$RUN_TMP/exit2.out")"
fi

# --- 7. The real corpus: reported, and the gate agrees with the validator ---
echo ""
echo "--- this repo's real corpus ---"
set +e
bash "$SCRIPT" "$REPO_ROOT" > "$RUN_TMP/real.out" 2>&1
rc=$?
set -e
if [[ $rc -eq 0 || $rc -eq 1 ]]; then
  pass "gate runs to a verdict (0 or 1) against the real corpus (exit $rc)"
else
  fail "gate errored (exit $rc) against the real corpus: $(cat "$RUN_TMP/real.out")"
fi

echo ""
echo "Results: $PASS passed, $FAIL failed"
[[ $FAIL -eq 0 ]]