#!/usr/bin/env bash set -euo pipefail REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" SCRIPT="$REPO_ROOT/scripts/check-provenance-corpus.sh" VALIDATOR_DIR="plugins/kyberforge/.apm/skills/factory-audit/scripts" PASS=0 FAIL=0 pass() { echo " PASS: $1"; PASS=$((PASS + 1)); } fail() { echo " FAIL: $1"; FAIL=$((FAIL + 1)); } FIXTURES=() cleanup() { [[ ${#FIXTURES[@]} -eq 0 ]] || rm -rf "${FIXTURES[@]}"; } trap cleanup EXIT # Per-run scratch for captured output, for the reason check-scope-walkup-sync's # test gives: tests/run-tests.sh fans test scripts out concurrently. RUN_TMP="$(mktemp -d)" FIXTURES+=("$RUN_TMP") # A minimal REPO_ROOT: a .git entry (the validator's find_repo_root stops at # it), a copy of the real validator at its real relative path, and one plugin # holding a Research registry. Copying the real validator means the fixtures # exercise the actual FAIL/INFO/exit contract rather than a stub of it. make_repo() { local dir dir="$(mktemp -d)" FIXTURES+=("$dir") mkdir -p "$dir/.git" "$dir/$VALIDATOR_DIR" "$dir/plugins/p/docs/research/docs/t" cp -R "$REPO_ROOT/$VALIDATOR_DIR/." "$dir/$VALIDATOR_DIR/" cat > "$dir/plugins/p/docs/research/docs/t/sources.md" <<'EOF' # Sources ## known-slug **Status:** `extracted` EOF echo "$dir" } # make_skill make_skill() { local repo="$1" name="$2" slug="$3" research="$4" local skill="$repo/plugins/p/.apm/skills/$name" mkdir -p "$skill/references" cat > "$skill/SKILL.md" < "$skill/references/sources.md" < "$RUN_TMP/good.out" 2>&1; then pass "exits 0 when every skill validates" else fail "exited non-zero on a clean corpus: $(cat "$RUN_TMP/good.out")" fi # --- 2. A slug missing from the registry is a FAIL and is named --- echo "" echo "--- failing skill ---" R="$(make_repo)" make_skill "$R" good known-slug "$REGISTRY" make_skill "$R" bad missing-slug "$REGISTRY" set +e bash "$SCRIPT" "$R" > "$RUN_TMP/bad.out" 2>&1 rc=$? set -e if [[ $rc -eq 1 ]]; then pass "exits 1 when one skill has a slug missing from its registry" else fail "expected exit 1, got $rc: $(cat "$RUN_TMP/bad.out")" fi if grep -q "bad" "$RUN_TMP/bad.out" && ! grep -qE "Failing skills:.*good" "$RUN_TMP/bad.out"; then pass "summary line names the failing skill and not the passing one" else fail "summary did not name only the failing skill: $(cat "$RUN_TMP/bad.out")" fi # --- 3. INFO-only passes but the INFO is printed, not swallowed --- echo "" echo "--- INFO-only skill ---" R="$(make_repo)" make_skill "$R" info-only known-slug "plugins/p/docs/research/docs/gone/sources.md" if bash "$SCRIPT" "$R" > "$RUN_TMP/info.out" 2>&1; then pass "exits 0 when the only findings are INFO" else fail "INFO-only corpus failed the gate: $(cat "$RUN_TMP/info.out")" fi if grep -q "INFO" "$RUN_TMP/info.out"; then pass "INFO findings are printed" else fail "INFO finding was swallowed: $(cat "$RUN_TMP/info.out")" fi # --- 4. Zero skills discovered is an error, not a pass --- echo "" echo "--- zero skills ---" R="$(make_repo)" set +e bash "$SCRIPT" "$R" > "$RUN_TMP/zero.out" 2>&1 rc=$? set -e if [[ $rc -eq 2 ]]; then pass "exits 2 when no skill with references/sources.md is found" else fail "expected exit 2 for an empty corpus, got $rc: $(cat "$RUN_TMP/zero.out")" fi # --- 5. A missing validator is a gate error (exit 2), never a pass --- echo "" echo "--- missing validator ---" R="$(make_repo)" make_skill "$R" good known-slug "$REGISTRY" rm -rf "${R:?}/$VALIDATOR_DIR" set +e bash "$SCRIPT" "$R" > "$RUN_TMP/novalidator.out" 2>&1 rc=$? set -e if [[ $rc -eq 2 ]]; then pass "exits 2 when the validator is missing" else fail "expected exit 2 for a missing validator, got $rc: $(cat "$RUN_TMP/novalidator.out")" fi # --- 6. A validator exit 2 (unauditable input) is a gate error, not a FAIL --- echo "" echo "--- validator exit 2 ---" R="$(make_repo)" make_skill "$R" good known-slug "$REGISTRY" # Replace the entry point with a stub that reports "not auditable". printf '#!/usr/bin/env bash\necho "stub: not auditable" >&2\nexit 2\n' \ > "$R/$VALIDATOR_DIR/validate-provenance.sh" set +e bash "$SCRIPT" "$R" > "$RUN_TMP/exit2.out" 2>&1 rc=$? set -e if [[ $rc -eq 2 ]]; then pass "a validator exit 2 surfaces as gate exit 2, not as a skill FAIL" else fail "expected exit 2 to propagate, got $rc: $(cat "$RUN_TMP/exit2.out")" fi # --- 7. The real corpus: reported, and the gate agrees with the validator --- echo "" echo "--- this repo's real corpus ---" set +e bash "$SCRIPT" "$REPO_ROOT" > "$RUN_TMP/real.out" 2>&1 rc=$? set -e if [[ $rc -eq 0 ]]; then pass "real corpus is clean (exit 0)" else fail "real corpus did not validate clean (exit $rc): $(cat "$RUN_TMP/real.out")" fi # --- 8. Runs by absolute path from another cwd, with no argument --- echo "" echo "--- other cwd, no argument ---" set +e (cd "$RUN_TMP" && bash "$SCRIPT" > "$RUN_TMP/cwd.out" 2>&1) rc=$? set -e if [[ $rc -eq 0 ]]; then pass "derives REPO_ROOT from the script location, not the cwd" else fail "expected exit 0 from a foreign cwd, got $rc: $(cat "$RUN_TMP/cwd.out")" fi # --- 9. A skill dir without references/sources.md is skipped, not an error --- echo "" echo "--- skill without sources.md ---" R="$(make_repo)" make_skill "$R" good known-slug "$REGISTRY" mkdir -p "$R/plugins/p/.apm/skills/nosources" printf -- '---\nname: nosources\ndescription: x\n---\n' > "$R/plugins/p/.apm/skills/nosources/SKILL.md" set +e bash "$SCRIPT" "$R" > "$RUN_TMP/skip.out" 2>&1 rc=$? set -e if [[ $rc -eq 0 ]] && grep -q "1 skill(s) checked" "$RUN_TMP/skip.out" && ! grep -q "nosources" "$RUN_TMP/skip.out"; then pass "skill without sources.md is skipped silently and not counted" else fail "expected exit 0, 1 skill checked, no mention (got $rc): $(cat "$RUN_TMP/skip.out")" fi # --- 10. Multiple failing skills are all reported --- echo "" echo "--- multiple failing skills ---" R="$(make_repo)" make_skill "$R" good known-slug "$REGISTRY" make_skill "$R" bad1 missing-one "$REGISTRY" make_skill "$R" bad2 missing-two "$REGISTRY" set +e bash "$SCRIPT" "$R" > "$RUN_TMP/multi.out" 2>&1 rc=$? set -e if [[ $rc -eq 1 ]] && grep -qE "Failing skills:.*bad1" "$RUN_TMP/multi.out" \ && grep -qE "Failing skills:.*bad2" "$RUN_TMP/multi.out" \ && ! grep -qE "Failing skills:.*good" "$RUN_TMP/multi.out"; then pass "exits 1 and names every failing skill" else fail "expected exit 1 naming bad1 and bad2 (got $rc): $(cat "$RUN_TMP/multi.out")" fi # --- 11. An errored skill alongside a failing one: exit 2 wins, both named --- echo "" echo "--- errored + failing precedence ---" R="$(make_repo)" make_skill "$R" failing known-slug "$REGISTRY" make_skill "$R" broken known-slug "$REGISTRY" # Stub validator: FAIL for 'failing', "not auditable" for 'broken'. cat > "$R/$VALIDATOR_DIR/validate-provenance.sh" <<'EOF' #!/usr/bin/env bash case "$1" in */failing) echo "FAIL: stub"; exit 1 ;; */broken) echo "stub: not auditable" >&2; exit 2 ;; esac exit 0 EOF set +e bash "$SCRIPT" "$R" > "$RUN_TMP/prec.out" 2>&1 rc=$? set -e if [[ $rc -eq 2 ]] && grep -q "errored (could not audit): .*broken" "$RUN_TMP/prec.out" \ && grep -q "Failing skills: .*failing" "$RUN_TMP/prec.out"; then pass "exit 2 takes precedence over exit 1, and both are reported" else fail "expected exit 2 naming both (got $rc): $(cat "$RUN_TMP/prec.out")" fi echo "" echo "Results: $PASS passed, $FAIL failed" [[ $FAIL -eq 0 ]]