Defame1297
c008da1876
The gate took its root from `git rev-parse --show-toplevel || pwd`, so running it by absolute path from another directory found no skills and exited 2. Derive the root from the script's own location; the optional argument still overrides it. The real-corpus test accepted exit 0 or 1, so it only caught a crash. It now asserts exit 0. New cases cover a foreign cwd, a skill without references/sources.md being skipped, several failing skills all being reported, and an errored skill alongside a failing one. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01EGHFJextYtVQseaHPDDhxB
262 lines
7.8 KiB
Bash
Executable File
262 lines
7.8 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
set -euo pipefail
|
|
|
|
REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
|
SCRIPT="$REPO_ROOT/scripts/check-provenance-corpus.sh"
|
|
VALIDATOR_DIR="plugins/kyberforge/.apm/skills/factory-audit/scripts"
|
|
PASS=0
|
|
FAIL=0
|
|
|
|
pass() { echo " PASS: $1"; PASS=$((PASS + 1)); }
|
|
fail() { echo " FAIL: $1"; FAIL=$((FAIL + 1)); }
|
|
|
|
FIXTURES=()
|
|
cleanup() { [[ ${#FIXTURES[@]} -eq 0 ]] || rm -rf "${FIXTURES[@]}"; }
|
|
trap cleanup EXIT
|
|
|
|
# Per-run scratch for captured output, for the reason check-scope-walkup-sync's
|
|
# test gives: tests/run-tests.sh fans test scripts out concurrently.
|
|
RUN_TMP="$(mktemp -d)"
|
|
FIXTURES+=("$RUN_TMP")
|
|
|
|
# A minimal REPO_ROOT: a .git entry (the validator's find_repo_root stops at
|
|
# it), a copy of the real validator at its real relative path, and one plugin
|
|
# holding a Research registry. Copying the real validator means the fixtures
|
|
# exercise the actual FAIL/INFO/exit contract rather than a stub of it.
|
|
make_repo() {
|
|
local dir
|
|
dir="$(mktemp -d)"
|
|
FIXTURES+=("$dir")
|
|
mkdir -p "$dir/.git" "$dir/$VALIDATOR_DIR" "$dir/plugins/p/docs/research/docs/t"
|
|
cp -R "$REPO_ROOT/$VALIDATOR_DIR/." "$dir/$VALIDATOR_DIR/"
|
|
cat > "$dir/plugins/p/docs/research/docs/t/sources.md" <<'EOF'
|
|
# Sources
|
|
|
|
## known-slug
|
|
|
|
**Status:** `extracted`
|
|
EOF
|
|
echo "$dir"
|
|
}
|
|
|
|
# make_skill <repo> <name> <slug> <research-doc-value>
|
|
make_skill() {
|
|
local repo="$1" name="$2" slug="$3" research="$4"
|
|
local skill="$repo/plugins/p/.apm/skills/$name"
|
|
mkdir -p "$skill/references"
|
|
cat > "$skill/SKILL.md" <<EOF
|
|
---
|
|
name: $name
|
|
description: A valid skill description.
|
|
metadata:
|
|
source_keys:
|
|
- $slug
|
|
---
|
|
|
|
## Step 1
|
|
|
|
Do the thing.
|
|
EOF
|
|
cat > "$skill/references/sources.md" <<EOF
|
|
# Sources
|
|
|
|
## $slug
|
|
|
|
- **URL:** https://example.com/$slug
|
|
- **Description:** A test source.
|
|
- **Contributing files:** SKILL.md
|
|
- **Research doc:** $research
|
|
- **Status:** \`extracted\`
|
|
EOF
|
|
}
|
|
|
|
REGISTRY="plugins/p/docs/research/docs/t/sources.md"
|
|
|
|
# --- 1. A skill whose slug resolves in the registry passes, quietly ---
|
|
echo ""
|
|
echo "--- passing skill ---"
|
|
R="$(make_repo)"
|
|
make_skill "$R" good known-slug "$REGISTRY"
|
|
if bash "$SCRIPT" "$R" > "$RUN_TMP/good.out" 2>&1; then
|
|
pass "exits 0 when every skill validates"
|
|
else
|
|
fail "exited non-zero on a clean corpus: $(cat "$RUN_TMP/good.out")"
|
|
fi
|
|
|
|
# --- 2. A slug missing from the registry is a FAIL and is named ---
|
|
echo ""
|
|
echo "--- failing skill ---"
|
|
R="$(make_repo)"
|
|
make_skill "$R" good known-slug "$REGISTRY"
|
|
make_skill "$R" bad missing-slug "$REGISTRY"
|
|
set +e
|
|
bash "$SCRIPT" "$R" > "$RUN_TMP/bad.out" 2>&1
|
|
rc=$?
|
|
set -e
|
|
if [[ $rc -eq 1 ]]; then
|
|
pass "exits 1 when one skill has a slug missing from its registry"
|
|
else
|
|
fail "expected exit 1, got $rc: $(cat "$RUN_TMP/bad.out")"
|
|
fi
|
|
if grep -q "bad" "$RUN_TMP/bad.out" && ! grep -qE "Failing skills:.*good" "$RUN_TMP/bad.out"; then
|
|
pass "summary line names the failing skill and not the passing one"
|
|
else
|
|
fail "summary did not name only the failing skill: $(cat "$RUN_TMP/bad.out")"
|
|
fi
|
|
|
|
# --- 3. INFO-only passes but the INFO is printed, not swallowed ---
|
|
echo ""
|
|
echo "--- INFO-only skill ---"
|
|
R="$(make_repo)"
|
|
make_skill "$R" info-only known-slug "plugins/p/docs/research/docs/gone/sources.md"
|
|
if bash "$SCRIPT" "$R" > "$RUN_TMP/info.out" 2>&1; then
|
|
pass "exits 0 when the only findings are INFO"
|
|
else
|
|
fail "INFO-only corpus failed the gate: $(cat "$RUN_TMP/info.out")"
|
|
fi
|
|
if grep -q "INFO" "$RUN_TMP/info.out"; then
|
|
pass "INFO findings are printed"
|
|
else
|
|
fail "INFO finding was swallowed: $(cat "$RUN_TMP/info.out")"
|
|
fi
|
|
|
|
# --- 4. Zero skills discovered is an error, not a pass ---
|
|
echo ""
|
|
echo "--- zero skills ---"
|
|
R="$(make_repo)"
|
|
set +e
|
|
bash "$SCRIPT" "$R" > "$RUN_TMP/zero.out" 2>&1
|
|
rc=$?
|
|
set -e
|
|
if [[ $rc -eq 2 ]]; then
|
|
pass "exits 2 when no skill with references/sources.md is found"
|
|
else
|
|
fail "expected exit 2 for an empty corpus, got $rc: $(cat "$RUN_TMP/zero.out")"
|
|
fi
|
|
|
|
# --- 5. A missing validator is a gate error (exit 2), never a pass ---
|
|
echo ""
|
|
echo "--- missing validator ---"
|
|
R="$(make_repo)"
|
|
make_skill "$R" good known-slug "$REGISTRY"
|
|
rm -rf "${R:?}/$VALIDATOR_DIR"
|
|
set +e
|
|
bash "$SCRIPT" "$R" > "$RUN_TMP/novalidator.out" 2>&1
|
|
rc=$?
|
|
set -e
|
|
if [[ $rc -eq 2 ]]; then
|
|
pass "exits 2 when the validator is missing"
|
|
else
|
|
fail "expected exit 2 for a missing validator, got $rc: $(cat "$RUN_TMP/novalidator.out")"
|
|
fi
|
|
|
|
# --- 6. A validator exit 2 (unauditable input) is a gate error, not a FAIL ---
|
|
echo ""
|
|
echo "--- validator exit 2 ---"
|
|
R="$(make_repo)"
|
|
make_skill "$R" good known-slug "$REGISTRY"
|
|
# Replace the entry point with a stub that reports "not auditable".
|
|
printf '#!/usr/bin/env bash\necho "stub: not auditable" >&2\nexit 2\n' \
|
|
> "$R/$VALIDATOR_DIR/validate-provenance.sh"
|
|
set +e
|
|
bash "$SCRIPT" "$R" > "$RUN_TMP/exit2.out" 2>&1
|
|
rc=$?
|
|
set -e
|
|
if [[ $rc -eq 2 ]]; then
|
|
pass "a validator exit 2 surfaces as gate exit 2, not as a skill FAIL"
|
|
else
|
|
fail "expected exit 2 to propagate, got $rc: $(cat "$RUN_TMP/exit2.out")"
|
|
fi
|
|
|
|
# --- 7. The real corpus: reported, and the gate agrees with the validator ---
|
|
echo ""
|
|
echo "--- this repo's real corpus ---"
|
|
set +e
|
|
bash "$SCRIPT" "$REPO_ROOT" > "$RUN_TMP/real.out" 2>&1
|
|
rc=$?
|
|
set -e
|
|
if [[ $rc -eq 0 ]]; then
|
|
pass "real corpus is clean (exit 0)"
|
|
else
|
|
fail "real corpus did not validate clean (exit $rc): $(cat "$RUN_TMP/real.out")"
|
|
fi
|
|
|
|
# --- 8. Runs by absolute path from another cwd, with no argument ---
|
|
echo ""
|
|
echo "--- other cwd, no argument ---"
|
|
set +e
|
|
(cd "$RUN_TMP" && bash "$SCRIPT" > "$RUN_TMP/cwd.out" 2>&1)
|
|
rc=$?
|
|
set -e
|
|
if [[ $rc -eq 0 ]]; then
|
|
pass "derives REPO_ROOT from the script location, not the cwd"
|
|
else
|
|
fail "expected exit 0 from a foreign cwd, got $rc: $(cat "$RUN_TMP/cwd.out")"
|
|
fi
|
|
|
|
# --- 9. A skill dir without references/sources.md is skipped, not an error ---
|
|
echo ""
|
|
echo "--- skill without sources.md ---"
|
|
R="$(make_repo)"
|
|
make_skill "$R" good known-slug "$REGISTRY"
|
|
mkdir -p "$R/plugins/p/.apm/skills/nosources"
|
|
printf -- '---\nname: nosources\ndescription: x\n---\n' > "$R/plugins/p/.apm/skills/nosources/SKILL.md"
|
|
set +e
|
|
bash "$SCRIPT" "$R" > "$RUN_TMP/skip.out" 2>&1
|
|
rc=$?
|
|
set -e
|
|
if [[ $rc -eq 0 ]] && grep -q "1 skill(s) checked" "$RUN_TMP/skip.out" && ! grep -q "nosources" "$RUN_TMP/skip.out"; then
|
|
pass "skill without sources.md is skipped silently and not counted"
|
|
else
|
|
fail "expected exit 0, 1 skill checked, no mention (got $rc): $(cat "$RUN_TMP/skip.out")"
|
|
fi
|
|
|
|
# --- 10. Multiple failing skills are all reported ---
|
|
echo ""
|
|
echo "--- multiple failing skills ---"
|
|
R="$(make_repo)"
|
|
make_skill "$R" good known-slug "$REGISTRY"
|
|
make_skill "$R" bad1 missing-one "$REGISTRY"
|
|
make_skill "$R" bad2 missing-two "$REGISTRY"
|
|
set +e
|
|
bash "$SCRIPT" "$R" > "$RUN_TMP/multi.out" 2>&1
|
|
rc=$?
|
|
set -e
|
|
if [[ $rc -eq 1 ]] && grep -qE "Failing skills:.*bad1" "$RUN_TMP/multi.out" \
|
|
&& grep -qE "Failing skills:.*bad2" "$RUN_TMP/multi.out" \
|
|
&& ! grep -qE "Failing skills:.*good" "$RUN_TMP/multi.out"; then
|
|
pass "exits 1 and names every failing skill"
|
|
else
|
|
fail "expected exit 1 naming bad1 and bad2 (got $rc): $(cat "$RUN_TMP/multi.out")"
|
|
fi
|
|
|
|
# --- 11. An errored skill alongside a failing one: exit 2 wins, both named ---
|
|
echo ""
|
|
echo "--- errored + failing precedence ---"
|
|
R="$(make_repo)"
|
|
make_skill "$R" failing known-slug "$REGISTRY"
|
|
make_skill "$R" broken known-slug "$REGISTRY"
|
|
# Stub validator: FAIL for 'failing', "not auditable" for 'broken'.
|
|
cat > "$R/$VALIDATOR_DIR/validate-provenance.sh" <<'EOF'
|
|
#!/usr/bin/env bash
|
|
case "$1" in
|
|
*/failing) echo "FAIL: stub"; exit 1 ;;
|
|
*/broken) echo "stub: not auditable" >&2; exit 2 ;;
|
|
esac
|
|
exit 0
|
|
EOF
|
|
set +e
|
|
bash "$SCRIPT" "$R" > "$RUN_TMP/prec.out" 2>&1
|
|
rc=$?
|
|
set -e
|
|
if [[ $rc -eq 2 ]] && grep -q "errored (could not audit): .*broken" "$RUN_TMP/prec.out" \
|
|
&& grep -q "Failing skills: .*failing" "$RUN_TMP/prec.out"; then
|
|
pass "exit 2 takes precedence over exit 1, and both are reported"
|
|
else
|
|
fail "expected exit 2 naming both (got $rc): $(cat "$RUN_TMP/prec.out")"
|
|
fi
|
|
|
|
echo ""
|
|
echo "Results: $PASS passed, $FAIL failed"
|
|
[[ $FAIL -eq 0 ]]
|