fix(gates): run the provenance corpus gate from any cwd
The gate took its root from `git rev-parse --show-toplevel || pwd`, so running it by absolute path from another directory found no skills and exited 2. Derive the root from the script's own location; the optional argument still overrides it. The real-corpus test accepted exit 0 or 1, so it only caught a crash. It now asserts exit 0. New cases cover a foreign cwd, a skill without references/sources.md being skipped, several failing skills all being reported, and an errored skill alongside a failing one. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01EGHFJextYtVQseaHPDDhxB
This commit is contained in:
@@ -23,10 +23,10 @@ set -euo pipefail
|
|||||||
# could not be audited must not read as a skill that failed the audit.
|
# could not be audited must not read as a skill that failed the audit.
|
||||||
#
|
#
|
||||||
# The skill set is discovered by glob, not hardcoded, so a new skill is covered
|
# The skill set is discovered by glob, not hardcoded, so a new skill is covered
|
||||||
# the moment it grows a references/sources.md. Run from repo root or pass
|
# the moment it grows a references/sources.md. Runs from any cwd: REPO_ROOT defaults to the parent of this script's directory, or pass
|
||||||
# REPO_ROOT as arg.
|
# REPO_ROOT as arg.
|
||||||
|
|
||||||
REPO_ROOT="${1:-$(git rev-parse --show-toplevel 2>/dev/null || pwd)}"
|
REPO_ROOT="${1:-$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)}"
|
||||||
if [[ ! -d "$REPO_ROOT" ]]; then
|
if [[ ! -d "$REPO_ROOT" ]]; then
|
||||||
echo "Provenance corpus check failed: REPO_ROOT '$REPO_ROOT' is not a directory." >&2
|
echo "Provenance corpus check failed: REPO_ROOT '$REPO_ROOT' is not a directory." >&2
|
||||||
exit 2
|
exit 2
|
||||||
|
|||||||
@@ -175,10 +175,85 @@ set +e
|
|||||||
bash "$SCRIPT" "$REPO_ROOT" > "$RUN_TMP/real.out" 2>&1
|
bash "$SCRIPT" "$REPO_ROOT" > "$RUN_TMP/real.out" 2>&1
|
||||||
rc=$?
|
rc=$?
|
||||||
set -e
|
set -e
|
||||||
if [[ $rc -eq 0 || $rc -eq 1 ]]; then
|
if [[ $rc -eq 0 ]]; then
|
||||||
pass "gate runs to a verdict (0 or 1) against the real corpus (exit $rc)"
|
pass "real corpus is clean (exit 0)"
|
||||||
else
|
else
|
||||||
fail "gate errored (exit $rc) against the real corpus: $(cat "$RUN_TMP/real.out")"
|
fail "real corpus did not validate clean (exit $rc): $(cat "$RUN_TMP/real.out")"
|
||||||
|
fi
|
||||||
|
|
||||||
|
# --- 8. Runs by absolute path from another cwd, with no argument ---
|
||||||
|
echo ""
|
||||||
|
echo "--- other cwd, no argument ---"
|
||||||
|
set +e
|
||||||
|
(cd "$RUN_TMP" && bash "$SCRIPT" > "$RUN_TMP/cwd.out" 2>&1)
|
||||||
|
rc=$?
|
||||||
|
set -e
|
||||||
|
if [[ $rc -eq 0 ]]; then
|
||||||
|
pass "derives REPO_ROOT from the script location, not the cwd"
|
||||||
|
else
|
||||||
|
fail "expected exit 0 from a foreign cwd, got $rc: $(cat "$RUN_TMP/cwd.out")"
|
||||||
|
fi
|
||||||
|
|
||||||
|
# --- 9. A skill dir without references/sources.md is skipped, not an error ---
|
||||||
|
echo ""
|
||||||
|
echo "--- skill without sources.md ---"
|
||||||
|
R="$(make_repo)"
|
||||||
|
make_skill "$R" good known-slug "$REGISTRY"
|
||||||
|
mkdir -p "$R/plugins/p/.apm/skills/nosources"
|
||||||
|
printf -- '---\nname: nosources\ndescription: x\n---\n' > "$R/plugins/p/.apm/skills/nosources/SKILL.md"
|
||||||
|
set +e
|
||||||
|
bash "$SCRIPT" "$R" > "$RUN_TMP/skip.out" 2>&1
|
||||||
|
rc=$?
|
||||||
|
set -e
|
||||||
|
if [[ $rc -eq 0 ]] && grep -q "1 skill(s) checked" "$RUN_TMP/skip.out" && ! grep -q "nosources" "$RUN_TMP/skip.out"; then
|
||||||
|
pass "skill without sources.md is skipped silently and not counted"
|
||||||
|
else
|
||||||
|
fail "expected exit 0, 1 skill checked, no mention (got $rc): $(cat "$RUN_TMP/skip.out")"
|
||||||
|
fi
|
||||||
|
|
||||||
|
# --- 10. Multiple failing skills are all reported ---
|
||||||
|
echo ""
|
||||||
|
echo "--- multiple failing skills ---"
|
||||||
|
R="$(make_repo)"
|
||||||
|
make_skill "$R" good known-slug "$REGISTRY"
|
||||||
|
make_skill "$R" bad1 missing-one "$REGISTRY"
|
||||||
|
make_skill "$R" bad2 missing-two "$REGISTRY"
|
||||||
|
set +e
|
||||||
|
bash "$SCRIPT" "$R" > "$RUN_TMP/multi.out" 2>&1
|
||||||
|
rc=$?
|
||||||
|
set -e
|
||||||
|
if [[ $rc -eq 1 ]] && grep -qE "Failing skills:.*bad1" "$RUN_TMP/multi.out" \
|
||||||
|
&& grep -qE "Failing skills:.*bad2" "$RUN_TMP/multi.out" \
|
||||||
|
&& ! grep -qE "Failing skills:.*good" "$RUN_TMP/multi.out"; then
|
||||||
|
pass "exits 1 and names every failing skill"
|
||||||
|
else
|
||||||
|
fail "expected exit 1 naming bad1 and bad2 (got $rc): $(cat "$RUN_TMP/multi.out")"
|
||||||
|
fi
|
||||||
|
|
||||||
|
# --- 11. An errored skill alongside a failing one: exit 2 wins, both named ---
|
||||||
|
echo ""
|
||||||
|
echo "--- errored + failing precedence ---"
|
||||||
|
R="$(make_repo)"
|
||||||
|
make_skill "$R" failing known-slug "$REGISTRY"
|
||||||
|
make_skill "$R" broken known-slug "$REGISTRY"
|
||||||
|
# Stub validator: FAIL for 'failing', "not auditable" for 'broken'.
|
||||||
|
cat > "$R/$VALIDATOR_DIR/validate-provenance.sh" <<'EOF'
|
||||||
|
#!/usr/bin/env bash
|
||||||
|
case "$1" in
|
||||||
|
*/failing) echo "FAIL: stub"; exit 1 ;;
|
||||||
|
*/broken) echo "stub: not auditable" >&2; exit 2 ;;
|
||||||
|
esac
|
||||||
|
exit 0
|
||||||
|
EOF
|
||||||
|
set +e
|
||||||
|
bash "$SCRIPT" "$R" > "$RUN_TMP/prec.out" 2>&1
|
||||||
|
rc=$?
|
||||||
|
set -e
|
||||||
|
if [[ $rc -eq 2 ]] && grep -q "errored (could not audit): .*broken" "$RUN_TMP/prec.out" \
|
||||||
|
&& grep -q "Failing skills: .*failing" "$RUN_TMP/prec.out"; then
|
||||||
|
pass "exit 2 takes precedence over exit 1, and both are reported"
|
||||||
|
else
|
||||||
|
fail "expected exit 2 naming both (got $rc): $(cat "$RUN_TMP/prec.out")"
|
||||||
fi
|
fi
|
||||||
|
|
||||||
echo ""
|
echo ""
|
||||||
|
|||||||
Reference in New Issue
Block a user