Defame1297
044b2d3f08
validate.sh's detect_scope() and validate-provenance.sh's find_plugin_root() disagreed with new-agent.sh's already-correct, documented walk-up semantics on three points, each causing validate.sh to false-FAIL a legitimately-scaffolded project-scope agent pair: - a marker-less directory walked up into $HOME (no .git/apm.yml of its own) was classified as user scope instead of project scope - the .git-boundary branch returned the walked-to .git location instead of the conventional scope root, breaking any <root> that is a subdirectory of a larger git-tracked tree (monorepo package dirs) - the new conventional-root arithmetic introduced to fix the above two cases had no guard against non-conventional/hand-placed file paths, which could point it at the wrong ancestor Also adds scripts/check-scope-walkup-sync.sh, a behavioral drift-guard (per ADR-0014's no-cross-skill-path precedent) that cross-checks the four independently hand-ported walk-up implementations (validate.sh, validate-provenance.sh, new-agent.sh, new-skill.sh) against real fixture scaffolds, wired into .pre-commit-config.yaml at pre-push so future drift between the ports is caught automatically. Verified via bash tests/run-tests.sh (13/13) and targeted before/after reproduction of each bug this closes.
343 lines
16 KiB
Bash
Executable File
343 lines
16 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
set -euo pipefail
|
|
|
|
# Behavioral consistency check for the four independent, semantically-equivalent
|
|
# ports of "walk up from a directory looking for a scope-defining marker" living
|
|
# in this repo:
|
|
#
|
|
# - plugins/kyberforge/skills/agent-audit/scripts/validate.sh (Python: detect_scope)
|
|
# - plugins/kyberforge/skills/agent-audit/scripts/validate-provenance.sh (Python: find_plugin_root)
|
|
# - plugins/kyberforge/skills/agent-author/scripts/new-agent.sh (Bash: find_package_root)
|
|
# - plugins/kyberforge/skills/skill-author/scripts/new-skill.sh (Bash: find_package_root)
|
|
#
|
|
# Per ADR-0014's no-cross-skill-path rule, these can't be consolidated into a
|
|
# shared file (each skill's cache-install copies only its own files), so unlike
|
|
# check-vale-style-sync.sh (which diffs literal file copies) this can't be a
|
|
# text diff — the four implementations are hand-ported, not copied. Instead
|
|
# this builds a matrix of fixture directory trees and asserts the *observable
|
|
# behavior* agrees: whatever new-agent.sh/new-skill.sh actually create on disk,
|
|
# validate.sh/validate-provenance.sh must classify the same way when pointed at
|
|
# the result. Run from repo root or pass REPO_ROOT as arg.
|
|
|
|
REPO_ROOT="${1:-$(git rev-parse --show-toplevel 2>/dev/null || pwd)}"
|
|
if [[ ! -d "$REPO_ROOT" ]]; then
|
|
echo "Scope walk-up sync check failed: REPO_ROOT '$REPO_ROOT' is not a directory." >&2
|
|
exit 1
|
|
fi
|
|
REPO_ROOT="$(cd "$REPO_ROOT" && pwd)"
|
|
|
|
NEW_AGENT="$REPO_ROOT/plugins/kyberforge/skills/agent-author/scripts/new-agent.sh"
|
|
NEW_SKILL="$REPO_ROOT/plugins/kyberforge/skills/skill-author/scripts/new-skill.sh"
|
|
VALIDATE="$REPO_ROOT/plugins/kyberforge/skills/agent-audit/scripts/validate.sh"
|
|
VALIDATE_PROVENANCE="$REPO_ROOT/plugins/kyberforge/skills/agent-audit/scripts/validate-provenance.sh"
|
|
|
|
for f in "$NEW_AGENT" "$NEW_SKILL" "$VALIDATE" "$VALIDATE_PROVENANCE"; do
|
|
if [[ ! -f "$f" ]]; then
|
|
echo "Scope walk-up sync check: $f not found — kyberforge agent-author/agent-audit/skill-author skills not present, nothing to check." >&2
|
|
exit 0
|
|
fi
|
|
done
|
|
|
|
FAIL=0
|
|
err() { echo " FAIL: $1" >&2; FAIL=$((FAIL + 1)); }
|
|
ok() { echo " ok: $1"; }
|
|
|
|
FIXTURES=()
|
|
cleanup() { [[ ${#FIXTURES[@]} -eq 0 ]] || rm -rf "${FIXTURES[@]}"; }
|
|
trap cleanup EXIT
|
|
|
|
# Fill a new-agent.sh-scaffolded pair's FILL IN: placeholders with valid
|
|
# content, isolating the scope/counterpart-lookup question from unrelated
|
|
# content-quality FAILs when cross-checking against validate.sh.
|
|
fill_agent_pair() {
|
|
local file="$1" name="$2"
|
|
cat > "$file" <<EOF
|
|
---
|
|
name: ${name}
|
|
description: A valid agent description.
|
|
---
|
|
|
|
You are a test agent. When invoked, do the thing.
|
|
EOF
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Fixture 1: root exactly $HOME (no walk-up) — new-agent.sh's own documented
|
|
# user-scope case.
|
|
# ---------------------------------------------------------------------------
|
|
echo ""
|
|
echo "--- fixture: root exactly \$HOME ---"
|
|
F1_HOME="$(mktemp -d)"
|
|
FIXTURES+=("$F1_HOME")
|
|
NAME1="probe-home-exact"
|
|
if ! env HOME="$F1_HOME" bash "$NEW_AGENT" "$NAME1" "$F1_HOME" >/dev/null 2>&1; then
|
|
err "new-agent.sh failed to scaffold at root exactly \$HOME"
|
|
else
|
|
if [[ ! -f "$F1_HOME/.claude/agents/$NAME1.md" || ! -f "$F1_HOME/.copilot/agents/$NAME1.agent.md" ]]; then
|
|
err "new-agent.sh did not create the expected user-scope pair at \$HOME/.claude and \$HOME/.copilot"
|
|
else
|
|
fill_agent_pair "$F1_HOME/.claude/agents/$NAME1.md" "$NAME1"
|
|
fill_agent_pair "$F1_HOME/.copilot/agents/$NAME1.agent.md" "$NAME1"
|
|
if env HOME="$F1_HOME" bash "$VALIDATE" "$F1_HOME/.claude/agents/$NAME1.md" >/tmp/f1.out 2>&1; then
|
|
ok "validate.sh agrees: user scope, counterpart found under \$HOME/.copilot"
|
|
else
|
|
err "validate.sh disagreed with new-agent.sh's user-scope classification at root exactly \$HOME"
|
|
sed 's/^/ /' /tmp/f1.out
|
|
fi
|
|
fi
|
|
fi
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Fixture 2: nested marker-less directory under $HOME — the live-repro shape.
|
|
# new-agent.sh's own docs call this out as deliberately project scope, not
|
|
# user scope (a stray directory under $HOME can't be silently redirected into
|
|
# the shared global ~/.claude or ~/.copilot agent directories).
|
|
# ---------------------------------------------------------------------------
|
|
echo ""
|
|
echo "--- fixture: nested marker-less directory under \$HOME ---"
|
|
F2_HOME="$(mktemp -d)"
|
|
FIXTURES+=("$F2_HOME")
|
|
F2_NESTED="$F2_HOME/scratch/testdir"
|
|
mkdir -p "$F2_NESTED"
|
|
NAME2="probe-home-nested"
|
|
if ! env HOME="$F2_HOME" bash "$NEW_AGENT" "$NAME2" "$F2_NESTED" >/dev/null 2>&1; then
|
|
err "new-agent.sh failed to scaffold under a nested marker-less \$HOME subdirectory"
|
|
else
|
|
if [[ ! -f "$F2_NESTED/.claude/agents/$NAME2.md" || ! -f "$F2_NESTED/.github/agents/$NAME2.agent.md" ]]; then
|
|
err "new-agent.sh did not scaffold a project-scope pair at the nested dir (rooted at \$F2_NESTED, not \$HOME)"
|
|
elif [[ -f "$F2_HOME/.claude/agents/$NAME2.md" || -f "$F2_HOME/.copilot/agents/$NAME2.agent.md" ]]; then
|
|
err "new-agent.sh unexpectedly wrote into \$HOME/.claude or \$HOME/.copilot for a nested marker-less start dir"
|
|
else
|
|
ok "new-agent.sh: nested marker-less dir under \$HOME scaffolds project scope at the nested dir"
|
|
fill_agent_pair "$F2_NESTED/.claude/agents/$NAME2.md" "$NAME2"
|
|
fill_agent_pair "$F2_NESTED/.github/agents/$NAME2.agent.md" "$NAME2"
|
|
if env HOME="$F2_HOME" bash "$VALIDATE" "$F2_NESTED/.claude/agents/$NAME2.md" >/tmp/f2.out 2>&1; then
|
|
ok "validate.sh agrees: project scope, counterpart found at the nested dir (not \$HOME/.copilot)"
|
|
else
|
|
err "validate.sh disagreed with new-agent.sh: misclassified the nested marker-less \$HOME subdirectory"
|
|
sed 's/^/ /' /tmp/f2.out
|
|
fi
|
|
# new-skill.sh has no user/project distinction of its own (no $HOME
|
|
# awareness at all — see new-skill.sh's find_package_root), but it shares
|
|
# the same .git/apm.yml walk-up primitive. It must land its standalone
|
|
# scaffold at the given path too, not get redirected toward $HOME.
|
|
if env HOME="$F2_HOME" bash "$NEW_SKILL" probe-home-nested-skill "$F2_NESTED" >/tmp/f2skill.out 2>&1 \
|
|
&& [[ -d "$F2_NESTED/probe-home-nested-skill" ]]; then
|
|
ok "new-skill.sh agrees: standalone mode scaffolds at the nested dir, not redirected toward \$HOME"
|
|
else
|
|
err "new-skill.sh disagreed with new-agent.sh/validate.sh on the nested marker-less \$HOME subdirectory"
|
|
sed 's/^/ /' /tmp/f2skill.out
|
|
fi
|
|
fi
|
|
fi
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Fixture 3: a .git boundary between the probe dir and $HOME must stop the
|
|
# walk before it ever reaches $HOME (so it can't be misclassified as user
|
|
# scope via the home-boundary path).
|
|
# ---------------------------------------------------------------------------
|
|
echo ""
|
|
echo "--- fixture: .git boundary short-circuits before reaching \$HOME ---"
|
|
F3_HOME="$(mktemp -d)"
|
|
FIXTURES+=("$F3_HOME")
|
|
# .git sits directly at the probe root (the conventional two-segments-above
|
|
# location .claude/agents and .github/agents are placed relative to). This
|
|
# fixture only exercises what it's meant to: that a .git ancestor stops the
|
|
# walk before it ever reaches $HOME. Fixture 3b below covers .git sitting
|
|
# higher up than the probe root.
|
|
F3_PROBE="$F3_HOME/myrepo"
|
|
mkdir -p "$F3_PROBE/.git"
|
|
NAME3="probe-git-boundary"
|
|
if ! env HOME="$F3_HOME" bash "$NEW_AGENT" "$NAME3" "$F3_PROBE" >/dev/null 2>&1; then
|
|
err "new-agent.sh failed to scaffold at a dir with a .git ancestor short of \$HOME"
|
|
else
|
|
if [[ ! -f "$F3_PROBE/.claude/agents/$NAME3.md" || ! -f "$F3_PROBE/.github/agents/$NAME3.agent.md" ]]; then
|
|
err "new-agent.sh did not scaffold a project-scope pair at the probe dir"
|
|
else
|
|
fill_agent_pair "$F3_PROBE/.claude/agents/$NAME3.md" "$NAME3"
|
|
fill_agent_pair "$F3_PROBE/.github/agents/$NAME3.agent.md" "$NAME3"
|
|
if env HOME="$F3_HOME" bash "$VALIDATE" "$F3_PROBE/.claude/agents/$NAME3.md" >/tmp/f3.out 2>&1; then
|
|
ok "validate.sh agrees: .git boundary keeps this project scope, not promoted to user scope at \$HOME"
|
|
else
|
|
err "validate.sh disagreed with new-agent.sh on the .git-boundary-before-\$HOME fixture"
|
|
sed 's/^/ /' /tmp/f3.out
|
|
fi
|
|
fi
|
|
fi
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Fixture 3b: .git sits one level ABOVE the probe root — a subdirectory of a
|
|
# larger git-tracked tree (e.g. a monorepo package dir). new-agent.sh always
|
|
# places project-scope files at its ROOT argument, never at the walked-up
|
|
# .git location, so validate.sh must resolve scope to the probe root too, not
|
|
# to the ancestor where .git happened to be found.
|
|
# ---------------------------------------------------------------------------
|
|
echo ""
|
|
echo "--- fixture: .git ancestor sits above <root> (subdirectory of a larger git tree) ---"
|
|
F3B_REPO="$(mktemp -d)"
|
|
FIXTURES+=("$F3B_REPO")
|
|
mkdir -p "$F3B_REPO/.git"
|
|
F3B_PROBE="$F3B_REPO/subdir"
|
|
mkdir -p "$F3B_PROBE"
|
|
NAME3B="probe-git-above-root"
|
|
if ! bash "$NEW_AGENT" "$NAME3B" "$F3B_PROBE" >/dev/null 2>&1; then
|
|
err "new-agent.sh failed to scaffold at a dir one level below a .git ancestor"
|
|
else
|
|
if [[ ! -f "$F3B_PROBE/.claude/agents/$NAME3B.md" || ! -f "$F3B_PROBE/.github/agents/$NAME3B.agent.md" ]]; then
|
|
err "new-agent.sh did not scaffold a project-scope pair at the probe dir (rooted at \$F3B_PROBE, not the repo root)"
|
|
else
|
|
fill_agent_pair "$F3B_PROBE/.claude/agents/$NAME3B.md" "$NAME3B"
|
|
fill_agent_pair "$F3B_PROBE/.github/agents/$NAME3B.agent.md" "$NAME3B"
|
|
if bash "$VALIDATE" "$F3B_PROBE/.claude/agents/$NAME3B.md" >/tmp/f3b.out 2>&1; then
|
|
ok "validate.sh agrees: scope root is <root>, not the .git ancestor above it"
|
|
else
|
|
err "validate.sh disagreed with new-agent.sh: resolved scope to the .git ancestor instead of <root>"
|
|
sed 's/^/ /' /tmp/f3b.out
|
|
fi
|
|
fi
|
|
fi
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Fixture 4: a type-bearing apm.yml — plugin/APM scope. new-agent.sh and
|
|
# new-skill.sh must agree on the same package root, and validate.sh /
|
|
# validate-provenance.sh must both recognize it as plugin scope.
|
|
# ---------------------------------------------------------------------------
|
|
echo ""
|
|
echo "--- fixture: type-bearing apm.yml (plugin/APM scope) ---"
|
|
F4_ROOT="$(mktemp -d)"
|
|
FIXTURES+=("$F4_ROOT")
|
|
printf 'name: test-package\nversion: 0.1.0\ntype: skill\n' > "$F4_ROOT/apm.yml"
|
|
NAME4="probe-plugin"
|
|
if ! bash "$NEW_AGENT" "$NAME4" "$F4_ROOT" >/dev/null 2>&1; then
|
|
err "new-agent.sh failed to scaffold at a type-bearing apm.yml root"
|
|
elif [[ ! -f "$F4_ROOT/.apm/agents/$NAME4.agent.md" ]]; then
|
|
err "new-agent.sh did not scaffold plugin scope at the type-bearing apm.yml root"
|
|
else
|
|
ok "new-agent.sh: plugin scope at type-bearing apm.yml root"
|
|
if bash "$NEW_SKILL" probe-plugin-skill "$F4_ROOT" >/tmp/f4skill.out 2>&1 \
|
|
&& [[ -d "$F4_ROOT/.apm/skills/probe-plugin-skill" ]]; then
|
|
ok "new-skill.sh agrees: package mode at the same apm.yml root"
|
|
else
|
|
err "new-skill.sh disagreed with new-agent.sh on the type-bearing apm.yml root"
|
|
sed 's/^/ /' /tmp/f4skill.out
|
|
fi
|
|
fill_agent_pair "$F4_ROOT/.apm/agents/$NAME4.agent.md" "$NAME4"
|
|
if bash "$VALIDATE" "$F4_ROOT/.apm/agents/$NAME4.agent.md" >/tmp/f4validate.out 2>&1; then
|
|
ok "validate.sh agrees: plugin/APM scope, structural checks pass"
|
|
else
|
|
err "validate.sh disagreed with new-agent.sh: did not treat the type-bearing apm.yml root as plugin scope"
|
|
sed 's/^/ /' /tmp/f4validate.out
|
|
fi
|
|
# source_keys + a matching sources.md round-trips only if validate-provenance.sh
|
|
# resolves the SAME plugin root new-agent.sh/new-skill.sh did.
|
|
cat > "$F4_ROOT/.apm/agents/$NAME4.agent.md" <<EOF
|
|
---
|
|
name: ${NAME4}
|
|
description: A valid agent description.
|
|
source_keys:
|
|
- probe-source
|
|
---
|
|
|
|
You are a test agent.
|
|
EOF
|
|
cat > "$F4_ROOT/sources.md" <<EOF
|
|
# Sources
|
|
|
|
## probe-source
|
|
|
|
- **URL:** https://example.com/probe-source
|
|
- **Description:** A test source.
|
|
- **Contributing files:** .apm/agents/${NAME4}.agent.md
|
|
- **Research doc:** (none)
|
|
- **Status:** \`extracted\`
|
|
EOF
|
|
if bash "$VALIDATE_PROVENANCE" "$F4_ROOT/.apm/agents/$NAME4.agent.md" >/tmp/f4prov.out 2>&1; then
|
|
ok "validate-provenance.sh agrees: resolves the same plugin root, sources.md round-trips"
|
|
else
|
|
err "validate-provenance.sh disagreed on the plugin root for the type-bearing apm.yml fixture"
|
|
sed 's/^/ /' /tmp/f4prov.out
|
|
fi
|
|
fi
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Fixture 5: filesystem-boundary fallback — no $HOME relation, no marker
|
|
# anywhere. Both scripts must fall through to project scope, not user scope.
|
|
# ---------------------------------------------------------------------------
|
|
echo ""
|
|
echo "--- fixture: filesystem-boundary fallback (no \$HOME relation, no markers) ---"
|
|
F5_UNRELATED_HOME_PARENT="$(mktemp -d)"
|
|
FIXTURES+=("$F5_UNRELATED_HOME_PARENT")
|
|
F5_UNRELATED_HOME="$F5_UNRELATED_HOME_PARENT/never-reached-$$"
|
|
F5_ROOT="$(mktemp -d)/deep/proj"
|
|
mkdir -p "$F5_ROOT"
|
|
FIXTURES+=("$(dirname "$(dirname "$F5_ROOT")")")
|
|
NAME5="probe-fs-boundary"
|
|
if ! env HOME="$F5_UNRELATED_HOME" bash "$NEW_AGENT" "$NAME5" "$F5_ROOT" >/dev/null 2>&1; then
|
|
err "new-agent.sh failed to scaffold at the filesystem-boundary fixture"
|
|
else
|
|
if [[ ! -f "$F5_ROOT/.claude/agents/$NAME5.md" || ! -f "$F5_ROOT/.github/agents/$NAME5.agent.md" ]]; then
|
|
err "new-agent.sh did not scaffold project scope at the filesystem-boundary fixture"
|
|
else
|
|
fill_agent_pair "$F5_ROOT/.claude/agents/$NAME5.md" "$NAME5"
|
|
fill_agent_pair "$F5_ROOT/.github/agents/$NAME5.agent.md" "$NAME5"
|
|
if env HOME="$F5_UNRELATED_HOME" bash "$VALIDATE" "$F5_ROOT/.claude/agents/$NAME5.md" >/tmp/f5.out 2>&1; then
|
|
ok "validate.sh agrees: filesystem-boundary fallback resolves to project scope"
|
|
else
|
|
err "validate.sh disagreed with new-agent.sh on the filesystem-boundary fallback fixture"
|
|
sed 's/^/ /' /tmp/f5.out
|
|
fi
|
|
fi
|
|
fi
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Fixture 6: a type-bearing apm.yml ABOVE $HOME must not be reached by
|
|
# validate-provenance.sh's walk-up from a nested, marker-less dir under $HOME
|
|
# — matches new-agent.sh, which also stops at $HOME before ever looking that
|
|
# far up.
|
|
# ---------------------------------------------------------------------------
|
|
echo ""
|
|
echo "--- fixture: type-bearing apm.yml above \$HOME must not be reached ---"
|
|
F6_ANCESTOR="$(mktemp -d)"
|
|
FIXTURES+=("$F6_ANCESTOR")
|
|
printf 'name: outer-package\nversion: 0.1.0\ntype: skill\n' > "$F6_ANCESTOR/apm.yml"
|
|
F6_HOME="$F6_ANCESTOR/fakehome"
|
|
mkdir -p "$F6_HOME"
|
|
NAME6="probe-above-home"
|
|
if ! env HOME="$F6_HOME" bash "$NEW_AGENT" "$NAME6" "$F6_HOME" >/dev/null 2>&1; then
|
|
err "new-agent.sh failed to scaffold with a type-bearing apm.yml above \$HOME"
|
|
elif [[ -f "$F6_HOME/.apm/agents/$NAME6.agent.md" ]]; then
|
|
err "new-agent.sh walked past \$HOME and misclassified as plugin scope using the ancestor apm.yml"
|
|
elif [[ ! -f "$F6_HOME/.claude/agents/$NAME6.md" ]]; then
|
|
err "new-agent.sh did not scaffold user scope at root exactly \$HOME (with a type-bearing apm.yml above)"
|
|
else
|
|
ok "new-agent.sh: \$HOME boundary stops the walk before the ancestor apm.yml, user scope at \$HOME"
|
|
mkdir -p "$F6_HOME/.apm/agents"
|
|
cat > "$F6_HOME/.apm/agents/probe-prov.agent.md" <<'EOF'
|
|
---
|
|
name: probe-prov
|
|
description: A valid agent description.
|
|
source_keys:
|
|
- probe-source
|
|
---
|
|
|
|
You are a test agent.
|
|
EOF
|
|
# No sources.md exists anywhere under $F6_HOME or at the ancestor package
|
|
# root — if find_plugin_root walked past $HOME to the ancestor apm.yml,
|
|
# this would FAIL on Check 0 (source_keys declared but sources.md absent).
|
|
if env HOME="$F6_HOME" bash "$VALIDATE_PROVENANCE" "$F6_HOME/.apm/agents/probe-prov.agent.md" >/tmp/f6.out 2>&1 \
|
|
&& [[ -z "$(cat /tmp/f6.out)" ]]; then
|
|
ok "validate-provenance.sh agrees: \$HOME boundary stops the walk, exits 0 silently (not plugin scope)"
|
|
else
|
|
err "validate-provenance.sh walked past \$HOME to the ancestor apm.yml — disagrees with new-agent.sh"
|
|
sed 's/^/ /' /tmp/f6.out
|
|
fi
|
|
fi
|
|
|
|
echo ""
|
|
if [[ $FAIL -gt 0 ]]; then
|
|
echo "Scope walk-up sync check failed: $FAIL error(s). One of validate.sh's detect_scope, validate-provenance.sh's find_plugin_root, new-agent.sh's find_package_root, or new-skill.sh's find_package_root has drifted from the others' \$HOME/.git/apm.yml walk-up semantics. Re-read new-agent.sh's usage comment (the canonical description of the intended behavior) and bring the disagreeing script back in line." >&2
|
|
exit 1
|
|
fi
|
|
echo "Scope walk-up sync check passed: all four walk-up implementations agree on every fixture."
|