Defame1297/holocron
0
0
Fork 0
Code Issues 14 Pull Requests Actions Packages Projects Releases 3 Wiki Activity
Files
00daf285ec3b76ed8bb83454b6ee67513b770187
holocron/plugins/kyberforge/.apm/skills/agent-audit/tests/validate-provenance.bats
Defame1297 00daf285ec fix(kyberforge): tell an unparsable Contributing-files block from an explicit (none) 
parse_contributing_files documented that callers depend on None vs [], because a
parse failure returning [] would silently disable the check. Only check 8
honoured it; checks 4/5 (skill-audit) and 3/4 (agent-audit) used a truthiness
test, so an unreadable block disabled them without a word.

Two live corpus entries were skipping this way. A sweep of all 32 sources.md
found 134 entries, exactly 2 parsing to None, both in gitea-files: one heading
carried an inline parenthetical that defeated both regexes, and one (none) was
written without its leading bullet. Also pins EMPTY_SOURCE_KEYS_RE to the two
indents parse_source_keys actually reads.

agent-audit had no INFO tier at all, so it gains one rather than reporting a
check that could not run as a FAIL.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01EJJrm5YmacbwMdzZpXcoti
2026-08-31 19:46:05 +00:00

573 lines
18 KiB
Bash
Raw Blame History

#!/usr/bin/env bats
setup() {
REPO_ROOT="$(cd "$BATS_TEST_DIRNAME/../../../../../../" && pwd)"
load "$REPO_ROOT/tests/test_helper/bats-support/load"
load "$REPO_ROOT/tests/test_helper/bats-assert/load"
SCRIPT="$(cd "$BATS_TEST_DIRNAME/../scripts" && pwd)/validate-provenance.sh"
TMPDIR="$(mktemp -d)"
# Helper: create an APM package root at <root> (apm.yml with a top-level
# type: line — a real package manifest, not marketplace-only) plus a
# single vendor-neutral agent file at <root>/.apm/agents/<name>.agent.md.
make_package() {
local root="$1"
mkdir -p "$root/.apm/agents"
cat > "$root/apm.yml" <<EOF
name: test-package
version: 0.1.0
type: skill
EOF
}
# Helper: create a clean agent file (no source_keys)
make_clean_agent() {
local root="$1"
local name="${2:-my-agent}"
cat > "$root/.apm/agents/${name}.agent.md" <<EOF
---
name: ${name}
description: A valid agent description.
---
You are a test agent.
EOF
}
# Helper: create an agent file with source_keys
make_agent_with_source_keys() {
local root="$1"
local name="${2:-my-agent}"
local slug="${3:-my-source}"
cat > "$root/.apm/agents/${name}.agent.md" <<EOF
---
name: ${name}
description: A valid agent description.
source_keys:
- ${slug}
---
You are a test agent.
EOF
}
# Helper: create a valid sources.md with one entry
make_sources_md() {
local root="$1"
local slug="${2:-my-source}"
local contrib="${3:-.apm/agents/my-agent.agent.md}"
local research="${4:-(none)}"
cat > "$root/sources.md" <<EOF
# Sources
## ${slug}
- **URL:** https://example.com/${slug}
- **Description:** A test source.
- **Contributing files:** ${contrib}
- **Research doc:** ${research}
- **Status:** \`extracted\`
EOF
}
}
teardown() {
rm -rf "$TMPDIR"
}
# ---------------------------------------------------------------------------
# --help
# ---------------------------------------------------------------------------
@test "--help exits 0" {
run bash "$SCRIPT" --help
assert_success
assert_output --partial "Usage:"
}
# ---------------------------------------------------------------------------
# Non-plugin/APM scope → exit 0 silently
# ---------------------------------------------------------------------------
@test "non-plugin scope: no apm.yml in tree → exit 0, no output" {
local dir="$TMPDIR/no-package"
mkdir -p "$dir/.apm/agents"
cat > "$dir/.apm/agents/my-agent.agent.md" <<EOF
---
name: my-agent
description: A valid agent description.
source_keys:
- my-source
---
You are a test agent.
EOF
run bash "$SCRIPT" "$dir/.apm/agents/my-agent.agent.md"
assert_success
assert_output ""
}
@test "non-plugin scope: apm.yml present but type:-less (marketplace-only) → exit 0, no output" {
local dir="$TMPDIR/marketplace-only"
mkdir -p "$dir/.apm/agents"
cat > "$dir/apm.yml" <<EOF
name: root-marketplace
marketplace:
owner: someone
packages:
- ./packages/plugin-a
EOF
cat > "$dir/.apm/agents/my-agent.agent.md" <<EOF
---
name: my-agent
description: A valid agent description.
source_keys:
- my-source
---
You are a test agent.
EOF
run bash "$SCRIPT" "$dir/.apm/agents/my-agent.agent.md"
assert_success
assert_output ""
}
@test "non-plugin scope: bare plugin.json (no apm.yml) is no longer a scope signal → exit 0, no output" {
local dir="$TMPDIR/old-plugin-json-only"
mkdir -p "$dir/agents"
echo '{"name":"test-plugin","version":"0.1.0"}' > "$dir/plugin.json"
cat > "$dir/agents/my-agent.md" <<EOF
---
name: my-agent
description: A valid agent description.
source_keys:
- my-source
---
You are a test agent.
EOF
run bash "$SCRIPT" "$dir/agents/my-agent.md"
assert_success
assert_output ""
}
@test "non-plugin scope: walk-up stops at .git boundary before reaching an ancestor apm.yml" {
local dir="$TMPDIR/repo"
mkdir -p "$dir/.git" "$dir/.apm/agents"
cat > "$dir/apm.yml" <<EOF
name: test-package
version: 0.1.0
type: skill
EOF
mkdir -p "$dir/sub/.apm/agents"
cat > "$dir/sub/.apm/agents/my-agent.agent.md" <<EOF
---
name: my-agent
description: A valid agent description.
source_keys:
- my-source
---
You are a test agent.
EOF
# sub/ has no .git and no apm.yml of its own; the real package apm.yml
# lives at $dir, but $dir/.git means the walk from sub/ should stop at
# sub/ itself only if sub/ had a .git — here .git is at $dir, ABOVE
# sub/, so the walk from sub/ reaches $dir/apm.yml before any .git.
# This test instead verifies the walk finds that package root correctly
# (a positive case) — see the dedicated .git-stops-first test below for
# the negative case.
run bash "$SCRIPT" "$dir/sub/.apm/agents/my-agent.agent.md"
assert_failure
assert_output --partial "FAIL"
}
@test "non-plugin scope: \$HOME boundary stops the walk before reaching an ancestor apm.yml above \$HOME" {
# A type-bearing apm.yml sits ABOVE the fake $HOME — if find_plugin_root
# didn't stop at $HOME, it would walk past it and misclassify this
# user/project-scope file as plugin scope, which would then FAIL on
# Check 0 (source_keys declared but sources.md absent) since sources.md
# doesn't exist at that ancestor apm.yml's location either.
local dir="$TMPDIR/anc"
mkdir -p "$dir"
cat > "$dir/apm.yml" <<EOF
name: outer-package
version: 0.1.0
type: skill
EOF
local fake_home="$dir/fakehome"
mkdir -p "$fake_home/.apm/agents"
cat > "$fake_home/.apm/agents/my-agent.agent.md" <<EOF
---
name: my-agent
description: A valid agent description.
source_keys:
- my-source
---
You are a test agent.
EOF
run env HOME="$fake_home" bash "$SCRIPT" "$fake_home/.apm/agents/my-agent.agent.md"
assert_success
assert_output ""
}
@test "non-plugin scope: .git between the agent file and an ancestor apm.yml stops the walk first" {
local dir="$TMPDIR/repo2"
mkdir -p "$dir/.apm/agents"
cat > "$dir/apm.yml" <<EOF
name: test-package
version: 0.1.0
type: skill
EOF
mkdir -p "$dir/sub/.git" "$dir/sub/.apm/agents"
cat > "$dir/sub/.apm/agents/my-agent.agent.md" <<EOF
---
name: my-agent
description: A valid agent description.
source_keys:
- my-source
---
You are a test agent.
EOF
run bash "$SCRIPT" "$dir/sub/.apm/agents/my-agent.agent.md"
assert_success
assert_output ""
}
# ---------------------------------------------------------------------------
# Early exit: no sources.md, no source_keys → exit 0, no output
# ---------------------------------------------------------------------------
@test "clean pass: no sources.md and no source_keys → exit 0, no output" {
local root="$TMPDIR/package"
make_package "$root"
make_clean_agent "$root"
run bash "$SCRIPT" "$root/.apm/agents/my-agent.agent.md"
assert_success
assert_output ""
}
# ---------------------------------------------------------------------------
# Check 0: source_keys present but sources.md absent → FAIL
# ---------------------------------------------------------------------------
@test "FAIL: source_keys in agent file but sources.md absent" {
local root="$TMPDIR/package"
make_package "$root"
make_agent_with_source_keys "$root"
run bash "$SCRIPT" "$root/.apm/agents/my-agent.agent.md"
assert_failure
assert_output --partial "FAIL"
}
# ---------------------------------------------------------------------------
# Check 1: FILL IN: placeholder in sources.md → FAIL
# ---------------------------------------------------------------------------
@test "FAIL: FILL IN: placeholder in sources.md" {
local root="$TMPDIR/package"
make_package "$root"
make_agent_with_source_keys "$root"
cat > "$root/sources.md" <<EOF
# Sources
## my-source
- **URL:** FILL IN: add url
- **Description:** A test source.
- **Contributing files:** .apm/agents/my-agent.agent.md
- **Research doc:** (none)
- **Status:** \`extracted\`
EOF
run bash "$SCRIPT" "$root/.apm/agents/my-agent.agent.md"
assert_failure
assert_output --partial "FAIL"
}
@test "FILL IN: inside backticks in sources.md does not fail" {
local root="$TMPDIR/package"
make_package "$root"
make_agent_with_source_keys "$root"
make_sources_md "$root"
echo "Use \`FILL IN: value\` as example." >> "$root/sources.md"
run bash "$SCRIPT" "$root/.apm/agents/my-agent.agent.md"
assert_success
}
# ---------------------------------------------------------------------------
# Check 2: source_keys slug missing from sources.md → FAIL
# ---------------------------------------------------------------------------
@test "FAIL: source_keys slug in agent file not present as H2 in sources.md" {
local root="$TMPDIR/package"
make_package "$root"
make_agent_with_source_keys "$root" "my-agent" "my-source"
make_sources_md "$root" "different-source" "(none)" "(none)"
run bash "$SCRIPT" "$root/.apm/agents/my-agent.agent.md"
assert_failure
assert_output --partial "FAIL"
}
# ---------------------------------------------------------------------------
# Check 3: Contributing file path doesn't exist → FAIL
# ---------------------------------------------------------------------------
@test "FAIL: Contributing file listed in sources.md does not exist" {
local root="$TMPDIR/package"
make_package "$root"
make_agent_with_source_keys "$root"
make_sources_md "$root" "my-source" ".apm/agents/nonexistent.agent.md"
run bash "$SCRIPT" "$root/.apm/agents/my-agent.agent.md"
assert_failure
assert_output --partial "FAIL"
}
@test "pass: (none) in Contributing files is skipped" {
local root="$TMPDIR/package"
make_package "$root"
make_agent_with_source_keys "$root"
make_sources_md "$root" "my-source" "(none — not used directly)"
run bash "$SCRIPT" "$root/.apm/agents/my-agent.agent.md"
assert_success
}
# ---------------------------------------------------------------------------
# Check 5: Research doc field missing or placeholder → FAIL
# ---------------------------------------------------------------------------
@test "FAIL: Research doc field missing from sources.md entry" {
local root="$TMPDIR/package"
make_package "$root"
make_agent_with_source_keys "$root"
cat > "$root/sources.md" <<EOF
# Sources
## my-source
- **URL:** https://example.com/my-source
- **Description:** A test source.
- **Contributing files:** .apm/agents/my-agent.agent.md
- **Status:** \`extracted\`
EOF
run bash "$SCRIPT" "$root/.apm/agents/my-agent.agent.md"
assert_failure
assert_output --partial "FAIL"
}
@test "FAIL: Research doc field is FILL IN: placeholder" {
local root="$TMPDIR/package"
make_package "$root"
make_agent_with_source_keys "$root"
cat > "$root/sources.md" <<EOF
# Sources
## my-source
- **URL:** https://example.com/my-source
- **Description:** A test source.
- **Contributing files:** .apm/agents/my-agent.agent.md
- **Research doc:** FILL IN: path to research doc
- **Status:** \`extracted\`
EOF
run bash "$SCRIPT" "$root/.apm/agents/my-agent.agent.md"
assert_failure
assert_output --partial "FAIL"
}
# ---------------------------------------------------------------------------
# Check 4: Bidirectional — contributing file missing slug in source_keys → FAIL
# ---------------------------------------------------------------------------
@test "FAIL: Contributing file exists but does not list parent slug in source_keys" {
local root="$TMPDIR/package"
make_package "$root"
# agent file has source_keys: other-source (not my-source)
cat > "$root/.apm/agents/my-agent.agent.md" <<EOF
---
name: my-agent
description: A valid agent description.
source_keys:
- other-source
---
You are a test agent.
EOF
# sources.md says my-agent.agent.md contributed to my-source, but
# my-agent.agent.md doesn't list my-source
cat > "$root/sources.md" <<EOF
# Sources
## other-source
- **URL:** https://example.com/other-source
- **Description:** A test source.
- **Contributing files:** .apm/agents/my-agent.agent.md
- **Research doc:** (none)
- **Status:** \`extracted\`
## my-source
- **URL:** https://example.com/my-source
- **Description:** Another source.
- **Contributing files:** .apm/agents/my-agent.agent.md
- **Research doc:** (none)
- **Status:** \`extracted\`
EOF
run bash "$SCRIPT" "$root/.apm/agents/my-agent.agent.md"
assert_failure
assert_output --partial "FAIL"
}
# ---------------------------------------------------------------------------
# Clean full pass
# ---------------------------------------------------------------------------
@test "clean full pass: all checks satisfied" {
local root="$TMPDIR/package"
make_package "$root"
make_agent_with_source_keys "$root"
make_sources_md "$root"
run bash "$SCRIPT" "$root/.apm/agents/my-agent.agent.md"
assert_success
}
# ---------------------------------------------------------------------------
# Checks 3 and 4: None ("could not parse") is NOT [] ("explicitly (none)")
#
# parse_contributing_files returns three distinguishable answers and checks 3
# and 4 have to honour all three. `[]` is the author writing "(none)" — the
# skip is correct and silent. None is a Contributing files block the parser
# cannot read, and skipping THAT silently disables both checks on the one entry
# least likely to be right, which is the failure mode the parser's own
# docstring warns about. The assertions below are therefore about the INFO
# appearing; a silent exit 0 is exactly the bug.
# ---------------------------------------------------------------------------
@test "INFO: an unparsable Contributing files block names the slug instead of skipping checks 3 and 4 silently" {
local root="$TMPDIR/package"
make_package "$root"
make_agent_with_source_keys "$root"
cat > "$root/sources.md" <<EOF
# Sources
## my-source
- **URL:** https://example.com/my-source
- **Description:** A test source.
- **Research doc:** (none)
**Contributing files:**
* .apm/agents/ghost.agent.md (asterisk bullets are not the bullet form)
- **Status:** \`extracted\`
EOF
run bash "$SCRIPT" "$root/.apm/agents/my-agent.agent.md"
assert_success
assert_output --partial "INFO"
assert_output --partial "Contributing-file checks skipped for 'my-source' — the Contributing files block could not be parsed"
}
@test "INFO: an entry with no Contributing files field at all is reported, not skipped silently" {
local root="$TMPDIR/package"
make_package "$root"
make_agent_with_source_keys "$root"
cat > "$root/sources.md" <<EOF
# Sources
## my-source
- **URL:** https://example.com/my-source
- **Description:** A test source.
- **Research doc:** (none)
- **Status:** \`extracted\`
EOF
run bash "$SCRIPT" "$root/.apm/agents/my-agent.agent.md"
assert_success
assert_output --partial "INFO"
assert_output --partial "Contributing-file checks skipped for 'my-source' — the Contributing files block could not be parsed"
}
@test "checks 3 and 4 skipped silently: an explicit '(none)' emits no INFO" {
local root="$TMPDIR/package"
make_package "$root"
make_agent_with_source_keys "$root"
make_sources_md "$root" "my-source" "(none — not used directly)"
run bash "$SCRIPT" "$root/.apm/agents/my-agent.agent.md"
assert_success
assert_output ""
}
@test "checks 3 and 4 still run: a parseable Contributing files list is not diverted to the INFO" {
local root="$TMPDIR/package"
make_package "$root"
make_agent_with_source_keys "$root"
make_sources_md "$root" "my-source" ".apm/agents/nonexistent.agent.md"
run bash "$SCRIPT" "$root/.apm/agents/my-agent.agent.md"
assert_failure
assert_output --partial "FAIL"
assert_output --partial "Contributing file '.apm/agents/nonexistent.agent.md' does not exist"
refute_output --partial "could not be parsed"
}
# ---------------------------------------------------------------------------
# The INFO tier itself: kind-aware printing and a kind-aware exit code
#
# INFO is new here — before it, findings was a 5-tuple and print_findings
# stamped every entry FAIL. The two cases below pin the tier rather than any
# one check: an INFO must print under the INFO prefix and leave the exit code
# at 0, and a real FAIL must keep printing under the FAIL prefix and still exit
# non-zero even when an INFO is sitting in the same findings list.
# ---------------------------------------------------------------------------
@test "INFO tier: an INFO alone prints as INFO with a Note and does not set a failing exit code" {
local root="$TMPDIR/package"
make_package "$root"
make_agent_with_source_keys "$root"
cat > "$root/sources.md" <<EOF
# Sources
## my-source
- **URL:** https://example.com/my-source
- **Description:** A test source.
- **Research doc:** (none)
- **Status:** \`extracted\`
EOF
run bash "$SCRIPT" "$root/.apm/agents/my-agent.agent.md"
assert_success
assert_output --partial "INFO Contributing-file checks skipped for 'my-source'"
assert_output --partial "Note:"
refute_output --partial "FAIL"
}
@test "FAIL tier: a genuine FAIL alongside an INFO still prints as FAIL and exits non-zero" {
local root="$TMPDIR/package"
make_package "$root"
make_agent_with_source_keys "$root"
cat > "$root/sources.md" <<EOF
# Sources
## my-source
- **URL:** https://example.com/my-source
- **Description:** A test source.
- **Contributing files:** .apm/agents/nonexistent.agent.md
- **Research doc:** (none)
- **Status:** \`extracted\`
## ghost-source
- **URL:** https://example.com/ghost-source
- **Description:** Another test source.
- **Research doc:** (none)
- **Status:** \`extracted\`
EOF
run bash "$SCRIPT" "$root/.apm/agents/my-agent.agent.md"
assert_failure
assert_output --partial "FAIL Contributing file '.apm/agents/nonexistent.agent.md' does not exist"
assert_output --partial "Why:"
assert_output --partial "INFO Contributing-file checks skipped for 'ghost-source'"
assert_output --partial "Note:"
}
Reference in New Issue View Git Blame Copy Permalink