Why: .pre-commit-hooks.yaml and its release-tag gate served external consumers that do not exist. No repo on the Gitea instance pins these hooks, and the README names apm as the only supported install path. The mechanism was also already failing: skill-size-check.sh changed after v2.0.1 with no tag cut, and the gate cannot fire through Gitea's merge button. (Simplification audit finding 36.) Implementation Notes: - Delete .pre-commit-hooks.yaml, scripts/check-release-needed.sh, tests/test-check-release-needed.sh and tests/test-vale-hooks-consumer.sh, and remove the check-release-needed pre-push hook. The repo: local skill-size-check and vale-audit-prefilter-* hooks are unchanged. - ADR-0014 is amended, not retired: its runtime decision to bundle Vale inside factory-audit stands. The amendment keeps the entry[0]-only constraint (LESSONS.md:101,105) in case the export returns. ADR-0025 gets a pointer. - test-vale-wrap.sh: drop case 33 (the cross-manifest drift check) and case 28's hook-scope half, which read the published manifest. Case 32 now also requires each hook to select every tracked file of its class, which keeps case 33's one-plugin-narrowing guard, with a mutation test. - test-skill-size-check.sh and test-adr0020-contract.sh now assert the hook contract and verbose: true on .pre-commit-config.yaml only. - gates.md: pre-push count goes from 9 to 8 authored hooks (11 to 10 reported), and the Release table, the External consumers section and the two-manifest scope table are removed. README and script/test comments no longer describe the export as live. The resolver comment is edited identically in both copies. - The v1.0.0/v2.0.0/v2.0.1 tags are left in place; they are inert. ADR: 0014 Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
2063 lines
67 KiB
Bash
2063 lines
67 KiB
Bash
#!/usr/bin/env bats
|
|
|
|
setup() {
|
|
REPO_ROOT="$(cd "$BATS_TEST_DIRNAME/../../../../../../" && pwd)"
|
|
load "$REPO_ROOT/tests/test_helper/bats-support/load"
|
|
load "$REPO_ROOT/tests/test_helper/bats-assert/load"
|
|
|
|
SCRIPT="$(cd "$BATS_TEST_DIRNAME/../scripts" && pwd)/validate-provenance.sh"
|
|
TMPDIR="$(mktemp -d)"
|
|
|
|
# Helper: create a minimal skill directory with no sources.md and no source_keys
|
|
make_clean_skill() {
|
|
local dir="$1"
|
|
local name
|
|
name="$(basename "$dir")"
|
|
mkdir -p "$dir"
|
|
cat > "$dir/SKILL.md" <<EOF
|
|
---
|
|
name: $name
|
|
description: A valid skill description.
|
|
---
|
|
|
|
## Step 1
|
|
|
|
Do the thing.
|
|
EOF
|
|
}
|
|
|
|
# Helper: create a skill with source_keys in SKILL.md
|
|
make_skill_with_source_keys() {
|
|
local dir="$1"
|
|
local name
|
|
name="$(basename "$dir")"
|
|
mkdir -p "$dir"
|
|
cat > "$dir/SKILL.md" <<EOF
|
|
---
|
|
name: $name
|
|
description: A valid skill description.
|
|
metadata:
|
|
source_keys:
|
|
- my-source
|
|
---
|
|
|
|
## Step 1
|
|
|
|
Do the thing.
|
|
EOF
|
|
}
|
|
|
|
# Helper: create a valid sources.md with one entry
|
|
make_sources_md() {
|
|
local dir="$1"
|
|
local slug="${2:-my-source}"
|
|
local contrib="${3:-SKILL.md}"
|
|
local research="${4:-(none)}"
|
|
mkdir -p "$dir/references"
|
|
cat > "$dir/references/sources.md" <<EOF
|
|
# Sources
|
|
|
|
## ${slug}
|
|
|
|
- **URL:** https://example.com/${slug}
|
|
- **Description:** A test source.
|
|
- **Contributing files:** ${contrib}
|
|
- **Research doc:** ${research}
|
|
- **Status:** \`extracted\`
|
|
EOF
|
|
}
|
|
|
|
# Helper: a sources.md whose Description is wrapped across two physical
|
|
# lines — the shape check 9's parser used to truncate at the first newline.
|
|
make_wrapped_sources_md() {
|
|
local dir="$1"
|
|
mkdir -p "$dir/references"
|
|
cat > "$dir/references/sources.md" <<'EOF'
|
|
# Sources
|
|
|
|
## my-source
|
|
|
|
- **URL:** https://example.com/my-source
|
|
- **Description:** A test source, informing the dispatch table's shape with
|
|
no forge-specific content drawn directly from it beyond that.
|
|
- **Contributing files:** SKILL.md
|
|
- **Research doc:** (none)
|
|
- **Status:** `extracted`
|
|
EOF
|
|
}
|
|
|
|
# Helper: turn dir into a real git repo with one commit of its current
|
|
# contents, and a refs/remotes/origin/main pointing at that same commit.
|
|
# Check 9 diffs the skill's references/sources.md against `git merge-base
|
|
# HEAD origin/main` by default; this makes that resolve to a commit whose
|
|
# sources.md is byte-identical to the working tree, so check 9 has
|
|
# nothing to report there — exactly what a real repo looks like the
|
|
# instant after a clean commit. Fixtures that go on to test something
|
|
# else entirely (checks 3, 4, 5, 7, 8...) call this once, at the point
|
|
# their skill's own sources.md is in its final state, so a completely
|
|
# clean run stays completely clean.
|
|
commit_as_base() {
|
|
local dir="$1"
|
|
git -C "$dir" init -q >/dev/null 2>&1
|
|
git -C "$dir" -c user.email=test@example.com -c user.name=test add -A >/dev/null 2>&1
|
|
git -C "$dir" -c user.email=test@example.com -c user.name=test commit -q -m base >/dev/null 2>&1
|
|
git -C "$dir" update-ref refs/remotes/origin/main HEAD >/dev/null 2>&1
|
|
}
|
|
|
|
# Helper: create a fake repo (a real git repo, one commit, makes
|
|
# find_repo_root stop there) holding one skill whose single sources.md
|
|
# slug points at the given Research doc value. Checks 7 and 8 only run
|
|
# for a skill inside a checkout, so every upstream case needs this shape;
|
|
# the research doc itself is written per test into
|
|
# "$repo/docs/research/sources.md" — which check 9 does not examine, so
|
|
# a test overwriting it after this helper runs does not disturb check 9.
|
|
make_upstream_skill() {
|
|
local repo="$1"
|
|
local research="${2:-docs/research/sources.md}"
|
|
local skill="$repo/my-skill"
|
|
mkdir -p "$skill/references" "$repo/docs/research"
|
|
cat > "$skill/SKILL.md" <<EOF
|
|
---
|
|
name: my-skill
|
|
description: A valid skill description.
|
|
metadata:
|
|
source_keys:
|
|
- my-source
|
|
---
|
|
|
|
## Step 1
|
|
|
|
Do the thing.
|
|
EOF
|
|
cat > "$skill/references/sources.md" <<EOF
|
|
# Sources
|
|
|
|
## my-source
|
|
|
|
- **URL:** https://example.com/my-source
|
|
- **Description:** A test source.
|
|
- **Contributing files:** SKILL.md
|
|
- **Research doc:** ${research}
|
|
- **Status:** \`extracted\`
|
|
EOF
|
|
commit_as_base "$repo"
|
|
}
|
|
}
|
|
|
|
teardown() {
|
|
rm -rf "$TMPDIR"
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Cycle 1 — --help
|
|
# ---------------------------------------------------------------------------
|
|
|
|
@test "--help exits 0" {
|
|
run bash "$SCRIPT" --help
|
|
assert_success
|
|
assert_output --partial "Usage:"
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Cycle 2 — Early exit: no sources.md, no source_keys → exit 0, no output
|
|
# ---------------------------------------------------------------------------
|
|
|
|
@test "clean pass: no sources.md and no source_keys anywhere → exit 0, no output" {
|
|
local skill="$TMPDIR/my-skill"
|
|
make_clean_skill "$skill"
|
|
run bash "$SCRIPT" "$skill"
|
|
assert_success
|
|
assert_output ""
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Cycle 3 — Check 0: source_keys present but no sources.md → FAIL
|
|
# ---------------------------------------------------------------------------
|
|
|
|
@test "FAIL: source_keys in SKILL.md but sources.md absent" {
|
|
local skill="$TMPDIR/my-skill"
|
|
make_skill_with_source_keys "$skill"
|
|
run bash "$SCRIPT" "$skill"
|
|
assert_failure
|
|
assert_output --partial "FAIL"
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Cycle 4 — Check 1: FILL IN: placeholder in sources.md → FAIL
|
|
# ---------------------------------------------------------------------------
|
|
|
|
@test "FAIL: FILL IN: placeholder in sources.md" {
|
|
local skill="$TMPDIR/my-skill"
|
|
make_skill_with_source_keys "$skill"
|
|
mkdir -p "$skill/references"
|
|
cat > "$skill/references/sources.md" <<EOF
|
|
# Sources
|
|
|
|
## my-source
|
|
|
|
- **URL:** FILL IN: add url
|
|
- **Description:** A test source.
|
|
- **Contributing files:** SKILL.md
|
|
- **Research doc:** (none)
|
|
- **Status:** \`extracted\`
|
|
EOF
|
|
run bash "$SCRIPT" "$skill"
|
|
assert_failure
|
|
assert_output --partial "FAIL"
|
|
}
|
|
|
|
@test "FILL IN: inside backticks in sources.md does not fail" {
|
|
local skill="$TMPDIR/my-skill"
|
|
make_skill_with_source_keys "$skill"
|
|
make_sources_md "$skill"
|
|
echo "Use \`FILL IN: value\` as example." >> "$skill/references/sources.md"
|
|
run bash "$SCRIPT" "$skill"
|
|
assert_success
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Cycle 5 — Check 2: source_keys slug missing from sources.md → FAIL
|
|
# ---------------------------------------------------------------------------
|
|
|
|
@test "FAIL: source_keys slug in SKILL.md not present as H2 in sources.md" {
|
|
local skill="$TMPDIR/my-skill"
|
|
make_skill_with_source_keys "$skill"
|
|
mkdir -p "$skill/references"
|
|
cat > "$skill/references/sources.md" <<EOF
|
|
# Sources
|
|
|
|
## different-source
|
|
|
|
- **URL:** https://example.com/different-source
|
|
- **Description:** A different source.
|
|
- **Contributing files:** SKILL.md
|
|
- **Research doc:** (none)
|
|
- **Status:** \`extracted\`
|
|
EOF
|
|
run bash "$SCRIPT" "$skill"
|
|
assert_failure
|
|
assert_output --partial "FAIL"
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Cycle 6 — Check 4: Contributing file path doesn't exist → FAIL
|
|
# ---------------------------------------------------------------------------
|
|
|
|
@test "FAIL: Contributing file listed in sources.md does not exist" {
|
|
local skill="$TMPDIR/my-skill"
|
|
make_skill_with_source_keys "$skill"
|
|
make_sources_md "$skill" "my-source" "references/nonexistent.md"
|
|
run bash "$SCRIPT" "$skill"
|
|
assert_failure
|
|
assert_output --partial "FAIL"
|
|
}
|
|
|
|
@test "pass: (none) in Contributing files is skipped" {
|
|
local skill="$TMPDIR/my-skill"
|
|
make_skill_with_source_keys "$skill"
|
|
make_sources_md "$skill" "my-source" "(none — not used directly)"
|
|
run bash "$SCRIPT" "$skill"
|
|
assert_success
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Cycle 7 — Check 6: Research doc field missing → FAIL
|
|
# ---------------------------------------------------------------------------
|
|
|
|
@test "FAIL: Research doc field missing from sources.md entry" {
|
|
local skill="$TMPDIR/my-skill"
|
|
make_skill_with_source_keys "$skill"
|
|
mkdir -p "$skill/references"
|
|
cat > "$skill/references/sources.md" <<EOF
|
|
# Sources
|
|
|
|
## my-source
|
|
|
|
- **URL:** https://example.com/my-source
|
|
- **Description:** A test source.
|
|
- **Contributing files:** SKILL.md
|
|
- **Status:** \`extracted\`
|
|
EOF
|
|
run bash "$SCRIPT" "$skill"
|
|
assert_failure
|
|
assert_output --partial "FAIL"
|
|
}
|
|
|
|
@test "FAIL: Research doc field is FILL IN: placeholder" {
|
|
local skill="$TMPDIR/my-skill"
|
|
make_skill_with_source_keys "$skill"
|
|
mkdir -p "$skill/references"
|
|
cat > "$skill/references/sources.md" <<EOF
|
|
# Sources
|
|
|
|
## my-source
|
|
|
|
- **URL:** https://example.com/my-source
|
|
- **Description:** A test source.
|
|
- **Contributing files:** SKILL.md
|
|
- **Research doc:** FILL IN: path to research doc
|
|
- **Status:** \`extracted\`
|
|
EOF
|
|
run bash "$SCRIPT" "$skill"
|
|
assert_failure
|
|
assert_output --partial "FAIL"
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Cycle 8 — Check 5: Bidirectional mismatch → FAIL
|
|
# ---------------------------------------------------------------------------
|
|
|
|
@test "FAIL: Contributing file exists but does not list parent slug in source_keys" {
|
|
local skill="$TMPDIR/my-skill"
|
|
make_skill_with_source_keys "$skill"
|
|
make_sources_md "$skill" "my-source" "SKILL.md"
|
|
# SKILL.md has source_keys: my-source, but let's change it to NOT have my-source
|
|
cat > "$skill/SKILL.md" <<EOF
|
|
---
|
|
name: my-skill
|
|
description: A valid skill description.
|
|
metadata:
|
|
source_keys:
|
|
- other-source
|
|
---
|
|
|
|
## Step 1
|
|
|
|
Do the thing.
|
|
EOF
|
|
mkdir -p "$skill/references"
|
|
cat > "$skill/references/sources.md" <<EOF
|
|
# Sources
|
|
|
|
## other-source
|
|
|
|
- **URL:** https://example.com/other-source
|
|
- **Description:** A test source.
|
|
- **Contributing files:** SKILL.md
|
|
- **Research doc:** (none)
|
|
- **Status:** \`extracted\`
|
|
EOF
|
|
# Now add my-source that references SKILL.md but SKILL.md doesn't back-reference it
|
|
cat >> "$skill/references/sources.md" <<EOF
|
|
|
|
## my-source
|
|
|
|
- **URL:** https://example.com/my-source
|
|
- **Description:** Another source.
|
|
- **Contributing files:** SKILL.md
|
|
- **Research doc:** (none)
|
|
- **Status:** \`extracted\`
|
|
EOF
|
|
run bash "$SCRIPT" "$skill"
|
|
assert_failure
|
|
assert_output --partial "FAIL"
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Cycle 9 — Check 3: references/*.md with no source_keys → INFO (exit 0)
|
|
# ---------------------------------------------------------------------------
|
|
|
|
@test "INFO: references doc with no source_keys frontmatter emits INFO but exits 0" {
|
|
local skill="$TMPDIR/my-skill"
|
|
make_skill_with_source_keys "$skill"
|
|
make_sources_md "$skill"
|
|
mkdir -p "$skill/references"
|
|
cat > "$skill/references/extra.md" <<EOF
|
|
# Extra Reference
|
|
|
|
No frontmatter here.
|
|
EOF
|
|
run bash "$SCRIPT" "$skill"
|
|
assert_success
|
|
assert_output --partial "INFO"
|
|
}
|
|
|
|
@test "pass: references doc with source_keys all matching sources.md exits 0" {
|
|
local skill="$TMPDIR/my-skill"
|
|
make_skill_with_source_keys "$skill"
|
|
make_sources_md "$skill"
|
|
mkdir -p "$skill/references"
|
|
cat > "$skill/references/extra.md" <<EOF
|
|
---
|
|
source_keys:
|
|
- my-source
|
|
---
|
|
|
|
# Extra Reference
|
|
|
|
Content here.
|
|
EOF
|
|
run bash "$SCRIPT" "$skill"
|
|
assert_success
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Cycle 10 — Clean full pass: valid sources.md, all source_keys match, files exist
|
|
# ---------------------------------------------------------------------------
|
|
|
|
@test "clean full pass: all checks satisfied" {
|
|
local skill="$TMPDIR/my-skill"
|
|
make_skill_with_source_keys "$skill"
|
|
make_sources_md "$skill"
|
|
run bash "$SCRIPT" "$skill"
|
|
assert_success
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Cycle 11 — Check 7: Upstream forward: slug in sources.md not in research doc → INFO
|
|
# ---------------------------------------------------------------------------
|
|
|
|
@test "INFO: slug in sources.md not found in research doc → INFO, exits 0" {
|
|
local skill="$TMPDIR/my-skill"
|
|
make_skill_with_source_keys "$skill"
|
|
|
|
# Create a research doc that does NOT have the slug
|
|
local research_dir="$TMPDIR/research"
|
|
mkdir -p "$research_dir"
|
|
cat > "$research_dir/sources.md" <<EOF
|
|
# Research
|
|
|
|
## different-slug
|
|
|
|
- **Contributing files:** (none)
|
|
- **Status:** \`extracted\`
|
|
EOF
|
|
|
|
# Use a path relative to repo root — we'll place research doc inside TMPDIR
|
|
# and reference it as absolute for test purposes.
|
|
# The script finds repo root by walking up from skill-dir until .git is found.
|
|
# Since TMPDIR won't have .git, we simulate a repo structure.
|
|
local fake_repo="$TMPDIR/fakerepo"
|
|
mkdir -p "$fake_repo"
|
|
touch "$fake_repo/.git" # fake .git marker
|
|
|
|
local skill2="$fake_repo/my-skill"
|
|
mkdir -p "$skill2"
|
|
cat > "$skill2/SKILL.md" <<EOF
|
|
---
|
|
name: my-skill
|
|
description: A valid skill description.
|
|
metadata:
|
|
source_keys:
|
|
- my-source
|
|
---
|
|
|
|
## Step 1
|
|
|
|
Do the thing.
|
|
EOF
|
|
|
|
mkdir -p "$skill2/references"
|
|
mkdir -p "$fake_repo/docs/research"
|
|
cat > "$fake_repo/docs/research/sources.md" <<EOF
|
|
# Research
|
|
|
|
## different-slug
|
|
|
|
- **Contributing files:** (none)
|
|
- **Status:** \`extracted\`
|
|
EOF
|
|
|
|
cat > "$skill2/references/sources.md" <<EOF
|
|
# Sources
|
|
|
|
## my-source
|
|
|
|
- **URL:** https://example.com/my-source
|
|
- **Description:** A test source.
|
|
- **Contributing files:** SKILL.md
|
|
- **Research doc:** docs/research/sources.md
|
|
- **Status:** \`extracted\`
|
|
EOF
|
|
|
|
run bash "$SCRIPT" "$skill2"
|
|
assert_success
|
|
assert_output --partial "INFO"
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Cycle 12 — Check 8: Upstream reverse: extracted slug in research doc not in sources.md → FAIL
|
|
# ---------------------------------------------------------------------------
|
|
|
|
@test "FAIL: extracted non-(none) slug in research doc missing from skill sources.md" {
|
|
local fake_repo="$TMPDIR/fakerepo"
|
|
mkdir -p "$fake_repo"
|
|
touch "$fake_repo/.git"
|
|
|
|
local skill="$fake_repo/my-skill"
|
|
mkdir -p "$skill"
|
|
cat > "$skill/SKILL.md" <<EOF
|
|
---
|
|
name: my-skill
|
|
description: A valid skill description.
|
|
metadata:
|
|
source_keys:
|
|
- my-source
|
|
---
|
|
|
|
## Step 1
|
|
|
|
Do the thing.
|
|
EOF
|
|
|
|
mkdir -p "$skill/references"
|
|
mkdir -p "$fake_repo/docs/research"
|
|
|
|
# Research doc has my-source (extracted, with a contributing file) AND extra-source (also extracted)
|
|
cat > "$fake_repo/docs/research/sources.md" <<EOF
|
|
# Research
|
|
|
|
## my-source
|
|
|
|
- **Contributing files:** some-skill/SKILL.md
|
|
- **Status:** \`extracted\`
|
|
|
|
## extra-source
|
|
|
|
- **Contributing files:** some-skill/references/extra.md
|
|
- **Status:** \`extracted\`
|
|
EOF
|
|
|
|
cat > "$skill/references/sources.md" <<EOF
|
|
# Sources
|
|
|
|
## my-source
|
|
|
|
- **URL:** https://example.com/my-source
|
|
- **Description:** A test source.
|
|
- **Contributing files:** SKILL.md
|
|
- **Research doc:** docs/research/sources.md
|
|
- **Status:** \`extracted\`
|
|
EOF
|
|
|
|
run bash "$SCRIPT" "$skill"
|
|
assert_failure
|
|
assert_output --partial "FAIL"
|
|
}
|
|
|
|
@test "pass: extracted slug in research doc with (none) contributing files is not required in sources.md" {
|
|
local fake_repo="$TMPDIR/fakerepo"
|
|
mkdir -p "$fake_repo"
|
|
touch "$fake_repo/.git"
|
|
|
|
local skill="$fake_repo/my-skill"
|
|
mkdir -p "$skill"
|
|
cat > "$skill/SKILL.md" <<EOF
|
|
---
|
|
name: my-skill
|
|
description: A valid skill description.
|
|
metadata:
|
|
source_keys:
|
|
- my-source
|
|
---
|
|
|
|
## Step 1
|
|
|
|
Do the thing.
|
|
EOF
|
|
|
|
mkdir -p "$skill/references"
|
|
mkdir -p "$fake_repo/docs/research"
|
|
|
|
cat > "$fake_repo/docs/research/sources.md" <<EOF
|
|
# Research
|
|
|
|
## my-source
|
|
|
|
- **Contributing files:** some-skill/SKILL.md
|
|
- **Status:** \`extracted\`
|
|
|
|
## extra-source
|
|
|
|
- **Contributing files:** (none — not relevant)
|
|
- **Status:** \`extracted\`
|
|
EOF
|
|
|
|
cat > "$skill/references/sources.md" <<EOF
|
|
# Sources
|
|
|
|
## my-source
|
|
|
|
- **URL:** https://example.com/my-source
|
|
- **Description:** A test source.
|
|
- **Contributing files:** SKILL.md
|
|
- **Research doc:** docs/research/sources.md
|
|
- **Status:** \`extracted\`
|
|
EOF
|
|
|
|
run bash "$SCRIPT" "$skill"
|
|
assert_success
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Cycle 13 — parse_contributing_files: None ("could not parse") is NOT []
|
|
# ("explicitly (none)")
|
|
#
|
|
# Check 8 reads [] as "the research doc deliberately records no contributing
|
|
# files" and SKIPS the slug on that basis. A block the parser cannot read must
|
|
# therefore return None, so the slug stays exposed to check 8. Each case below
|
|
# asserts that CONSEQUENCE — check 8 firing on the unreadable entry — not the
|
|
# parser's return value, because returning [] is exactly the shape that makes
|
|
# the check silently do nothing while still exiting 0.
|
|
# ---------------------------------------------------------------------------
|
|
|
|
@test "check 8 runs: bullet form with '*' asterisk bullets is unparsable, not '(none)'" {
|
|
local fake_repo="$TMPDIR/fakerepo"
|
|
make_upstream_skill "$fake_repo"
|
|
|
|
cat > "$fake_repo/docs/research/sources.md" <<EOF
|
|
# Research
|
|
|
|
## my-source
|
|
|
|
- **Contributing files:** some-skill/SKILL.md
|
|
- **Status:** \`extracted\`
|
|
|
|
## extra-source
|
|
|
|
**Contributing files:**
|
|
* some-skill/references/extra.md
|
|
|
|
- **Status:** \`extracted\`
|
|
EOF
|
|
|
|
run bash "$SCRIPT" "$fake_repo/my-skill"
|
|
assert_failure
|
|
assert_output --partial "FAIL"
|
|
assert_output --partial "extra-source"
|
|
}
|
|
|
|
@test "check 8 runs: bullet form with a numbered list is unparsable, not '(none)'" {
|
|
local fake_repo="$TMPDIR/fakerepo"
|
|
make_upstream_skill "$fake_repo"
|
|
|
|
cat > "$fake_repo/docs/research/sources.md" <<EOF
|
|
# Research
|
|
|
|
## my-source
|
|
|
|
- **Contributing files:** some-skill/SKILL.md
|
|
- **Status:** \`extracted\`
|
|
|
|
## extra-source
|
|
|
|
**Contributing files:**
|
|
1. some-skill/references/extra.md
|
|
|
|
- **Status:** \`extracted\`
|
|
EOF
|
|
|
|
run bash "$SCRIPT" "$fake_repo/my-skill"
|
|
assert_failure
|
|
assert_output --partial "FAIL"
|
|
assert_output --partial "extra-source"
|
|
}
|
|
|
|
@test "check 8 runs: bullet form followed by prose is unparsable, not '(none)'" {
|
|
local fake_repo="$TMPDIR/fakerepo"
|
|
make_upstream_skill "$fake_repo"
|
|
|
|
cat > "$fake_repo/docs/research/sources.md" <<EOF
|
|
# Research
|
|
|
|
## my-source
|
|
|
|
- **Contributing files:** some-skill/SKILL.md
|
|
- **Status:** \`extracted\`
|
|
|
|
## extra-source
|
|
|
|
**Contributing files:**
|
|
See the table below for the file list.
|
|
|
|
- **Status:** \`extracted\`
|
|
EOF
|
|
|
|
run bash "$SCRIPT" "$fake_repo/my-skill"
|
|
assert_failure
|
|
assert_output --partial "FAIL"
|
|
assert_output --partial "extra-source"
|
|
}
|
|
|
|
@test "check 8 runs: bullet form heading with a blank line and nothing after is unparsable, not '(none)'" {
|
|
local fake_repo="$TMPDIR/fakerepo"
|
|
make_upstream_skill "$fake_repo"
|
|
|
|
cat > "$fake_repo/docs/research/sources.md" <<EOF
|
|
# Research
|
|
|
|
## my-source
|
|
|
|
- **Contributing files:** some-skill/SKILL.md
|
|
- **Status:** \`extracted\`
|
|
|
|
## extra-source
|
|
|
|
- **Status:** \`extracted\`
|
|
|
|
**Contributing files:**
|
|
|
|
EOF
|
|
|
|
run bash "$SCRIPT" "$fake_repo/my-skill"
|
|
assert_failure
|
|
assert_output --partial "FAIL"
|
|
assert_output --partial "extra-source"
|
|
}
|
|
|
|
@test "check 8 runs: inline form whose whole value is a parenthetical is unparsable, not '(none)'" {
|
|
local fake_repo="$TMPDIR/fakerepo"
|
|
make_upstream_skill "$fake_repo"
|
|
|
|
cat > "$fake_repo/docs/research/sources.md" <<EOF
|
|
# Research
|
|
|
|
## my-source
|
|
|
|
- **Contributing files:** some-skill/SKILL.md
|
|
- **Status:** \`extracted\`
|
|
|
|
## extra-source
|
|
|
|
- **Contributing files:** (see notes below)
|
|
- **Status:** \`extracted\`
|
|
EOF
|
|
|
|
run bash "$SCRIPT" "$fake_repo/my-skill"
|
|
assert_failure
|
|
assert_output --partial "FAIL"
|
|
assert_output --partial "extra-source"
|
|
}
|
|
|
|
@test "check 4 runs: bullet form with '-' hyphen bullets still parses each path" {
|
|
local skill="$TMPDIR/my-skill"
|
|
make_skill_with_source_keys "$skill"
|
|
mkdir -p "$skill/references"
|
|
cat > "$skill/references/sources.md" <<EOF
|
|
# Sources
|
|
|
|
## my-source
|
|
|
|
- **URL:** https://example.com/my-source
|
|
- **Description:** A test source.
|
|
|
|
**Contributing files:**
|
|
- SKILL.md (the dispatch table, with a comma in the note)
|
|
- references/nonexistent.md (why this one matters)
|
|
|
|
- **Research doc:** (none)
|
|
- **Status:** \`extracted\`
|
|
EOF
|
|
run bash "$SCRIPT" "$skill"
|
|
assert_failure
|
|
assert_output --partial "Contributing file 'references/nonexistent.md' does not exist"
|
|
}
|
|
|
|
@test "check 4 runs: inline comma-separated form still parses each path" {
|
|
local skill="$TMPDIR/my-skill"
|
|
make_skill_with_source_keys "$skill"
|
|
make_sources_md "$skill" "my-source" "SKILL.md, references/nonexistent.md"
|
|
run bash "$SCRIPT" "$skill"
|
|
assert_failure
|
|
assert_output --partial "Contributing file 'references/nonexistent.md' does not exist"
|
|
}
|
|
|
|
@test "check 8 skips: bullet form '- (none)' is an explicit declaration" {
|
|
local fake_repo="$TMPDIR/fakerepo"
|
|
make_upstream_skill "$fake_repo"
|
|
|
|
cat > "$fake_repo/docs/research/sources.md" <<EOF
|
|
# Research
|
|
|
|
## my-source
|
|
|
|
- **Contributing files:** some-skill/SKILL.md
|
|
- **Status:** \`extracted\`
|
|
|
|
## extra-source
|
|
|
|
**Contributing files:**
|
|
- (none — nothing was extracted from this section)
|
|
|
|
- **Status:** \`extracted\`
|
|
EOF
|
|
|
|
run bash "$SCRIPT" "$fake_repo/my-skill"
|
|
assert_success
|
|
}
|
|
|
|
@test "check 8 skips: inline bare '(none)' is an explicit declaration" {
|
|
local fake_repo="$TMPDIR/fakerepo"
|
|
make_upstream_skill "$fake_repo"
|
|
|
|
cat > "$fake_repo/docs/research/sources.md" <<EOF
|
|
# Research
|
|
|
|
## my-source
|
|
|
|
- **Contributing files:** some-skill/SKILL.md
|
|
- **Status:** \`extracted\`
|
|
|
|
## extra-source
|
|
|
|
- **Contributing files:** (none)
|
|
- **Status:** \`extracted\`
|
|
EOF
|
|
|
|
run bash "$SCRIPT" "$fake_repo/my-skill"
|
|
assert_success
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Cycle 14 — Check 3 (#111): an explicit 'source_keys: []' is a house-authored
|
|
# declaration, a bare 'source_keys:' is truncation
|
|
#
|
|
# Three states, three outcomes. Only the middle one — declared empty — is
|
|
# silent; collapsing any pair of them is the defect #111 filed.
|
|
# ---------------------------------------------------------------------------
|
|
|
|
@test "check 3 silent: references doc with top-level 'source_keys: []' emits nothing" {
|
|
local skill="$TMPDIR/my-skill"
|
|
make_skill_with_source_keys "$skill"
|
|
make_sources_md "$skill"
|
|
cat > "$skill/references/extra.md" <<EOF
|
|
---
|
|
source_keys: []
|
|
---
|
|
|
|
# Extra Reference
|
|
|
|
House-authored, no external source.
|
|
EOF
|
|
commit_as_base "$skill"
|
|
run bash "$SCRIPT" "$skill"
|
|
assert_success
|
|
assert_output ""
|
|
}
|
|
|
|
@test "check 3 silent: references doc with 'source_keys: []' nested under metadata: emits nothing" {
|
|
local skill="$TMPDIR/my-skill"
|
|
make_skill_with_source_keys "$skill"
|
|
make_sources_md "$skill"
|
|
cat > "$skill/references/extra.md" <<EOF
|
|
---
|
|
metadata:
|
|
source_keys: []
|
|
---
|
|
|
|
# Extra Reference
|
|
|
|
House-authored, no external source.
|
|
EOF
|
|
commit_as_base "$skill"
|
|
run bash "$SCRIPT" "$skill"
|
|
assert_success
|
|
assert_output ""
|
|
}
|
|
|
|
@test "INFO: references doc with a bare 'source_keys:' and no value still emits INFO" {
|
|
local skill="$TMPDIR/my-skill"
|
|
make_skill_with_source_keys "$skill"
|
|
make_sources_md "$skill"
|
|
cat > "$skill/references/extra.md" <<EOF
|
|
---
|
|
source_keys:
|
|
---
|
|
|
|
# Extra Reference
|
|
|
|
Truncated frontmatter — this is not a decision.
|
|
EOF
|
|
run bash "$SCRIPT" "$skill"
|
|
assert_success
|
|
assert_output --partial "INFO"
|
|
assert_output --partial "No source_keys frontmatter"
|
|
}
|
|
|
|
@test "INFO: references doc with frontmatter but no source_keys key still emits INFO" {
|
|
local skill="$TMPDIR/my-skill"
|
|
make_skill_with_source_keys "$skill"
|
|
make_sources_md "$skill"
|
|
cat > "$skill/references/extra.md" <<EOF
|
|
---
|
|
title: Extra Reference
|
|
---
|
|
|
|
# Extra Reference
|
|
|
|
Never said either way.
|
|
EOF
|
|
run bash "$SCRIPT" "$skill"
|
|
assert_success
|
|
assert_output --partial "INFO"
|
|
assert_output --partial "No source_keys frontmatter"
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Cycle 15 — Checks 7 and 8: Research doc annotation stripping, and the INFO
|
|
# that replaced the silent skip
|
|
#
|
|
# A Research doc value is very often a path PLUS a section annotation, and
|
|
# os.path.isfile() is false for every such string. Before the strip, checks 7
|
|
# and 8 skipped SILENTLY on those — so the negative assertions here (an INFO
|
|
# that must appear) are what distinguishes a running check from a dead one.
|
|
# ---------------------------------------------------------------------------
|
|
|
|
@test "check 7 runs: '§' section annotation is stripped before the path is resolved" {
|
|
local fake_repo="$TMPDIR/fakerepo"
|
|
make_upstream_skill "$fake_repo" 'docs/research/sources.md § "Some Section"'
|
|
|
|
cat > "$fake_repo/docs/research/sources.md" <<EOF
|
|
# Research
|
|
|
|
## different-slug
|
|
|
|
- **Contributing files:** (none)
|
|
- **Status:** \`extracted\`
|
|
EOF
|
|
|
|
run bash "$SCRIPT" "$fake_repo/my-skill"
|
|
assert_success
|
|
assert_output --partial "Slug 'my-source' not found as H2 in research doc 'docs/research/sources.md'"
|
|
refute_output --partial "§"
|
|
}
|
|
|
|
@test "check 7 runs: '→' section annotation is stripped before the path is resolved" {
|
|
local fake_repo="$TMPDIR/fakerepo"
|
|
make_upstream_skill "$fake_repo" 'docs/research/sources.md → `## Pushing`'
|
|
|
|
cat > "$fake_repo/docs/research/sources.md" <<EOF
|
|
# Research
|
|
|
|
## different-slug
|
|
|
|
- **Contributing files:** (none)
|
|
- **Status:** \`extracted\`
|
|
EOF
|
|
|
|
run bash "$SCRIPT" "$fake_repo/my-skill"
|
|
assert_success
|
|
assert_output --partial "Slug 'my-source' not found as H2 in research doc 'docs/research/sources.md'"
|
|
refute_output --partial "→"
|
|
}
|
|
|
|
@test "check 7 runs: parenthetical annotation is stripped before the path is resolved" {
|
|
local fake_repo="$TMPDIR/fakerepo"
|
|
make_upstream_skill "$fake_repo" "docs/research/sources.md (whole-document reference)"
|
|
|
|
cat > "$fake_repo/docs/research/sources.md" <<EOF
|
|
# Research
|
|
|
|
## different-slug
|
|
|
|
- **Contributing files:** (none)
|
|
- **Status:** \`extracted\`
|
|
EOF
|
|
|
|
run bash "$SCRIPT" "$fake_repo/my-skill"
|
|
assert_success
|
|
assert_output --partial "Slug 'my-source' not found as H2 in research doc 'docs/research/sources.md'"
|
|
refute_output --partial "whole-document reference"
|
|
}
|
|
|
|
@test "check 7 runs: a bare path with no annotation survives the strip intact" {
|
|
local fake_repo="$TMPDIR/fakerepo"
|
|
make_upstream_skill "$fake_repo" "docs/research/sources.md"
|
|
|
|
cat > "$fake_repo/docs/research/sources.md" <<EOF
|
|
# Research
|
|
|
|
## my-source
|
|
|
|
- **Contributing files:** (none)
|
|
- **Status:** \`extracted\`
|
|
EOF
|
|
|
|
run bash "$SCRIPT" "$fake_repo/my-skill"
|
|
assert_success
|
|
assert_output ""
|
|
}
|
|
|
|
@test "checks 7 and 8 skipped silently: Research doc '(none)' is recognised before the strip" {
|
|
local fake_repo="$TMPDIR/fakerepo"
|
|
make_upstream_skill "$fake_repo" "(none)"
|
|
|
|
run bash "$SCRIPT" "$fake_repo/my-skill"
|
|
assert_success
|
|
assert_output ""
|
|
}
|
|
|
|
@test "checks 7 and 8 skipped silently: bare 'none — reason' is recognised as a declaration" {
|
|
local fake_repo="$TMPDIR/fakerepo"
|
|
make_upstream_skill "$fake_repo" "none — org convention, no upstream research doc"
|
|
|
|
run bash "$SCRIPT" "$fake_repo/my-skill"
|
|
assert_success
|
|
assert_output ""
|
|
}
|
|
|
|
@test "INFO: a stripped path that does not resolve names the slug instead of skipping silently" {
|
|
local fake_repo="$TMPDIR/fakerepo"
|
|
make_upstream_skill "$fake_repo" 'docs/research/missing.md § "Some Section"'
|
|
|
|
run bash "$SCRIPT" "$fake_repo/my-skill"
|
|
assert_success
|
|
assert_output --partial "INFO"
|
|
assert_output --partial "Upstream checks skipped for 'my-source' — research doc 'docs/research/missing.md' does not exist"
|
|
}
|
|
|
|
@test "INFO: a Research doc value that is entirely annotation names the slug instead of skipping silently" {
|
|
local fake_repo="$TMPDIR/fakerepo"
|
|
make_upstream_skill "$fake_repo" '§ "Some Section"'
|
|
|
|
run bash "$SCRIPT" "$fake_repo/my-skill"
|
|
assert_success
|
|
assert_output --partial "INFO"
|
|
assert_output --partial "Upstream checks skipped for 'my-source' — Research doc value names no path"
|
|
}
|
|
|
|
@test "INFO: no repo root above the skill directory names the slug instead of skipping silently" {
|
|
local skill="$TMPDIR/my-skill"
|
|
make_skill_with_source_keys "$skill"
|
|
make_sources_md "$skill" "my-source" "SKILL.md" "docs/research/sources.md"
|
|
|
|
run bash "$SCRIPT" "$skill"
|
|
assert_success
|
|
assert_output --partial "INFO"
|
|
assert_output --partial "Upstream checks skipped for 'my-source' — no repo root above the skill directory"
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Cycle 16 — Checks 4 and 5: None ("could not parse") is NOT [] ("explicitly
|
|
# (none)"), on the sources.md side this time
|
|
#
|
|
# Cycle 13 pinned the distinction for check 8, which reads the parser's output
|
|
# against a RESEARCH doc. Checks 4 and 5 read it against the skill's own
|
|
# sources.md and honoured neither half: a truthiness test collapsed None into
|
|
# [], so an unreadable Contributing files block disabled both checks and the
|
|
# script still exited 0 with no output — the failure mode
|
|
# parse_contributing_files' docstring names in as many words. The assertions
|
|
# below are therefore about the INFO appearing; a silent exit 0 is exactly the
|
|
# bug.
|
|
# ---------------------------------------------------------------------------
|
|
|
|
@test "INFO: an unparsable Contributing files block names the slug instead of skipping checks 4 and 5 silently" {
|
|
local skill="$TMPDIR/my-skill"
|
|
make_skill_with_source_keys "$skill"
|
|
mkdir -p "$skill/references"
|
|
cat > "$skill/references/sources.md" <<EOF
|
|
# Sources
|
|
|
|
## my-source
|
|
|
|
- **URL:** https://example.com/my-source
|
|
- **Description:** A test source.
|
|
- **Research doc:** (none)
|
|
**Contributing files:**
|
|
* references/ghost.md (asterisk bullets are not the bullet form)
|
|
- **Status:** \`extracted\`
|
|
EOF
|
|
run bash "$SCRIPT" "$skill"
|
|
assert_success
|
|
assert_output --partial "INFO"
|
|
assert_output --partial "Contributing-file checks skipped for 'my-source' — the Contributing files block could not be parsed"
|
|
}
|
|
|
|
@test "INFO: an entry with no Contributing files field at all is reported, not skipped silently" {
|
|
local skill="$TMPDIR/my-skill"
|
|
make_skill_with_source_keys "$skill"
|
|
mkdir -p "$skill/references"
|
|
cat > "$skill/references/sources.md" <<EOF
|
|
# Sources
|
|
|
|
## my-source
|
|
|
|
- **URL:** https://example.com/my-source
|
|
- **Description:** A test source.
|
|
- **Research doc:** (none)
|
|
- **Status:** \`extracted\`
|
|
EOF
|
|
run bash "$SCRIPT" "$skill"
|
|
assert_success
|
|
assert_output --partial "INFO"
|
|
assert_output --partial "Contributing-file checks skipped for 'my-source' — the Contributing files block could not be parsed"
|
|
}
|
|
|
|
@test "checks 4 and 5 skipped silently: an explicit '(none)' emits no INFO" {
|
|
local skill="$TMPDIR/my-skill"
|
|
make_skill_with_source_keys "$skill"
|
|
make_sources_md "$skill" "my-source" "(none — not used directly)"
|
|
commit_as_base "$skill"
|
|
run bash "$SCRIPT" "$skill"
|
|
assert_success
|
|
assert_output ""
|
|
}
|
|
|
|
@test "checks 4 and 5 still run: a parseable Contributing files list is not diverted to the INFO" {
|
|
local skill="$TMPDIR/my-skill"
|
|
make_skill_with_source_keys "$skill"
|
|
make_sources_md "$skill" "my-source" "references/nonexistent.md"
|
|
run bash "$SCRIPT" "$skill"
|
|
assert_failure
|
|
assert_output --partial "FAIL"
|
|
assert_output --partial "Contributing file 'references/nonexistent.md' does not exist"
|
|
refute_output --partial "could not be parsed"
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Cycle 17 — Check 3 (#111): 'source_keys: []' only declares anything in the
|
|
# two positions parse_source_keys() actually reads
|
|
#
|
|
# The declaration and the parse have to agree on WHERE the key lives. They did
|
|
# not: the declaration regex accepted any indent, so a `source_keys: []` buried
|
|
# under an unrelated key — a position parse_source_keys() never reads — passed
|
|
# as a house-authored declaration and silenced the INFO for a file that had
|
|
# declared nothing.
|
|
# ---------------------------------------------------------------------------
|
|
|
|
@test "INFO: 'source_keys: []' nested under an unrelated key is not a declaration" {
|
|
local skill="$TMPDIR/my-skill"
|
|
make_skill_with_source_keys "$skill"
|
|
make_sources_md "$skill"
|
|
cat > "$skill/references/extra.md" <<EOF
|
|
---
|
|
title: Extra Reference
|
|
unrelated:
|
|
nested:
|
|
source_keys: []
|
|
---
|
|
|
|
# Extra Reference
|
|
|
|
The empty list is nested where parse_source_keys never looks.
|
|
EOF
|
|
run bash "$SCRIPT" "$skill"
|
|
assert_success
|
|
assert_output --partial "INFO"
|
|
assert_output --partial "No source_keys frontmatter"
|
|
}
|
|
|
|
@test "INFO: a four-space-indented 'source_keys: []' is not a declaration either" {
|
|
local skill="$TMPDIR/my-skill"
|
|
make_skill_with_source_keys "$skill"
|
|
make_sources_md "$skill"
|
|
cat > "$skill/references/extra.md" <<EOF
|
|
---
|
|
metadata:
|
|
source_keys: []
|
|
---
|
|
|
|
# Extra Reference
|
|
|
|
Two spaces is the position parse_source_keys reads under metadata:, not four.
|
|
EOF
|
|
run bash "$SCRIPT" "$skill"
|
|
assert_success
|
|
assert_output --partial "INFO"
|
|
assert_output --partial "No source_keys frontmatter"
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Cycle 20 — F: checks 7 and 8 apply only to a research SOURCE INDEX, and
|
|
# every skip announces itself
|
|
# ---------------------------------------------------------------------------
|
|
|
|
@test "F: a topic-doc Research doc is reported as not applicable, not as a missing slug" {
|
|
local fake_repo="$TMPDIR/fakerepo"
|
|
make_upstream_skill "$fake_repo" "docs/research/remotes.md"
|
|
cat > "$fake_repo/docs/research/remotes.md" <<EOF
|
|
# Remotes
|
|
|
|
## Core Philosophy
|
|
|
|
Prose about remotes.
|
|
EOF
|
|
|
|
run bash "$SCRIPT" "$fake_repo/my-skill"
|
|
assert_success
|
|
assert_output --partial "Upstream checks not applicable for 'my-source'"
|
|
assert_output --partial "is a topic document, not a source index"
|
|
refute_output --partial "not found as H2 in research doc"
|
|
}
|
|
|
|
@test "F: check 7 still runs when the Research doc IS a source index" {
|
|
local fake_repo="$TMPDIR/fakerepo"
|
|
make_upstream_skill "$fake_repo"
|
|
cat > "$fake_repo/docs/research/sources.md" <<EOF
|
|
# Sources
|
|
|
|
## different-slug
|
|
|
|
- **Contributing files:** (none)
|
|
- **Status:** \`extracted\`
|
|
EOF
|
|
|
|
run bash "$SCRIPT" "$fake_repo/my-skill"
|
|
assert_success
|
|
assert_output --partial "Slug 'my-source' not found as H2 in research doc 'docs/research/sources.md'"
|
|
refute_output --partial "not applicable"
|
|
}
|
|
|
|
@test "F: check 8 announces the slug it skipped for a non-extracted Status" {
|
|
local fake_repo="$TMPDIR/fakerepo"
|
|
make_upstream_skill "$fake_repo"
|
|
cat > "$fake_repo/docs/research/sources.md" <<EOF
|
|
# Sources
|
|
|
|
## my-source
|
|
|
|
- **Contributing files:** some-skill/SKILL.md
|
|
- **Status:** \`extracted\`
|
|
|
|
## extra-source
|
|
|
|
- **Contributing files:** some-skill/references/extra.md
|
|
- **Status:** \`referenced\`
|
|
EOF
|
|
|
|
run bash "$SCRIPT" "$fake_repo/my-skill"
|
|
assert_success
|
|
assert_output --partial "Check 8 skipped for research-doc slug 'extra-source'"
|
|
assert_output --partial "its Status is \`referenced\`, not \`extracted\`"
|
|
}
|
|
|
|
@test "F: a Status with a trailing note after the backticked value still reads as extracted" {
|
|
local fake_repo="$TMPDIR/fakerepo"
|
|
make_upstream_skill "$fake_repo"
|
|
cat > "$fake_repo/docs/research/sources.md" <<EOF
|
|
# Sources
|
|
|
|
## my-source
|
|
|
|
- **Contributing files:** some-skill/SKILL.md
|
|
- **Status:** \`extracted\`
|
|
|
|
## extra-source
|
|
|
|
- **Contributing files:** some-skill/references/extra.md
|
|
- **Status:** \`extracted\` — partial fetch, section 3 only
|
|
EOF
|
|
|
|
run bash "$SCRIPT" "$fake_repo/my-skill"
|
|
assert_failure
|
|
assert_output --partial "Research doc slug 'extra-source' missing from skill sources.md"
|
|
}
|
|
|
|
@test "F: a bullet-form Status still reads as extracted" {
|
|
local fake_repo="$TMPDIR/fakerepo"
|
|
make_upstream_skill "$fake_repo"
|
|
cat > "$fake_repo/docs/research/sources.md" <<EOF
|
|
# Sources
|
|
|
|
## my-source
|
|
|
|
- **Contributing files:** some-skill/SKILL.md
|
|
- **Status:** \`extracted\`
|
|
|
|
## extra-source
|
|
|
|
- **Contributing files:** some-skill/references/extra.md
|
|
|
|
**Status:**
|
|
- \`extracted\`
|
|
EOF
|
|
|
|
run bash "$SCRIPT" "$fake_repo/my-skill"
|
|
assert_failure
|
|
assert_output --partial "Research doc slug 'extra-source' missing from skill sources.md"
|
|
}
|
|
|
|
@test "F: a research-doc slug with no Status line at all is announced, not skipped silently" {
|
|
local fake_repo="$TMPDIR/fakerepo"
|
|
make_upstream_skill "$fake_repo"
|
|
cat > "$fake_repo/docs/research/sources.md" <<EOF
|
|
# Sources
|
|
|
|
## my-source
|
|
|
|
- **Contributing files:** some-skill/SKILL.md
|
|
- **Status:** \`extracted\`
|
|
|
|
## extra-source
|
|
|
|
- **Contributing files:** some-skill/references/extra.md
|
|
EOF
|
|
|
|
run bash "$SCRIPT" "$fake_repo/my-skill"
|
|
assert_success
|
|
assert_output --partial "Check 8 skipped for research-doc slug 'extra-source'"
|
|
assert_output --partial "its Status is absent, not \`extracted\`"
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Cycle 21 — G1: a bad target is a hard error, not a silent pass
|
|
#
|
|
# PORT NOTE RESOLVED (factory-audit merge): this block used to assert
|
|
# "not a directory" for the nonexistent target while the agent suite asserted
|
|
# "no such file" for its own, because each pre-merge script knew what shape it
|
|
# was owed. The merged entry point classifies before it complains, so neither
|
|
# old wording survives at THIS site and the two cases are no longer the same
|
|
# case:
|
|
#
|
|
# * a path that exists as nothing this script recognises (and a path that does
|
|
# not exist at all, when its name is not *.agent.md and its parent is not
|
|
# agents/) is "matches neither a skill directory nor an agent file";
|
|
# * a path that IS a directory but holds no SKILL.md gets its own, more
|
|
# specific line naming the missing SKILL.md.
|
|
#
|
|
# The agent suite's "no such file" wording did NOT move — a nonexistent
|
|
# *.agent.md path still classifies as an agent and is rejected by agent mode's
|
|
# own precondition — so that side stays verbatim over there. Both wordings are
|
|
# now asserted exactly rather than partially-matched, because the invariant both
|
|
# sides pin is exit 2 with a message that names the path, never a silent exit 0
|
|
# and never a bare code with no output.
|
|
# ---------------------------------------------------------------------------
|
|
|
|
@test "G1: a nonexistent directory is a hard error (exit 2), not a silent exit 0" {
|
|
run bash "$SCRIPT" "$TMPDIR/does-not-exist"
|
|
[ "$status" -eq 2 ]
|
|
assert_output --partial "Error: '$TMPDIR/does-not-exist' matches neither a skill directory nor an agent file."
|
|
}
|
|
|
|
@test "G1: a directory with no SKILL.md is a hard error (exit 2), not a silent exit 0" {
|
|
# Distinct from the case above on purpose: a directory IS half of a skill
|
|
# target, so the detector can say something sharper than "neither shape" and
|
|
# does. The exit code is unchanged from the pre-merge test, which already
|
|
# pinned 2; only the wording moved, from the suite's own "not a skill
|
|
# directory" precondition to the dispatcher's classification message.
|
|
mkdir -p "$TMPDIR/not-a-skill/references"
|
|
run bash "$SCRIPT" "$TMPDIR/not-a-skill"
|
|
[ "$status" -eq 2 ]
|
|
assert_output --partial "Error: '$TMPDIR/not-a-skill' is a directory with no SKILL.md in it."
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Cycle 22 — G2: a UTF-8 BOM does not disable check 2
|
|
# ---------------------------------------------------------------------------
|
|
|
|
@test "G2: a BOM at the head of SKILL.md does not silently disable check 2" {
|
|
local skill="$TMPDIR/my-skill"
|
|
mkdir -p "$skill/references"
|
|
printf '\xef\xbb\xbf' > "$skill/SKILL.md"
|
|
cat >> "$skill/SKILL.md" <<EOF
|
|
---
|
|
name: my-skill
|
|
description: A valid skill description.
|
|
metadata:
|
|
source_keys:
|
|
- my-source
|
|
---
|
|
|
|
## Step 1
|
|
|
|
Do the thing.
|
|
EOF
|
|
cat > "$skill/references/sources.md" <<EOF
|
|
# Sources
|
|
|
|
## different-source
|
|
|
|
- **URL:** https://example.com/different-source
|
|
- **Description:** A test source.
|
|
- **Contributing files:** references/sources.md
|
|
- **Research doc:** (none)
|
|
- **Status:** \`extracted\`
|
|
EOF
|
|
|
|
run bash "$SCRIPT" "$skill"
|
|
assert_failure
|
|
assert_output --partial "source_keys slug 'my-source' not found in sources.md"
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Cycle 23 — G3: reads are UTF-8 and an unreadable file is a finding
|
|
# ---------------------------------------------------------------------------
|
|
|
|
@test "G3: an em dash under LC_ALL=C is read, not turned into a traceback or a silent pass" {
|
|
local skill="$TMPDIR/my-skill"
|
|
make_skill_with_source_keys "$skill"
|
|
make_sources_md "$skill"
|
|
cat > "$skill/references/topic.md" <<EOF
|
|
---
|
|
source_keys:
|
|
- ghost-source
|
|
---
|
|
|
|
Prose — with an em dash.
|
|
EOF
|
|
|
|
LC_ALL=C PYTHONUTF8=0 PYTHONCOERCECLOCALE=0 run bash "$SCRIPT" "$skill"
|
|
assert_failure
|
|
assert_output --partial "source_keys slug 'ghost-source' not found in sources.md"
|
|
refute_output --partial "Traceback"
|
|
}
|
|
|
|
@test "G3: a file that is genuinely not UTF-8 is a FAIL, not a clean pass" {
|
|
local skill="$TMPDIR/my-skill"
|
|
make_skill_with_source_keys "$skill"
|
|
make_sources_md "$skill"
|
|
printf -- '---\nsource_keys:\n - my-source\n---\n\nLatin-1 byte: \xe9\n' \
|
|
> "$skill/references/topic.md"
|
|
|
|
run bash "$SCRIPT" "$skill"
|
|
assert_failure
|
|
assert_output --partial "not valid UTF-8"
|
|
assert_output --partial "references/topic.md"
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Cycle 24 — G4: check 3 walks references/ recursively
|
|
# ---------------------------------------------------------------------------
|
|
|
|
@test "G4: a source_keys file in references/<subdir>/ is validated, not skipped" {
|
|
local skill="$TMPDIR/my-skill"
|
|
make_skill_with_source_keys "$skill"
|
|
make_sources_md "$skill"
|
|
mkdir -p "$skill/references/nested"
|
|
cat > "$skill/references/nested/topic.md" <<EOF
|
|
---
|
|
source_keys:
|
|
- ghost-source
|
|
---
|
|
|
|
Nested prose.
|
|
EOF
|
|
|
|
run bash "$SCRIPT" "$skill"
|
|
assert_failure
|
|
assert_output --partial "source_keys slug 'ghost-source' not found in sources.md"
|
|
assert_output --partial "references/nested/topic.md"
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Cycle 25 — G5: a placeholder at end of line is still a placeholder
|
|
# ---------------------------------------------------------------------------
|
|
|
|
@test "G5: 'FILL IN:' at end of line is caught by check 1" {
|
|
local skill="$TMPDIR/my-skill"
|
|
make_skill_with_source_keys "$skill"
|
|
mkdir -p "$skill/references"
|
|
cat > "$skill/references/sources.md" <<EOF
|
|
# Sources
|
|
|
|
## my-source
|
|
|
|
- **URL:** https://example.com/my-source
|
|
- **Description:** FILL IN:
|
|
- **Contributing files:** SKILL.md
|
|
- **Research doc:** (none)
|
|
- **Status:** \`extracted\`
|
|
EOF
|
|
|
|
run bash "$SCRIPT" "$skill"
|
|
assert_failure
|
|
assert_output --partial "Unfilled FILL IN: placeholder"
|
|
}
|
|
|
|
@test "G5: a Research doc value that is a bare 'FILL IN:' is caught by check 6" {
|
|
local skill="$TMPDIR/my-skill"
|
|
make_skill_with_source_keys "$skill"
|
|
make_sources_md "$skill" "my-source" "SKILL.md" "FILL IN:"
|
|
run bash "$SCRIPT" "$skill"
|
|
assert_failure
|
|
assert_output --partial "Research doc field is empty or placeholder"
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Cycle 26 — G6/G7: duplicated entries and duplicated fields are announced
|
|
# ---------------------------------------------------------------------------
|
|
|
|
@test "G6: a duplicate '## slug' block is announced, not half-checked in silence" {
|
|
local skill="$TMPDIR/my-skill"
|
|
make_skill_with_source_keys "$skill"
|
|
mkdir -p "$skill/references"
|
|
cat > "$skill/references/sources.md" <<EOF
|
|
# Sources
|
|
|
|
## my-source
|
|
|
|
- **URL:** https://example.com/my-source
|
|
- **Description:** A test source.
|
|
- **Contributing files:** SKILL.md
|
|
- **Research doc:** (none)
|
|
- **Status:** \`extracted\`
|
|
|
|
## my-source
|
|
|
|
- **URL:** https://example.com/my-source-again
|
|
- **Description:** The same slug a second time.
|
|
- **Contributing files:** references/nonexistent.md
|
|
- **Research doc:** (none)
|
|
- **Status:** \`extracted\`
|
|
EOF
|
|
|
|
run bash "$SCRIPT" "$skill"
|
|
assert_success
|
|
assert_output --partial "Duplicate '## my-source' entry in sources.md"
|
|
assert_output --partial "only the first block is checked"
|
|
}
|
|
|
|
@test "G7: a second '- **Research doc:**' line in one entry is announced" {
|
|
local skill="$TMPDIR/my-skill"
|
|
make_skill_with_source_keys "$skill"
|
|
mkdir -p "$skill/references"
|
|
cat > "$skill/references/sources.md" <<EOF
|
|
# Sources
|
|
|
|
## my-source
|
|
|
|
- **URL:** https://example.com/my-source
|
|
- **Description:** A test source.
|
|
- **Contributing files:** SKILL.md
|
|
- **Research doc:** (none)
|
|
- **Research doc:** docs/research/sources.md
|
|
- **Status:** \`extracted\`
|
|
EOF
|
|
|
|
run bash "$SCRIPT" "$skill"
|
|
assert_success
|
|
assert_output --partial "Multiple '- **Research doc:**' lines for 'my-source'"
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Cycle 27 — G8: usage and environment errors exit 2, never 1
|
|
#
|
|
# PORT NOTE RESOLVED (factory-audit merge): the two suites were written against
|
|
# two separate scripts and disagreed about the no-argument wording — this file
|
|
# asserted "skill-dir is required", the agent suite "agent-file is required".
|
|
# Neither survives. One entry point takes both target shapes, so its no-argument
|
|
# message names both, and both suites now assert that one sentence. Exit 2 and a
|
|
# message (not a silent exit 1, and not a bare code with no output) is the
|
|
# invariant both sides were really pinning, and it is still pinned from both.
|
|
#
|
|
# Note the asymmetry with scripts/validate.sh, which exits 1 on no argument:
|
|
# only validate-provenance.sh carries the exit-2 usage tier, and this test is
|
|
# what holds it there.
|
|
# ---------------------------------------------------------------------------
|
|
|
|
@test "G8: a missing argument exits 2, not 1" {
|
|
run bash "$SCRIPT"
|
|
[ "$status" -eq 2 ]
|
|
assert_output --partial "Error: a skill directory or an agent file is required."
|
|
# The usage block has to follow the error, or "required" names no shape the
|
|
# caller can act on.
|
|
assert_output --partial "Usage: validate-provenance.sh"
|
|
}
|
|
|
|
@test "G8: an extra positional argument is rejected, not silently ignored" {
|
|
local skill="$TMPDIR/my-skill"
|
|
make_clean_skill "$skill"
|
|
run bash "$SCRIPT" "$skill" extra
|
|
[ "$status" -eq 2 ]
|
|
assert_output --partial "expected exactly one argument"
|
|
}
|
|
|
|
@test "G8: a missing python3 is reported by name and exits 2, not 127" {
|
|
local skill="$TMPDIR/my-skill"
|
|
make_clean_skill "$skill"
|
|
local stub="$TMPDIR/emptybin"
|
|
mkdir -p "$stub"
|
|
# dirname and basename are deliberately ABSENT, and that absence is load-
|
|
# bearing. The invariant: nothing external is needed to reach the python3
|
|
# preflight. The entry point resolves SCRIPT_DIR and classifies the target
|
|
# with _kf_dirname/_kf_basename/_kf_parent_name — pure-bash replacements
|
|
# that exist for precisely this reason — and the mode libraries build their
|
|
# Python bodies with `read` heredocs. Widening this list to keep a test
|
|
# green would silently retire that guarantee: on a PATH with neither
|
|
# coreutils nor python3 the script would die at 127 naming `dirname`
|
|
# instead of naming the dependency it actually needs, which is the failure
|
|
# tests/test-adr0020-contract.sh assertion 2 exists to prevent. bash, cat
|
|
# and sed stay so the stub is a DENY OF python3 ALONE rather than a test of
|
|
# "no PATH at all" — the diagnostic under test here is the python3 one.
|
|
for cmd in bash cat sed; do
|
|
ln -s "$(command -v "$cmd")" "$stub/$cmd"
|
|
done
|
|
PATH="$stub" run bash "$SCRIPT" "$skill"
|
|
[ "$status" -eq 2 ]
|
|
assert_output --partial "python3 is required"
|
|
}
|
|
|
|
@test "G3: an unreadable file is reported even when there is nothing else to validate" {
|
|
local skill="$TMPDIR/my-skill"
|
|
make_clean_skill "$skill"
|
|
mkdir -p "$skill/references"
|
|
printf -- '---\nsource_keys:\n - my-source\n---\n\nLatin-1 byte: \xe9\n' \
|
|
> "$skill/references/topic.md"
|
|
|
|
run bash "$SCRIPT" "$skill"
|
|
assert_failure
|
|
assert_output --partial "not valid UTF-8"
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Cycle 28 — Check 9: a changed Description or Contributing files claim is an
|
|
# INFO, never a FAIL — the script can tell the text changed, not whether the
|
|
# (possibly stronger) new wording is still true.
|
|
# ---------------------------------------------------------------------------
|
|
|
|
@test "check 9: Description text changed since base ref fires an INFO naming the slug and field" {
|
|
local skill="$TMPDIR/my-skill"
|
|
make_skill_with_source_keys "$skill"
|
|
make_sources_md "$skill"
|
|
commit_as_base "$skill"
|
|
|
|
# Rewritten in the working tree only, never committed — exactly the
|
|
# shape of the bug check 9 exists to flag: a hedge upgraded to a
|
|
# confident claim with nothing else in the entry touched.
|
|
sed -i 's/^- \*\*Description:\*\* A test source\.$/- **Description:** A test source that Grounds the dispatch table directly./' \
|
|
"$skill/references/sources.md"
|
|
|
|
run bash "$SCRIPT" "$skill"
|
|
assert_success
|
|
assert_output --partial "INFO"
|
|
assert_output --partial "'Description' changed for 'my-source'"
|
|
}
|
|
|
|
@test "check 9: Contributing files text changed since base ref fires an INFO naming the slug and field" {
|
|
local skill="$TMPDIR/my-skill"
|
|
make_skill_with_source_keys "$skill"
|
|
make_sources_md "$skill"
|
|
commit_as_base "$skill"
|
|
|
|
# Only an annotation is added. strip_note() in parse_contributing_files()
|
|
# removes it before checks 4 and 5 compare paths, so those stay clean —
|
|
# check 9 compares the raw field text, not the parsed path list, and
|
|
# this is still a real wording change worth a human re-reading it.
|
|
sed -i 's/^- \*\*Contributing files:\*\* SKILL\.md$/- **Contributing files:** SKILL.md (the dispatch table)/' \
|
|
"$skill/references/sources.md"
|
|
|
|
run bash "$SCRIPT" "$skill"
|
|
assert_success
|
|
assert_output --partial "INFO"
|
|
assert_output --partial "'Contributing files' changed for 'my-source'"
|
|
refute_output --partial "does not exist"
|
|
refute_output --partial "does not list"
|
|
}
|
|
|
|
@test "check 9: an entry unchanged since base ref produces no check-9 finding" {
|
|
local skill="$TMPDIR/my-skill"
|
|
make_skill_with_source_keys "$skill"
|
|
make_sources_md "$skill"
|
|
commit_as_base "$skill"
|
|
|
|
run bash "$SCRIPT" "$skill"
|
|
assert_success
|
|
assert_output ""
|
|
}
|
|
|
|
@test "check 9: a brand-new entry absent at the base ref is a creation, not a change" {
|
|
local skill="$TMPDIR/my-skill"
|
|
make_skill_with_source_keys "$skill"
|
|
make_sources_md "$skill"
|
|
commit_as_base "$skill"
|
|
|
|
# Added in the working tree only, after the commit above. It has no
|
|
# earlier revision to diff against, so check 9 must stay silent about
|
|
# it — a brand-new entry is a creation, not a rewrite of an existing
|
|
# claim, and flagging it would be exactly the false-positive shape the
|
|
# rejected literal-text approaches produced.
|
|
cat >> "$skill/references/sources.md" <<EOF
|
|
|
|
## new-source
|
|
|
|
- **URL:** https://example.com/new-source
|
|
- **Description:** A brand-new source, never committed.
|
|
- **Contributing files:** (none)
|
|
- **Research doc:** (none)
|
|
- **Status:** \`extracted\`
|
|
EOF
|
|
|
|
run bash "$SCRIPT" "$skill"
|
|
assert_success
|
|
assert_output ""
|
|
}
|
|
|
|
@test "check 9: no repo root emits the one graceful INFO, not a silent skip or a crash" {
|
|
local skill="$TMPDIR/my-skill"
|
|
make_skill_with_source_keys "$skill"
|
|
make_sources_md "$skill"
|
|
# Deliberately no commit_as_base — $TMPDIR has no ancestor .git, so
|
|
# find_repo_root() returns None.
|
|
|
|
run bash "$SCRIPT" "$skill"
|
|
assert_success
|
|
assert_output --partial "INFO"
|
|
assert_output --partial "Check 9 skipped — no repo root above the skill directory"
|
|
}
|
|
|
|
@test "check 9: an unresolvable base ref emits the one graceful INFO, not a silent skip or a crash" {
|
|
local skill="$TMPDIR/my-skill"
|
|
make_skill_with_source_keys "$skill"
|
|
make_sources_md "$skill"
|
|
commit_as_base "$skill"
|
|
# A repo with no origin/main and no override: `git merge-base HEAD
|
|
# origin/main` fails outright.
|
|
git -C "$skill" update-ref -d refs/remotes/origin/main >/dev/null 2>&1
|
|
|
|
run bash "$SCRIPT" "$skill"
|
|
assert_success
|
|
assert_output --partial "INFO"
|
|
assert_output --partial "Check 9 skipped — no base ref could be resolved"
|
|
}
|
|
|
|
@test "check 9: --base-ref overrides a default origin/main that resolves to something else" {
|
|
local skill="$TMPDIR/my-skill"
|
|
make_skill_with_source_keys "$skill"
|
|
make_sources_md "$skill"
|
|
commit_as_base "$skill"
|
|
local old_sha
|
|
old_sha="$(git -C "$skill" rev-parse HEAD)"
|
|
|
|
# A SECOND commit carrying the rewritten claim, with origin/main moved onto
|
|
# it. The default base ref therefore resolves — to a commit that matches the
|
|
# working tree — so a silent default run proves there was a default here to
|
|
# override. Deleting origin/main instead, as this test used to, proved only
|
|
# that the flag works when nothing else does.
|
|
sed -i 's/^- \*\*Description:\*\* A test source\.$/- **Description:** A rewritten claim./' \
|
|
"$skill/references/sources.md"
|
|
git -C "$skill" -c user.email=test@example.com -c user.name=test commit -aqm rewrite >/dev/null 2>&1
|
|
git -C "$skill" update-ref refs/remotes/origin/main HEAD >/dev/null 2>&1
|
|
|
|
run bash "$SCRIPT" "$skill"
|
|
assert_success
|
|
assert_output ""
|
|
|
|
run bash "$SCRIPT" "$skill" "--base-ref=$old_sha"
|
|
assert_success
|
|
assert_output --partial "'Description' changed for 'my-source'"
|
|
}
|
|
|
|
@test "check 9: VALIDATE_PROVENANCE_BASE_REF sets the base ref when no flag is given" {
|
|
local skill="$TMPDIR/my-skill"
|
|
make_skill_with_source_keys "$skill"
|
|
make_sources_md "$skill"
|
|
commit_as_base "$skill"
|
|
local old_sha
|
|
old_sha="$(git -C "$skill" rev-parse HEAD)"
|
|
sed -i 's/^- \*\*Description:\*\* A test source\.$/- **Description:** A rewritten claim./' \
|
|
"$skill/references/sources.md"
|
|
git -C "$skill" -c user.email=test@example.com -c user.name=test commit -aqm rewrite >/dev/null 2>&1
|
|
git -C "$skill" update-ref refs/remotes/origin/main HEAD >/dev/null 2>&1
|
|
|
|
run env VALIDATE_PROVENANCE_BASE_REF="$old_sha" bash "$SCRIPT" "$skill"
|
|
assert_success
|
|
assert_output --partial "'Description' changed for 'my-source'"
|
|
}
|
|
|
|
@test "check 9: the --base-ref flag wins over the environment variable" {
|
|
local skill="$TMPDIR/my-skill"
|
|
make_skill_with_source_keys "$skill"
|
|
make_sources_md "$skill"
|
|
commit_as_base "$skill"
|
|
local old_sha
|
|
old_sha="$(git -C "$skill" rev-parse HEAD)"
|
|
sed -i 's/^- \*\*Description:\*\* A test source\.$/- **Description:** A rewritten claim./' \
|
|
"$skill/references/sources.md"
|
|
git -C "$skill" -c user.email=test@example.com -c user.name=test commit -aqm rewrite >/dev/null 2>&1
|
|
local new_sha
|
|
new_sha="$(git -C "$skill" rev-parse HEAD)"
|
|
|
|
# The environment names the old commit (which would fire), the flag names
|
|
# the new one (which would not). The usage text promises the flag wins.
|
|
run env VALIDATE_PROVENANCE_BASE_REF="$old_sha" bash "$SCRIPT" "$skill" "--base-ref=$new_sha"
|
|
assert_success
|
|
assert_output ""
|
|
}
|
|
|
|
@test "check 9: an EMPTY --base-ref is still 'given' and wins over the environment variable" {
|
|
local skill="$TMPDIR/my-skill"
|
|
make_skill_with_source_keys "$skill"
|
|
make_sources_md "$skill"
|
|
commit_as_base "$skill"
|
|
local old_sha
|
|
old_sha="$(git -C "$skill" rev-parse HEAD)"
|
|
sed -i 's/^- \*\*Description:\*\* A test source\.$/- **Description:** A rewritten claim./' \
|
|
"$skill/references/sources.md"
|
|
git -C "$skill" -c user.email=test@example.com -c user.name=test commit -aqm rewrite >/dev/null 2>&1
|
|
git -C "$skill" update-ref refs/remotes/origin/main HEAD >/dev/null 2>&1
|
|
|
|
# `--base-ref=` selects the DEFAULT resolution (origin/main, which now
|
|
# matches the working tree), so nothing fires. Under the old `:-` spelling
|
|
# the empty value read as absent and the environment variable won, firing
|
|
# the INFO and contradicting the usage text.
|
|
run env VALIDATE_PROVENANCE_BASE_REF="$old_sha" bash "$SCRIPT" "$skill" "--base-ref="
|
|
assert_success
|
|
assert_output ""
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Cycle 29 — Check 9: a WRAPPED field value. The parser compared only the first
|
|
# physical line, so a rewrite confined to a continuation line — the exact
|
|
# hedge-to-confident-claim shape #118 exists to catch — produced no finding at
|
|
# all, while a pure re-wrap produced a false one.
|
|
# ---------------------------------------------------------------------------
|
|
|
|
@test "check 9: a rewrite confined to a wrapped Description's CONTINUATION line fires an INFO" {
|
|
local skill="$TMPDIR/my-skill"
|
|
make_skill_with_source_keys "$skill"
|
|
make_wrapped_sources_md "$skill"
|
|
commit_as_base "$skill"
|
|
|
|
# Hedge to confident claim, on the second physical line only. This is the
|
|
# regression check 9 was written for, and the one it could not see.
|
|
sed -i "s|^ no forge-specific content drawn directly from it beyond that\.\$| it grounds Step 2's dispatch table in full.|" \
|
|
"$skill/references/sources.md"
|
|
|
|
run bash "$SCRIPT" "$skill"
|
|
assert_success
|
|
assert_output --partial "'Description' changed for 'my-source'"
|
|
}
|
|
|
|
@test "check 9: re-wrapping a Description with no wording change produces no finding" {
|
|
local skill="$TMPDIR/my-skill"
|
|
make_skill_with_source_keys "$skill"
|
|
make_wrapped_sources_md "$skill"
|
|
commit_as_base "$skill"
|
|
|
|
# Same words, different line breaks. normalize_field_text()'s docstring
|
|
# promises this is invisible; it was not, because the value was truncated
|
|
# at its first newline before the whitespace collapse ever ran.
|
|
python3 - "$skill/references/sources.md" <<'PY'
|
|
import sys
|
|
path = sys.argv[1]
|
|
text = open(path).read()
|
|
old = ("- **Description:** A test source, informing the dispatch table's shape with\n"
|
|
" no forge-specific content drawn directly from it beyond that.")
|
|
new = ("- **Description:** A test source, informing the dispatch\n"
|
|
" table's shape with no forge-specific content drawn\n"
|
|
" directly from it beyond that.")
|
|
assert old in text
|
|
open(path, 'w').write(text.replace(old, new))
|
|
PY
|
|
|
|
run bash "$SCRIPT" "$skill"
|
|
assert_success
|
|
assert_output ""
|
|
}
|
|
|
|
@test "check 9: the bullet form of Contributing files is compared, not skipped" {
|
|
local skill="$TMPDIR/my-skill"
|
|
make_skill_with_source_keys "$skill"
|
|
mkdir -p "$skill/references"
|
|
cat > "$skill/references/sources.md" <<'EOF'
|
|
# Sources
|
|
|
|
## my-source
|
|
|
|
- **URL:** https://example.com/my-source
|
|
- **Description:** A test source.
|
|
|
|
**Contributing files:**
|
|
- SKILL.md (the dispatch table)
|
|
- references/other.md (the rubric)
|
|
|
|
- **Research doc:** (none)
|
|
- **Status:** `extracted`
|
|
EOF
|
|
printf -- '---\nsource_keys:\n - my-source\n---\n\nnotes\n' > "$skill/references/other.md"
|
|
commit_as_base "$skill"
|
|
|
|
# The change is in the SECOND bullet. The old loop joined bullets in
|
|
# document order too, but broke on any wrapped one — and no test covered
|
|
# this branch at all, both existing check-9 tests using the inline form.
|
|
sed -i 's|^- references/other.md (the rubric)$|- references/other.md (the whole rubric, verbatim)|' \
|
|
"$skill/references/sources.md"
|
|
|
|
run bash "$SCRIPT" "$skill"
|
|
assert_success
|
|
assert_output --partial "'Contributing files' changed for 'my-source'"
|
|
}
|
|
|
|
@test "check 9: a wrapped bullet does not silently drop the bullets after it" {
|
|
local skill="$TMPDIR/my-skill"
|
|
make_skill_with_source_keys "$skill"
|
|
mkdir -p "$skill/references"
|
|
cat > "$skill/references/sources.md" <<'EOF'
|
|
# Sources
|
|
|
|
## my-source
|
|
|
|
- **URL:** https://example.com/my-source
|
|
- **Description:** A test source.
|
|
|
|
**Contributing files:**
|
|
- SKILL.md (the dispatch table, and the gates
|
|
common to every branch of it)
|
|
- references/other.md (the rubric)
|
|
|
|
- **Research doc:** (none)
|
|
- **Status:** `extracted`
|
|
EOF
|
|
printf -- '---\nsource_keys:\n - my-source\n---\n\nnotes\n' > "$skill/references/other.md"
|
|
commit_as_base "$skill"
|
|
|
|
# The old loop broke at the wrapped continuation line, so everything from
|
|
# here down was never part of the compared value — a change to the last
|
|
# bullet was invisible.
|
|
sed -i 's|^- references/other.md (the rubric)$|- references/other.md (rewritten claim)|' \
|
|
"$skill/references/sources.md"
|
|
|
|
run bash "$SCRIPT" "$skill"
|
|
assert_success
|
|
assert_output --partial "'Contributing files' changed for 'my-source'"
|
|
}
|
|
|
|
@test "check 9: a field present at the base ref and deleted since is announced" {
|
|
local skill="$TMPDIR/my-skill"
|
|
make_skill_with_source_keys "$skill"
|
|
make_sources_md "$skill"
|
|
commit_as_base "$skill"
|
|
|
|
# No other check in this script requires a Description, so a deleted one
|
|
# used to leave no finding anywhere: a claim could be withdrawn as
|
|
# invisibly as it could be strengthened.
|
|
sed -i '/^- \*\*Description:\*\*/d' "$skill/references/sources.md"
|
|
|
|
run bash "$SCRIPT" "$skill"
|
|
assert_success
|
|
assert_output --partial "'Description' removed for 'my-source'"
|
|
}
|
|
|
|
@test "check 9: a sources.md untracked at the base ref is announced, not silently skipped" {
|
|
local repo="$TMPDIR/repo"
|
|
local skill="$repo/tracked-skill"
|
|
mkdir -p "$skill"
|
|
make_skill_with_source_keys "$skill"
|
|
make_sources_md "$skill"
|
|
commit_as_base "$repo"
|
|
|
|
# A copy of the same skill at a path git does not know — the everyday case
|
|
# being an installed, gitignored .claude/skills/ tree. The base ref
|
|
# resolves fine; `git show <ref>:<path>` does not. Treating that as
|
|
# "creation, nothing to flag" made the whole check vanish without a word,
|
|
# so the same directory reported findings at one path and silence at the
|
|
# other.
|
|
cp -r "$skill" "$repo/untracked-copy"
|
|
|
|
run bash "$SCRIPT" "$repo/untracked-copy"
|
|
assert_success
|
|
assert_output --partial "INFO"
|
|
assert_output --partial "is not tracked at"
|
|
assert_output --partial "Check 9 did not run for any slug in this skill."
|
|
}
|
|
|
|
@test "check 9: an invalid --base-ref value is reported as unresolvable, not a crash" {
|
|
local skill="$TMPDIR/my-skill"
|
|
make_skill_with_source_keys "$skill"
|
|
make_sources_md "$skill"
|
|
commit_as_base "$skill"
|
|
|
|
run bash "$SCRIPT" "$skill" "--base-ref=not-a-real-ref"
|
|
assert_success
|
|
assert_output --partial "Check 9 skipped — no base ref could be resolved"
|
|
assert_output --partial "not-a-real-ref"
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Auto-detection — the merged provenance entry point classifies its own target
|
|
#
|
|
# NEW with the factory-audit merge, and new behaviour rather than a ported
|
|
# case: scripts/validate-provenance.sh is now ONE entry point for both artifact
|
|
# types and works out from the target which rubric to run. A DIRECTORY holding
|
|
# SKILL.md is a skill; a FILE named *.agent.md, or sitting under .apm/agents/,
|
|
# is an agent.
|
|
#
|
|
# The two rubrics are not near-copies of one another — the skill side has a
|
|
# check 9 the agent side has none of, and reads sources.md from the skill's own
|
|
# references/ rather than from the package root — so a misclassification is not
|
|
# a near miss. It runs a set of checks that cannot apply and skips the set that
|
|
# can, at exit 0.
|
|
#
|
|
# This file is the SKILL half plus the neither-shape rejection;
|
|
# validate-provenance-agent.bats holds the agent half. Same script in both.
|
|
# ---------------------------------------------------------------------------
|
|
|
|
@test "auto-detect: a directory holding SKILL.md is checked in SKILL provenance mode" {
|
|
local skill="$TMPDIR/my-skill"
|
|
make_skill_with_source_keys "$skill"
|
|
make_sources_md "$skill"
|
|
# No commit_as_base, so check 9 has no repo root and announces the skip.
|
|
run bash "$SCRIPT" "$skill"
|
|
assert_success
|
|
# Check 9 exists only on the skill side — the agent rubric has no check 9 at
|
|
# any tier — so naming it is positive proof the SKILL rubric ran, rather
|
|
# than merely that nothing crashed.
|
|
assert_output --partial "Check 9 skipped — no repo root above the skill directory"
|
|
}
|
|
|
|
@test "auto-detect: a SKILL.md FILE path is checked in SKILL provenance mode, not rejected" {
|
|
# NEW with the merge and additive: pre-merge, handing the SKILL.md itself to
|
|
# skill-audit's validate-provenance.sh hit the "not a directory" precondition
|
|
# and died. It matters because pre-commit `files:` hooks match FILES — the
|
|
# vale-audit-prefilter-skill hook's regex ends in /SKILL\.md$ — so
|
|
# every hook-driven invocation hands over a SKILL.md path, never its
|
|
# directory. The entry point rewrites the token to the directory in place.
|
|
local skill="$TMPDIR/my-skill"
|
|
make_skill_with_source_keys "$skill"
|
|
make_sources_md "$skill"
|
|
run bash "$SCRIPT" "$skill/SKILL.md"
|
|
assert_success
|
|
# Same positive proof as the directory case: check 9 is skill-only.
|
|
assert_output --partial "Check 9 skipped — no repo root above the skill directory"
|
|
}
|
|
|
|
@test "auto-detect: a SKILL.md FILE path and its directory produce the same verdict" {
|
|
# The rewrite must be transparent, not merely non-fatal. If the two spellings
|
|
# of the same target could disagree, a pre-commit run and a hand run would
|
|
# report differently on one skill and neither would be obviously wrong.
|
|
local skill="$TMPDIR/my-skill"
|
|
make_skill_with_source_keys "$skill"
|
|
make_sources_md "$skill"
|
|
|
|
run bash "$SCRIPT" "$skill"
|
|
local dir_status="$status"
|
|
local dir_output="$output"
|
|
|
|
run bash "$SCRIPT" "$skill/SKILL.md"
|
|
[ "$status" -eq "$dir_status" ]
|
|
[ "$output" = "$dir_output" ]
|
|
# Guard against the comparison being satisfied by two empty runs: this
|
|
# fixture has a check-9 INFO to print, so silence here means neither
|
|
# spelling ran the rubric.
|
|
refute_output ""
|
|
}
|
|
|
|
@test "exit contract: SKILL mode exits 0 WITH output when the only findings are INFO" {
|
|
# The two modes' exit contracts are DIFFERENT and the merge must not quietly
|
|
# unify them. Skill mode is allowed to be chatty on a passing run: INFO
|
|
# findings print and the status stays 0. Agent mode's opposite half — a
|
|
# passing run prints nothing at all — is pinned from its own side in
|
|
# validate-provenance-agent.bats.
|
|
#
|
|
# Both halves are asserted explicitly and on purpose. A merge that collapsed
|
|
# one contract into the other would still satisfy whichever side was left
|
|
# unasserted, so a single-sided pin would go green on exactly the defect it
|
|
# was written to catch.
|
|
local skill="$TMPDIR/my-skill"
|
|
make_skill_with_source_keys "$skill"
|
|
make_sources_md "$skill"
|
|
run bash "$SCRIPT" "$skill"
|
|
[ "$status" -eq 0 ]
|
|
refute_output ""
|
|
assert_output --partial "INFO"
|
|
refute_output --partial "FAIL"
|
|
}
|
|
|
|
@test "auto-detect: a provenance target that is neither a skill directory nor an agent file FAILs, naming what it was handed" {
|
|
# The detector must not guess. Guessing here is worse than in validate.sh:
|
|
# this script's whole not-in-scope path is a SILENT exit 0, so a wrong guess
|
|
# followed by "provenance does not apply at this scope" is indistinguishable
|
|
# from a clean pass — which is the exact confusion the exit-2 tier above was
|
|
# created to end.
|
|
#
|
|
# A plain .txt file is neither shape under any reading of the contract: not
|
|
# a directory holding SKILL.md, not *.agent.md, not under .apm/agents/.
|
|
local dir="$TMPDIR/neither"
|
|
mkdir -p "$dir"
|
|
echo "not an artifact of either kind" > "$dir/notes.txt"
|
|
run bash "$SCRIPT" "$dir/notes.txt"
|
|
assert_failure
|
|
refute_output ""
|
|
assert_output --partial "$dir/notes.txt"
|
|
}
|