Files
holocron/plugins/kyberforge/.apm/skills/factory-audit/tests/validate-provenance-skill.bats
Defame1297 2c4b6d2615 fix(kyberforge): harden Research doc and Basis parsing in the validator
Review of PR 139 found list-rejection and confinement holes that let the
exact malformed entries the grammar forbids pass check 7.

- Reject comma, space-separated and backticked path lists, so
  `a/sources.md (x), b/topic.md` no longer exits 0 unchecked.
- FAIL absolute paths and any path whose realpath leaves the repo, for
  both `Research doc:` and `Basis:`.
- Anchor `(removed in <sha>)` to the end of the value with a 7-40 hex
  sha. The sha is format-checked only, not resolved with git cat-file.
- Read `* ` bullets and `- **X**` bullets correctly under a `**Basis:**`
  header, and strip backticks from Basis paths.
- Stop the semicolon rule firing on annotation prose, and stop `none`
  matching `none/foo.md`.
- Update the stale field messages to the new grammar and report an empty
  field as empty, not missing.
- Skip a removed Basis silently when there is no repo root.

Adds 40 tests. Each guarded line was mutated in place and every mutant
is caught.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01EGHFJextYtVQseaHPDDhxB
2026-09-21 19:40:28 +00:00

2226 lines
79 KiB
Bash

#!/usr/bin/env bats
setup() {
REPO_ROOT="$(cd "$BATS_TEST_DIRNAME/../../../../../../" && pwd)"
load "$REPO_ROOT/tests/test_helper/bats-support/load"
load "$REPO_ROOT/tests/test_helper/bats-assert/load"
SCRIPT="$(cd "$BATS_TEST_DIRNAME/../scripts" && pwd)/validate-provenance.sh"
TMPDIR="$(mktemp -d)"
# Helper: create a minimal skill directory with no sources.md and no source_keys
make_clean_skill() {
local dir="$1"
local name
name="$(basename "$dir")"
mkdir -p "$dir"
cat > "$dir/SKILL.md" <<EOF
---
name: $name
description: A valid skill description.
---
## Step 1
Do the thing.
EOF
}
# Helper: create a skill with source_keys in SKILL.md
make_skill_with_source_keys() {
local dir="$1"
local name
name="$(basename "$dir")"
mkdir -p "$dir"
cat > "$dir/SKILL.md" <<EOF
---
name: $name
description: A valid skill description.
metadata:
source_keys:
- my-source
---
## Step 1
Do the thing.
EOF
}
# Helper: create a valid sources.md with one entry
make_sources_md() {
local dir="$1"
local slug="${2:-my-source}"
local contrib="${3:-SKILL.md}"
local research="${4:-(none)}"
# A 'none' Research doc must name its Basis (#121); SKILL.md exists in
# every fixture skill and, once commit_as_base has run, is a repo path.
local basis_line=""
if [[ "$research" == *none* ]]; then basis_line="- **Basis:** SKILL.md"; fi
mkdir -p "$dir/references"
cat > "$dir/references/sources.md" <<EOF
# Sources
## ${slug}
- **URL:** https://example.com/${slug}
- **Description:** A test source.
- **Contributing files:** ${contrib}
- **Research doc:** ${research}
${basis_line}
- **Status:** \`extracted\`
EOF
}
# Helper: a sources.md whose Description is wrapped across two physical
# lines — the shape check 9's parser used to truncate at the first newline.
make_wrapped_sources_md() {
local dir="$1"
mkdir -p "$dir/references"
cat > "$dir/references/sources.md" <<'EOF'
# Sources
## my-source
- **URL:** https://example.com/my-source
- **Description:** A test source, informing the dispatch table's shape with
no forge-specific content drawn directly from it beyond that.
- **Contributing files:** SKILL.md
- **Research doc:** (none)
- **Basis:** SKILL.md
- **Status:** `extracted`
EOF
}
# Helper: turn dir into a real git repo with one commit of its current
# contents, and a refs/remotes/origin/main pointing at that same commit.
# Check 9 diffs the skill's references/sources.md against `git merge-base
# HEAD origin/main` by default; this makes that resolve to a commit whose
# sources.md is byte-identical to the working tree, so check 9 has
# nothing to report there — exactly what a real repo looks like the
# instant after a clean commit. Fixtures that go on to test something
# else entirely (checks 3, 4, 5, 7, 8...) call this once, at the point
# their skill's own sources.md is in its final state, so a completely
# clean run stays completely clean.
commit_as_base() {
local dir="$1"
git -C "$dir" init -q >/dev/null 2>&1
git -C "$dir" -c user.email=test@example.com -c user.name=test add -A >/dev/null 2>&1
git -C "$dir" -c user.email=test@example.com -c user.name=test commit -q -m base >/dev/null 2>&1
git -C "$dir" update-ref refs/remotes/origin/main HEAD >/dev/null 2>&1
}
# Helper: create a fake repo (a real git repo, one commit, makes
# find_repo_root stop there) holding one skill whose single sources.md
# slug points at the given Research doc value. Check 7 only runs
# for a skill inside a checkout, so every upstream case needs this shape;
# the research doc itself is written per test into
# "$repo/docs/research/sources.md" — which check 9 does not examine, so
# a test overwriting it after this helper runs does not disturb check 9.
make_upstream_skill() {
local repo="$1"
local research="${2:-docs/research/sources.md}"
local basis_line=""
if [[ "$research" == *none* ]]; then basis_line="- **Basis:** my-skill/SKILL.md"; fi
local skill="$repo/my-skill"
mkdir -p "$skill/references" "$repo/docs/research"
cat > "$skill/SKILL.md" <<EOF
---
name: my-skill
description: A valid skill description.
metadata:
source_keys:
- my-source
---
## Step 1
Do the thing.
EOF
cat > "$skill/references/sources.md" <<EOF
# Sources
## my-source
- **URL:** https://example.com/my-source
- **Description:** A test source.
- **Contributing files:** SKILL.md
- **Research doc:** ${research}
${basis_line}
- **Status:** \`extracted\`
EOF
commit_as_base "$repo"
}
}
teardown() {
rm -rf "$TMPDIR"
}
# ---------------------------------------------------------------------------
# Cycle 1 — --help
# ---------------------------------------------------------------------------
@test "--help exits 0" {
run bash "$SCRIPT" --help
assert_success
assert_output --partial "Usage:"
}
# ---------------------------------------------------------------------------
# Cycle 2 — Early exit: no sources.md, no source_keys → exit 0, no output
# ---------------------------------------------------------------------------
@test "clean pass: no sources.md and no source_keys anywhere → exit 0, no output" {
local skill="$TMPDIR/my-skill"
make_clean_skill "$skill"
run bash "$SCRIPT" "$skill"
assert_success
assert_output ""
}
# ---------------------------------------------------------------------------
# Cycle 3 — Check 0: source_keys present but no sources.md → FAIL
# ---------------------------------------------------------------------------
@test "FAIL: source_keys in SKILL.md but sources.md absent" {
local skill="$TMPDIR/my-skill"
make_skill_with_source_keys "$skill"
run bash "$SCRIPT" "$skill"
assert_failure
assert_output --partial "FAIL"
}
# ---------------------------------------------------------------------------
# Cycle 4 — Check 1: FILL IN: placeholder in sources.md → FAIL
# ---------------------------------------------------------------------------
@test "FAIL: FILL IN: placeholder in sources.md" {
local skill="$TMPDIR/my-skill"
make_skill_with_source_keys "$skill"
mkdir -p "$skill/references"
cat > "$skill/references/sources.md" <<EOF
# Sources
## my-source
- **URL:** FILL IN: add url
- **Description:** A test source.
- **Contributing files:** SKILL.md
- **Research doc:** (none)
- **Basis:** SKILL.md
- **Status:** \`extracted\`
EOF
run bash "$SCRIPT" "$skill"
assert_failure
assert_output --partial "FAIL"
}
@test "FILL IN: inside backticks in sources.md does not fail" {
local skill="$TMPDIR/my-skill"
make_skill_with_source_keys "$skill"
make_sources_md "$skill"
echo "Use \`FILL IN: value\` as example." >> "$skill/references/sources.md"
run bash "$SCRIPT" "$skill"
assert_success
}
# ---------------------------------------------------------------------------
# Cycle 5 — Check 2: source_keys slug missing from sources.md → FAIL
# ---------------------------------------------------------------------------
@test "FAIL: source_keys slug in SKILL.md not present as H2 in sources.md" {
local skill="$TMPDIR/my-skill"
make_skill_with_source_keys "$skill"
mkdir -p "$skill/references"
cat > "$skill/references/sources.md" <<EOF
# Sources
## different-source
- **URL:** https://example.com/different-source
- **Description:** A different source.
- **Contributing files:** SKILL.md
- **Research doc:** (none)
- **Basis:** SKILL.md
- **Status:** \`extracted\`
EOF
run bash "$SCRIPT" "$skill"
assert_failure
assert_output --partial "FAIL"
}
# ---------------------------------------------------------------------------
# Cycle 6 — Check 4: Contributing file path doesn't exist → FAIL
# ---------------------------------------------------------------------------
@test "FAIL: Contributing file listed in sources.md does not exist" {
local skill="$TMPDIR/my-skill"
make_skill_with_source_keys "$skill"
make_sources_md "$skill" "my-source" "references/nonexistent.md"
run bash "$SCRIPT" "$skill"
assert_failure
assert_output --partial "FAIL"
}
@test "pass: (none) in Contributing files is skipped" {
local skill="$TMPDIR/my-skill"
make_skill_with_source_keys "$skill"
make_sources_md "$skill" "my-source" "(none — not used directly)"
run bash "$SCRIPT" "$skill"
assert_success
}
# ---------------------------------------------------------------------------
# Cycle 7 — Check 6: Research doc field missing → FAIL
# ---------------------------------------------------------------------------
@test "FAIL: Research doc field missing from sources.md entry" {
local skill="$TMPDIR/my-skill"
make_skill_with_source_keys "$skill"
mkdir -p "$skill/references"
cat > "$skill/references/sources.md" <<EOF
# Sources
## my-source
- **URL:** https://example.com/my-source
- **Description:** A test source.
- **Contributing files:** SKILL.md
- **Status:** \`extracted\`
EOF
run bash "$SCRIPT" "$skill"
assert_failure
assert_output --partial "FAIL"
}
@test "FAIL: Research doc field is FILL IN: placeholder" {
local skill="$TMPDIR/my-skill"
make_skill_with_source_keys "$skill"
mkdir -p "$skill/references"
cat > "$skill/references/sources.md" <<EOF
# Sources
## my-source
- **URL:** https://example.com/my-source
- **Description:** A test source.
- **Contributing files:** SKILL.md
- **Research doc:** FILL IN: path to research doc
- **Status:** \`extracted\`
EOF
run bash "$SCRIPT" "$skill"
assert_failure
assert_output --partial "FAIL"
}
# ---------------------------------------------------------------------------
# Cycle 8 — Check 5: Bidirectional mismatch → FAIL
# ---------------------------------------------------------------------------
@test "FAIL: Contributing file exists but does not list parent slug in source_keys" {
local skill="$TMPDIR/my-skill"
make_skill_with_source_keys "$skill"
make_sources_md "$skill" "my-source" "SKILL.md"
# SKILL.md has source_keys: my-source, but let's change it to NOT have my-source
cat > "$skill/SKILL.md" <<EOF
---
name: my-skill
description: A valid skill description.
metadata:
source_keys:
- other-source
---
## Step 1
Do the thing.
EOF
mkdir -p "$skill/references"
cat > "$skill/references/sources.md" <<EOF
# Sources
## other-source
- **URL:** https://example.com/other-source
- **Description:** A test source.
- **Contributing files:** SKILL.md
- **Research doc:** (none)
- **Basis:** SKILL.md
- **Status:** \`extracted\`
EOF
# Now add my-source that references SKILL.md but SKILL.md doesn't back-reference it
cat >> "$skill/references/sources.md" <<EOF
## my-source
- **URL:** https://example.com/my-source
- **Description:** Another source.
- **Contributing files:** SKILL.md
- **Research doc:** (none)
- **Basis:** SKILL.md
- **Status:** \`extracted\`
EOF
run bash "$SCRIPT" "$skill"
assert_failure
assert_output --partial "FAIL"
}
# ---------------------------------------------------------------------------
# Cycle 9 — Check 3: references/*.md with no source_keys → INFO (exit 0)
# ---------------------------------------------------------------------------
@test "INFO: references doc with no source_keys frontmatter emits INFO but exits 0" {
local skill="$TMPDIR/my-skill"
make_skill_with_source_keys "$skill"
make_sources_md "$skill"
mkdir -p "$skill/references"
cat > "$skill/references/extra.md" <<EOF
# Extra Reference
No frontmatter here.
EOF
run bash "$SCRIPT" "$skill"
assert_success
assert_output --partial "INFO"
}
@test "pass: references doc with source_keys all matching sources.md exits 0" {
local skill="$TMPDIR/my-skill"
make_skill_with_source_keys "$skill"
make_sources_md "$skill"
mkdir -p "$skill/references"
cat > "$skill/references/extra.md" <<EOF
---
source_keys:
- my-source
---
# Extra Reference
Content here.
EOF
run bash "$SCRIPT" "$skill"
assert_success
}
# ---------------------------------------------------------------------------
# Cycle 10 — Clean full pass: valid sources.md, all source_keys match, files exist
# ---------------------------------------------------------------------------
@test "clean full pass: all checks satisfied" {
local skill="$TMPDIR/my-skill"
make_skill_with_source_keys "$skill"
make_sources_md "$skill"
run bash "$SCRIPT" "$skill"
assert_success
}
# ---------------------------------------------------------------------------
# Cycle 11 — Check 7: Upstream forward: slug in sources.md not in research doc → INFO
# ---------------------------------------------------------------------------
@test "FAIL: slug in sources.md not found in the research registry → FAIL (was INFO before #121)" {
local skill="$TMPDIR/my-skill"
make_skill_with_source_keys "$skill"
# Create a research doc that does NOT have the slug
local research_dir="$TMPDIR/research"
mkdir -p "$research_dir"
cat > "$research_dir/sources.md" <<EOF
# Research
## different-slug
- **Contributing files:** (none)
- **Status:** \`extracted\`
EOF
# Use a path relative to repo root — we'll place research doc inside TMPDIR
# and reference it as absolute for test purposes.
# The script finds repo root by walking up from skill-dir until .git is found.
# Since TMPDIR won't have .git, we simulate a repo structure.
local fake_repo="$TMPDIR/fakerepo"
mkdir -p "$fake_repo"
touch "$fake_repo/.git" # fake .git marker
local skill2="$fake_repo/my-skill"
mkdir -p "$skill2"
cat > "$skill2/SKILL.md" <<EOF
---
name: my-skill
description: A valid skill description.
metadata:
source_keys:
- my-source
---
## Step 1
Do the thing.
EOF
mkdir -p "$skill2/references"
mkdir -p "$fake_repo/docs/research"
cat > "$fake_repo/docs/research/sources.md" <<EOF
# Research
## different-slug
- **Contributing files:** (none)
- **Status:** \`extracted\`
EOF
cat > "$skill2/references/sources.md" <<EOF
# Sources
## my-source
- **URL:** https://example.com/my-source
- **Description:** A test source.
- **Contributing files:** SKILL.md
- **Research doc:** docs/research/sources.md
- **Status:** \`extracted\`
EOF
run bash "$SCRIPT" "$skill2"
assert_failure
assert_output --partial "FAIL"
assert_output --partial "Slug 'my-source' not found as H2 in research doc 'docs/research/sources.md'"
}
@test "check 4 runs: bullet form with '-' hyphen bullets still parses each path" {
local skill="$TMPDIR/my-skill"
make_skill_with_source_keys "$skill"
mkdir -p "$skill/references"
cat > "$skill/references/sources.md" <<EOF
# Sources
## my-source
- **URL:** https://example.com/my-source
- **Description:** A test source.
**Contributing files:**
- SKILL.md (the dispatch table, with a comma in the note)
- references/nonexistent.md (why this one matters)
- **Research doc:** (none)
- **Basis:** SKILL.md
- **Status:** \`extracted\`
EOF
run bash "$SCRIPT" "$skill"
assert_failure
assert_output --partial "Contributing file 'references/nonexistent.md' does not exist"
}
@test "check 4 runs: inline comma-separated form still parses each path" {
local skill="$TMPDIR/my-skill"
make_skill_with_source_keys "$skill"
make_sources_md "$skill" "my-source" "SKILL.md, references/nonexistent.md"
run bash "$SCRIPT" "$skill"
assert_failure
assert_output --partial "Contributing file 'references/nonexistent.md' does not exist"
}
# ---------------------------------------------------------------------------
# Cycle 14 — Check 3 (#111): an explicit 'source_keys: []' is a house-authored
# declaration, a bare 'source_keys:' is truncation
#
# Three states, three outcomes. Only the middle one — declared empty — is
# silent; collapsing any pair of them is the defect #111 filed.
# ---------------------------------------------------------------------------
@test "check 3 silent: references doc with top-level 'source_keys: []' emits nothing" {
local skill="$TMPDIR/my-skill"
make_skill_with_source_keys "$skill"
make_sources_md "$skill"
cat > "$skill/references/extra.md" <<EOF
---
source_keys: []
---
# Extra Reference
House-authored, no external source.
EOF
commit_as_base "$skill"
run bash "$SCRIPT" "$skill"
assert_success
assert_output ""
}
@test "check 3 silent: references doc with 'source_keys: []' nested under metadata: emits nothing" {
local skill="$TMPDIR/my-skill"
make_skill_with_source_keys "$skill"
make_sources_md "$skill"
cat > "$skill/references/extra.md" <<EOF
---
metadata:
source_keys: []
---
# Extra Reference
House-authored, no external source.
EOF
commit_as_base "$skill"
run bash "$SCRIPT" "$skill"
assert_success
assert_output ""
}
@test "INFO: references doc with a bare 'source_keys:' and no value still emits INFO" {
local skill="$TMPDIR/my-skill"
make_skill_with_source_keys "$skill"
make_sources_md "$skill"
cat > "$skill/references/extra.md" <<EOF
---
source_keys:
---
# Extra Reference
Truncated frontmatter — this is not a decision.
EOF
run bash "$SCRIPT" "$skill"
assert_success
assert_output --partial "INFO"
assert_output --partial "No source_keys frontmatter"
}
@test "INFO: references doc with frontmatter but no source_keys key still emits INFO" {
local skill="$TMPDIR/my-skill"
make_skill_with_source_keys "$skill"
make_sources_md "$skill"
cat > "$skill/references/extra.md" <<EOF
---
title: Extra Reference
---
# Extra Reference
Never said either way.
EOF
run bash "$SCRIPT" "$skill"
assert_success
assert_output --partial "INFO"
assert_output --partial "No source_keys frontmatter"
}
# ---------------------------------------------------------------------------
# Cycle 15 — Check 7: Research doc annotation stripping, and the INFO
# that replaced the silent skip
#
# A Research doc value is very often a path PLUS a section annotation, and
# os.path.isfile() is false for every such string. Before the strip, checks 7
# and 8 skipped SILENTLY on those — so the negative assertions here (an INFO
# that must appear) are what distinguishes a running check from a dead one.
# ---------------------------------------------------------------------------
@test "check 7 runs: '§' section annotation is stripped before the path is resolved" {
local fake_repo="$TMPDIR/fakerepo"
make_upstream_skill "$fake_repo" 'docs/research/sources.md § "Some Section"'
cat > "$fake_repo/docs/research/sources.md" <<EOF
# Research
## different-slug
- **Contributing files:** (none)
- **Status:** \`extracted\`
EOF
run bash "$SCRIPT" "$fake_repo/my-skill"
assert_failure
assert_output --partial "Slug 'my-source' not found as H2 in research doc 'docs/research/sources.md'"
refute_output --partial "§"
}
@test "check 7 runs: '→' section annotation is stripped before the path is resolved" {
local fake_repo="$TMPDIR/fakerepo"
make_upstream_skill "$fake_repo" 'docs/research/sources.md → `## Pushing`'
cat > "$fake_repo/docs/research/sources.md" <<EOF
# Research
## different-slug
- **Contributing files:** (none)
- **Status:** \`extracted\`
EOF
run bash "$SCRIPT" "$fake_repo/my-skill"
assert_failure
assert_output --partial "Slug 'my-source' not found as H2 in research doc 'docs/research/sources.md'"
refute_output --partial "→"
}
@test "check 7 runs: parenthetical annotation is stripped before the path is resolved" {
local fake_repo="$TMPDIR/fakerepo"
make_upstream_skill "$fake_repo" "docs/research/sources.md (whole-document reference)"
cat > "$fake_repo/docs/research/sources.md" <<EOF
# Research
## different-slug
- **Contributing files:** (none)
- **Status:** \`extracted\`
EOF
run bash "$SCRIPT" "$fake_repo/my-skill"
assert_failure
assert_output --partial "Slug 'my-source' not found as H2 in research doc 'docs/research/sources.md'"
refute_output --partial "whole-document reference"
}
@test "check 7 runs: a bare path with no annotation survives the strip intact" {
local fake_repo="$TMPDIR/fakerepo"
make_upstream_skill "$fake_repo" "docs/research/sources.md"
cat > "$fake_repo/docs/research/sources.md" <<EOF
# Research
## my-source
- **Contributing files:** (none)
- **Status:** \`extracted\`
EOF
run bash "$SCRIPT" "$fake_repo/my-skill"
assert_success
assert_output ""
}
@test "check 7 skipped silently: Research doc '(none)' is recognised before the strip" {
local fake_repo="$TMPDIR/fakerepo"
make_upstream_skill "$fake_repo" "(none)"
run bash "$SCRIPT" "$fake_repo/my-skill"
assert_success
assert_output ""
}
@test "check 7 skipped silently: bare 'none — reason' is recognised as a declaration" {
local fake_repo="$TMPDIR/fakerepo"
make_upstream_skill "$fake_repo" "none — org convention, no upstream research doc"
run bash "$SCRIPT" "$fake_repo/my-skill"
assert_success
assert_output ""
}
@test "INFO: a stripped path that does not resolve names the slug instead of skipping silently" {
local fake_repo="$TMPDIR/fakerepo"
make_upstream_skill "$fake_repo" 'docs/research/missing.md § "Some Section"'
run bash "$SCRIPT" "$fake_repo/my-skill"
assert_success
assert_output --partial "INFO"
assert_output --partial "Upstream checks skipped for 'my-source' — research doc 'docs/research/missing.md' does not exist"
}
@test "INFO: a Research doc value that is entirely annotation names the slug instead of skipping silently" {
local fake_repo="$TMPDIR/fakerepo"
make_upstream_skill "$fake_repo" '§ "Some Section"'
run bash "$SCRIPT" "$fake_repo/my-skill"
assert_success
assert_output --partial "INFO"
assert_output --partial "Upstream checks skipped for 'my-source' — Research doc value names no path"
}
@test "INFO: no repo root above the skill directory names the slug instead of skipping silently" {
local skill="$TMPDIR/my-skill"
make_skill_with_source_keys "$skill"
make_sources_md "$skill" "my-source" "SKILL.md" "docs/research/sources.md"
run bash "$SCRIPT" "$skill"
assert_success
assert_output --partial "INFO"
assert_output --partial "Upstream checks skipped for 'my-source' — no repo root above the skill directory"
}
# ---------------------------------------------------------------------------
# Cycle 16 — Checks 4 and 5: None ("could not parse") is NOT [] ("explicitly
# (none)"), on the sources.md side this time
#
# Cycle 13 pinned the distinction for check 8 (since retired), which read the parser's output
# against a RESEARCH doc. Checks 4 and 5 read it against the skill's own
# sources.md and honoured neither half: a truthiness test collapsed None into
# [], so an unreadable Contributing files block disabled both checks and the
# script still exited 0 with no output — the failure mode
# parse_contributing_files' docstring names in as many words. The assertions
# below are therefore about the INFO appearing; a silent exit 0 is exactly the
# bug.
# ---------------------------------------------------------------------------
@test "INFO: an unparsable Contributing files block names the slug instead of skipping checks 4 and 5 silently" {
local skill="$TMPDIR/my-skill"
make_skill_with_source_keys "$skill"
mkdir -p "$skill/references"
cat > "$skill/references/sources.md" <<EOF
# Sources
## my-source
- **URL:** https://example.com/my-source
- **Description:** A test source.
- **Research doc:** (none)
- **Basis:** SKILL.md
**Contributing files:**
* references/ghost.md (asterisk bullets are not the bullet form)
- **Status:** \`extracted\`
EOF
run bash "$SCRIPT" "$skill"
assert_success
assert_output --partial "INFO"
assert_output --partial "Contributing-file checks skipped for 'my-source' — the Contributing files block could not be parsed"
}
@test "INFO: an entry with no Contributing files field at all is reported, not skipped silently" {
local skill="$TMPDIR/my-skill"
make_skill_with_source_keys "$skill"
mkdir -p "$skill/references"
cat > "$skill/references/sources.md" <<EOF
# Sources
## my-source
- **URL:** https://example.com/my-source
- **Description:** A test source.
- **Research doc:** (none)
- **Basis:** SKILL.md
- **Status:** \`extracted\`
EOF
run bash "$SCRIPT" "$skill"
assert_success
assert_output --partial "INFO"
assert_output --partial "Contributing-file checks skipped for 'my-source' — the Contributing files block could not be parsed"
}
@test "checks 4 and 5 skipped silently: an explicit '(none)' emits no INFO" {
local skill="$TMPDIR/my-skill"
make_skill_with_source_keys "$skill"
make_sources_md "$skill" "my-source" "(none — not used directly)"
commit_as_base "$skill"
run bash "$SCRIPT" "$skill"
assert_success
assert_output ""
}
@test "checks 4 and 5 still run: a parseable Contributing files list is not diverted to the INFO" {
local skill="$TMPDIR/my-skill"
make_skill_with_source_keys "$skill"
make_sources_md "$skill" "my-source" "references/nonexistent.md"
run bash "$SCRIPT" "$skill"
assert_failure
assert_output --partial "FAIL"
assert_output --partial "Contributing file 'references/nonexistent.md' does not exist"
refute_output --partial "could not be parsed"
}
# ---------------------------------------------------------------------------
# Cycle 17 — Check 3 (#111): 'source_keys: []' only declares anything in the
# two positions parse_source_keys() actually reads
#
# The declaration and the parse have to agree on WHERE the key lives. They did
# not: the declaration regex accepted any indent, so a `source_keys: []` buried
# under an unrelated key — a position parse_source_keys() never reads — passed
# as a house-authored declaration and silenced the INFO for a file that had
# declared nothing.
# ---------------------------------------------------------------------------
@test "INFO: 'source_keys: []' nested under an unrelated key is not a declaration" {
local skill="$TMPDIR/my-skill"
make_skill_with_source_keys "$skill"
make_sources_md "$skill"
cat > "$skill/references/extra.md" <<EOF
---
title: Extra Reference
unrelated:
nested:
source_keys: []
---
# Extra Reference
The empty list is nested where parse_source_keys never looks.
EOF
run bash "$SCRIPT" "$skill"
assert_success
assert_output --partial "INFO"
assert_output --partial "No source_keys frontmatter"
}
@test "INFO: a four-space-indented 'source_keys: []' is not a declaration either" {
local skill="$TMPDIR/my-skill"
make_skill_with_source_keys "$skill"
make_sources_md "$skill"
cat > "$skill/references/extra.md" <<EOF
---
metadata:
source_keys: []
---
# Extra Reference
Two spaces is the position parse_source_keys reads under metadata:, not four.
EOF
run bash "$SCRIPT" "$skill"
assert_success
assert_output --partial "INFO"
assert_output --partial "No source_keys frontmatter"
}
# ---------------------------------------------------------------------------
# Cycle 20 — F: check 7 applies only to a Research registry, and
# every skip announces itself
# ---------------------------------------------------------------------------
@test "F: a topic-doc Research doc is a FAIL naming the registry, not a missing slug (#121)" {
local fake_repo="$TMPDIR/fakerepo"
make_upstream_skill "$fake_repo" "docs/research/remotes.md"
cat > "$fake_repo/docs/research/remotes.md" <<EOF
# Remotes
## Core Philosophy
Prose about remotes.
EOF
run bash "$SCRIPT" "$fake_repo/my-skill"
assert_failure
assert_output --partial "is a topic document, not a Research registry"
refute_output --partial "not found as H2 in research doc"
}
@test "F: check 7 still runs when the Research doc IS a source index" {
local fake_repo="$TMPDIR/fakerepo"
make_upstream_skill "$fake_repo"
cat > "$fake_repo/docs/research/sources.md" <<EOF
# Sources
## different-slug
- **Contributing files:** (none)
- **Status:** \`extracted\`
EOF
run bash "$SCRIPT" "$fake_repo/my-skill"
assert_failure
assert_output --partial "Slug 'my-source' not found as H2 in research doc 'docs/research/sources.md'"
refute_output --partial "not applicable"
}
# ---------------------------------------------------------------------------
# Cycle 21 — G1: a bad target is a hard error, not a silent pass
#
# PORT NOTE RESOLVED (factory-audit merge): this block used to assert
# "not a directory" for the nonexistent target while the agent suite asserted
# "no such file" for its own, because each pre-merge script knew what shape it
# was owed. The merged entry point classifies before it complains, so neither
# old wording survives at THIS site and the two cases are no longer the same
# case:
#
# * a path that exists as nothing this script recognises (and a path that does
# not exist at all, when its name is not *.agent.md and its parent is not
# agents/) is "matches neither a skill directory nor an agent file";
# * a path that IS a directory but holds no SKILL.md gets its own, more
# specific line naming the missing SKILL.md.
#
# The agent suite's "no such file" wording did NOT move — a nonexistent
# *.agent.md path still classifies as an agent and is rejected by agent mode's
# own precondition — so that side stays verbatim over there. Both wordings are
# now asserted exactly rather than partially-matched, because the invariant both
# sides pin is exit 2 with a message that names the path, never a silent exit 0
# and never a bare code with no output.
# ---------------------------------------------------------------------------
@test "G1: a nonexistent directory is a hard error (exit 2), not a silent exit 0" {
run bash "$SCRIPT" "$TMPDIR/does-not-exist"
[ "$status" -eq 2 ]
assert_output --partial "Error: '$TMPDIR/does-not-exist' matches neither a skill directory nor an agent file."
}
@test "G1: a directory with no SKILL.md is a hard error (exit 2), not a silent exit 0" {
# Distinct from the case above on purpose: a directory IS half of a skill
# target, so the detector can say something sharper than "neither shape" and
# does. The exit code is unchanged from the pre-merge test, which already
# pinned 2; only the wording moved, from the suite's own "not a skill
# directory" precondition to the dispatcher's classification message.
mkdir -p "$TMPDIR/not-a-skill/references"
run bash "$SCRIPT" "$TMPDIR/not-a-skill"
[ "$status" -eq 2 ]
assert_output --partial "Error: '$TMPDIR/not-a-skill' is a directory with no SKILL.md in it."
}
# ---------------------------------------------------------------------------
# Cycle 22 — G2: a UTF-8 BOM does not disable check 2
# ---------------------------------------------------------------------------
@test "G2: a BOM at the head of SKILL.md does not silently disable check 2" {
local skill="$TMPDIR/my-skill"
mkdir -p "$skill/references"
printf '\xef\xbb\xbf' > "$skill/SKILL.md"
cat >> "$skill/SKILL.md" <<EOF
---
name: my-skill
description: A valid skill description.
metadata:
source_keys:
- my-source
---
## Step 1
Do the thing.
EOF
cat > "$skill/references/sources.md" <<EOF
# Sources
## different-source
- **URL:** https://example.com/different-source
- **Description:** A test source.
- **Contributing files:** references/sources.md
- **Research doc:** (none)
- **Basis:** SKILL.md
- **Status:** \`extracted\`
EOF
run bash "$SCRIPT" "$skill"
assert_failure
assert_output --partial "source_keys slug 'my-source' not found in sources.md"
}
# ---------------------------------------------------------------------------
# Cycle 23 — G3: reads are UTF-8 and an unreadable file is a finding
# ---------------------------------------------------------------------------
@test "G3: an em dash under LC_ALL=C is read, not turned into a traceback or a silent pass" {
local skill="$TMPDIR/my-skill"
make_skill_with_source_keys "$skill"
make_sources_md "$skill"
cat > "$skill/references/topic.md" <<EOF
---
source_keys:
- ghost-source
---
Prose — with an em dash.
EOF
LC_ALL=C PYTHONUTF8=0 PYTHONCOERCECLOCALE=0 run bash "$SCRIPT" "$skill"
assert_failure
assert_output --partial "source_keys slug 'ghost-source' not found in sources.md"
refute_output --partial "Traceback"
}
@test "G3: a file that is genuinely not UTF-8 is a FAIL, not a clean pass" {
local skill="$TMPDIR/my-skill"
make_skill_with_source_keys "$skill"
make_sources_md "$skill"
printf -- '---\nsource_keys:\n - my-source\n---\n\nLatin-1 byte: \xe9\n' \
> "$skill/references/topic.md"
run bash "$SCRIPT" "$skill"
assert_failure
assert_output --partial "not valid UTF-8"
assert_output --partial "references/topic.md"
}
# ---------------------------------------------------------------------------
# Cycle 24 — G4: check 3 walks references/ recursively
# ---------------------------------------------------------------------------
@test "G4: a source_keys file in references/<subdir>/ is validated, not skipped" {
local skill="$TMPDIR/my-skill"
make_skill_with_source_keys "$skill"
make_sources_md "$skill"
mkdir -p "$skill/references/nested"
cat > "$skill/references/nested/topic.md" <<EOF
---
source_keys:
- ghost-source
---
Nested prose.
EOF
run bash "$SCRIPT" "$skill"
assert_failure
assert_output --partial "source_keys slug 'ghost-source' not found in sources.md"
assert_output --partial "references/nested/topic.md"
}
# ---------------------------------------------------------------------------
# Cycle 25 — G5: a placeholder at end of line is still a placeholder
# ---------------------------------------------------------------------------
@test "G5: 'FILL IN:' at end of line is caught by check 1" {
local skill="$TMPDIR/my-skill"
make_skill_with_source_keys "$skill"
mkdir -p "$skill/references"
cat > "$skill/references/sources.md" <<EOF
# Sources
## my-source
- **URL:** https://example.com/my-source
- **Description:** FILL IN:
- **Contributing files:** SKILL.md
- **Research doc:** (none)
- **Basis:** SKILL.md
- **Status:** \`extracted\`
EOF
run bash "$SCRIPT" "$skill"
assert_failure
assert_output --partial "Unfilled FILL IN: placeholder"
}
@test "G5: a Research doc value that is a bare 'FILL IN:' is caught by check 6" {
local skill="$TMPDIR/my-skill"
make_skill_with_source_keys "$skill"
make_sources_md "$skill" "my-source" "SKILL.md" "FILL IN:"
run bash "$SCRIPT" "$skill"
assert_failure
assert_output --partial "Research doc field is empty or placeholder"
}
# ---------------------------------------------------------------------------
# Cycle 26 — G6/G7: duplicated entries and duplicated fields are announced
# ---------------------------------------------------------------------------
@test "G6: a duplicate '## slug' block is announced, not half-checked in silence" {
local skill="$TMPDIR/my-skill"
make_skill_with_source_keys "$skill"
mkdir -p "$skill/references"
cat > "$skill/references/sources.md" <<EOF
# Sources
## my-source
- **URL:** https://example.com/my-source
- **Description:** A test source.
- **Contributing files:** SKILL.md
- **Research doc:** (none)
- **Basis:** SKILL.md
- **Status:** \`extracted\`
## my-source
- **URL:** https://example.com/my-source-again
- **Description:** The same slug a second time.
- **Contributing files:** references/nonexistent.md
- **Research doc:** (none)
- **Basis:** SKILL.md
- **Status:** \`extracted\`
EOF
run bash "$SCRIPT" "$skill"
assert_success
assert_output --partial "Duplicate '## my-source' entry in sources.md"
assert_output --partial "only the first block is checked"
}
@test "G7: a second '- **Research doc:**' line in one entry FAILs (Research doc is single-valued, #121)" {
local skill="$TMPDIR/my-skill"
make_skill_with_source_keys "$skill"
mkdir -p "$skill/references"
cat > "$skill/references/sources.md" <<EOF
# Sources
## my-source
- **URL:** https://example.com/my-source
- **Description:** A test source.
- **Contributing files:** SKILL.md
- **Research doc:** (none)
- **Basis:** SKILL.md
- **Research doc:** docs/research/sources.md
- **Status:** \`extracted\`
EOF
run bash "$SCRIPT" "$skill"
assert_failure
assert_output --partial "Multiple '- **Research doc:**' lines for 'my-source'"
}
# ---------------------------------------------------------------------------
# Cycle 27 — G8: usage and environment errors exit 2, never 1
#
# PORT NOTE RESOLVED (factory-audit merge): the two suites were written against
# two separate scripts and disagreed about the no-argument wording — this file
# asserted "skill-dir is required", the agent suite "agent-file is required".
# Neither survives. One entry point takes both target shapes, so its no-argument
# message names both, and both suites now assert that one sentence. Exit 2 and a
# message (not a silent exit 1, and not a bare code with no output) is the
# invariant both sides were really pinning, and it is still pinned from both.
#
# Note the asymmetry with scripts/validate.sh, which exits 1 on no argument:
# only validate-provenance.sh carries the exit-2 usage tier, and this test is
# what holds it there.
# ---------------------------------------------------------------------------
@test "G8: a missing argument exits 2, not 1" {
run bash "$SCRIPT"
[ "$status" -eq 2 ]
assert_output --partial "Error: a skill directory or an agent file is required."
# The usage block has to follow the error, or "required" names no shape the
# caller can act on.
assert_output --partial "Usage: validate-provenance.sh"
}
@test "G8: an extra positional argument is rejected, not silently ignored" {
local skill="$TMPDIR/my-skill"
make_clean_skill "$skill"
run bash "$SCRIPT" "$skill" extra
[ "$status" -eq 2 ]
assert_output --partial "expected exactly one argument"
}
@test "G8: a missing python3 is reported by name and exits 2, not 127" {
local skill="$TMPDIR/my-skill"
make_clean_skill "$skill"
local stub="$TMPDIR/emptybin"
mkdir -p "$stub"
# dirname and basename are deliberately ABSENT, and that absence is load-
# bearing. The invariant: nothing external is needed to reach the python3
# preflight. The entry point resolves SCRIPT_DIR and classifies the target
# with _kf_dirname/_kf_basename/_kf_parent_name — pure-bash replacements
# that exist for precisely this reason — and the mode libraries build their
# Python bodies with `read` heredocs. Widening this list to keep a test
# green would silently retire that guarantee: on a PATH with neither
# coreutils nor python3 the script would die at 127 naming `dirname`
# instead of naming the dependency it actually needs, which is the failure
# tests/test-adr0020-contract.sh assertion 2 exists to prevent. bash, cat
# and sed stay so the stub is a DENY OF python3 ALONE rather than a test of
# "no PATH at all" — the diagnostic under test here is the python3 one.
for cmd in bash cat sed; do
ln -s "$(command -v "$cmd")" "$stub/$cmd"
done
PATH="$stub" run bash "$SCRIPT" "$skill"
[ "$status" -eq 2 ]
assert_output --partial "python3 is required"
}
@test "G3: an unreadable file is reported even when there is nothing else to validate" {
local skill="$TMPDIR/my-skill"
make_clean_skill "$skill"
mkdir -p "$skill/references"
printf -- '---\nsource_keys:\n - my-source\n---\n\nLatin-1 byte: \xe9\n' \
> "$skill/references/topic.md"
run bash "$SCRIPT" "$skill"
assert_failure
assert_output --partial "not valid UTF-8"
}
# ---------------------------------------------------------------------------
# Cycle 28 — Check 9: a changed Description or Contributing files claim is an
# INFO, never a FAIL — the script can tell the text changed, not whether the
# (possibly stronger) new wording is still true.
# ---------------------------------------------------------------------------
@test "check 9: Description text changed since base ref fires an INFO naming the slug and field" {
local skill="$TMPDIR/my-skill"
make_skill_with_source_keys "$skill"
make_sources_md "$skill"
commit_as_base "$skill"
# Rewritten in the working tree only, never committed — exactly the
# shape of the bug check 9 exists to flag: a hedge upgraded to a
# confident claim with nothing else in the entry touched.
sed -i 's/^- \*\*Description:\*\* A test source\.$/- **Description:** A test source that Grounds the dispatch table directly./' \
"$skill/references/sources.md"
run bash "$SCRIPT" "$skill"
assert_success
assert_output --partial "INFO"
assert_output --partial "'Description' changed for 'my-source'"
}
@test "check 9: Contributing files text changed since base ref fires an INFO naming the slug and field" {
local skill="$TMPDIR/my-skill"
make_skill_with_source_keys "$skill"
make_sources_md "$skill"
commit_as_base "$skill"
# Only an annotation is added. strip_note() in parse_contributing_files()
# removes it before checks 4 and 5 compare paths, so those stay clean —
# check 9 compares the raw field text, not the parsed path list, and
# this is still a real wording change worth a human re-reading it.
sed -i 's/^- \*\*Contributing files:\*\* SKILL\.md$/- **Contributing files:** SKILL.md (the dispatch table)/' \
"$skill/references/sources.md"
run bash "$SCRIPT" "$skill"
assert_success
assert_output --partial "INFO"
assert_output --partial "'Contributing files' changed for 'my-source'"
refute_output --partial "does not exist"
refute_output --partial "does not list"
}
@test "check 9: an entry unchanged since base ref produces no check-9 finding" {
local skill="$TMPDIR/my-skill"
make_skill_with_source_keys "$skill"
make_sources_md "$skill"
commit_as_base "$skill"
run bash "$SCRIPT" "$skill"
assert_success
assert_output ""
}
@test "check 9: a brand-new entry absent at the base ref is a creation, not a change" {
local skill="$TMPDIR/my-skill"
make_skill_with_source_keys "$skill"
make_sources_md "$skill"
commit_as_base "$skill"
# Added in the working tree only, after the commit above. It has no
# earlier revision to diff against, so check 9 must stay silent about
# it — a brand-new entry is a creation, not a rewrite of an existing
# claim, and flagging it would be exactly the false-positive shape the
# rejected literal-text approaches produced.
cat >> "$skill/references/sources.md" <<EOF
## new-source
- **URL:** https://example.com/new-source
- **Description:** A brand-new source, never committed.
- **Contributing files:** (none)
- **Research doc:** (none)
- **Basis:** SKILL.md
- **Status:** \`extracted\`
EOF
run bash "$SCRIPT" "$skill"
assert_success
assert_output ""
}
@test "check 9: no repo root emits the one graceful INFO, not a silent skip or a crash" {
local skill="$TMPDIR/my-skill"
make_skill_with_source_keys "$skill"
make_sources_md "$skill"
# Deliberately no commit_as_base — $TMPDIR has no ancestor .git, so
# find_repo_root() returns None.
run bash "$SCRIPT" "$skill"
assert_success
assert_output --partial "INFO"
assert_output --partial "Check 9 skipped — no repo root above the skill directory"
}
@test "check 9: an unresolvable base ref emits the one graceful INFO, not a silent skip or a crash" {
local skill="$TMPDIR/my-skill"
make_skill_with_source_keys "$skill"
make_sources_md "$skill"
commit_as_base "$skill"
# A repo with no origin/main and no override: `git merge-base HEAD
# origin/main` fails outright.
git -C "$skill" update-ref -d refs/remotes/origin/main >/dev/null 2>&1
run bash "$SCRIPT" "$skill"
assert_success
assert_output --partial "INFO"
assert_output --partial "Check 9 skipped — no base ref could be resolved"
}
@test "check 9: --base-ref overrides a default origin/main that resolves to something else" {
local skill="$TMPDIR/my-skill"
make_skill_with_source_keys "$skill"
make_sources_md "$skill"
commit_as_base "$skill"
local old_sha
old_sha="$(git -C "$skill" rev-parse HEAD)"
# A SECOND commit carrying the rewritten claim, with origin/main moved onto
# it. The default base ref therefore resolves — to a commit that matches the
# working tree — so a silent default run proves there was a default here to
# override. Deleting origin/main instead, as this test used to, proved only
# that the flag works when nothing else does.
sed -i 's/^- \*\*Description:\*\* A test source\.$/- **Description:** A rewritten claim./' \
"$skill/references/sources.md"
git -C "$skill" -c user.email=test@example.com -c user.name=test commit -aqm rewrite >/dev/null 2>&1
git -C "$skill" update-ref refs/remotes/origin/main HEAD >/dev/null 2>&1
run bash "$SCRIPT" "$skill"
assert_success
assert_output ""
run bash "$SCRIPT" "$skill" "--base-ref=$old_sha"
assert_success
assert_output --partial "'Description' changed for 'my-source'"
}
@test "check 9: VALIDATE_PROVENANCE_BASE_REF sets the base ref when no flag is given" {
local skill="$TMPDIR/my-skill"
make_skill_with_source_keys "$skill"
make_sources_md "$skill"
commit_as_base "$skill"
local old_sha
old_sha="$(git -C "$skill" rev-parse HEAD)"
sed -i 's/^- \*\*Description:\*\* A test source\.$/- **Description:** A rewritten claim./' \
"$skill/references/sources.md"
git -C "$skill" -c user.email=test@example.com -c user.name=test commit -aqm rewrite >/dev/null 2>&1
git -C "$skill" update-ref refs/remotes/origin/main HEAD >/dev/null 2>&1
run env VALIDATE_PROVENANCE_BASE_REF="$old_sha" bash "$SCRIPT" "$skill"
assert_success
assert_output --partial "'Description' changed for 'my-source'"
}
@test "check 9: the --base-ref flag wins over the environment variable" {
local skill="$TMPDIR/my-skill"
make_skill_with_source_keys "$skill"
make_sources_md "$skill"
commit_as_base "$skill"
local old_sha
old_sha="$(git -C "$skill" rev-parse HEAD)"
sed -i 's/^- \*\*Description:\*\* A test source\.$/- **Description:** A rewritten claim./' \
"$skill/references/sources.md"
git -C "$skill" -c user.email=test@example.com -c user.name=test commit -aqm rewrite >/dev/null 2>&1
local new_sha
new_sha="$(git -C "$skill" rev-parse HEAD)"
# The environment names the old commit (which would fire), the flag names
# the new one (which would not). The usage text promises the flag wins.
run env VALIDATE_PROVENANCE_BASE_REF="$old_sha" bash "$SCRIPT" "$skill" "--base-ref=$new_sha"
assert_success
assert_output ""
}
@test "check 9: an EMPTY --base-ref is still 'given' and wins over the environment variable" {
local skill="$TMPDIR/my-skill"
make_skill_with_source_keys "$skill"
make_sources_md "$skill"
commit_as_base "$skill"
local old_sha
old_sha="$(git -C "$skill" rev-parse HEAD)"
sed -i 's/^- \*\*Description:\*\* A test source\.$/- **Description:** A rewritten claim./' \
"$skill/references/sources.md"
git -C "$skill" -c user.email=test@example.com -c user.name=test commit -aqm rewrite >/dev/null 2>&1
git -C "$skill" update-ref refs/remotes/origin/main HEAD >/dev/null 2>&1
# `--base-ref=` selects the DEFAULT resolution (origin/main, which now
# matches the working tree), so nothing fires. Under the old `:-` spelling
# the empty value read as absent and the environment variable won, firing
# the INFO and contradicting the usage text.
run env VALIDATE_PROVENANCE_BASE_REF="$old_sha" bash "$SCRIPT" "$skill" "--base-ref="
assert_success
assert_output ""
}
# ---------------------------------------------------------------------------
# Cycle 29 — Check 9: a WRAPPED field value. The parser compared only the first
# physical line, so a rewrite confined to a continuation line — the exact
# hedge-to-confident-claim shape #118 exists to catch — produced no finding at
# all, while a pure re-wrap produced a false one.
# ---------------------------------------------------------------------------
@test "check 9: a rewrite confined to a wrapped Description's CONTINUATION line fires an INFO" {
local skill="$TMPDIR/my-skill"
make_skill_with_source_keys "$skill"
make_wrapped_sources_md "$skill"
commit_as_base "$skill"
# Hedge to confident claim, on the second physical line only. This is the
# regression check 9 was written for, and the one it could not see.
sed -i "s|^ no forge-specific content drawn directly from it beyond that\.\$| it grounds Step 2's dispatch table in full.|" \
"$skill/references/sources.md"
run bash "$SCRIPT" "$skill"
assert_success
assert_output --partial "'Description' changed for 'my-source'"
}
@test "check 9: re-wrapping a Description with no wording change produces no finding" {
local skill="$TMPDIR/my-skill"
make_skill_with_source_keys "$skill"
make_wrapped_sources_md "$skill"
commit_as_base "$skill"
# Same words, different line breaks. normalize_field_text()'s docstring
# promises this is invisible; it was not, because the value was truncated
# at its first newline before the whitespace collapse ever ran.
python3 - "$skill/references/sources.md" <<'PY'
import sys
path = sys.argv[1]
text = open(path).read()
old = ("- **Description:** A test source, informing the dispatch table's shape with\n"
" no forge-specific content drawn directly from it beyond that.")
new = ("- **Description:** A test source, informing the dispatch\n"
" table's shape with no forge-specific content drawn\n"
" directly from it beyond that.")
assert old in text
open(path, 'w').write(text.replace(old, new))
PY
run bash "$SCRIPT" "$skill"
assert_success
assert_output ""
}
@test "check 9: the bullet form of Contributing files is compared, not skipped" {
local skill="$TMPDIR/my-skill"
make_skill_with_source_keys "$skill"
mkdir -p "$skill/references"
cat > "$skill/references/sources.md" <<'EOF'
# Sources
## my-source
- **URL:** https://example.com/my-source
- **Description:** A test source.
**Contributing files:**
- SKILL.md (the dispatch table)
- references/other.md (the rubric)
- **Research doc:** (none)
- **Basis:** SKILL.md
- **Status:** `extracted`
EOF
printf -- '---\nsource_keys:\n - my-source\n---\n\nnotes\n' > "$skill/references/other.md"
commit_as_base "$skill"
# The change is in the SECOND bullet. The old loop joined bullets in
# document order too, but broke on any wrapped one — and no test covered
# this branch at all, both existing check-9 tests using the inline form.
sed -i 's|^- references/other.md (the rubric)$|- references/other.md (the whole rubric, verbatim)|' \
"$skill/references/sources.md"
run bash "$SCRIPT" "$skill"
assert_success
assert_output --partial "'Contributing files' changed for 'my-source'"
}
@test "check 9: a wrapped bullet does not silently drop the bullets after it" {
local skill="$TMPDIR/my-skill"
make_skill_with_source_keys "$skill"
mkdir -p "$skill/references"
cat > "$skill/references/sources.md" <<'EOF'
# Sources
## my-source
- **URL:** https://example.com/my-source
- **Description:** A test source.
**Contributing files:**
- SKILL.md (the dispatch table, and the gates
common to every branch of it)
- references/other.md (the rubric)
- **Research doc:** (none)
- **Basis:** SKILL.md
- **Status:** `extracted`
EOF
printf -- '---\nsource_keys:\n - my-source\n---\n\nnotes\n' > "$skill/references/other.md"
commit_as_base "$skill"
# The old loop broke at the wrapped continuation line, so everything from
# here down was never part of the compared value — a change to the last
# bullet was invisible.
sed -i 's|^- references/other.md (the rubric)$|- references/other.md (rewritten claim)|' \
"$skill/references/sources.md"
run bash "$SCRIPT" "$skill"
assert_success
assert_output --partial "'Contributing files' changed for 'my-source'"
}
@test "check 9: a field present at the base ref and deleted since is announced" {
local skill="$TMPDIR/my-skill"
make_skill_with_source_keys "$skill"
make_sources_md "$skill"
commit_as_base "$skill"
# No other check in this script requires a Description, so a deleted one
# used to leave no finding anywhere: a claim could be withdrawn as
# invisibly as it could be strengthened.
sed -i '/^- \*\*Description:\*\*/d' "$skill/references/sources.md"
run bash "$SCRIPT" "$skill"
assert_success
assert_output --partial "'Description' removed for 'my-source'"
}
@test "check 9: a sources.md untracked at the base ref is announced, not silently skipped" {
local repo="$TMPDIR/repo"
local skill="$repo/tracked-skill"
mkdir -p "$skill"
make_skill_with_source_keys "$skill"
make_sources_md "$skill"
sed -i 's#^- \*\*Basis:\*\* SKILL.md#- **Basis:** tracked-skill/SKILL.md#' "$skill/references/sources.md"
commit_as_base "$repo"
# A copy of the same skill at a path git does not know — the everyday case
# being an installed, gitignored .claude/skills/ tree. The base ref
# resolves fine; `git show <ref>:<path>` does not. Treating that as
# "creation, nothing to flag" made the whole check vanish without a word,
# so the same directory reported findings at one path and silence at the
# other.
cp -r "$skill" "$repo/untracked-copy"
run bash "$SCRIPT" "$repo/untracked-copy"
assert_success
assert_output --partial "INFO"
assert_output --partial "is not tracked at"
assert_output --partial "Check 9 did not run for any slug in this skill."
}
@test "check 9: an invalid --base-ref value is reported as unresolvable, not a crash" {
local skill="$TMPDIR/my-skill"
make_skill_with_source_keys "$skill"
make_sources_md "$skill"
commit_as_base "$skill"
run bash "$SCRIPT" "$skill" "--base-ref=not-a-real-ref"
assert_success
assert_output --partial "Check 9 skipped — no base ref could be resolved"
assert_output --partial "not-a-real-ref"
}
# ---------------------------------------------------------------------------
# Auto-detection — the merged provenance entry point classifies its own target
#
# NEW with the factory-audit merge, and new behaviour rather than a ported
# case: scripts/validate-provenance.sh is now ONE entry point for both artifact
# types and works out from the target which rubric to run. A DIRECTORY holding
# SKILL.md is a skill; a FILE named *.agent.md, or sitting under .apm/agents/,
# is an agent.
#
# The two rubrics are not near-copies of one another — the skill side has a
# check 9 the agent side has none of, and reads sources.md from the skill's own
# references/ rather than from the package root — so a misclassification is not
# a near miss. It runs a set of checks that cannot apply and skips the set that
# can, at exit 0.
#
# This file is the SKILL half plus the neither-shape rejection;
# validate-provenance-agent.bats holds the agent half. Same script in both.
# ---------------------------------------------------------------------------
@test "auto-detect: a directory holding SKILL.md is checked in SKILL provenance mode" {
local skill="$TMPDIR/my-skill"
make_skill_with_source_keys "$skill"
make_sources_md "$skill"
# No commit_as_base, so check 9 has no repo root and announces the skip.
run bash "$SCRIPT" "$skill"
assert_success
# Check 9 exists only on the skill side — the agent rubric has no check 9 at
# any tier — so naming it is positive proof the SKILL rubric ran, rather
# than merely that nothing crashed.
assert_output --partial "Check 9 skipped — no repo root above the skill directory"
}
@test "auto-detect: a SKILL.md FILE path is checked in SKILL provenance mode, not rejected" {
# NEW with the merge and additive: pre-merge, handing the SKILL.md itself to
# skill-audit's validate-provenance.sh hit the "not a directory" precondition
# and died. It matters because pre-commit `files:` hooks match FILES — the
# vale-audit-prefilter-skill hook's regex ends in /SKILL\.md$ — so
# every hook-driven invocation hands over a SKILL.md path, never its
# directory. The entry point rewrites the token to the directory in place.
local skill="$TMPDIR/my-skill"
make_skill_with_source_keys "$skill"
make_sources_md "$skill"
run bash "$SCRIPT" "$skill/SKILL.md"
assert_success
# Same positive proof as the directory case: check 9 is skill-only.
assert_output --partial "Check 9 skipped — no repo root above the skill directory"
}
@test "auto-detect: a SKILL.md FILE path and its directory produce the same verdict" {
# The rewrite must be transparent, not merely non-fatal. If the two spellings
# of the same target could disagree, a pre-commit run and a hand run would
# report differently on one skill and neither would be obviously wrong.
local skill="$TMPDIR/my-skill"
make_skill_with_source_keys "$skill"
make_sources_md "$skill"
run bash "$SCRIPT" "$skill"
local dir_status="$status"
local dir_output="$output"
run bash "$SCRIPT" "$skill/SKILL.md"
[ "$status" -eq "$dir_status" ]
[ "$output" = "$dir_output" ]
# Guard against the comparison being satisfied by two empty runs: this
# fixture has a check-9 INFO to print, so silence here means neither
# spelling ran the rubric.
refute_output ""
}
@test "exit contract: SKILL mode exits 0 WITH output when the only findings are INFO" {
# The two modes' exit contracts are DIFFERENT and the merge must not quietly
# unify them. Skill mode is allowed to be chatty on a passing run: INFO
# findings print and the status stays 0. Agent mode's opposite half — a
# passing run prints nothing at all — is pinned from its own side in
# validate-provenance-agent.bats.
#
# Both halves are asserted explicitly and on purpose. A merge that collapsed
# one contract into the other would still satisfy whichever side was left
# unasserted, so a single-sided pin would go green on exactly the defect it
# was written to catch.
local skill="$TMPDIR/my-skill"
make_skill_with_source_keys "$skill"
make_sources_md "$skill"
run bash "$SCRIPT" "$skill"
[ "$status" -eq 0 ]
refute_output ""
assert_output --partial "INFO"
refute_output --partial "FAIL"
}
@test "auto-detect: a provenance target that is neither a skill directory nor an agent file FAILs, naming what it was handed" {
# The detector must not guess. Guessing here is worse than in validate.sh:
# this script's whole not-in-scope path is a SILENT exit 0, so a wrong guess
# followed by "provenance does not apply at this scope" is indistinguishable
# from a clean pass — which is the exact confusion the exit-2 tier above was
# created to end.
#
# A plain .txt file is neither shape under any reading of the contract: not
# a directory holding SKILL.md, not *.agent.md, not under .apm/agents/.
local dir="$TMPDIR/neither"
mkdir -p "$dir"
echo "not an artifact of either kind" > "$dir/notes.txt"
run bash "$SCRIPT" "$dir/notes.txt"
assert_failure
refute_output ""
assert_output --partial "$dir/notes.txt"
}
# ---------------------------------------------------------------------------
# Cycle 28 — #121: `Research doc:` names exactly one Research registry;
# an entry with no registry writes `none` plus `Basis:`
#
# Helper: a fake repo holding one skill whose single entry is written verbatim
# from the fields passed in — make_upstream_skill only varies the Research doc
# value, and the Basis cases need a second field. The registry lives at
# docs/research/sources.md and a basis file at docs/basis.md.
# ---------------------------------------------------------------------------
make_entry_skill() {
local repo="$1"
local fields="$2"
local skill="$repo/my-skill"
mkdir -p "$skill/references" "$repo/docs/research"
cat > "$skill/SKILL.md" <<EOF
---
name: my-skill
description: A valid skill description.
metadata:
source_keys:
- my-source
---
## Step 1
Do the thing.
EOF
printf '%s\n' "# Basis" > "$repo/docs/basis.md"
printf '%s\n' "# Other" > "$repo/docs/other-basis.md"
{
printf '# Sources\n\n## my-source\n\n'
printf '%s\n' '- **URL:** https://example.com/my-source'
printf '%s\n' '- **Description:** A test source.'
printf '%s\n' '- **Contributing files:** SKILL.md'
printf '%s\n' "$fields"
printf '%s\n' '- **Status:** `extracted`'
} > "$skill/references/sources.md"
cat > "$repo/docs/research/sources.md" <<EOF
# Research
## my-source
- **Contributing files:** (none)
- **Status:** \`extracted\`
EOF
commit_as_base "$repo"
}
@test "#121 FAIL: a brace-expansion Research doc names more than one path" {
local repo="$TMPDIR/fakerepo"
make_entry_skill "$repo" '- **Research doc:** docs/research/{sources,other}.md'
run bash "$SCRIPT" "$repo/my-skill"
assert_failure
assert_output --partial "Research doc names more than one path"
}
@test "#121 FAIL: a comma-separated Research doc names more than one path" {
local repo="$TMPDIR/fakerepo"
make_entry_skill "$repo" '- **Research doc:** docs/research/sources.md, docs/research/other.md'
run bash "$SCRIPT" "$repo/my-skill"
assert_failure
assert_output --partial "Research doc names more than one path"
}
@test "#121 FAIL: a semicolon-separated pair of annotated paths names more than one path" {
local repo="$TMPDIR/fakerepo"
make_entry_skill "$repo" '- **Research doc:** docs/research/sources.md (Releases section); docs/research/other.md (`delete_release` gotcha)'
run bash "$SCRIPT" "$repo/my-skill"
assert_failure
assert_output --partial "Research doc names more than one path"
}
@test "#121 pass: a comma or semicolon INSIDE the annotation is prose, not a list" {
local repo="$TMPDIR/fakerepo"
make_entry_skill "$repo" '- **Research doc:** docs/research/sources.md (cross-cutting; no dedicated section, see notes)'
run bash "$SCRIPT" "$repo/my-skill"
assert_success
assert_output ""
}
@test "#121 FAIL: repeated Research doc lines are a list, not an INFO" {
local repo="$TMPDIR/fakerepo"
make_entry_skill "$repo" "$(printf '%s\n%s' '- **Research doc:** docs/research/sources.md' '- **Research doc:** docs/research/other.md')"
run bash "$SCRIPT" "$repo/my-skill"
assert_failure
assert_output --partial "Multiple '- **Research doc:**' lines for 'my-source'"
}
@test "#121 pass: 'none' with one Basis bullet resolves and is clean" {
local repo="$TMPDIR/fakerepo"
make_entry_skill "$repo" "$(printf '%s\n%s' '- **Research doc:** none' '- **Basis:** docs/basis.md')"
run bash "$SCRIPT" "$repo/my-skill"
assert_success
assert_output ""
}
@test "#121 pass: 'none — reason' carries a trailing annotation and still counts as none" {
local repo="$TMPDIR/fakerepo"
make_entry_skill "$repo" "$(printf '%s\n%s' '- **Research doc:** none — org convention, not a research corpus entry' '- **Basis:** docs/basis.md')"
run bash "$SCRIPT" "$repo/my-skill"
assert_success
assert_output ""
}
@test "#121 pass: repeated single-path Basis bullets are each checked" {
local repo="$TMPDIR/fakerepo"
make_entry_skill "$repo" "$(printf '%s\n%s\n%s' '- **Research doc:** none' '- **Basis:** docs/basis.md' '- **Basis:** docs/other-basis.md')"
run bash "$SCRIPT" "$repo/my-skill"
assert_success
assert_output ""
}
@test "#121 FAIL: a second Basis bullet naming a missing path fails even when the first resolves" {
local repo="$TMPDIR/fakerepo"
make_entry_skill "$repo" "$(printf '%s\n%s\n%s' '- **Research doc:** none' '- **Basis:** docs/basis.md' '- **Basis:** docs/gone.md')"
run bash "$SCRIPT" "$repo/my-skill"
assert_failure
assert_output --partial "Basis path 'docs/gone.md' does not exist"
}
@test "#121 pass: a Basis header followed by '- ' path bullets is read too" {
local repo="$TMPDIR/fakerepo"
make_entry_skill "$repo" "$(printf '%s\n%s\n%s\n%s' '- **Research doc:** none' '**Basis:**' '- docs/basis.md' '- docs/other-basis.md')"
run bash "$SCRIPT" "$repo/my-skill"
assert_success
assert_output ""
}
@test "#121 FAIL: 'none' with no Basis declares nothing" {
local repo="$TMPDIR/fakerepo"
make_entry_skill "$repo" '- **Research doc:** none'
run bash "$SCRIPT" "$repo/my-skill"
assert_failure
assert_output --partial "Basis missing for 'my-source'"
}
@test "#121 FAIL: a Basis path that does not exist" {
local repo="$TMPDIR/fakerepo"
make_entry_skill "$repo" "$(printf '%s\n%s' '- **Research doc:** none' '- **Basis:** docs/gone.md')"
run bash "$SCRIPT" "$repo/my-skill"
assert_failure
assert_output --partial "Basis path 'docs/gone.md' does not exist"
}
@test "#121 FAIL: one Basis value naming several paths is a list" {
local repo="$TMPDIR/fakerepo"
make_entry_skill "$repo" "$(printf '%s\n%s' '- **Research doc:** none' '- **Basis:** docs/basis.md, docs/other-basis.md')"
run bash "$SCRIPT" "$repo/my-skill"
assert_failure
assert_output --partial "Basis value names more than one path"
}
@test "#121 pass: a '(removed in <sha>)' Basis skips the existence check" {
local repo="$TMPDIR/fakerepo"
make_entry_skill "$repo" "$(printf '%s\n%s\n%s' '- **Research doc:** none' '- **Basis:** docs/basis.md' '- **Basis:** docs/deleted-adr.md (removed in 5b80f30)')"
run bash "$SCRIPT" "$repo/my-skill"
assert_success
assert_output ""
}
@test "#121 FAIL: a Basis annotated with something other than '(removed in <sha>)' is still existence-checked" {
local repo="$TMPDIR/fakerepo"
make_entry_skill "$repo" "$(printf '%s\n%s' '- **Research doc:** none' '- **Basis:** docs/deleted-adr.md (removed later)')"
run bash "$SCRIPT" "$repo/my-skill"
assert_failure
assert_output --partial "Basis path 'docs/deleted-adr.md' does not exist"
}
@test "#121 pass: a Basis with a trailing annotation resolves its path" {
local repo="$TMPDIR/fakerepo"
make_entry_skill "$repo" "$(printf '%s\n%s' '- **Research doc:** none' '- **Basis:** docs/basis.md (org convention file, not a corpus entry)')"
run bash "$SCRIPT" "$repo/my-skill"
assert_success
assert_output ""
}
@test "#121 FAIL: check 7 — a slug missing from a resolved registry is a FAIL, not an INFO" {
local repo="$TMPDIR/fakerepo"
make_entry_skill "$repo" '- **Research doc:** docs/research/sources.md'
cat > "$repo/docs/research/sources.md" <<EOF
# Research
## different-slug
- **Contributing files:** (none)
- **Status:** \`extracted\`
EOF
run bash "$SCRIPT" "$repo/my-skill"
assert_failure
assert_output --partial "FAIL"
assert_output --partial "Slug 'my-source' not found as H2 in research doc 'docs/research/sources.md'"
}
@test "#121 FAIL: a Research doc that resolves to a topic document is wrong, not 'not applicable'" {
local repo="$TMPDIR/fakerepo"
make_entry_skill "$repo" '- **Research doc:** docs/research/remotes.md (whole-document reference)'
printf '# Remotes\n\n## Core Philosophy\n\nProse.\n' > "$repo/docs/research/remotes.md"
run bash "$SCRIPT" "$repo/my-skill"
assert_failure
assert_output --partial "is a topic document, not a Research registry"
refute_output --partial "not applicable"
}
@test "#121 INFO: an unresolvable Research doc path stays INFO and exits 0" {
local repo="$TMPDIR/fakerepo"
make_entry_skill "$repo" '- **Research doc:** docs/research/gone/sources.md'
run bash "$SCRIPT" "$repo/my-skill"
assert_success
assert_output --partial "INFO"
assert_output --partial "does not exist"
}
@test "#121 INFO: a Basis cannot be existence-checked with no repo root, and says so" {
local skill="$TMPDIR/my-skill"
make_skill_with_source_keys "$skill"
mkdir -p "$skill/references"
cat > "$skill/references/sources.md" <<EOF
# Sources
## my-source
- **URL:** https://example.com/my-source
- **Description:** A test source.
- **Contributing files:** SKILL.md
- **Research doc:** none
- **Basis:** docs/basis.md
- **Status:** \`extracted\`
EOF
run bash "$SCRIPT" "$skill"
assert_success
assert_output --partial "Basis check skipped for 'my-source'"
}
# --- Parser parity: the spelling of a field must not change what is read ----
@test "#121 parity: an inline Research doc with no leading hyphen is read, not reported missing" {
local repo="$TMPDIR/fakerepo"
make_entry_skill "$repo" '**Research doc:** docs/research/sources.md'
run bash "$SCRIPT" "$repo/my-skill"
assert_success
refute_output --partial "Research doc field missing"
assert_output ""
}
@test "#121 parity: a Research doc header with one '- ' bullet is read" {
local repo="$TMPDIR/fakerepo"
make_entry_skill "$repo" "$(printf '%s\n%s' '**Research doc:**' '- docs/research/sources.md')"
run bash "$SCRIPT" "$repo/my-skill"
assert_success
refute_output --partial "Research doc field missing"
assert_output ""
}
# --- #121 review round: list detection, repo confinement, parser edge cases --
# Helper: one-line Research doc / Basis fixtures over make_entry_skill.
rd_fixture() { make_entry_skill "$TMPDIR/fakerepo" "$1"; }
basis_fixture() { make_entry_skill "$TMPDIR/fakerepo" "$(printf '%s\n%s' '- **Research doc:** none' "$1")"; }
@test "#121 FAIL: a brace-only Research doc (no comma) names more than one path" {
rd_fixture '- **Research doc:** docs/research/{sources}.md'
run bash "$SCRIPT" "$TMPDIR/fakerepo/my-skill"
assert_failure
assert_output --partial "Research doc names more than one path"
}
@test "#121 FAIL: a bare 'a.md; b.md' Research doc names more than one path" {
rd_fixture '- **Research doc:** docs/research/sources.md; docs/other-basis.md'
run bash "$SCRIPT" "$TMPDIR/fakerepo/my-skill"
assert_failure
assert_output --partial "Research doc names more than one path"
}
@test "#121 FAIL: an annotated first path followed by ', second-path' is a list" {
rd_fixture '- **Research doc:** docs/research/sources.md (x), docs/research/topic.md'
run bash "$SCRIPT" "$TMPDIR/fakerepo/my-skill"
assert_failure
assert_output --partial "Research doc names more than one path"
}
@test "#121 FAIL: a space-separated pair of Research docs is a list" {
rd_fixture '- **Research doc:** docs/research/sources.md docs/other-basis.md'
run bash "$SCRIPT" "$TMPDIR/fakerepo/my-skill"
assert_failure
assert_output --partial "Research doc names more than one path"
}
@test "#121 FAIL: a space-separated pair of backticked Research docs is a list" {
rd_fixture '- **Research doc:** `docs/research/sources.md` `docs/other-basis.md`'
run bash "$SCRIPT" "$TMPDIR/fakerepo/my-skill"
assert_failure
assert_output --partial "Research doc names more than one path"
}
@test "#121 pass: a single backticked Research doc path is unwrapped before resolving" {
rd_fixture '- **Research doc:** `docs/research/sources.md`'
run bash "$SCRIPT" "$TMPDIR/fakerepo/my-skill"
assert_success
assert_output ""
}
@test "#121 pass: a ';' inside an annotation that holds a path is prose (path part only is checked)" {
rd_fixture '- **Research doc:** docs/research/sources.md (digested; docs/other-basis.md)'
run bash "$SCRIPT" "$TMPDIR/fakerepo/my-skill"
assert_success
assert_output ""
}
@test "#121 FAIL: a bare 'a.md; b.md' Basis names more than one path" {
basis_fixture '- **Basis:** docs/basis.md; docs/other-basis.md'
run bash "$SCRIPT" "$TMPDIR/fakerepo/my-skill"
assert_failure
assert_output --partial "Basis value names more than one path"
}
@test "#121 FAIL: a brace Basis names more than one path" {
basis_fixture '- **Basis:** docs/{basis,other-basis}.md'
run bash "$SCRIPT" "$TMPDIR/fakerepo/my-skill"
assert_failure
assert_output --partial "Basis value names more than one path"
}
@test "#121 FAIL: a brace-only Basis names more than one path" {
basis_fixture '- **Basis:** docs/{basis}.md'
run bash "$SCRIPT" "$TMPDIR/fakerepo/my-skill"
assert_failure
assert_output --partial "Basis value names more than one path"
}
@test "#121 FAIL: a space-separated Basis pair names more than one path" {
basis_fixture '- **Basis:** docs/basis.md docs/other-basis.md'
run bash "$SCRIPT" "$TMPDIR/fakerepo/my-skill"
assert_failure
assert_output --partial "Basis value names more than one path"
}
@test "#121 pass: a backticked Basis path is unwrapped before resolving" {
basis_fixture '- **Basis:** `docs/basis.md`'
run bash "$SCRIPT" "$TMPDIR/fakerepo/my-skill"
assert_success
assert_output ""
}
@test "#121 FAIL: an absolute Research doc path is outside the repo" {
rd_fixture '- **Research doc:** /etc/passwd'
run bash "$SCRIPT" "$TMPDIR/fakerepo/my-skill"
assert_failure
assert_output --partial "outside the repository"
}
@test "#121 FAIL: a '..' Research doc escape is outside the repo" {
rd_fixture '- **Research doc:** ../outside/sources.md'
mkdir -p "$TMPDIR/outside"
printf '# R\n\n## my-source\n' > "$TMPDIR/outside/sources.md"
run bash "$SCRIPT" "$TMPDIR/fakerepo/my-skill"
assert_failure
assert_output --partial "outside the repository"
}
@test "#121 FAIL: an absolute Basis path is outside the repo" {
basis_fixture '- **Basis:** /etc/passwd'
run bash "$SCRIPT" "$TMPDIR/fakerepo/my-skill"
assert_failure
assert_output --partial "outside the repository"
}
@test "#121 FAIL: a '..' Basis escape is outside the repo even though the file exists" {
basis_fixture '- **Basis:** ../outside.md'
printf 'x\n' > "$TMPDIR/outside.md"
run bash "$SCRIPT" "$TMPDIR/fakerepo/my-skill"
assert_failure
assert_output --partial "outside the repository"
}
@test "#121 FAIL: '(removed in abc)' is too short a sha to skip the check" {
basis_fixture '- **Basis:** docs/deleted-adr.md (removed in abc)'
run bash "$SCRIPT" "$TMPDIR/fakerepo/my-skill"
assert_failure
assert_output --partial "Basis path 'docs/deleted-adr.md' does not exist"
}
@test "#121 FAIL: '(removed in <sha>)' followed by more text is not the annotation" {
basis_fixture '- **Basis:** docs/deleted-adr.md (removed in 5b80f30) but really still here'
run bash "$SCRIPT" "$TMPDIR/fakerepo/my-skill"
assert_failure
assert_output --partial "Basis path 'docs/deleted-adr.md' does not exist"
}
@test "#121 pass: '(removed in <sha>)' Basis with no repo root is skipped silently" {
local skill="$TMPDIR/my-skill"
make_skill_with_source_keys "$skill"
mkdir -p "$skill/references"
cat > "$skill/references/sources.md" <<EOF
# Sources
## my-source
- **URL:** https://example.com/my-source
- **Description:** A test source.
- **Contributing files:** SKILL.md
- **Research doc:** none
- **Basis:** docs/gone.md (removed in 5b80f30)
- **Status:** \`extracted\`
EOF
run bash "$SCRIPT" "$skill"
assert_success
refute_output --partial "Basis check skipped"
}
@test "#121 parity: a '- **X**' bullet under a Basis header is a value, not the next field" {
make_entry_skill "$TMPDIR/fakerepo" "$(printf '%s\n%s\n%s' '- **Research doc:** none' '**Basis:**' '- **docs/gone.md**')"
run bash "$SCRIPT" "$TMPDIR/fakerepo/my-skill"
assert_failure
assert_output --partial "does not exist"
refute_output --partial "Basis missing"
}
@test "#121 parity: '* ' bullets under a Basis header are read" {
make_entry_skill "$TMPDIR/fakerepo" "$(printf '%s\n%s\n%s\n%s' '- **Research doc:** none' '**Basis:**' '* docs/basis.md' '* docs/gone.md')"
run bash "$SCRIPT" "$TMPDIR/fakerepo/my-skill"
assert_failure
assert_output --partial "Basis path 'docs/gone.md' does not exist"
}
@test "#121 'none/foo.md' is a path, not a 'none' declaration" {
rd_fixture '- **Research doc:** none/foo.md'
run bash "$SCRIPT" "$TMPDIR/fakerepo/my-skill"
refute_output --partial "Basis missing"
}
@test "#121 'none-of-these.md' is a path, not a 'none' declaration" {
rd_fixture '- **Research doc:** none-of-these.md'
run bash "$SCRIPT" "$TMPDIR/fakerepo/my-skill"
refute_output --partial "Basis missing"
}
@test "#121 pass: 'None' and 'NONE' are recognised case-insensitively" {
local v
for v in None NONE; do
make_entry_skill "$TMPDIR/fakerepo" "$(printf '%s\n%s' "- **Research doc:** $v" '- **Basis:** docs/basis.md')"
run bash "$SCRIPT" "$TMPDIR/fakerepo/my-skill"
assert_success
assert_output ""
done
}
@test "#121 FAIL: an empty Basis value is empty, not missing" {
basis_fixture '- **Basis:**'
run bash "$SCRIPT" "$TMPDIR/fakerepo/my-skill"
assert_failure
assert_output --partial "Basis is empty or placeholder"
refute_output --partial "Basis missing"
}
@test "#121 FAIL: a 'FILL IN:' Basis is a placeholder" {
basis_fixture '- **Basis:** FILL IN: repo path'
run bash "$SCRIPT" "$TMPDIR/fakerepo/my-skill"
assert_failure
assert_output --partial "Basis is empty or placeholder"
}
@test "#121 FAIL: an empty inline Research doc says empty, not missing" {
rd_fixture '- **Research doc:**'
run bash "$SCRIPT" "$TMPDIR/fakerepo/my-skill"
assert_failure
assert_output --partial "Research doc field is empty or placeholder"
refute_output --partial "Research doc field missing"
}
@test "#121 FAIL: a missing Research doc advises the new grammar, not '<path-or-(none)>'" {
rd_fixture ''
run bash "$SCRIPT" "$TMPDIR/fakerepo/my-skill"
assert_failure
assert_output --partial "Research doc field missing"
refute_output --partial "path-or-(none)"
assert_output --partial "Basis"
}
@test "#121 parity: an inline Basis with no leading hyphen is read" {
basis_fixture '**Basis:** docs/gone.md'
run bash "$SCRIPT" "$TMPDIR/fakerepo/my-skill"
assert_failure
assert_output --partial "Basis path 'docs/gone.md' does not exist"
refute_output --partial "Basis missing"
}
@test "#121 FAIL: 'a.md;b.md' with no space is a list, for Research doc" {
rd_fixture '- **Research doc:** docs/research/sources.md;docs/basis.md'
run bash "$SCRIPT" "$TMPDIR/fakerepo/my-skill"
assert_failure
assert_output --partial "Research doc names more than one path"
}
@test "#121 FAIL: 'a.md;b.md' with no space is a list, for Basis" {
basis_fixture '- **Basis:** docs/basis.md;docs/other-basis.md'
run bash "$SCRIPT" "$TMPDIR/fakerepo/my-skill"
assert_failure
assert_output --partial "Basis value names more than one path"
}
@test "#121 FAIL: an absolute Basis path is outside the repo even when it points inside the checkout" {
make_entry_skill "$TMPDIR/fakerepo" "$(printf '%s\n%s' '- **Research doc:** none' "- **Basis:** $TMPDIR/fakerepo/docs/basis.md")"
run bash "$SCRIPT" "$TMPDIR/fakerepo/my-skill"
assert_failure
assert_output --partial "outside the repository"
}
@test "#121 FAIL: an absolute Research doc path is outside the repo even when it points inside the checkout" {
rd_fixture "- **Research doc:** $TMPDIR/fakerepo/docs/research/sources.md"
run bash "$SCRIPT" "$TMPDIR/fakerepo/my-skill"
assert_failure
assert_output --partial "outside the repository"
}
@test "#121 parity: a '* **Basis:**' bullet spelling is read" {
make_entry_skill "$TMPDIR/fakerepo" "$(printf '%s\n%s' '- **Research doc:** none' '* **Basis:** docs/gone.md')"
run bash "$SCRIPT" "$TMPDIR/fakerepo/my-skill"
assert_failure
assert_output --partial "Basis path 'docs/gone.md' does not exist"
}
@test "#121 pass: a comma inside a section-marker annotation is prose, not a list" {
rd_fixture '- **Research doc:** docs/research/sources.md § "Foo, bar and baz"'
run bash "$SCRIPT" "$TMPDIR/fakerepo/my-skill"
assert_success
assert_output ""
}