validate-provenance.sh assumed `Research doc:` names a research sources.md whose H2 headings are the source slugs, but 29 corpus entries named topic docs and 6 values were not a single path, so checks 7 and 8 reported INFO for 36 entries and nothing ever failed. `Research doc:` now takes exactly one path. An entry with no registry writes `none` plus one `- **Basis:** <path>` bullet per path; each Basis path is existence-checked unless annotated `(removed in <sha>)`. - Check 7 FAILs when a resolved registry lacks the slug, when the value is a topic doc, or when it is a list. An unresolvable path stays INFO. - Check 8 is retired: one registry serves many skills, so requiring every registry slug in each skill's sources.md is unsatisfiable. - The Research doc and Basis parsers accept the inline, bullet and header-plus-bullets spellings, so a differently spelled field is no longer read as absent. Refs: #121 ADR: 0028 Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01EGHFJextYtVQseaHPDDhxB
1951 lines
69 KiB
Bash
1951 lines
69 KiB
Bash
#!/usr/bin/env bats
|
|
|
|
setup() {
|
|
REPO_ROOT="$(cd "$BATS_TEST_DIRNAME/../../../../../../" && pwd)"
|
|
load "$REPO_ROOT/tests/test_helper/bats-support/load"
|
|
load "$REPO_ROOT/tests/test_helper/bats-assert/load"
|
|
|
|
SCRIPT="$(cd "$BATS_TEST_DIRNAME/../scripts" && pwd)/validate-provenance.sh"
|
|
TMPDIR="$(mktemp -d)"
|
|
|
|
# Helper: create a minimal skill directory with no sources.md and no source_keys
|
|
make_clean_skill() {
|
|
local dir="$1"
|
|
local name
|
|
name="$(basename "$dir")"
|
|
mkdir -p "$dir"
|
|
cat > "$dir/SKILL.md" <<EOF
|
|
---
|
|
name: $name
|
|
description: A valid skill description.
|
|
---
|
|
|
|
## Step 1
|
|
|
|
Do the thing.
|
|
EOF
|
|
}
|
|
|
|
# Helper: create a skill with source_keys in SKILL.md
|
|
make_skill_with_source_keys() {
|
|
local dir="$1"
|
|
local name
|
|
name="$(basename "$dir")"
|
|
mkdir -p "$dir"
|
|
cat > "$dir/SKILL.md" <<EOF
|
|
---
|
|
name: $name
|
|
description: A valid skill description.
|
|
metadata:
|
|
source_keys:
|
|
- my-source
|
|
---
|
|
|
|
## Step 1
|
|
|
|
Do the thing.
|
|
EOF
|
|
}
|
|
|
|
# Helper: create a valid sources.md with one entry
|
|
make_sources_md() {
|
|
local dir="$1"
|
|
local slug="${2:-my-source}"
|
|
local contrib="${3:-SKILL.md}"
|
|
local research="${4:-(none)}"
|
|
# A 'none' Research doc must name its Basis (#121); SKILL.md exists in
|
|
# every fixture skill and, once commit_as_base has run, is a repo path.
|
|
local basis_line=""
|
|
if [[ "$research" == *none* ]]; then basis_line="- **Basis:** SKILL.md"; fi
|
|
mkdir -p "$dir/references"
|
|
cat > "$dir/references/sources.md" <<EOF
|
|
# Sources
|
|
|
|
## ${slug}
|
|
|
|
- **URL:** https://example.com/${slug}
|
|
- **Description:** A test source.
|
|
- **Contributing files:** ${contrib}
|
|
- **Research doc:** ${research}
|
|
${basis_line}
|
|
- **Status:** \`extracted\`
|
|
EOF
|
|
}
|
|
|
|
# Helper: a sources.md whose Description is wrapped across two physical
|
|
# lines — the shape check 9's parser used to truncate at the first newline.
|
|
make_wrapped_sources_md() {
|
|
local dir="$1"
|
|
mkdir -p "$dir/references"
|
|
cat > "$dir/references/sources.md" <<'EOF'
|
|
# Sources
|
|
|
|
## my-source
|
|
|
|
- **URL:** https://example.com/my-source
|
|
- **Description:** A test source, informing the dispatch table's shape with
|
|
no forge-specific content drawn directly from it beyond that.
|
|
- **Contributing files:** SKILL.md
|
|
- **Research doc:** (none)
|
|
- **Basis:** SKILL.md
|
|
- **Status:** `extracted`
|
|
EOF
|
|
}
|
|
|
|
# Helper: turn dir into a real git repo with one commit of its current
|
|
# contents, and a refs/remotes/origin/main pointing at that same commit.
|
|
# Check 9 diffs the skill's references/sources.md against `git merge-base
|
|
# HEAD origin/main` by default; this makes that resolve to a commit whose
|
|
# sources.md is byte-identical to the working tree, so check 9 has
|
|
# nothing to report there — exactly what a real repo looks like the
|
|
# instant after a clean commit. Fixtures that go on to test something
|
|
# else entirely (checks 3, 4, 5, 7, 8...) call this once, at the point
|
|
# their skill's own sources.md is in its final state, so a completely
|
|
# clean run stays completely clean.
|
|
commit_as_base() {
|
|
local dir="$1"
|
|
git -C "$dir" init -q >/dev/null 2>&1
|
|
git -C "$dir" -c user.email=test@example.com -c user.name=test add -A >/dev/null 2>&1
|
|
git -C "$dir" -c user.email=test@example.com -c user.name=test commit -q -m base >/dev/null 2>&1
|
|
git -C "$dir" update-ref refs/remotes/origin/main HEAD >/dev/null 2>&1
|
|
}
|
|
|
|
# Helper: create a fake repo (a real git repo, one commit, makes
|
|
# find_repo_root stop there) holding one skill whose single sources.md
|
|
# slug points at the given Research doc value. Check 7 only runs
|
|
# for a skill inside a checkout, so every upstream case needs this shape;
|
|
# the research doc itself is written per test into
|
|
# "$repo/docs/research/sources.md" — which check 9 does not examine, so
|
|
# a test overwriting it after this helper runs does not disturb check 9.
|
|
make_upstream_skill() {
|
|
local repo="$1"
|
|
local research="${2:-docs/research/sources.md}"
|
|
local basis_line=""
|
|
if [[ "$research" == *none* ]]; then basis_line="- **Basis:** my-skill/SKILL.md"; fi
|
|
local skill="$repo/my-skill"
|
|
mkdir -p "$skill/references" "$repo/docs/research"
|
|
cat > "$skill/SKILL.md" <<EOF
|
|
---
|
|
name: my-skill
|
|
description: A valid skill description.
|
|
metadata:
|
|
source_keys:
|
|
- my-source
|
|
---
|
|
|
|
## Step 1
|
|
|
|
Do the thing.
|
|
EOF
|
|
cat > "$skill/references/sources.md" <<EOF
|
|
# Sources
|
|
|
|
## my-source
|
|
|
|
- **URL:** https://example.com/my-source
|
|
- **Description:** A test source.
|
|
- **Contributing files:** SKILL.md
|
|
- **Research doc:** ${research}
|
|
${basis_line}
|
|
- **Status:** \`extracted\`
|
|
EOF
|
|
commit_as_base "$repo"
|
|
}
|
|
}
|
|
|
|
teardown() {
|
|
rm -rf "$TMPDIR"
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Cycle 1 — --help
|
|
# ---------------------------------------------------------------------------
|
|
|
|
@test "--help exits 0" {
|
|
run bash "$SCRIPT" --help
|
|
assert_success
|
|
assert_output --partial "Usage:"
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Cycle 2 — Early exit: no sources.md, no source_keys → exit 0, no output
|
|
# ---------------------------------------------------------------------------
|
|
|
|
@test "clean pass: no sources.md and no source_keys anywhere → exit 0, no output" {
|
|
local skill="$TMPDIR/my-skill"
|
|
make_clean_skill "$skill"
|
|
run bash "$SCRIPT" "$skill"
|
|
assert_success
|
|
assert_output ""
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Cycle 3 — Check 0: source_keys present but no sources.md → FAIL
|
|
# ---------------------------------------------------------------------------
|
|
|
|
@test "FAIL: source_keys in SKILL.md but sources.md absent" {
|
|
local skill="$TMPDIR/my-skill"
|
|
make_skill_with_source_keys "$skill"
|
|
run bash "$SCRIPT" "$skill"
|
|
assert_failure
|
|
assert_output --partial "FAIL"
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Cycle 4 — Check 1: FILL IN: placeholder in sources.md → FAIL
|
|
# ---------------------------------------------------------------------------
|
|
|
|
@test "FAIL: FILL IN: placeholder in sources.md" {
|
|
local skill="$TMPDIR/my-skill"
|
|
make_skill_with_source_keys "$skill"
|
|
mkdir -p "$skill/references"
|
|
cat > "$skill/references/sources.md" <<EOF
|
|
# Sources
|
|
|
|
## my-source
|
|
|
|
- **URL:** FILL IN: add url
|
|
- **Description:** A test source.
|
|
- **Contributing files:** SKILL.md
|
|
- **Research doc:** (none)
|
|
- **Basis:** SKILL.md
|
|
- **Status:** \`extracted\`
|
|
EOF
|
|
run bash "$SCRIPT" "$skill"
|
|
assert_failure
|
|
assert_output --partial "FAIL"
|
|
}
|
|
|
|
@test "FILL IN: inside backticks in sources.md does not fail" {
|
|
local skill="$TMPDIR/my-skill"
|
|
make_skill_with_source_keys "$skill"
|
|
make_sources_md "$skill"
|
|
echo "Use \`FILL IN: value\` as example." >> "$skill/references/sources.md"
|
|
run bash "$SCRIPT" "$skill"
|
|
assert_success
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Cycle 5 — Check 2: source_keys slug missing from sources.md → FAIL
|
|
# ---------------------------------------------------------------------------
|
|
|
|
@test "FAIL: source_keys slug in SKILL.md not present as H2 in sources.md" {
|
|
local skill="$TMPDIR/my-skill"
|
|
make_skill_with_source_keys "$skill"
|
|
mkdir -p "$skill/references"
|
|
cat > "$skill/references/sources.md" <<EOF
|
|
# Sources
|
|
|
|
## different-source
|
|
|
|
- **URL:** https://example.com/different-source
|
|
- **Description:** A different source.
|
|
- **Contributing files:** SKILL.md
|
|
- **Research doc:** (none)
|
|
- **Basis:** SKILL.md
|
|
- **Status:** \`extracted\`
|
|
EOF
|
|
run bash "$SCRIPT" "$skill"
|
|
assert_failure
|
|
assert_output --partial "FAIL"
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Cycle 6 — Check 4: Contributing file path doesn't exist → FAIL
|
|
# ---------------------------------------------------------------------------
|
|
|
|
@test "FAIL: Contributing file listed in sources.md does not exist" {
|
|
local skill="$TMPDIR/my-skill"
|
|
make_skill_with_source_keys "$skill"
|
|
make_sources_md "$skill" "my-source" "references/nonexistent.md"
|
|
run bash "$SCRIPT" "$skill"
|
|
assert_failure
|
|
assert_output --partial "FAIL"
|
|
}
|
|
|
|
@test "pass: (none) in Contributing files is skipped" {
|
|
local skill="$TMPDIR/my-skill"
|
|
make_skill_with_source_keys "$skill"
|
|
make_sources_md "$skill" "my-source" "(none — not used directly)"
|
|
run bash "$SCRIPT" "$skill"
|
|
assert_success
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Cycle 7 — Check 6: Research doc field missing → FAIL
|
|
# ---------------------------------------------------------------------------
|
|
|
|
@test "FAIL: Research doc field missing from sources.md entry" {
|
|
local skill="$TMPDIR/my-skill"
|
|
make_skill_with_source_keys "$skill"
|
|
mkdir -p "$skill/references"
|
|
cat > "$skill/references/sources.md" <<EOF
|
|
# Sources
|
|
|
|
## my-source
|
|
|
|
- **URL:** https://example.com/my-source
|
|
- **Description:** A test source.
|
|
- **Contributing files:** SKILL.md
|
|
- **Status:** \`extracted\`
|
|
EOF
|
|
run bash "$SCRIPT" "$skill"
|
|
assert_failure
|
|
assert_output --partial "FAIL"
|
|
}
|
|
|
|
@test "FAIL: Research doc field is FILL IN: placeholder" {
|
|
local skill="$TMPDIR/my-skill"
|
|
make_skill_with_source_keys "$skill"
|
|
mkdir -p "$skill/references"
|
|
cat > "$skill/references/sources.md" <<EOF
|
|
# Sources
|
|
|
|
## my-source
|
|
|
|
- **URL:** https://example.com/my-source
|
|
- **Description:** A test source.
|
|
- **Contributing files:** SKILL.md
|
|
- **Research doc:** FILL IN: path to research doc
|
|
- **Status:** \`extracted\`
|
|
EOF
|
|
run bash "$SCRIPT" "$skill"
|
|
assert_failure
|
|
assert_output --partial "FAIL"
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Cycle 8 — Check 5: Bidirectional mismatch → FAIL
|
|
# ---------------------------------------------------------------------------
|
|
|
|
@test "FAIL: Contributing file exists but does not list parent slug in source_keys" {
|
|
local skill="$TMPDIR/my-skill"
|
|
make_skill_with_source_keys "$skill"
|
|
make_sources_md "$skill" "my-source" "SKILL.md"
|
|
# SKILL.md has source_keys: my-source, but let's change it to NOT have my-source
|
|
cat > "$skill/SKILL.md" <<EOF
|
|
---
|
|
name: my-skill
|
|
description: A valid skill description.
|
|
metadata:
|
|
source_keys:
|
|
- other-source
|
|
---
|
|
|
|
## Step 1
|
|
|
|
Do the thing.
|
|
EOF
|
|
mkdir -p "$skill/references"
|
|
cat > "$skill/references/sources.md" <<EOF
|
|
# Sources
|
|
|
|
## other-source
|
|
|
|
- **URL:** https://example.com/other-source
|
|
- **Description:** A test source.
|
|
- **Contributing files:** SKILL.md
|
|
- **Research doc:** (none)
|
|
- **Basis:** SKILL.md
|
|
- **Status:** \`extracted\`
|
|
EOF
|
|
# Now add my-source that references SKILL.md but SKILL.md doesn't back-reference it
|
|
cat >> "$skill/references/sources.md" <<EOF
|
|
|
|
## my-source
|
|
|
|
- **URL:** https://example.com/my-source
|
|
- **Description:** Another source.
|
|
- **Contributing files:** SKILL.md
|
|
- **Research doc:** (none)
|
|
- **Basis:** SKILL.md
|
|
- **Status:** \`extracted\`
|
|
EOF
|
|
run bash "$SCRIPT" "$skill"
|
|
assert_failure
|
|
assert_output --partial "FAIL"
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Cycle 9 — Check 3: references/*.md with no source_keys → INFO (exit 0)
|
|
# ---------------------------------------------------------------------------
|
|
|
|
@test "INFO: references doc with no source_keys frontmatter emits INFO but exits 0" {
|
|
local skill="$TMPDIR/my-skill"
|
|
make_skill_with_source_keys "$skill"
|
|
make_sources_md "$skill"
|
|
mkdir -p "$skill/references"
|
|
cat > "$skill/references/extra.md" <<EOF
|
|
# Extra Reference
|
|
|
|
No frontmatter here.
|
|
EOF
|
|
run bash "$SCRIPT" "$skill"
|
|
assert_success
|
|
assert_output --partial "INFO"
|
|
}
|
|
|
|
@test "pass: references doc with source_keys all matching sources.md exits 0" {
|
|
local skill="$TMPDIR/my-skill"
|
|
make_skill_with_source_keys "$skill"
|
|
make_sources_md "$skill"
|
|
mkdir -p "$skill/references"
|
|
cat > "$skill/references/extra.md" <<EOF
|
|
---
|
|
source_keys:
|
|
- my-source
|
|
---
|
|
|
|
# Extra Reference
|
|
|
|
Content here.
|
|
EOF
|
|
run bash "$SCRIPT" "$skill"
|
|
assert_success
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Cycle 10 — Clean full pass: valid sources.md, all source_keys match, files exist
|
|
# ---------------------------------------------------------------------------
|
|
|
|
@test "clean full pass: all checks satisfied" {
|
|
local skill="$TMPDIR/my-skill"
|
|
make_skill_with_source_keys "$skill"
|
|
make_sources_md "$skill"
|
|
run bash "$SCRIPT" "$skill"
|
|
assert_success
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Cycle 11 — Check 7: Upstream forward: slug in sources.md not in research doc → INFO
|
|
# ---------------------------------------------------------------------------
|
|
|
|
@test "FAIL: slug in sources.md not found in the research registry → FAIL (was INFO before #121)" {
|
|
local skill="$TMPDIR/my-skill"
|
|
make_skill_with_source_keys "$skill"
|
|
|
|
# Create a research doc that does NOT have the slug
|
|
local research_dir="$TMPDIR/research"
|
|
mkdir -p "$research_dir"
|
|
cat > "$research_dir/sources.md" <<EOF
|
|
# Research
|
|
|
|
## different-slug
|
|
|
|
- **Contributing files:** (none)
|
|
- **Status:** \`extracted\`
|
|
EOF
|
|
|
|
# Use a path relative to repo root — we'll place research doc inside TMPDIR
|
|
# and reference it as absolute for test purposes.
|
|
# The script finds repo root by walking up from skill-dir until .git is found.
|
|
# Since TMPDIR won't have .git, we simulate a repo structure.
|
|
local fake_repo="$TMPDIR/fakerepo"
|
|
mkdir -p "$fake_repo"
|
|
touch "$fake_repo/.git" # fake .git marker
|
|
|
|
local skill2="$fake_repo/my-skill"
|
|
mkdir -p "$skill2"
|
|
cat > "$skill2/SKILL.md" <<EOF
|
|
---
|
|
name: my-skill
|
|
description: A valid skill description.
|
|
metadata:
|
|
source_keys:
|
|
- my-source
|
|
---
|
|
|
|
## Step 1
|
|
|
|
Do the thing.
|
|
EOF
|
|
|
|
mkdir -p "$skill2/references"
|
|
mkdir -p "$fake_repo/docs/research"
|
|
cat > "$fake_repo/docs/research/sources.md" <<EOF
|
|
# Research
|
|
|
|
## different-slug
|
|
|
|
- **Contributing files:** (none)
|
|
- **Status:** \`extracted\`
|
|
EOF
|
|
|
|
cat > "$skill2/references/sources.md" <<EOF
|
|
# Sources
|
|
|
|
## my-source
|
|
|
|
- **URL:** https://example.com/my-source
|
|
- **Description:** A test source.
|
|
- **Contributing files:** SKILL.md
|
|
- **Research doc:** docs/research/sources.md
|
|
- **Status:** \`extracted\`
|
|
EOF
|
|
|
|
run bash "$SCRIPT" "$skill2"
|
|
assert_failure
|
|
assert_output --partial "FAIL"
|
|
assert_output --partial "Slug 'my-source' not found as H2 in research doc 'docs/research/sources.md'"
|
|
}
|
|
|
|
@test "check 4 runs: bullet form with '-' hyphen bullets still parses each path" {
|
|
local skill="$TMPDIR/my-skill"
|
|
make_skill_with_source_keys "$skill"
|
|
mkdir -p "$skill/references"
|
|
cat > "$skill/references/sources.md" <<EOF
|
|
# Sources
|
|
|
|
## my-source
|
|
|
|
- **URL:** https://example.com/my-source
|
|
- **Description:** A test source.
|
|
|
|
**Contributing files:**
|
|
- SKILL.md (the dispatch table, with a comma in the note)
|
|
- references/nonexistent.md (why this one matters)
|
|
|
|
- **Research doc:** (none)
|
|
- **Basis:** SKILL.md
|
|
- **Status:** \`extracted\`
|
|
EOF
|
|
run bash "$SCRIPT" "$skill"
|
|
assert_failure
|
|
assert_output --partial "Contributing file 'references/nonexistent.md' does not exist"
|
|
}
|
|
|
|
@test "check 4 runs: inline comma-separated form still parses each path" {
|
|
local skill="$TMPDIR/my-skill"
|
|
make_skill_with_source_keys "$skill"
|
|
make_sources_md "$skill" "my-source" "SKILL.md, references/nonexistent.md"
|
|
run bash "$SCRIPT" "$skill"
|
|
assert_failure
|
|
assert_output --partial "Contributing file 'references/nonexistent.md' does not exist"
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Cycle 14 — Check 3 (#111): an explicit 'source_keys: []' is a house-authored
|
|
# declaration, a bare 'source_keys:' is truncation
|
|
#
|
|
# Three states, three outcomes. Only the middle one — declared empty — is
|
|
# silent; collapsing any pair of them is the defect #111 filed.
|
|
# ---------------------------------------------------------------------------
|
|
|
|
@test "check 3 silent: references doc with top-level 'source_keys: []' emits nothing" {
|
|
local skill="$TMPDIR/my-skill"
|
|
make_skill_with_source_keys "$skill"
|
|
make_sources_md "$skill"
|
|
cat > "$skill/references/extra.md" <<EOF
|
|
---
|
|
source_keys: []
|
|
---
|
|
|
|
# Extra Reference
|
|
|
|
House-authored, no external source.
|
|
EOF
|
|
commit_as_base "$skill"
|
|
run bash "$SCRIPT" "$skill"
|
|
assert_success
|
|
assert_output ""
|
|
}
|
|
|
|
@test "check 3 silent: references doc with 'source_keys: []' nested under metadata: emits nothing" {
|
|
local skill="$TMPDIR/my-skill"
|
|
make_skill_with_source_keys "$skill"
|
|
make_sources_md "$skill"
|
|
cat > "$skill/references/extra.md" <<EOF
|
|
---
|
|
metadata:
|
|
source_keys: []
|
|
---
|
|
|
|
# Extra Reference
|
|
|
|
House-authored, no external source.
|
|
EOF
|
|
commit_as_base "$skill"
|
|
run bash "$SCRIPT" "$skill"
|
|
assert_success
|
|
assert_output ""
|
|
}
|
|
|
|
@test "INFO: references doc with a bare 'source_keys:' and no value still emits INFO" {
|
|
local skill="$TMPDIR/my-skill"
|
|
make_skill_with_source_keys "$skill"
|
|
make_sources_md "$skill"
|
|
cat > "$skill/references/extra.md" <<EOF
|
|
---
|
|
source_keys:
|
|
---
|
|
|
|
# Extra Reference
|
|
|
|
Truncated frontmatter — this is not a decision.
|
|
EOF
|
|
run bash "$SCRIPT" "$skill"
|
|
assert_success
|
|
assert_output --partial "INFO"
|
|
assert_output --partial "No source_keys frontmatter"
|
|
}
|
|
|
|
@test "INFO: references doc with frontmatter but no source_keys key still emits INFO" {
|
|
local skill="$TMPDIR/my-skill"
|
|
make_skill_with_source_keys "$skill"
|
|
make_sources_md "$skill"
|
|
cat > "$skill/references/extra.md" <<EOF
|
|
---
|
|
title: Extra Reference
|
|
---
|
|
|
|
# Extra Reference
|
|
|
|
Never said either way.
|
|
EOF
|
|
run bash "$SCRIPT" "$skill"
|
|
assert_success
|
|
assert_output --partial "INFO"
|
|
assert_output --partial "No source_keys frontmatter"
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Cycle 15 — Check 7: Research doc annotation stripping, and the INFO
|
|
# that replaced the silent skip
|
|
#
|
|
# A Research doc value is very often a path PLUS a section annotation, and
|
|
# os.path.isfile() is false for every such string. Before the strip, checks 7
|
|
# and 8 skipped SILENTLY on those — so the negative assertions here (an INFO
|
|
# that must appear) are what distinguishes a running check from a dead one.
|
|
# ---------------------------------------------------------------------------
|
|
|
|
@test "check 7 runs: '§' section annotation is stripped before the path is resolved" {
|
|
local fake_repo="$TMPDIR/fakerepo"
|
|
make_upstream_skill "$fake_repo" 'docs/research/sources.md § "Some Section"'
|
|
|
|
cat > "$fake_repo/docs/research/sources.md" <<EOF
|
|
# Research
|
|
|
|
## different-slug
|
|
|
|
- **Contributing files:** (none)
|
|
- **Status:** \`extracted\`
|
|
EOF
|
|
|
|
run bash "$SCRIPT" "$fake_repo/my-skill"
|
|
assert_failure
|
|
assert_output --partial "Slug 'my-source' not found as H2 in research doc 'docs/research/sources.md'"
|
|
refute_output --partial "§"
|
|
}
|
|
|
|
@test "check 7 runs: '→' section annotation is stripped before the path is resolved" {
|
|
local fake_repo="$TMPDIR/fakerepo"
|
|
make_upstream_skill "$fake_repo" 'docs/research/sources.md → `## Pushing`'
|
|
|
|
cat > "$fake_repo/docs/research/sources.md" <<EOF
|
|
# Research
|
|
|
|
## different-slug
|
|
|
|
- **Contributing files:** (none)
|
|
- **Status:** \`extracted\`
|
|
EOF
|
|
|
|
run bash "$SCRIPT" "$fake_repo/my-skill"
|
|
assert_failure
|
|
assert_output --partial "Slug 'my-source' not found as H2 in research doc 'docs/research/sources.md'"
|
|
refute_output --partial "→"
|
|
}
|
|
|
|
@test "check 7 runs: parenthetical annotation is stripped before the path is resolved" {
|
|
local fake_repo="$TMPDIR/fakerepo"
|
|
make_upstream_skill "$fake_repo" "docs/research/sources.md (whole-document reference)"
|
|
|
|
cat > "$fake_repo/docs/research/sources.md" <<EOF
|
|
# Research
|
|
|
|
## different-slug
|
|
|
|
- **Contributing files:** (none)
|
|
- **Status:** \`extracted\`
|
|
EOF
|
|
|
|
run bash "$SCRIPT" "$fake_repo/my-skill"
|
|
assert_failure
|
|
assert_output --partial "Slug 'my-source' not found as H2 in research doc 'docs/research/sources.md'"
|
|
refute_output --partial "whole-document reference"
|
|
}
|
|
|
|
@test "check 7 runs: a bare path with no annotation survives the strip intact" {
|
|
local fake_repo="$TMPDIR/fakerepo"
|
|
make_upstream_skill "$fake_repo" "docs/research/sources.md"
|
|
|
|
cat > "$fake_repo/docs/research/sources.md" <<EOF
|
|
# Research
|
|
|
|
## my-source
|
|
|
|
- **Contributing files:** (none)
|
|
- **Status:** \`extracted\`
|
|
EOF
|
|
|
|
run bash "$SCRIPT" "$fake_repo/my-skill"
|
|
assert_success
|
|
assert_output ""
|
|
}
|
|
|
|
@test "check 7 skipped silently: Research doc '(none)' is recognised before the strip" {
|
|
local fake_repo="$TMPDIR/fakerepo"
|
|
make_upstream_skill "$fake_repo" "(none)"
|
|
|
|
run bash "$SCRIPT" "$fake_repo/my-skill"
|
|
assert_success
|
|
assert_output ""
|
|
}
|
|
|
|
@test "check 7 skipped silently: bare 'none — reason' is recognised as a declaration" {
|
|
local fake_repo="$TMPDIR/fakerepo"
|
|
make_upstream_skill "$fake_repo" "none — org convention, no upstream research doc"
|
|
|
|
run bash "$SCRIPT" "$fake_repo/my-skill"
|
|
assert_success
|
|
assert_output ""
|
|
}
|
|
|
|
@test "INFO: a stripped path that does not resolve names the slug instead of skipping silently" {
|
|
local fake_repo="$TMPDIR/fakerepo"
|
|
make_upstream_skill "$fake_repo" 'docs/research/missing.md § "Some Section"'
|
|
|
|
run bash "$SCRIPT" "$fake_repo/my-skill"
|
|
assert_success
|
|
assert_output --partial "INFO"
|
|
assert_output --partial "Upstream checks skipped for 'my-source' — research doc 'docs/research/missing.md' does not exist"
|
|
}
|
|
|
|
@test "INFO: a Research doc value that is entirely annotation names the slug instead of skipping silently" {
|
|
local fake_repo="$TMPDIR/fakerepo"
|
|
make_upstream_skill "$fake_repo" '§ "Some Section"'
|
|
|
|
run bash "$SCRIPT" "$fake_repo/my-skill"
|
|
assert_success
|
|
assert_output --partial "INFO"
|
|
assert_output --partial "Upstream checks skipped for 'my-source' — Research doc value names no path"
|
|
}
|
|
|
|
@test "INFO: no repo root above the skill directory names the slug instead of skipping silently" {
|
|
local skill="$TMPDIR/my-skill"
|
|
make_skill_with_source_keys "$skill"
|
|
make_sources_md "$skill" "my-source" "SKILL.md" "docs/research/sources.md"
|
|
|
|
run bash "$SCRIPT" "$skill"
|
|
assert_success
|
|
assert_output --partial "INFO"
|
|
assert_output --partial "Upstream checks skipped for 'my-source' — no repo root above the skill directory"
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Cycle 16 — Checks 4 and 5: None ("could not parse") is NOT [] ("explicitly
|
|
# (none)"), on the sources.md side this time
|
|
#
|
|
# Cycle 13 pinned the distinction for check 8 (since retired), which read the parser's output
|
|
# against a RESEARCH doc. Checks 4 and 5 read it against the skill's own
|
|
# sources.md and honoured neither half: a truthiness test collapsed None into
|
|
# [], so an unreadable Contributing files block disabled both checks and the
|
|
# script still exited 0 with no output — the failure mode
|
|
# parse_contributing_files' docstring names in as many words. The assertions
|
|
# below are therefore about the INFO appearing; a silent exit 0 is exactly the
|
|
# bug.
|
|
# ---------------------------------------------------------------------------
|
|
|
|
@test "INFO: an unparsable Contributing files block names the slug instead of skipping checks 4 and 5 silently" {
|
|
local skill="$TMPDIR/my-skill"
|
|
make_skill_with_source_keys "$skill"
|
|
mkdir -p "$skill/references"
|
|
cat > "$skill/references/sources.md" <<EOF
|
|
# Sources
|
|
|
|
## my-source
|
|
|
|
- **URL:** https://example.com/my-source
|
|
- **Description:** A test source.
|
|
- **Research doc:** (none)
|
|
- **Basis:** SKILL.md
|
|
**Contributing files:**
|
|
* references/ghost.md (asterisk bullets are not the bullet form)
|
|
- **Status:** \`extracted\`
|
|
EOF
|
|
run bash "$SCRIPT" "$skill"
|
|
assert_success
|
|
assert_output --partial "INFO"
|
|
assert_output --partial "Contributing-file checks skipped for 'my-source' — the Contributing files block could not be parsed"
|
|
}
|
|
|
|
@test "INFO: an entry with no Contributing files field at all is reported, not skipped silently" {
|
|
local skill="$TMPDIR/my-skill"
|
|
make_skill_with_source_keys "$skill"
|
|
mkdir -p "$skill/references"
|
|
cat > "$skill/references/sources.md" <<EOF
|
|
# Sources
|
|
|
|
## my-source
|
|
|
|
- **URL:** https://example.com/my-source
|
|
- **Description:** A test source.
|
|
- **Research doc:** (none)
|
|
- **Basis:** SKILL.md
|
|
- **Status:** \`extracted\`
|
|
EOF
|
|
run bash "$SCRIPT" "$skill"
|
|
assert_success
|
|
assert_output --partial "INFO"
|
|
assert_output --partial "Contributing-file checks skipped for 'my-source' — the Contributing files block could not be parsed"
|
|
}
|
|
|
|
@test "checks 4 and 5 skipped silently: an explicit '(none)' emits no INFO" {
|
|
local skill="$TMPDIR/my-skill"
|
|
make_skill_with_source_keys "$skill"
|
|
make_sources_md "$skill" "my-source" "(none — not used directly)"
|
|
commit_as_base "$skill"
|
|
run bash "$SCRIPT" "$skill"
|
|
assert_success
|
|
assert_output ""
|
|
}
|
|
|
|
@test "checks 4 and 5 still run: a parseable Contributing files list is not diverted to the INFO" {
|
|
local skill="$TMPDIR/my-skill"
|
|
make_skill_with_source_keys "$skill"
|
|
make_sources_md "$skill" "my-source" "references/nonexistent.md"
|
|
run bash "$SCRIPT" "$skill"
|
|
assert_failure
|
|
assert_output --partial "FAIL"
|
|
assert_output --partial "Contributing file 'references/nonexistent.md' does not exist"
|
|
refute_output --partial "could not be parsed"
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Cycle 17 — Check 3 (#111): 'source_keys: []' only declares anything in the
|
|
# two positions parse_source_keys() actually reads
|
|
#
|
|
# The declaration and the parse have to agree on WHERE the key lives. They did
|
|
# not: the declaration regex accepted any indent, so a `source_keys: []` buried
|
|
# under an unrelated key — a position parse_source_keys() never reads — passed
|
|
# as a house-authored declaration and silenced the INFO for a file that had
|
|
# declared nothing.
|
|
# ---------------------------------------------------------------------------
|
|
|
|
@test "INFO: 'source_keys: []' nested under an unrelated key is not a declaration" {
|
|
local skill="$TMPDIR/my-skill"
|
|
make_skill_with_source_keys "$skill"
|
|
make_sources_md "$skill"
|
|
cat > "$skill/references/extra.md" <<EOF
|
|
---
|
|
title: Extra Reference
|
|
unrelated:
|
|
nested:
|
|
source_keys: []
|
|
---
|
|
|
|
# Extra Reference
|
|
|
|
The empty list is nested where parse_source_keys never looks.
|
|
EOF
|
|
run bash "$SCRIPT" "$skill"
|
|
assert_success
|
|
assert_output --partial "INFO"
|
|
assert_output --partial "No source_keys frontmatter"
|
|
}
|
|
|
|
@test "INFO: a four-space-indented 'source_keys: []' is not a declaration either" {
|
|
local skill="$TMPDIR/my-skill"
|
|
make_skill_with_source_keys "$skill"
|
|
make_sources_md "$skill"
|
|
cat > "$skill/references/extra.md" <<EOF
|
|
---
|
|
metadata:
|
|
source_keys: []
|
|
---
|
|
|
|
# Extra Reference
|
|
|
|
Two spaces is the position parse_source_keys reads under metadata:, not four.
|
|
EOF
|
|
run bash "$SCRIPT" "$skill"
|
|
assert_success
|
|
assert_output --partial "INFO"
|
|
assert_output --partial "No source_keys frontmatter"
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Cycle 20 — F: check 7 applies only to a Research registry, and
|
|
# every skip announces itself
|
|
# ---------------------------------------------------------------------------
|
|
|
|
@test "F: a topic-doc Research doc is a FAIL naming the registry, not a missing slug (#121)" {
|
|
local fake_repo="$TMPDIR/fakerepo"
|
|
make_upstream_skill "$fake_repo" "docs/research/remotes.md"
|
|
cat > "$fake_repo/docs/research/remotes.md" <<EOF
|
|
# Remotes
|
|
|
|
## Core Philosophy
|
|
|
|
Prose about remotes.
|
|
EOF
|
|
|
|
run bash "$SCRIPT" "$fake_repo/my-skill"
|
|
assert_failure
|
|
assert_output --partial "is a topic document, not a Research registry"
|
|
refute_output --partial "not found as H2 in research doc"
|
|
}
|
|
|
|
@test "F: check 7 still runs when the Research doc IS a source index" {
|
|
local fake_repo="$TMPDIR/fakerepo"
|
|
make_upstream_skill "$fake_repo"
|
|
cat > "$fake_repo/docs/research/sources.md" <<EOF
|
|
# Sources
|
|
|
|
## different-slug
|
|
|
|
- **Contributing files:** (none)
|
|
- **Status:** \`extracted\`
|
|
EOF
|
|
|
|
run bash "$SCRIPT" "$fake_repo/my-skill"
|
|
assert_failure
|
|
assert_output --partial "Slug 'my-source' not found as H2 in research doc 'docs/research/sources.md'"
|
|
refute_output --partial "not applicable"
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Cycle 21 — G1: a bad target is a hard error, not a silent pass
|
|
#
|
|
# PORT NOTE RESOLVED (factory-audit merge): this block used to assert
|
|
# "not a directory" for the nonexistent target while the agent suite asserted
|
|
# "no such file" for its own, because each pre-merge script knew what shape it
|
|
# was owed. The merged entry point classifies before it complains, so neither
|
|
# old wording survives at THIS site and the two cases are no longer the same
|
|
# case:
|
|
#
|
|
# * a path that exists as nothing this script recognises (and a path that does
|
|
# not exist at all, when its name is not *.agent.md and its parent is not
|
|
# agents/) is "matches neither a skill directory nor an agent file";
|
|
# * a path that IS a directory but holds no SKILL.md gets its own, more
|
|
# specific line naming the missing SKILL.md.
|
|
#
|
|
# The agent suite's "no such file" wording did NOT move — a nonexistent
|
|
# *.agent.md path still classifies as an agent and is rejected by agent mode's
|
|
# own precondition — so that side stays verbatim over there. Both wordings are
|
|
# now asserted exactly rather than partially-matched, because the invariant both
|
|
# sides pin is exit 2 with a message that names the path, never a silent exit 0
|
|
# and never a bare code with no output.
|
|
# ---------------------------------------------------------------------------
|
|
|
|
@test "G1: a nonexistent directory is a hard error (exit 2), not a silent exit 0" {
|
|
run bash "$SCRIPT" "$TMPDIR/does-not-exist"
|
|
[ "$status" -eq 2 ]
|
|
assert_output --partial "Error: '$TMPDIR/does-not-exist' matches neither a skill directory nor an agent file."
|
|
}
|
|
|
|
@test "G1: a directory with no SKILL.md is a hard error (exit 2), not a silent exit 0" {
|
|
# Distinct from the case above on purpose: a directory IS half of a skill
|
|
# target, so the detector can say something sharper than "neither shape" and
|
|
# does. The exit code is unchanged from the pre-merge test, which already
|
|
# pinned 2; only the wording moved, from the suite's own "not a skill
|
|
# directory" precondition to the dispatcher's classification message.
|
|
mkdir -p "$TMPDIR/not-a-skill/references"
|
|
run bash "$SCRIPT" "$TMPDIR/not-a-skill"
|
|
[ "$status" -eq 2 ]
|
|
assert_output --partial "Error: '$TMPDIR/not-a-skill' is a directory with no SKILL.md in it."
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Cycle 22 — G2: a UTF-8 BOM does not disable check 2
|
|
# ---------------------------------------------------------------------------
|
|
|
|
@test "G2: a BOM at the head of SKILL.md does not silently disable check 2" {
|
|
local skill="$TMPDIR/my-skill"
|
|
mkdir -p "$skill/references"
|
|
printf '\xef\xbb\xbf' > "$skill/SKILL.md"
|
|
cat >> "$skill/SKILL.md" <<EOF
|
|
---
|
|
name: my-skill
|
|
description: A valid skill description.
|
|
metadata:
|
|
source_keys:
|
|
- my-source
|
|
---
|
|
|
|
## Step 1
|
|
|
|
Do the thing.
|
|
EOF
|
|
cat > "$skill/references/sources.md" <<EOF
|
|
# Sources
|
|
|
|
## different-source
|
|
|
|
- **URL:** https://example.com/different-source
|
|
- **Description:** A test source.
|
|
- **Contributing files:** references/sources.md
|
|
- **Research doc:** (none)
|
|
- **Basis:** SKILL.md
|
|
- **Status:** \`extracted\`
|
|
EOF
|
|
|
|
run bash "$SCRIPT" "$skill"
|
|
assert_failure
|
|
assert_output --partial "source_keys slug 'my-source' not found in sources.md"
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Cycle 23 — G3: reads are UTF-8 and an unreadable file is a finding
|
|
# ---------------------------------------------------------------------------
|
|
|
|
@test "G3: an em dash under LC_ALL=C is read, not turned into a traceback or a silent pass" {
|
|
local skill="$TMPDIR/my-skill"
|
|
make_skill_with_source_keys "$skill"
|
|
make_sources_md "$skill"
|
|
cat > "$skill/references/topic.md" <<EOF
|
|
---
|
|
source_keys:
|
|
- ghost-source
|
|
---
|
|
|
|
Prose — with an em dash.
|
|
EOF
|
|
|
|
LC_ALL=C PYTHONUTF8=0 PYTHONCOERCECLOCALE=0 run bash "$SCRIPT" "$skill"
|
|
assert_failure
|
|
assert_output --partial "source_keys slug 'ghost-source' not found in sources.md"
|
|
refute_output --partial "Traceback"
|
|
}
|
|
|
|
@test "G3: a file that is genuinely not UTF-8 is a FAIL, not a clean pass" {
|
|
local skill="$TMPDIR/my-skill"
|
|
make_skill_with_source_keys "$skill"
|
|
make_sources_md "$skill"
|
|
printf -- '---\nsource_keys:\n - my-source\n---\n\nLatin-1 byte: \xe9\n' \
|
|
> "$skill/references/topic.md"
|
|
|
|
run bash "$SCRIPT" "$skill"
|
|
assert_failure
|
|
assert_output --partial "not valid UTF-8"
|
|
assert_output --partial "references/topic.md"
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Cycle 24 — G4: check 3 walks references/ recursively
|
|
# ---------------------------------------------------------------------------
|
|
|
|
@test "G4: a source_keys file in references/<subdir>/ is validated, not skipped" {
|
|
local skill="$TMPDIR/my-skill"
|
|
make_skill_with_source_keys "$skill"
|
|
make_sources_md "$skill"
|
|
mkdir -p "$skill/references/nested"
|
|
cat > "$skill/references/nested/topic.md" <<EOF
|
|
---
|
|
source_keys:
|
|
- ghost-source
|
|
---
|
|
|
|
Nested prose.
|
|
EOF
|
|
|
|
run bash "$SCRIPT" "$skill"
|
|
assert_failure
|
|
assert_output --partial "source_keys slug 'ghost-source' not found in sources.md"
|
|
assert_output --partial "references/nested/topic.md"
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Cycle 25 — G5: a placeholder at end of line is still a placeholder
|
|
# ---------------------------------------------------------------------------
|
|
|
|
@test "G5: 'FILL IN:' at end of line is caught by check 1" {
|
|
local skill="$TMPDIR/my-skill"
|
|
make_skill_with_source_keys "$skill"
|
|
mkdir -p "$skill/references"
|
|
cat > "$skill/references/sources.md" <<EOF
|
|
# Sources
|
|
|
|
## my-source
|
|
|
|
- **URL:** https://example.com/my-source
|
|
- **Description:** FILL IN:
|
|
- **Contributing files:** SKILL.md
|
|
- **Research doc:** (none)
|
|
- **Basis:** SKILL.md
|
|
- **Status:** \`extracted\`
|
|
EOF
|
|
|
|
run bash "$SCRIPT" "$skill"
|
|
assert_failure
|
|
assert_output --partial "Unfilled FILL IN: placeholder"
|
|
}
|
|
|
|
@test "G5: a Research doc value that is a bare 'FILL IN:' is caught by check 6" {
|
|
local skill="$TMPDIR/my-skill"
|
|
make_skill_with_source_keys "$skill"
|
|
make_sources_md "$skill" "my-source" "SKILL.md" "FILL IN:"
|
|
run bash "$SCRIPT" "$skill"
|
|
assert_failure
|
|
assert_output --partial "Research doc field is empty or placeholder"
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Cycle 26 — G6/G7: duplicated entries and duplicated fields are announced
|
|
# ---------------------------------------------------------------------------
|
|
|
|
@test "G6: a duplicate '## slug' block is announced, not half-checked in silence" {
|
|
local skill="$TMPDIR/my-skill"
|
|
make_skill_with_source_keys "$skill"
|
|
mkdir -p "$skill/references"
|
|
cat > "$skill/references/sources.md" <<EOF
|
|
# Sources
|
|
|
|
## my-source
|
|
|
|
- **URL:** https://example.com/my-source
|
|
- **Description:** A test source.
|
|
- **Contributing files:** SKILL.md
|
|
- **Research doc:** (none)
|
|
- **Basis:** SKILL.md
|
|
- **Status:** \`extracted\`
|
|
|
|
## my-source
|
|
|
|
- **URL:** https://example.com/my-source-again
|
|
- **Description:** The same slug a second time.
|
|
- **Contributing files:** references/nonexistent.md
|
|
- **Research doc:** (none)
|
|
- **Basis:** SKILL.md
|
|
- **Status:** \`extracted\`
|
|
EOF
|
|
|
|
run bash "$SCRIPT" "$skill"
|
|
assert_success
|
|
assert_output --partial "Duplicate '## my-source' entry in sources.md"
|
|
assert_output --partial "only the first block is checked"
|
|
}
|
|
|
|
@test "G7: a second '- **Research doc:**' line in one entry FAILs (Research doc is single-valued, #121)" {
|
|
local skill="$TMPDIR/my-skill"
|
|
make_skill_with_source_keys "$skill"
|
|
mkdir -p "$skill/references"
|
|
cat > "$skill/references/sources.md" <<EOF
|
|
# Sources
|
|
|
|
## my-source
|
|
|
|
- **URL:** https://example.com/my-source
|
|
- **Description:** A test source.
|
|
- **Contributing files:** SKILL.md
|
|
- **Research doc:** (none)
|
|
- **Basis:** SKILL.md
|
|
- **Research doc:** docs/research/sources.md
|
|
- **Status:** \`extracted\`
|
|
EOF
|
|
|
|
run bash "$SCRIPT" "$skill"
|
|
assert_failure
|
|
assert_output --partial "Multiple '- **Research doc:**' lines for 'my-source'"
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Cycle 27 — G8: usage and environment errors exit 2, never 1
|
|
#
|
|
# PORT NOTE RESOLVED (factory-audit merge): the two suites were written against
|
|
# two separate scripts and disagreed about the no-argument wording — this file
|
|
# asserted "skill-dir is required", the agent suite "agent-file is required".
|
|
# Neither survives. One entry point takes both target shapes, so its no-argument
|
|
# message names both, and both suites now assert that one sentence. Exit 2 and a
|
|
# message (not a silent exit 1, and not a bare code with no output) is the
|
|
# invariant both sides were really pinning, and it is still pinned from both.
|
|
#
|
|
# Note the asymmetry with scripts/validate.sh, which exits 1 on no argument:
|
|
# only validate-provenance.sh carries the exit-2 usage tier, and this test is
|
|
# what holds it there.
|
|
# ---------------------------------------------------------------------------
|
|
|
|
@test "G8: a missing argument exits 2, not 1" {
|
|
run bash "$SCRIPT"
|
|
[ "$status" -eq 2 ]
|
|
assert_output --partial "Error: a skill directory or an agent file is required."
|
|
# The usage block has to follow the error, or "required" names no shape the
|
|
# caller can act on.
|
|
assert_output --partial "Usage: validate-provenance.sh"
|
|
}
|
|
|
|
@test "G8: an extra positional argument is rejected, not silently ignored" {
|
|
local skill="$TMPDIR/my-skill"
|
|
make_clean_skill "$skill"
|
|
run bash "$SCRIPT" "$skill" extra
|
|
[ "$status" -eq 2 ]
|
|
assert_output --partial "expected exactly one argument"
|
|
}
|
|
|
|
@test "G8: a missing python3 is reported by name and exits 2, not 127" {
|
|
local skill="$TMPDIR/my-skill"
|
|
make_clean_skill "$skill"
|
|
local stub="$TMPDIR/emptybin"
|
|
mkdir -p "$stub"
|
|
# dirname and basename are deliberately ABSENT, and that absence is load-
|
|
# bearing. The invariant: nothing external is needed to reach the python3
|
|
# preflight. The entry point resolves SCRIPT_DIR and classifies the target
|
|
# with _kf_dirname/_kf_basename/_kf_parent_name — pure-bash replacements
|
|
# that exist for precisely this reason — and the mode libraries build their
|
|
# Python bodies with `read` heredocs. Widening this list to keep a test
|
|
# green would silently retire that guarantee: on a PATH with neither
|
|
# coreutils nor python3 the script would die at 127 naming `dirname`
|
|
# instead of naming the dependency it actually needs, which is the failure
|
|
# tests/test-adr0020-contract.sh assertion 2 exists to prevent. bash, cat
|
|
# and sed stay so the stub is a DENY OF python3 ALONE rather than a test of
|
|
# "no PATH at all" — the diagnostic under test here is the python3 one.
|
|
for cmd in bash cat sed; do
|
|
ln -s "$(command -v "$cmd")" "$stub/$cmd"
|
|
done
|
|
PATH="$stub" run bash "$SCRIPT" "$skill"
|
|
[ "$status" -eq 2 ]
|
|
assert_output --partial "python3 is required"
|
|
}
|
|
|
|
@test "G3: an unreadable file is reported even when there is nothing else to validate" {
|
|
local skill="$TMPDIR/my-skill"
|
|
make_clean_skill "$skill"
|
|
mkdir -p "$skill/references"
|
|
printf -- '---\nsource_keys:\n - my-source\n---\n\nLatin-1 byte: \xe9\n' \
|
|
> "$skill/references/topic.md"
|
|
|
|
run bash "$SCRIPT" "$skill"
|
|
assert_failure
|
|
assert_output --partial "not valid UTF-8"
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Cycle 28 — Check 9: a changed Description or Contributing files claim is an
|
|
# INFO, never a FAIL — the script can tell the text changed, not whether the
|
|
# (possibly stronger) new wording is still true.
|
|
# ---------------------------------------------------------------------------
|
|
|
|
@test "check 9: Description text changed since base ref fires an INFO naming the slug and field" {
|
|
local skill="$TMPDIR/my-skill"
|
|
make_skill_with_source_keys "$skill"
|
|
make_sources_md "$skill"
|
|
commit_as_base "$skill"
|
|
|
|
# Rewritten in the working tree only, never committed — exactly the
|
|
# shape of the bug check 9 exists to flag: a hedge upgraded to a
|
|
# confident claim with nothing else in the entry touched.
|
|
sed -i 's/^- \*\*Description:\*\* A test source\.$/- **Description:** A test source that Grounds the dispatch table directly./' \
|
|
"$skill/references/sources.md"
|
|
|
|
run bash "$SCRIPT" "$skill"
|
|
assert_success
|
|
assert_output --partial "INFO"
|
|
assert_output --partial "'Description' changed for 'my-source'"
|
|
}
|
|
|
|
@test "check 9: Contributing files text changed since base ref fires an INFO naming the slug and field" {
|
|
local skill="$TMPDIR/my-skill"
|
|
make_skill_with_source_keys "$skill"
|
|
make_sources_md "$skill"
|
|
commit_as_base "$skill"
|
|
|
|
# Only an annotation is added. strip_note() in parse_contributing_files()
|
|
# removes it before checks 4 and 5 compare paths, so those stay clean —
|
|
# check 9 compares the raw field text, not the parsed path list, and
|
|
# this is still a real wording change worth a human re-reading it.
|
|
sed -i 's/^- \*\*Contributing files:\*\* SKILL\.md$/- **Contributing files:** SKILL.md (the dispatch table)/' \
|
|
"$skill/references/sources.md"
|
|
|
|
run bash "$SCRIPT" "$skill"
|
|
assert_success
|
|
assert_output --partial "INFO"
|
|
assert_output --partial "'Contributing files' changed for 'my-source'"
|
|
refute_output --partial "does not exist"
|
|
refute_output --partial "does not list"
|
|
}
|
|
|
|
@test "check 9: an entry unchanged since base ref produces no check-9 finding" {
|
|
local skill="$TMPDIR/my-skill"
|
|
make_skill_with_source_keys "$skill"
|
|
make_sources_md "$skill"
|
|
commit_as_base "$skill"
|
|
|
|
run bash "$SCRIPT" "$skill"
|
|
assert_success
|
|
assert_output ""
|
|
}
|
|
|
|
@test "check 9: a brand-new entry absent at the base ref is a creation, not a change" {
|
|
local skill="$TMPDIR/my-skill"
|
|
make_skill_with_source_keys "$skill"
|
|
make_sources_md "$skill"
|
|
commit_as_base "$skill"
|
|
|
|
# Added in the working tree only, after the commit above. It has no
|
|
# earlier revision to diff against, so check 9 must stay silent about
|
|
# it — a brand-new entry is a creation, not a rewrite of an existing
|
|
# claim, and flagging it would be exactly the false-positive shape the
|
|
# rejected literal-text approaches produced.
|
|
cat >> "$skill/references/sources.md" <<EOF
|
|
|
|
## new-source
|
|
|
|
- **URL:** https://example.com/new-source
|
|
- **Description:** A brand-new source, never committed.
|
|
- **Contributing files:** (none)
|
|
- **Research doc:** (none)
|
|
- **Basis:** SKILL.md
|
|
- **Status:** \`extracted\`
|
|
EOF
|
|
|
|
run bash "$SCRIPT" "$skill"
|
|
assert_success
|
|
assert_output ""
|
|
}
|
|
|
|
@test "check 9: no repo root emits the one graceful INFO, not a silent skip or a crash" {
|
|
local skill="$TMPDIR/my-skill"
|
|
make_skill_with_source_keys "$skill"
|
|
make_sources_md "$skill"
|
|
# Deliberately no commit_as_base — $TMPDIR has no ancestor .git, so
|
|
# find_repo_root() returns None.
|
|
|
|
run bash "$SCRIPT" "$skill"
|
|
assert_success
|
|
assert_output --partial "INFO"
|
|
assert_output --partial "Check 9 skipped — no repo root above the skill directory"
|
|
}
|
|
|
|
@test "check 9: an unresolvable base ref emits the one graceful INFO, not a silent skip or a crash" {
|
|
local skill="$TMPDIR/my-skill"
|
|
make_skill_with_source_keys "$skill"
|
|
make_sources_md "$skill"
|
|
commit_as_base "$skill"
|
|
# A repo with no origin/main and no override: `git merge-base HEAD
|
|
# origin/main` fails outright.
|
|
git -C "$skill" update-ref -d refs/remotes/origin/main >/dev/null 2>&1
|
|
|
|
run bash "$SCRIPT" "$skill"
|
|
assert_success
|
|
assert_output --partial "INFO"
|
|
assert_output --partial "Check 9 skipped — no base ref could be resolved"
|
|
}
|
|
|
|
@test "check 9: --base-ref overrides a default origin/main that resolves to something else" {
|
|
local skill="$TMPDIR/my-skill"
|
|
make_skill_with_source_keys "$skill"
|
|
make_sources_md "$skill"
|
|
commit_as_base "$skill"
|
|
local old_sha
|
|
old_sha="$(git -C "$skill" rev-parse HEAD)"
|
|
|
|
# A SECOND commit carrying the rewritten claim, with origin/main moved onto
|
|
# it. The default base ref therefore resolves — to a commit that matches the
|
|
# working tree — so a silent default run proves there was a default here to
|
|
# override. Deleting origin/main instead, as this test used to, proved only
|
|
# that the flag works when nothing else does.
|
|
sed -i 's/^- \*\*Description:\*\* A test source\.$/- **Description:** A rewritten claim./' \
|
|
"$skill/references/sources.md"
|
|
git -C "$skill" -c user.email=test@example.com -c user.name=test commit -aqm rewrite >/dev/null 2>&1
|
|
git -C "$skill" update-ref refs/remotes/origin/main HEAD >/dev/null 2>&1
|
|
|
|
run bash "$SCRIPT" "$skill"
|
|
assert_success
|
|
assert_output ""
|
|
|
|
run bash "$SCRIPT" "$skill" "--base-ref=$old_sha"
|
|
assert_success
|
|
assert_output --partial "'Description' changed for 'my-source'"
|
|
}
|
|
|
|
@test "check 9: VALIDATE_PROVENANCE_BASE_REF sets the base ref when no flag is given" {
|
|
local skill="$TMPDIR/my-skill"
|
|
make_skill_with_source_keys "$skill"
|
|
make_sources_md "$skill"
|
|
commit_as_base "$skill"
|
|
local old_sha
|
|
old_sha="$(git -C "$skill" rev-parse HEAD)"
|
|
sed -i 's/^- \*\*Description:\*\* A test source\.$/- **Description:** A rewritten claim./' \
|
|
"$skill/references/sources.md"
|
|
git -C "$skill" -c user.email=test@example.com -c user.name=test commit -aqm rewrite >/dev/null 2>&1
|
|
git -C "$skill" update-ref refs/remotes/origin/main HEAD >/dev/null 2>&1
|
|
|
|
run env VALIDATE_PROVENANCE_BASE_REF="$old_sha" bash "$SCRIPT" "$skill"
|
|
assert_success
|
|
assert_output --partial "'Description' changed for 'my-source'"
|
|
}
|
|
|
|
@test "check 9: the --base-ref flag wins over the environment variable" {
|
|
local skill="$TMPDIR/my-skill"
|
|
make_skill_with_source_keys "$skill"
|
|
make_sources_md "$skill"
|
|
commit_as_base "$skill"
|
|
local old_sha
|
|
old_sha="$(git -C "$skill" rev-parse HEAD)"
|
|
sed -i 's/^- \*\*Description:\*\* A test source\.$/- **Description:** A rewritten claim./' \
|
|
"$skill/references/sources.md"
|
|
git -C "$skill" -c user.email=test@example.com -c user.name=test commit -aqm rewrite >/dev/null 2>&1
|
|
local new_sha
|
|
new_sha="$(git -C "$skill" rev-parse HEAD)"
|
|
|
|
# The environment names the old commit (which would fire), the flag names
|
|
# the new one (which would not). The usage text promises the flag wins.
|
|
run env VALIDATE_PROVENANCE_BASE_REF="$old_sha" bash "$SCRIPT" "$skill" "--base-ref=$new_sha"
|
|
assert_success
|
|
assert_output ""
|
|
}
|
|
|
|
@test "check 9: an EMPTY --base-ref is still 'given' and wins over the environment variable" {
|
|
local skill="$TMPDIR/my-skill"
|
|
make_skill_with_source_keys "$skill"
|
|
make_sources_md "$skill"
|
|
commit_as_base "$skill"
|
|
local old_sha
|
|
old_sha="$(git -C "$skill" rev-parse HEAD)"
|
|
sed -i 's/^- \*\*Description:\*\* A test source\.$/- **Description:** A rewritten claim./' \
|
|
"$skill/references/sources.md"
|
|
git -C "$skill" -c user.email=test@example.com -c user.name=test commit -aqm rewrite >/dev/null 2>&1
|
|
git -C "$skill" update-ref refs/remotes/origin/main HEAD >/dev/null 2>&1
|
|
|
|
# `--base-ref=` selects the DEFAULT resolution (origin/main, which now
|
|
# matches the working tree), so nothing fires. Under the old `:-` spelling
|
|
# the empty value read as absent and the environment variable won, firing
|
|
# the INFO and contradicting the usage text.
|
|
run env VALIDATE_PROVENANCE_BASE_REF="$old_sha" bash "$SCRIPT" "$skill" "--base-ref="
|
|
assert_success
|
|
assert_output ""
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Cycle 29 — Check 9: a WRAPPED field value. The parser compared only the first
|
|
# physical line, so a rewrite confined to a continuation line — the exact
|
|
# hedge-to-confident-claim shape #118 exists to catch — produced no finding at
|
|
# all, while a pure re-wrap produced a false one.
|
|
# ---------------------------------------------------------------------------
|
|
|
|
@test "check 9: a rewrite confined to a wrapped Description's CONTINUATION line fires an INFO" {
|
|
local skill="$TMPDIR/my-skill"
|
|
make_skill_with_source_keys "$skill"
|
|
make_wrapped_sources_md "$skill"
|
|
commit_as_base "$skill"
|
|
|
|
# Hedge to confident claim, on the second physical line only. This is the
|
|
# regression check 9 was written for, and the one it could not see.
|
|
sed -i "s|^ no forge-specific content drawn directly from it beyond that\.\$| it grounds Step 2's dispatch table in full.|" \
|
|
"$skill/references/sources.md"
|
|
|
|
run bash "$SCRIPT" "$skill"
|
|
assert_success
|
|
assert_output --partial "'Description' changed for 'my-source'"
|
|
}
|
|
|
|
@test "check 9: re-wrapping a Description with no wording change produces no finding" {
|
|
local skill="$TMPDIR/my-skill"
|
|
make_skill_with_source_keys "$skill"
|
|
make_wrapped_sources_md "$skill"
|
|
commit_as_base "$skill"
|
|
|
|
# Same words, different line breaks. normalize_field_text()'s docstring
|
|
# promises this is invisible; it was not, because the value was truncated
|
|
# at its first newline before the whitespace collapse ever ran.
|
|
python3 - "$skill/references/sources.md" <<'PY'
|
|
import sys
|
|
path = sys.argv[1]
|
|
text = open(path).read()
|
|
old = ("- **Description:** A test source, informing the dispatch table's shape with\n"
|
|
" no forge-specific content drawn directly from it beyond that.")
|
|
new = ("- **Description:** A test source, informing the dispatch\n"
|
|
" table's shape with no forge-specific content drawn\n"
|
|
" directly from it beyond that.")
|
|
assert old in text
|
|
open(path, 'w').write(text.replace(old, new))
|
|
PY
|
|
|
|
run bash "$SCRIPT" "$skill"
|
|
assert_success
|
|
assert_output ""
|
|
}
|
|
|
|
@test "check 9: the bullet form of Contributing files is compared, not skipped" {
|
|
local skill="$TMPDIR/my-skill"
|
|
make_skill_with_source_keys "$skill"
|
|
mkdir -p "$skill/references"
|
|
cat > "$skill/references/sources.md" <<'EOF'
|
|
# Sources
|
|
|
|
## my-source
|
|
|
|
- **URL:** https://example.com/my-source
|
|
- **Description:** A test source.
|
|
|
|
**Contributing files:**
|
|
- SKILL.md (the dispatch table)
|
|
- references/other.md (the rubric)
|
|
|
|
- **Research doc:** (none)
|
|
- **Basis:** SKILL.md
|
|
- **Status:** `extracted`
|
|
EOF
|
|
printf -- '---\nsource_keys:\n - my-source\n---\n\nnotes\n' > "$skill/references/other.md"
|
|
commit_as_base "$skill"
|
|
|
|
# The change is in the SECOND bullet. The old loop joined bullets in
|
|
# document order too, but broke on any wrapped one — and no test covered
|
|
# this branch at all, both existing check-9 tests using the inline form.
|
|
sed -i 's|^- references/other.md (the rubric)$|- references/other.md (the whole rubric, verbatim)|' \
|
|
"$skill/references/sources.md"
|
|
|
|
run bash "$SCRIPT" "$skill"
|
|
assert_success
|
|
assert_output --partial "'Contributing files' changed for 'my-source'"
|
|
}
|
|
|
|
@test "check 9: a wrapped bullet does not silently drop the bullets after it" {
|
|
local skill="$TMPDIR/my-skill"
|
|
make_skill_with_source_keys "$skill"
|
|
mkdir -p "$skill/references"
|
|
cat > "$skill/references/sources.md" <<'EOF'
|
|
# Sources
|
|
|
|
## my-source
|
|
|
|
- **URL:** https://example.com/my-source
|
|
- **Description:** A test source.
|
|
|
|
**Contributing files:**
|
|
- SKILL.md (the dispatch table, and the gates
|
|
common to every branch of it)
|
|
- references/other.md (the rubric)
|
|
|
|
- **Research doc:** (none)
|
|
- **Basis:** SKILL.md
|
|
- **Status:** `extracted`
|
|
EOF
|
|
printf -- '---\nsource_keys:\n - my-source\n---\n\nnotes\n' > "$skill/references/other.md"
|
|
commit_as_base "$skill"
|
|
|
|
# The old loop broke at the wrapped continuation line, so everything from
|
|
# here down was never part of the compared value — a change to the last
|
|
# bullet was invisible.
|
|
sed -i 's|^- references/other.md (the rubric)$|- references/other.md (rewritten claim)|' \
|
|
"$skill/references/sources.md"
|
|
|
|
run bash "$SCRIPT" "$skill"
|
|
assert_success
|
|
assert_output --partial "'Contributing files' changed for 'my-source'"
|
|
}
|
|
|
|
@test "check 9: a field present at the base ref and deleted since is announced" {
|
|
local skill="$TMPDIR/my-skill"
|
|
make_skill_with_source_keys "$skill"
|
|
make_sources_md "$skill"
|
|
commit_as_base "$skill"
|
|
|
|
# No other check in this script requires a Description, so a deleted one
|
|
# used to leave no finding anywhere: a claim could be withdrawn as
|
|
# invisibly as it could be strengthened.
|
|
sed -i '/^- \*\*Description:\*\*/d' "$skill/references/sources.md"
|
|
|
|
run bash "$SCRIPT" "$skill"
|
|
assert_success
|
|
assert_output --partial "'Description' removed for 'my-source'"
|
|
}
|
|
|
|
@test "check 9: a sources.md untracked at the base ref is announced, not silently skipped" {
|
|
local repo="$TMPDIR/repo"
|
|
local skill="$repo/tracked-skill"
|
|
mkdir -p "$skill"
|
|
make_skill_with_source_keys "$skill"
|
|
make_sources_md "$skill"
|
|
sed -i 's#^- \*\*Basis:\*\* SKILL.md#- **Basis:** tracked-skill/SKILL.md#' "$skill/references/sources.md"
|
|
commit_as_base "$repo"
|
|
|
|
# A copy of the same skill at a path git does not know — the everyday case
|
|
# being an installed, gitignored .claude/skills/ tree. The base ref
|
|
# resolves fine; `git show <ref>:<path>` does not. Treating that as
|
|
# "creation, nothing to flag" made the whole check vanish without a word,
|
|
# so the same directory reported findings at one path and silence at the
|
|
# other.
|
|
cp -r "$skill" "$repo/untracked-copy"
|
|
|
|
run bash "$SCRIPT" "$repo/untracked-copy"
|
|
assert_success
|
|
assert_output --partial "INFO"
|
|
assert_output --partial "is not tracked at"
|
|
assert_output --partial "Check 9 did not run for any slug in this skill."
|
|
}
|
|
|
|
@test "check 9: an invalid --base-ref value is reported as unresolvable, not a crash" {
|
|
local skill="$TMPDIR/my-skill"
|
|
make_skill_with_source_keys "$skill"
|
|
make_sources_md "$skill"
|
|
commit_as_base "$skill"
|
|
|
|
run bash "$SCRIPT" "$skill" "--base-ref=not-a-real-ref"
|
|
assert_success
|
|
assert_output --partial "Check 9 skipped — no base ref could be resolved"
|
|
assert_output --partial "not-a-real-ref"
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Auto-detection — the merged provenance entry point classifies its own target
|
|
#
|
|
# NEW with the factory-audit merge, and new behaviour rather than a ported
|
|
# case: scripts/validate-provenance.sh is now ONE entry point for both artifact
|
|
# types and works out from the target which rubric to run. A DIRECTORY holding
|
|
# SKILL.md is a skill; a FILE named *.agent.md, or sitting under .apm/agents/,
|
|
# is an agent.
|
|
#
|
|
# The two rubrics are not near-copies of one another — the skill side has a
|
|
# check 9 the agent side has none of, and reads sources.md from the skill's own
|
|
# references/ rather than from the package root — so a misclassification is not
|
|
# a near miss. It runs a set of checks that cannot apply and skips the set that
|
|
# can, at exit 0.
|
|
#
|
|
# This file is the SKILL half plus the neither-shape rejection;
|
|
# validate-provenance-agent.bats holds the agent half. Same script in both.
|
|
# ---------------------------------------------------------------------------
|
|
|
|
@test "auto-detect: a directory holding SKILL.md is checked in SKILL provenance mode" {
|
|
local skill="$TMPDIR/my-skill"
|
|
make_skill_with_source_keys "$skill"
|
|
make_sources_md "$skill"
|
|
# No commit_as_base, so check 9 has no repo root and announces the skip.
|
|
run bash "$SCRIPT" "$skill"
|
|
assert_success
|
|
# Check 9 exists only on the skill side — the agent rubric has no check 9 at
|
|
# any tier — so naming it is positive proof the SKILL rubric ran, rather
|
|
# than merely that nothing crashed.
|
|
assert_output --partial "Check 9 skipped — no repo root above the skill directory"
|
|
}
|
|
|
|
@test "auto-detect: a SKILL.md FILE path is checked in SKILL provenance mode, not rejected" {
|
|
# NEW with the merge and additive: pre-merge, handing the SKILL.md itself to
|
|
# skill-audit's validate-provenance.sh hit the "not a directory" precondition
|
|
# and died. It matters because pre-commit `files:` hooks match FILES — the
|
|
# vale-audit-prefilter-skill hook's regex ends in /SKILL\.md$ — so
|
|
# every hook-driven invocation hands over a SKILL.md path, never its
|
|
# directory. The entry point rewrites the token to the directory in place.
|
|
local skill="$TMPDIR/my-skill"
|
|
make_skill_with_source_keys "$skill"
|
|
make_sources_md "$skill"
|
|
run bash "$SCRIPT" "$skill/SKILL.md"
|
|
assert_success
|
|
# Same positive proof as the directory case: check 9 is skill-only.
|
|
assert_output --partial "Check 9 skipped — no repo root above the skill directory"
|
|
}
|
|
|
|
@test "auto-detect: a SKILL.md FILE path and its directory produce the same verdict" {
|
|
# The rewrite must be transparent, not merely non-fatal. If the two spellings
|
|
# of the same target could disagree, a pre-commit run and a hand run would
|
|
# report differently on one skill and neither would be obviously wrong.
|
|
local skill="$TMPDIR/my-skill"
|
|
make_skill_with_source_keys "$skill"
|
|
make_sources_md "$skill"
|
|
|
|
run bash "$SCRIPT" "$skill"
|
|
local dir_status="$status"
|
|
local dir_output="$output"
|
|
|
|
run bash "$SCRIPT" "$skill/SKILL.md"
|
|
[ "$status" -eq "$dir_status" ]
|
|
[ "$output" = "$dir_output" ]
|
|
# Guard against the comparison being satisfied by two empty runs: this
|
|
# fixture has a check-9 INFO to print, so silence here means neither
|
|
# spelling ran the rubric.
|
|
refute_output ""
|
|
}
|
|
|
|
@test "exit contract: SKILL mode exits 0 WITH output when the only findings are INFO" {
|
|
# The two modes' exit contracts are DIFFERENT and the merge must not quietly
|
|
# unify them. Skill mode is allowed to be chatty on a passing run: INFO
|
|
# findings print and the status stays 0. Agent mode's opposite half — a
|
|
# passing run prints nothing at all — is pinned from its own side in
|
|
# validate-provenance-agent.bats.
|
|
#
|
|
# Both halves are asserted explicitly and on purpose. A merge that collapsed
|
|
# one contract into the other would still satisfy whichever side was left
|
|
# unasserted, so a single-sided pin would go green on exactly the defect it
|
|
# was written to catch.
|
|
local skill="$TMPDIR/my-skill"
|
|
make_skill_with_source_keys "$skill"
|
|
make_sources_md "$skill"
|
|
run bash "$SCRIPT" "$skill"
|
|
[ "$status" -eq 0 ]
|
|
refute_output ""
|
|
assert_output --partial "INFO"
|
|
refute_output --partial "FAIL"
|
|
}
|
|
|
|
@test "auto-detect: a provenance target that is neither a skill directory nor an agent file FAILs, naming what it was handed" {
|
|
# The detector must not guess. Guessing here is worse than in validate.sh:
|
|
# this script's whole not-in-scope path is a SILENT exit 0, so a wrong guess
|
|
# followed by "provenance does not apply at this scope" is indistinguishable
|
|
# from a clean pass — which is the exact confusion the exit-2 tier above was
|
|
# created to end.
|
|
#
|
|
# A plain .txt file is neither shape under any reading of the contract: not
|
|
# a directory holding SKILL.md, not *.agent.md, not under .apm/agents/.
|
|
local dir="$TMPDIR/neither"
|
|
mkdir -p "$dir"
|
|
echo "not an artifact of either kind" > "$dir/notes.txt"
|
|
run bash "$SCRIPT" "$dir/notes.txt"
|
|
assert_failure
|
|
refute_output ""
|
|
assert_output --partial "$dir/notes.txt"
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Cycle 28 — #121: `Research doc:` names exactly one Research registry;
|
|
# an entry with no registry writes `none` plus `Basis:`
|
|
#
|
|
# Helper: a fake repo holding one skill whose single entry is written verbatim
|
|
# from the fields passed in — make_upstream_skill only varies the Research doc
|
|
# value, and the Basis cases need a second field. The registry lives at
|
|
# docs/research/sources.md and a basis file at docs/basis.md.
|
|
# ---------------------------------------------------------------------------
|
|
|
|
make_entry_skill() {
|
|
local repo="$1"
|
|
local fields="$2"
|
|
local skill="$repo/my-skill"
|
|
mkdir -p "$skill/references" "$repo/docs/research"
|
|
cat > "$skill/SKILL.md" <<EOF
|
|
---
|
|
name: my-skill
|
|
description: A valid skill description.
|
|
metadata:
|
|
source_keys:
|
|
- my-source
|
|
---
|
|
|
|
## Step 1
|
|
|
|
Do the thing.
|
|
EOF
|
|
printf '%s\n' "# Basis" > "$repo/docs/basis.md"
|
|
printf '%s\n' "# Other" > "$repo/docs/other-basis.md"
|
|
{
|
|
printf '# Sources\n\n## my-source\n\n'
|
|
printf '%s\n' '- **URL:** https://example.com/my-source'
|
|
printf '%s\n' '- **Description:** A test source.'
|
|
printf '%s\n' '- **Contributing files:** SKILL.md'
|
|
printf '%s\n' "$fields"
|
|
printf '%s\n' '- **Status:** `extracted`'
|
|
} > "$skill/references/sources.md"
|
|
cat > "$repo/docs/research/sources.md" <<EOF
|
|
# Research
|
|
|
|
## my-source
|
|
|
|
- **Contributing files:** (none)
|
|
- **Status:** \`extracted\`
|
|
EOF
|
|
commit_as_base "$repo"
|
|
}
|
|
|
|
@test "#121 FAIL: a brace-expansion Research doc names more than one path" {
|
|
local repo="$TMPDIR/fakerepo"
|
|
make_entry_skill "$repo" '- **Research doc:** docs/research/{sources,other}.md'
|
|
run bash "$SCRIPT" "$repo/my-skill"
|
|
assert_failure
|
|
assert_output --partial "Research doc names more than one path"
|
|
}
|
|
|
|
@test "#121 FAIL: a comma-separated Research doc names more than one path" {
|
|
local repo="$TMPDIR/fakerepo"
|
|
make_entry_skill "$repo" '- **Research doc:** docs/research/sources.md, docs/research/other.md'
|
|
run bash "$SCRIPT" "$repo/my-skill"
|
|
assert_failure
|
|
assert_output --partial "Research doc names more than one path"
|
|
}
|
|
|
|
@test "#121 FAIL: a semicolon-separated pair of annotated paths names more than one path" {
|
|
local repo="$TMPDIR/fakerepo"
|
|
make_entry_skill "$repo" '- **Research doc:** docs/research/sources.md (Releases section); docs/research/other.md (`delete_release` gotcha)'
|
|
run bash "$SCRIPT" "$repo/my-skill"
|
|
assert_failure
|
|
assert_output --partial "Research doc names more than one path"
|
|
}
|
|
|
|
@test "#121 pass: a comma or semicolon INSIDE the annotation is prose, not a list" {
|
|
local repo="$TMPDIR/fakerepo"
|
|
make_entry_skill "$repo" '- **Research doc:** docs/research/sources.md (cross-cutting; no dedicated section, see notes)'
|
|
run bash "$SCRIPT" "$repo/my-skill"
|
|
assert_success
|
|
assert_output ""
|
|
}
|
|
|
|
@test "#121 FAIL: repeated Research doc lines are a list, not an INFO" {
|
|
local repo="$TMPDIR/fakerepo"
|
|
make_entry_skill "$repo" "$(printf '%s\n%s' '- **Research doc:** docs/research/sources.md' '- **Research doc:** docs/research/other.md')"
|
|
run bash "$SCRIPT" "$repo/my-skill"
|
|
assert_failure
|
|
assert_output --partial "Multiple '- **Research doc:**' lines for 'my-source'"
|
|
}
|
|
|
|
@test "#121 pass: 'none' with one Basis bullet resolves and is clean" {
|
|
local repo="$TMPDIR/fakerepo"
|
|
make_entry_skill "$repo" "$(printf '%s\n%s' '- **Research doc:** none' '- **Basis:** docs/basis.md')"
|
|
run bash "$SCRIPT" "$repo/my-skill"
|
|
assert_success
|
|
assert_output ""
|
|
}
|
|
|
|
@test "#121 pass: 'none — reason' carries a trailing annotation and still counts as none" {
|
|
local repo="$TMPDIR/fakerepo"
|
|
make_entry_skill "$repo" "$(printf '%s\n%s' '- **Research doc:** none — org convention, not a research corpus entry' '- **Basis:** docs/basis.md')"
|
|
run bash "$SCRIPT" "$repo/my-skill"
|
|
assert_success
|
|
assert_output ""
|
|
}
|
|
|
|
@test "#121 pass: repeated single-path Basis bullets are each checked" {
|
|
local repo="$TMPDIR/fakerepo"
|
|
make_entry_skill "$repo" "$(printf '%s\n%s\n%s' '- **Research doc:** none' '- **Basis:** docs/basis.md' '- **Basis:** docs/other-basis.md')"
|
|
run bash "$SCRIPT" "$repo/my-skill"
|
|
assert_success
|
|
assert_output ""
|
|
}
|
|
|
|
@test "#121 FAIL: a second Basis bullet naming a missing path fails even when the first resolves" {
|
|
local repo="$TMPDIR/fakerepo"
|
|
make_entry_skill "$repo" "$(printf '%s\n%s\n%s' '- **Research doc:** none' '- **Basis:** docs/basis.md' '- **Basis:** docs/gone.md')"
|
|
run bash "$SCRIPT" "$repo/my-skill"
|
|
assert_failure
|
|
assert_output --partial "Basis path 'docs/gone.md' does not exist"
|
|
}
|
|
|
|
@test "#121 pass: a Basis header followed by '- ' path bullets is read too" {
|
|
local repo="$TMPDIR/fakerepo"
|
|
make_entry_skill "$repo" "$(printf '%s\n%s\n%s\n%s' '- **Research doc:** none' '**Basis:**' '- docs/basis.md' '- docs/other-basis.md')"
|
|
run bash "$SCRIPT" "$repo/my-skill"
|
|
assert_success
|
|
assert_output ""
|
|
}
|
|
|
|
@test "#121 FAIL: 'none' with no Basis declares nothing" {
|
|
local repo="$TMPDIR/fakerepo"
|
|
make_entry_skill "$repo" '- **Research doc:** none'
|
|
run bash "$SCRIPT" "$repo/my-skill"
|
|
assert_failure
|
|
assert_output --partial "Basis missing for 'my-source'"
|
|
}
|
|
|
|
@test "#121 FAIL: a Basis path that does not exist" {
|
|
local repo="$TMPDIR/fakerepo"
|
|
make_entry_skill "$repo" "$(printf '%s\n%s' '- **Research doc:** none' '- **Basis:** docs/gone.md')"
|
|
run bash "$SCRIPT" "$repo/my-skill"
|
|
assert_failure
|
|
assert_output --partial "Basis path 'docs/gone.md' does not exist"
|
|
}
|
|
|
|
@test "#121 FAIL: one Basis value naming several paths is a list" {
|
|
local repo="$TMPDIR/fakerepo"
|
|
make_entry_skill "$repo" "$(printf '%s\n%s' '- **Research doc:** none' '- **Basis:** docs/basis.md, docs/other-basis.md')"
|
|
run bash "$SCRIPT" "$repo/my-skill"
|
|
assert_failure
|
|
assert_output --partial "Basis value names more than one path"
|
|
}
|
|
|
|
@test "#121 pass: a '(removed in <sha>)' Basis skips the existence check" {
|
|
local repo="$TMPDIR/fakerepo"
|
|
make_entry_skill "$repo" "$(printf '%s\n%s\n%s' '- **Research doc:** none' '- **Basis:** docs/basis.md' '- **Basis:** docs/deleted-adr.md (removed in 5b80f30)')"
|
|
run bash "$SCRIPT" "$repo/my-skill"
|
|
assert_success
|
|
assert_output ""
|
|
}
|
|
|
|
@test "#121 FAIL: a Basis annotated with something other than '(removed in <sha>)' is still existence-checked" {
|
|
local repo="$TMPDIR/fakerepo"
|
|
make_entry_skill "$repo" "$(printf '%s\n%s' '- **Research doc:** none' '- **Basis:** docs/deleted-adr.md (removed later)')"
|
|
run bash "$SCRIPT" "$repo/my-skill"
|
|
assert_failure
|
|
assert_output --partial "Basis path 'docs/deleted-adr.md' does not exist"
|
|
}
|
|
|
|
@test "#121 pass: a Basis with a trailing annotation resolves its path" {
|
|
local repo="$TMPDIR/fakerepo"
|
|
make_entry_skill "$repo" "$(printf '%s\n%s' '- **Research doc:** none' '- **Basis:** docs/basis.md (org convention file, not a corpus entry)')"
|
|
run bash "$SCRIPT" "$repo/my-skill"
|
|
assert_success
|
|
assert_output ""
|
|
}
|
|
|
|
@test "#121 FAIL: check 7 — a slug missing from a resolved registry is a FAIL, not an INFO" {
|
|
local repo="$TMPDIR/fakerepo"
|
|
make_entry_skill "$repo" '- **Research doc:** docs/research/sources.md'
|
|
cat > "$repo/docs/research/sources.md" <<EOF
|
|
# Research
|
|
|
|
## different-slug
|
|
|
|
- **Contributing files:** (none)
|
|
- **Status:** \`extracted\`
|
|
EOF
|
|
run bash "$SCRIPT" "$repo/my-skill"
|
|
assert_failure
|
|
assert_output --partial "FAIL"
|
|
assert_output --partial "Slug 'my-source' not found as H2 in research doc 'docs/research/sources.md'"
|
|
}
|
|
|
|
@test "#121 FAIL: a Research doc that resolves to a topic document is wrong, not 'not applicable'" {
|
|
local repo="$TMPDIR/fakerepo"
|
|
make_entry_skill "$repo" '- **Research doc:** docs/research/remotes.md (whole-document reference)'
|
|
printf '# Remotes\n\n## Core Philosophy\n\nProse.\n' > "$repo/docs/research/remotes.md"
|
|
run bash "$SCRIPT" "$repo/my-skill"
|
|
assert_failure
|
|
assert_output --partial "is a topic document, not a Research registry"
|
|
refute_output --partial "not applicable"
|
|
}
|
|
|
|
@test "#121 INFO: an unresolvable Research doc path stays INFO and exits 0" {
|
|
local repo="$TMPDIR/fakerepo"
|
|
make_entry_skill "$repo" '- **Research doc:** docs/research/gone/sources.md'
|
|
run bash "$SCRIPT" "$repo/my-skill"
|
|
assert_success
|
|
assert_output --partial "INFO"
|
|
assert_output --partial "does not exist"
|
|
}
|
|
|
|
@test "#121 INFO: a Basis cannot be existence-checked with no repo root, and says so" {
|
|
local skill="$TMPDIR/my-skill"
|
|
make_skill_with_source_keys "$skill"
|
|
mkdir -p "$skill/references"
|
|
cat > "$skill/references/sources.md" <<EOF
|
|
# Sources
|
|
|
|
## my-source
|
|
|
|
- **URL:** https://example.com/my-source
|
|
- **Description:** A test source.
|
|
- **Contributing files:** SKILL.md
|
|
- **Research doc:** none
|
|
- **Basis:** docs/basis.md
|
|
- **Status:** \`extracted\`
|
|
EOF
|
|
run bash "$SCRIPT" "$skill"
|
|
assert_success
|
|
assert_output --partial "Basis check skipped for 'my-source'"
|
|
}
|
|
|
|
# --- Parser parity: the spelling of a field must not change what is read ----
|
|
|
|
@test "#121 parity: an inline Research doc with no leading hyphen is read, not reported missing" {
|
|
local repo="$TMPDIR/fakerepo"
|
|
make_entry_skill "$repo" '**Research doc:** docs/research/sources.md'
|
|
run bash "$SCRIPT" "$repo/my-skill"
|
|
assert_success
|
|
refute_output --partial "Research doc field missing"
|
|
assert_output ""
|
|
}
|
|
|
|
@test "#121 parity: a Research doc header with one '- ' bullet is read" {
|
|
local repo="$TMPDIR/fakerepo"
|
|
make_entry_skill "$repo" "$(printf '%s\n%s' '**Research doc:**' '- docs/research/sources.md')"
|
|
run bash "$SCRIPT" "$repo/my-skill"
|
|
assert_success
|
|
refute_output --partial "Research doc field missing"
|
|
assert_output ""
|
|
}
|