test(kyberforge): pin the provenance checker's three new behaviours
27a7669 changed validate-provenance.sh in three ways and tested none of
them: the bullet-form parser returning None rather than [] on unparsable
input, the source_keys: [] house-authored declaration, and the stripping
of section annotations off a Research doc path.
All three are checks that previously failed by staying silent, which is
the failure mode this repo keeps rediscovering. Shipping them untested
left nothing to catch a regression back to silence.
The 22 cases assert consequences rather than return values -- that check
8 runs on an unparsable block, that an unresolvable path emits an INFO
naming its slug -- and each was verified against a deliberate mutation
of the behaviour it covers.
Addresses #111.This commit is contained in:
@@ -64,6 +64,43 @@ EOF
|
||||
- **Contributing files:** ${contrib}
|
||||
- **Research doc:** ${research}
|
||||
- **Status:** \`extracted\`
|
||||
EOF
|
||||
}
|
||||
|
||||
# Helper: create a fake repo (a .git marker makes find_repo_root stop there)
|
||||
# holding one skill whose single sources.md slug points at the given
|
||||
# Research doc value. Checks 7 and 8 only run for a skill inside a checkout,
|
||||
# so every upstream case needs this shape; the research doc itself is
|
||||
# written per test into "$repo/docs/research/my-research.md".
|
||||
make_upstream_skill() {
|
||||
local repo="$1"
|
||||
local research="${2:-docs/research/my-research.md}"
|
||||
local skill="$repo/my-skill"
|
||||
mkdir -p "$skill/references" "$repo/docs/research"
|
||||
touch "$repo/.git"
|
||||
cat > "$skill/SKILL.md" <<EOF
|
||||
---
|
||||
name: my-skill
|
||||
description: A valid skill description.
|
||||
metadata:
|
||||
source_keys:
|
||||
- my-source
|
||||
---
|
||||
|
||||
## Step 1
|
||||
|
||||
Do the thing.
|
||||
EOF
|
||||
cat > "$skill/references/sources.md" <<EOF
|
||||
# Sources
|
||||
|
||||
## my-source
|
||||
|
||||
- **URL:** https://example.com/my-source
|
||||
- **Description:** A test source.
|
||||
- **Contributing files:** SKILL.md
|
||||
- **Research doc:** ${research}
|
||||
- **Status:** \`extracted\`
|
||||
EOF
|
||||
}
|
||||
}
|
||||
@@ -512,3 +549,439 @@ EOF
|
||||
run bash "$SCRIPT" "$skill"
|
||||
assert_success
|
||||
}
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Cycle 13 — parse_contributing_files: None ("could not parse") is NOT []
|
||||
# ("explicitly (none)")
|
||||
#
|
||||
# Check 8 reads [] as "the research doc deliberately records no contributing
|
||||
# files" and SKIPS the slug on that basis. A block the parser cannot read must
|
||||
# therefore return None, so the slug stays exposed to check 8. Each case below
|
||||
# asserts that CONSEQUENCE — check 8 firing on the unreadable entry — not the
|
||||
# parser's return value, because returning [] is exactly the shape that makes
|
||||
# the check silently do nothing while still exiting 0.
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
@test "check 8 runs: bullet form with '*' asterisk bullets is unparsable, not '(none)'" {
|
||||
local fake_repo="$TMPDIR/fakerepo"
|
||||
make_upstream_skill "$fake_repo"
|
||||
|
||||
cat > "$fake_repo/docs/research/my-research.md" <<EOF
|
||||
# Research
|
||||
|
||||
## my-source
|
||||
|
||||
- **Contributing files:** some-skill/SKILL.md
|
||||
- **Status:** \`extracted\`
|
||||
|
||||
## extra-source
|
||||
|
||||
**Contributing files:**
|
||||
* some-skill/references/extra.md
|
||||
|
||||
- **Status:** \`extracted\`
|
||||
EOF
|
||||
|
||||
run bash "$SCRIPT" "$fake_repo/my-skill"
|
||||
assert_failure
|
||||
assert_output --partial "FAIL"
|
||||
assert_output --partial "extra-source"
|
||||
}
|
||||
|
||||
@test "check 8 runs: bullet form with a numbered list is unparsable, not '(none)'" {
|
||||
local fake_repo="$TMPDIR/fakerepo"
|
||||
make_upstream_skill "$fake_repo"
|
||||
|
||||
cat > "$fake_repo/docs/research/my-research.md" <<EOF
|
||||
# Research
|
||||
|
||||
## my-source
|
||||
|
||||
- **Contributing files:** some-skill/SKILL.md
|
||||
- **Status:** \`extracted\`
|
||||
|
||||
## extra-source
|
||||
|
||||
**Contributing files:**
|
||||
1. some-skill/references/extra.md
|
||||
|
||||
- **Status:** \`extracted\`
|
||||
EOF
|
||||
|
||||
run bash "$SCRIPT" "$fake_repo/my-skill"
|
||||
assert_failure
|
||||
assert_output --partial "FAIL"
|
||||
assert_output --partial "extra-source"
|
||||
}
|
||||
|
||||
@test "check 8 runs: bullet form followed by prose is unparsable, not '(none)'" {
|
||||
local fake_repo="$TMPDIR/fakerepo"
|
||||
make_upstream_skill "$fake_repo"
|
||||
|
||||
cat > "$fake_repo/docs/research/my-research.md" <<EOF
|
||||
# Research
|
||||
|
||||
## my-source
|
||||
|
||||
- **Contributing files:** some-skill/SKILL.md
|
||||
- **Status:** \`extracted\`
|
||||
|
||||
## extra-source
|
||||
|
||||
**Contributing files:**
|
||||
See the table below for the file list.
|
||||
|
||||
- **Status:** \`extracted\`
|
||||
EOF
|
||||
|
||||
run bash "$SCRIPT" "$fake_repo/my-skill"
|
||||
assert_failure
|
||||
assert_output --partial "FAIL"
|
||||
assert_output --partial "extra-source"
|
||||
}
|
||||
|
||||
@test "check 8 runs: bullet form heading with a blank line and nothing after is unparsable, not '(none)'" {
|
||||
local fake_repo="$TMPDIR/fakerepo"
|
||||
make_upstream_skill "$fake_repo"
|
||||
|
||||
cat > "$fake_repo/docs/research/my-research.md" <<EOF
|
||||
# Research
|
||||
|
||||
## my-source
|
||||
|
||||
- **Contributing files:** some-skill/SKILL.md
|
||||
- **Status:** \`extracted\`
|
||||
|
||||
## extra-source
|
||||
|
||||
- **Status:** \`extracted\`
|
||||
|
||||
**Contributing files:**
|
||||
|
||||
EOF
|
||||
|
||||
run bash "$SCRIPT" "$fake_repo/my-skill"
|
||||
assert_failure
|
||||
assert_output --partial "FAIL"
|
||||
assert_output --partial "extra-source"
|
||||
}
|
||||
|
||||
@test "check 8 runs: inline form whose whole value is a parenthetical is unparsable, not '(none)'" {
|
||||
local fake_repo="$TMPDIR/fakerepo"
|
||||
make_upstream_skill "$fake_repo"
|
||||
|
||||
cat > "$fake_repo/docs/research/my-research.md" <<EOF
|
||||
# Research
|
||||
|
||||
## my-source
|
||||
|
||||
- **Contributing files:** some-skill/SKILL.md
|
||||
- **Status:** \`extracted\`
|
||||
|
||||
## extra-source
|
||||
|
||||
- **Contributing files:** (see notes below)
|
||||
- **Status:** \`extracted\`
|
||||
EOF
|
||||
|
||||
run bash "$SCRIPT" "$fake_repo/my-skill"
|
||||
assert_failure
|
||||
assert_output --partial "FAIL"
|
||||
assert_output --partial "extra-source"
|
||||
}
|
||||
|
||||
@test "check 4 runs: bullet form with '-' hyphen bullets still parses each path" {
|
||||
local skill="$TMPDIR/my-skill"
|
||||
make_skill_with_source_keys "$skill"
|
||||
mkdir -p "$skill/references"
|
||||
cat > "$skill/references/sources.md" <<EOF
|
||||
# Sources
|
||||
|
||||
## my-source
|
||||
|
||||
- **URL:** https://example.com/my-source
|
||||
- **Description:** A test source.
|
||||
|
||||
**Contributing files:**
|
||||
- SKILL.md (the dispatch table, with a comma in the note)
|
||||
- references/nonexistent.md (why this one matters)
|
||||
|
||||
- **Research doc:** (none)
|
||||
- **Status:** \`extracted\`
|
||||
EOF
|
||||
run bash "$SCRIPT" "$skill"
|
||||
assert_failure
|
||||
assert_output --partial "Contributing file 'references/nonexistent.md' does not exist"
|
||||
}
|
||||
|
||||
@test "check 4 runs: inline comma-separated form still parses each path" {
|
||||
local skill="$TMPDIR/my-skill"
|
||||
make_skill_with_source_keys "$skill"
|
||||
make_sources_md "$skill" "my-source" "SKILL.md, references/nonexistent.md"
|
||||
run bash "$SCRIPT" "$skill"
|
||||
assert_failure
|
||||
assert_output --partial "Contributing file 'references/nonexistent.md' does not exist"
|
||||
}
|
||||
|
||||
@test "check 8 skips: bullet form '- (none)' is an explicit declaration" {
|
||||
local fake_repo="$TMPDIR/fakerepo"
|
||||
make_upstream_skill "$fake_repo"
|
||||
|
||||
cat > "$fake_repo/docs/research/my-research.md" <<EOF
|
||||
# Research
|
||||
|
||||
## my-source
|
||||
|
||||
- **Contributing files:** some-skill/SKILL.md
|
||||
- **Status:** \`extracted\`
|
||||
|
||||
## extra-source
|
||||
|
||||
**Contributing files:**
|
||||
- (none — nothing was extracted from this section)
|
||||
|
||||
- **Status:** \`extracted\`
|
||||
EOF
|
||||
|
||||
run bash "$SCRIPT" "$fake_repo/my-skill"
|
||||
assert_success
|
||||
}
|
||||
|
||||
@test "check 8 skips: inline bare '(none)' is an explicit declaration" {
|
||||
local fake_repo="$TMPDIR/fakerepo"
|
||||
make_upstream_skill "$fake_repo"
|
||||
|
||||
cat > "$fake_repo/docs/research/my-research.md" <<EOF
|
||||
# Research
|
||||
|
||||
## my-source
|
||||
|
||||
- **Contributing files:** some-skill/SKILL.md
|
||||
- **Status:** \`extracted\`
|
||||
|
||||
## extra-source
|
||||
|
||||
- **Contributing files:** (none)
|
||||
- **Status:** \`extracted\`
|
||||
EOF
|
||||
|
||||
run bash "$SCRIPT" "$fake_repo/my-skill"
|
||||
assert_success
|
||||
}
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Cycle 14 — Check 3 (#111): an explicit 'source_keys: []' is a house-authored
|
||||
# declaration, a bare 'source_keys:' is truncation
|
||||
#
|
||||
# Three states, three outcomes. Only the middle one — declared empty — is
|
||||
# silent; collapsing any pair of them is the defect #111 filed.
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
@test "check 3 silent: references doc with top-level 'source_keys: []' emits nothing" {
|
||||
local skill="$TMPDIR/my-skill"
|
||||
make_skill_with_source_keys "$skill"
|
||||
make_sources_md "$skill"
|
||||
cat > "$skill/references/extra.md" <<EOF
|
||||
---
|
||||
source_keys: []
|
||||
---
|
||||
|
||||
# Extra Reference
|
||||
|
||||
House-authored, no external source.
|
||||
EOF
|
||||
run bash "$SCRIPT" "$skill"
|
||||
assert_success
|
||||
assert_output ""
|
||||
}
|
||||
|
||||
@test "check 3 silent: references doc with 'source_keys: []' nested under metadata: emits nothing" {
|
||||
local skill="$TMPDIR/my-skill"
|
||||
make_skill_with_source_keys "$skill"
|
||||
make_sources_md "$skill"
|
||||
cat > "$skill/references/extra.md" <<EOF
|
||||
---
|
||||
metadata:
|
||||
source_keys: []
|
||||
---
|
||||
|
||||
# Extra Reference
|
||||
|
||||
House-authored, no external source.
|
||||
EOF
|
||||
run bash "$SCRIPT" "$skill"
|
||||
assert_success
|
||||
assert_output ""
|
||||
}
|
||||
|
||||
@test "INFO: references doc with a bare 'source_keys:' and no value still emits INFO" {
|
||||
local skill="$TMPDIR/my-skill"
|
||||
make_skill_with_source_keys "$skill"
|
||||
make_sources_md "$skill"
|
||||
cat > "$skill/references/extra.md" <<EOF
|
||||
---
|
||||
source_keys:
|
||||
---
|
||||
|
||||
# Extra Reference
|
||||
|
||||
Truncated frontmatter — this is not a decision.
|
||||
EOF
|
||||
run bash "$SCRIPT" "$skill"
|
||||
assert_success
|
||||
assert_output --partial "INFO"
|
||||
assert_output --partial "No source_keys frontmatter"
|
||||
}
|
||||
|
||||
@test "INFO: references doc with frontmatter but no source_keys key still emits INFO" {
|
||||
local skill="$TMPDIR/my-skill"
|
||||
make_skill_with_source_keys "$skill"
|
||||
make_sources_md "$skill"
|
||||
cat > "$skill/references/extra.md" <<EOF
|
||||
---
|
||||
title: Extra Reference
|
||||
---
|
||||
|
||||
# Extra Reference
|
||||
|
||||
Never said either way.
|
||||
EOF
|
||||
run bash "$SCRIPT" "$skill"
|
||||
assert_success
|
||||
assert_output --partial "INFO"
|
||||
assert_output --partial "No source_keys frontmatter"
|
||||
}
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Cycle 15 — Checks 7 and 8: Research doc annotation stripping, and the INFO
|
||||
# that replaced the silent skip
|
||||
#
|
||||
# A Research doc value is very often a path PLUS a section annotation, and
|
||||
# os.path.isfile() is false for every such string. Before the strip, checks 7
|
||||
# and 8 skipped SILENTLY on those — so the negative assertions here (an INFO
|
||||
# that must appear) are what distinguishes a running check from a dead one.
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
@test "check 7 runs: '§' section annotation is stripped before the path is resolved" {
|
||||
local fake_repo="$TMPDIR/fakerepo"
|
||||
make_upstream_skill "$fake_repo" 'docs/research/my-research.md § "Some Section"'
|
||||
|
||||
cat > "$fake_repo/docs/research/my-research.md" <<EOF
|
||||
# Research
|
||||
|
||||
## different-slug
|
||||
|
||||
- **Contributing files:** (none)
|
||||
- **Status:** \`extracted\`
|
||||
EOF
|
||||
|
||||
run bash "$SCRIPT" "$fake_repo/my-skill"
|
||||
assert_success
|
||||
assert_output --partial "Slug 'my-source' not found as H2 in research doc 'docs/research/my-research.md'"
|
||||
refute_output --partial "§"
|
||||
}
|
||||
|
||||
@test "check 7 runs: '→' section annotation is stripped before the path is resolved" {
|
||||
local fake_repo="$TMPDIR/fakerepo"
|
||||
make_upstream_skill "$fake_repo" 'docs/research/my-research.md → `## Pushing`'
|
||||
|
||||
cat > "$fake_repo/docs/research/my-research.md" <<EOF
|
||||
# Research
|
||||
|
||||
## different-slug
|
||||
|
||||
- **Contributing files:** (none)
|
||||
- **Status:** \`extracted\`
|
||||
EOF
|
||||
|
||||
run bash "$SCRIPT" "$fake_repo/my-skill"
|
||||
assert_success
|
||||
assert_output --partial "Slug 'my-source' not found as H2 in research doc 'docs/research/my-research.md'"
|
||||
refute_output --partial "→"
|
||||
}
|
||||
|
||||
@test "check 7 runs: parenthetical annotation is stripped before the path is resolved" {
|
||||
local fake_repo="$TMPDIR/fakerepo"
|
||||
make_upstream_skill "$fake_repo" "docs/research/my-research.md (whole-document reference)"
|
||||
|
||||
cat > "$fake_repo/docs/research/my-research.md" <<EOF
|
||||
# Research
|
||||
|
||||
## different-slug
|
||||
|
||||
- **Contributing files:** (none)
|
||||
- **Status:** \`extracted\`
|
||||
EOF
|
||||
|
||||
run bash "$SCRIPT" "$fake_repo/my-skill"
|
||||
assert_success
|
||||
assert_output --partial "Slug 'my-source' not found as H2 in research doc 'docs/research/my-research.md'"
|
||||
refute_output --partial "whole-document reference"
|
||||
}
|
||||
|
||||
@test "check 7 runs: a bare path with no annotation survives the strip intact" {
|
||||
local fake_repo="$TMPDIR/fakerepo"
|
||||
make_upstream_skill "$fake_repo" "docs/research/my-research.md"
|
||||
|
||||
cat > "$fake_repo/docs/research/my-research.md" <<EOF
|
||||
# Research
|
||||
|
||||
## my-source
|
||||
|
||||
- **Contributing files:** (none)
|
||||
- **Status:** \`extracted\`
|
||||
EOF
|
||||
|
||||
run bash "$SCRIPT" "$fake_repo/my-skill"
|
||||
assert_success
|
||||
assert_output ""
|
||||
}
|
||||
|
||||
@test "checks 7 and 8 skipped silently: Research doc '(none)' is recognised before the strip" {
|
||||
local fake_repo="$TMPDIR/fakerepo"
|
||||
make_upstream_skill "$fake_repo" "(none)"
|
||||
|
||||
run bash "$SCRIPT" "$fake_repo/my-skill"
|
||||
assert_success
|
||||
assert_output ""
|
||||
}
|
||||
|
||||
@test "checks 7 and 8 skipped silently: bare 'none — reason' is recognised as a declaration" {
|
||||
local fake_repo="$TMPDIR/fakerepo"
|
||||
make_upstream_skill "$fake_repo" "none — org convention, no upstream research doc"
|
||||
|
||||
run bash "$SCRIPT" "$fake_repo/my-skill"
|
||||
assert_success
|
||||
assert_output ""
|
||||
}
|
||||
|
||||
@test "INFO: a stripped path that does not resolve names the slug instead of skipping silently" {
|
||||
local fake_repo="$TMPDIR/fakerepo"
|
||||
make_upstream_skill "$fake_repo" 'docs/research/missing.md § "Some Section"'
|
||||
|
||||
run bash "$SCRIPT" "$fake_repo/my-skill"
|
||||
assert_success
|
||||
assert_output --partial "INFO"
|
||||
assert_output --partial "Upstream checks skipped for 'my-source' — research doc 'docs/research/missing.md' does not exist"
|
||||
}
|
||||
|
||||
@test "INFO: a Research doc value that is entirely annotation names the slug instead of skipping silently" {
|
||||
local fake_repo="$TMPDIR/fakerepo"
|
||||
make_upstream_skill "$fake_repo" '§ "Some Section"'
|
||||
|
||||
run bash "$SCRIPT" "$fake_repo/my-skill"
|
||||
assert_success
|
||||
assert_output --partial "INFO"
|
||||
assert_output --partial "Upstream checks skipped for 'my-source' — Research doc value names no path"
|
||||
}
|
||||
|
||||
@test "INFO: no repo root above the skill directory names the slug instead of skipping silently" {
|
||||
local skill="$TMPDIR/my-skill"
|
||||
make_skill_with_source_keys "$skill"
|
||||
make_sources_md "$skill" "my-source" "SKILL.md" "docs/research/my-research.md"
|
||||
|
||||
run bash "$SCRIPT" "$skill"
|
||||
assert_success
|
||||
assert_output --partial "INFO"
|
||||
assert_output --partial "Upstream checks skipped for 'my-source' — no repo root above the skill directory"
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user