diff --git a/plugins/kyberforge/skills/skill-audit/README.md b/plugins/kyberforge/skills/skill-audit/README.md index ad161fd..740a638 100644 --- a/plugins/kyberforge/skills/skill-audit/README.md +++ b/plugins/kyberforge/skills/skill-audit/README.md @@ -4,7 +4,7 @@ Audit a skill directory against the agentskills.io specification. Runs structura ## What it does -1. Runs `scripts/validate.sh` for structural checks (name format, description length, line count, placeholder detection, script rules) +1. Runs `scripts/validate.sh` and `scripts/validate-provenance.sh` for structural and provenance checks 2. Reads all files in the skill directory 3. Applies qualitative checks across seven dimensions 4. Outputs a compact findings report — findings only, grouped by dimension, each with Why and Fix — and a result block with handoff to /skill-improve @@ -23,8 +23,10 @@ Provide the path to the skill directory to audit when invoking. |------|---------| | `SKILL.md` | Skill instructions for agents | | `scripts/validate.sh` | Structural validator — checks name format, name matches directory, description length, line count, placeholder detection, script executable bit, and interactive-prompt detection | +| `scripts/validate-provenance.sh` | Provenance validator — checks sources.md completeness, source_keys/slug consistency, Contributing files existence, bidirectional linkage, Research doc: fields, and upstream research doc alignment | | `references/description-quality.md` | Spec-grounded rubric for description auditing — loaded when a finding is borderline | | `references/body-discipline.md` | Spec-grounded rubric for body discipline auditing — loaded when padding vs necessity is unclear | | `references/sources.md` | Provenance record — agentskills.io sources that informed this skill and which files each contributed to | | `tests/validate.bats` | Bats test suite for validate.sh | +| `tests/validate-provenance.bats` | Bats test suite for validate-provenance.sh | | `tests/README.md` | Setup instructions for bats-support and bats-assert test dependencies | diff --git a/plugins/kyberforge/skills/skill-audit/SKILL.md b/plugins/kyberforge/skills/skill-audit/SKILL.md index 75fca50..d40c0a1 100644 --- a/plugins/kyberforge/skills/skill-audit/SKILL.md +++ b/plugins/kyberforge/skills/skill-audit/SKILL.md @@ -30,10 +30,13 @@ metadata: ```bash bash scripts/validate.sh +bash scripts/validate-provenance.sh ``` Note any structural FAILs — they will appear in the report as a `### Structure` dimension. If the script cannot execute (python3 unavailable, Bash denied, or permission error), perform structural checks manually: name format, name matches directory, description length ≤1024 chars, SKILL.md ≤500 lines, no unfilled `FILL IN:` placeholders, scripts executable and free of interactive prompts. +Note any Provenance FAILs and INFO findings from `validate-provenance.sh` — they surface in the report as a `### Provenance` dimension (separate from `### Structure`). The script embeds full FAIL/INFO format with Why and Fix per finding; surface them verbatim. + ## Step 2 — Read all skill files Read every file in the skill directory: `SKILL.md`, `README.md` (if present), all files in `scripts/`, `references/`, `assets/`, and `tests/`. Skip binary files only. Do not skip text files — internal consistency checks require the full picture. @@ -109,7 +112,7 @@ Check each pattern is appropriate and correctly formed: Open with a coverage line listing every dimension checked: ```text -Checked: structure · description · body-discipline · patterns · file-structure · formatting · scripts · internal-consistency +Checked: structure · description · body-discipline · patterns · file-structure · formatting · scripts · internal-consistency · provenance ``` Then output only dimensions that have findings, grouped under H3 headings, FAILs before SUGGESTIONs within each dimension. Omit clean dimensions entirely — their absence confirms they passed. @@ -127,8 +130,13 @@ Close with a result block: ```text ## Result -PASS / PASS (N suggestions) / FAIL (N fails · M suggestions) +PASS +PASS (N suggestions) +PASS · P info +PASS (N suggestions) · P info +FAIL (N fails · M suggestions) +FAIL (N fails · M suggestions) · P info Run /skill-improve to address findings. ``` -Omit the `/skill-improve` line when there are no findings. Do not apply fixes — report and propose only. +INFO findings are observational — do not affect PASS/FAIL. Omit `· P info` when there are no INFO findings. Omit the `/skill-improve` line when there are no findings at all. Do not apply fixes — report and propose only. diff --git a/plugins/kyberforge/skills/skill-audit/scripts/validate-provenance.sh b/plugins/kyberforge/skills/skill-audit/scripts/validate-provenance.sh new file mode 100755 index 0000000..edfe211 --- /dev/null +++ b/plugins/kyberforge/skills/skill-audit/scripts/validate-provenance.sh @@ -0,0 +1,395 @@ +#!/usr/bin/env bash +set -euo pipefail + +usage() { + cat < + +Validate that a skill's sources provenance chain is complete and internally consistent. + +Arguments: + skill-dir Path to the skill directory to validate. + +Exit codes: + 0 All checks passed (or nothing to validate) + 1 One or more checks failed + +Checks performed: + 0 source_keys present but references/sources.md absent + 1 FILL IN: placeholders in sources.md + 2 source_keys in SKILL.md → slug exists in sources.md + 3 source_keys in references/*.md → slug exists in sources.md (INFO if no source_keys) + 4 Contributing files listed in sources.md exist on disk + 5 Contributing files back-reference the parent slug in their source_keys + 6 Research doc field present and not placeholder + 7 Slug in sources.md present in upstream research doc (INFO only) + 8 Extracted non-(none) slug in research doc present in sources.md +EOF +} + +if [[ "${1:-}" == "--help" || "${1:-}" == "-h" ]]; then + usage + exit 0 +fi + +if [[ $# -lt 1 ]]; then + echo "Error: skill-dir is required." >&2 + echo "" >&2 + usage >&2 + exit 1 +fi + +python3 -u - "$1" <<'PYTHON' +import sys +import os +import re + +skill_dir = os.path.abspath(sys.argv[1]) +sources_md_path = os.path.join(skill_dir, "references", "sources.md") +refs_dir = os.path.join(skill_dir, "references") + +# --- Helpers --- + +PLACEHOLDER_RE = re.compile(r'(?' to the '## {slug}' entry in references/sources.md." + ) + elif rd_value == "" or PLACEHOLDER_RE.search(rd_value): + emit_fail( + f"Research doc field is empty or placeholder", + f"references/sources.md (## {slug})", + f"The '## {slug}' entry has an unfilled Research doc value.", + f"Set '- **Research doc:**' to a real path relative to repo root, or '(none)' if not applicable." + ) + else: + # Check 7: Upstream forward — slug should appear in research doc + if repo_root and not rd_value.startswith("(none"): + rd_abs = os.path.join(repo_root, rd_value) + if os.path.isfile(rd_abs): + with open(rd_abs) as f: + rd_content = f.read() + rd_slugs = set(parse_h2_slugs(rd_content)) + if slug not in rd_slugs: + emit_info( + f"Slug '{slug}' not found as H2 in research doc '{rd_value}'", + f"references/sources.md (## {slug})", + f"The research doc '{rd_value}' does not have a '## {slug}' heading. " + f"The provenance link may be imprecise — the slug name in sources.md may differ from the research doc's heading." + ) + # Track for Check 8 + if rd_abs not in research_docs_seen: + research_docs_seen[rd_abs] = (rd_value, set()) + research_docs_seen[rd_abs][1].add(slug) + +# --- Check 8: Upstream reverse --- +for rd_abs, (rd_rel, known_slugs) in research_docs_seen.items(): + with open(rd_abs) as f: + rd_content = f.read() + for rd_slug in parse_h2_slugs(rd_content): + # Parse this slug's Contributing files and Status in the research doc + rd_cf = parse_contributing_files(rd_content, rd_slug) + rd_status = parse_status(rd_content, rd_slug) + # Skip if contributing files start with (none + if rd_cf and rd_cf.startswith("(none"): + continue + # Skip if status is not `extracted` + if rd_status != "`extracted`": + continue + # This slug should be in sources.md + if rd_slug not in sources_slugs: + emit_fail( + f"Research doc slug '{rd_slug}' missing from skill sources.md", + f"references/sources.md", + f"The research doc '{rd_rel}' has '## {rd_slug}' with status `extracted` and contributing files, " + f"but this skill's sources.md has no '## {rd_slug}' entry.", + f"Add '## {rd_slug}' to references/sources.md or mark it as '(none)' in the research doc's Contributing files." + ) + +print_findings() +sys.exit(1 if has_fail else 0) +PYTHON diff --git a/plugins/kyberforge/skills/skill-audit/tests/README.md b/plugins/kyberforge/skills/skill-audit/tests/README.md index ac0cfa7..e8d2466 100644 --- a/plugins/kyberforge/skills/skill-audit/tests/README.md +++ b/plugins/kyberforge/skills/skill-audit/tests/README.md @@ -26,3 +26,4 @@ bats plugins/kyberforge/skills/skill-audit/tests/ | File | Purpose | |------|---------| | `validate.bats` | Bats test suite for `scripts/validate.sh` | +| `validate-provenance.bats` | Bats test suite for `scripts/validate-provenance.sh` | diff --git a/plugins/kyberforge/skills/skill-audit/tests/validate-provenance.bats b/plugins/kyberforge/skills/skill-audit/tests/validate-provenance.bats new file mode 100644 index 0000000..d5619d1 --- /dev/null +++ b/plugins/kyberforge/skills/skill-audit/tests/validate-provenance.bats @@ -0,0 +1,514 @@ +#!/usr/bin/env bats + +setup() { + REPO_ROOT="$(cd "$BATS_TEST_DIRNAME/../../../../../" && pwd)" + load "$REPO_ROOT/tests/test_helper/bats-support/load" + load "$REPO_ROOT/tests/test_helper/bats-assert/load" + + SCRIPT="$(cd "$BATS_TEST_DIRNAME/../scripts" && pwd)/validate-provenance.sh" + TMPDIR="$(mktemp -d)" + + # Helper: create a minimal skill directory with no sources.md and no source_keys + make_clean_skill() { + local dir="$1" + local name + name="$(basename "$dir")" + mkdir -p "$dir" + cat > "$dir/SKILL.md" < "$dir/SKILL.md" < "$dir/references/sources.md" < "$skill/references/sources.md" <> "$skill/references/sources.md" + run bash "$SCRIPT" "$skill" + assert_success +} + +# --------------------------------------------------------------------------- +# Cycle 5 — Check 2: source_keys slug missing from sources.md → FAIL +# --------------------------------------------------------------------------- + +@test "FAIL: source_keys slug in SKILL.md not present as H2 in sources.md" { + local skill="$TMPDIR/my-skill" + make_skill_with_source_keys "$skill" + mkdir -p "$skill/references" + cat > "$skill/references/sources.md" < "$skill/references/sources.md" < "$skill/references/sources.md" < "$skill/SKILL.md" < "$skill/references/sources.md" <> "$skill/references/sources.md" < "$skill/references/extra.md" < "$skill/references/extra.md" < "$research_dir/my-research.md" < "$skill2/SKILL.md" < "$fake_repo/docs/research/my-research.md" < "$skill2/references/sources.md" < "$skill/SKILL.md" < "$fake_repo/docs/research/my-research.md" < "$skill/references/sources.md" < "$skill/SKILL.md" < "$fake_repo/docs/research/my-research.md" < "$skill/references/sources.md" <