chore: move skills and evals to plugins/bin, remove legacy root configs
Skills and evals migrated from .agents/ to plugins/bin/ plugin directory. Remove .mcp.json, provider-manifest.sh, and skills-lock.json legacy artifacts. Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
61
plugins/bin/evals/implement/write-docs/eval.yaml
Normal file
61
plugins/bin/evals/implement/write-docs/eval.yaml
Normal file
@@ -0,0 +1,61 @@
|
||||
skill_name: write-docs
|
||||
|
||||
trigger_tests:
|
||||
- id: explicit-trigger-document-module
|
||||
name: "Explicit trigger — document a script"
|
||||
query: "Write documentation for the install.sh script"
|
||||
should_trigger: true
|
||||
|
||||
- id: explicit-trigger-create-docs
|
||||
name: "Explicit trigger — create docs for a feature"
|
||||
query: "Create docs for this feature"
|
||||
should_trigger: true
|
||||
|
||||
- id: implicit-trigger-readme-update
|
||||
name: "Implicit trigger — outdated README section, no trigger phrase"
|
||||
query: "We need to update the README section for the auth module, the current one is outdated"
|
||||
should_trigger: true
|
||||
|
||||
- id: negative-trigger-prd
|
||||
name: "Negative — PRD request should route to to-prd"
|
||||
query: "Write a PRD for the new logging feature"
|
||||
should_trigger: false
|
||||
|
||||
- id: negative-trigger-write-skill
|
||||
name: "Negative — skill authoring request should route to write-skill"
|
||||
query: "Write a skill for generating documentation automatically"
|
||||
should_trigger: false
|
||||
|
||||
- id: negative-trigger-skill-file
|
||||
name: "Negative — SKILL.md update (skill files are self-describing)"
|
||||
query: "Document how the write-docs skill works by updating its SKILL.md"
|
||||
should_trigger: false
|
||||
|
||||
output_tests:
|
||||
- id: output-proposes-files-before-reading
|
||||
name: "Deterministic — candidates proposed or approval sought before reading files"
|
||||
type: deterministic
|
||||
prompt: "Write documentation for the config module"
|
||||
expected_output: "Skill proposes candidate files or asks the user to name specific files before reading any file content"
|
||||
assertions:
|
||||
- "Response proposes candidate file paths or asks the user to confirm which files to read before showing any extracted content"
|
||||
- "Response does not display extracted code content or API surface without first receiving file approval"
|
||||
|
||||
- id: output-gap-check-present
|
||||
name: "Deterministic — gap check step present before drafting"
|
||||
type: deterministic
|
||||
prompt: "Write documentation for the install.sh script, audience: developer"
|
||||
expected_output: "Skill presents extracted behaviour to the user and asks them to fill gaps before drafting any section"
|
||||
assertions:
|
||||
- "Response includes a gap check step that presents extracted behaviour and asks what the code does not explain"
|
||||
- "Response does not skip directly to a drafted documentation section without presenting extracted content first"
|
||||
|
||||
- id: output-never-invents-behaviour
|
||||
name: "LLM rubric — no invented behaviour, all claims sourced"
|
||||
type: llm-rubric
|
||||
prompt: "Document the src/config.py file for internal developers"
|
||||
expected_output: "Documentation where every claim is attributed to code content or explicit user input, with no invented explanations, assumptions about intent, or unverifiable behaviour claims."
|
||||
assertions:
|
||||
- "The skill explicitly derives each documented claim from a named source — a code line, spec section, or user statement — and does not add claims without attribution"
|
||||
- "The skill does not include descriptions of caller intent, design rationale, or future behaviour that are not present in the source material"
|
||||
- "If a behaviour is undocumentable (internal detail with no public spec), the skill notes it as out-of-scope rather than inventing an explanation"
|
||||
Reference in New Issue
Block a user