Defame1297
5dac52680f
Move all evals into each skill's own evals/ directory and test_scripts.sh into marketplace-architect/scripts/ so test artefacts live alongside the code they test. Also fix: hooks.json array→object, displayName title-case, marketplace.json owner placeholders, and .github/plugin/marketplace.json metadata-wrapper schema divergence. Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
86 lines
4.2 KiB
YAML
86 lines
4.2 KiB
YAML
skill_name: write-eval
|
|
|
|
trigger_tests:
|
|
- id: explicit-trigger-write-evals
|
|
name: "Explicit trigger — write evals"
|
|
query: "write evals for this skill"
|
|
should_trigger: true
|
|
|
|
- id: explicit-trigger-create-eval-yaml
|
|
name: "Explicit trigger — create eval.yaml"
|
|
query: "create eval.yaml for the tdd skill"
|
|
should_trigger: true
|
|
|
|
- id: implicit-trigger-test-coverage
|
|
name: "Implicit trigger — test coverage request"
|
|
query: "I need test coverage for the grill-me skill"
|
|
should_trigger: true
|
|
|
|
- id: negative-trigger-run-evals
|
|
name: "Negative — run evals (runner concern, not writer)"
|
|
query: "run my evals"
|
|
should_trigger: false
|
|
|
|
- id: negative-trigger-code-unit-tests
|
|
name: "Negative — unit tests for application code"
|
|
query: "write unit tests for my Python file"
|
|
should_trigger: false
|
|
|
|
- id: negative-trigger-debug-failing-eval
|
|
name: "Negative — debug failing eval"
|
|
query: "my eval is failing, help me debug it"
|
|
should_trigger: false
|
|
|
|
output_tests:
|
|
- id: deterministic-correct-output-path
|
|
name: "Deterministic — eval.yaml written to correct path"
|
|
type: deterministic
|
|
prompt: "write evals for the tdd skill"
|
|
expected_output: "eval.yaml written to .agents/evals/implement/tdd/eval.yaml containing skill_name: tdd"
|
|
assertions:
|
|
- "Output references the path .agents/evals/implement/tdd/eval.yaml"
|
|
- "Output file contains 'skill_name: tdd'"
|
|
|
|
- id: deterministic-all-five-types-present
|
|
name: "Deterministic — eval.yaml contains all five required test types"
|
|
type: deterministic
|
|
prompt: "create eval.yaml for the grill-me skill"
|
|
expected_output: "eval.yaml contains trigger_tests and output_tests sections with all five required test types represented"
|
|
assertions:
|
|
- "Output contains 'trigger_tests:'"
|
|
- "Output contains 'output_tests:'"
|
|
- "Output contains at least one entry with 'should_trigger: true'"
|
|
- "Output contains at least one entry with 'should_trigger: false'"
|
|
- "Output contains at least one entry with 'type: deterministic'"
|
|
- "Output contains at least one entry with 'type: llm-rubric'"
|
|
|
|
- id: deterministic-plan-shown-before-write
|
|
name: "Deterministic — test plan presented before file is written"
|
|
type: deterministic
|
|
prompt: "write evals for the diagnose skill"
|
|
expected_output: "Skill presents each proposed test case with its id, type, and query before writing any file, then requests confirmation"
|
|
assertions:
|
|
- "Response presents each proposed test case individually — showing at minimum the query and test type — before any file is written"
|
|
- "Response requests confirmation before proceeding to write"
|
|
|
|
- id: deterministic-merge-conflict-flagged
|
|
name: "Deterministic — conflict flagged in plan on re-run with existing eval"
|
|
type: deterministic
|
|
prompt: "write evals for the tdd skill — eval.yaml already exists at .agents/evals/implement/tdd/eval.yaml with a test case id 'explicit-trigger-basic'"
|
|
expected_output: "Skill identifies the existing eval.yaml, classifies the conflicting case as CONFLICT, and does not write until the user resolves it"
|
|
assertions:
|
|
- "Response indicates eval.yaml already exists at the target path"
|
|
- "Response labels the conflicting test case as CONFLICT or equivalent"
|
|
- "Response does not write the file before the user resolves the conflict"
|
|
|
|
- id: llm-rubric-assertion-quality
|
|
name: "LLM rubric — assertions are specific and verifiable"
|
|
type: llm-rubric
|
|
prompt: "write evals for the write-skill skill"
|
|
expected_output: "eval.yaml contains high-quality assertions that are specific, observable, and not vague"
|
|
assertions:
|
|
- "All assertions describe observable, verifiable conditions — not vague quality claims like 'output is good' or 'the response is helpful'"
|
|
- "Trigger test queries reflect realistic user phrasings, not just the exact skill description verbatim"
|
|
- "Negative trigger tests target adjacent tasks that share surface-level similarity with the skill's trigger"
|
|
- "Deterministic assertions are machine-checkable without LLM inference — presence of strings, path patterns, required sections"
|