holocron/plugins/kyberforge/skills/write-agent/evals/write-agent.yaml

skill_name: write-agent

trigger_tests:
  - id: explicit-basic
    name: Explicit trigger — basic invocation
    query: "write an agent for code review"
    should_trigger: true

  - id: explicit-plugin
    name: Explicit trigger — plugin agent
    query: "add an agent to the security-tools plugin"
    should_trigger: true

  - id: implicit-isolation
    name: Implicit trigger — isolated worker need
    query: "I need a specialized worker that handles DB migrations in isolation"
    should_trigger: true

  - id: negative-role-skill
    name: Negative — role skill (inline mode switch)
    query: "Add an architect role to my session"
    should_trigger: false

  - id: negative-plugin-create
    name: Negative — plugin scaffolding
    query: "Create a new plugin called security-tools"
    should_trigger: false

  - id: negative-edit-existing
    name: Negative — editing an existing agent
    query: "Update the description on my existing code-reviewer agent"
    should_trigger: false

  - id: negative-skill-authoring
    name: Negative — skill authoring inside a plugin
    query: "Write a skill for my refactor plugin"
    should_trigger: false

output_tests:
  - id: subagent-output-path
    name: Standalone subagent written to core/agents/
    type: deterministic
    prompt: "Create a standalone subagent called db-migrator that runs database migrations in an isolated context"
    expected_output: >
      The skill produces a file at core/agents/db-migrator.md containing valid YAML frontmatter
      with at minimum name and description fields, followed by a system prompt body.
    assertions:
      - "Output references the file path core/agents/db-migrator.md"
      - "Output does not reference .claude/agents/ as the write target"
      - "Generated file contains a --- frontmatter block with name: db-migrator"

  - id: plugin-agent-dual-file
    name: Plugin agent produces both .md and .agent.md
    type: deterministic
    prompt: "Add a cross-tool agent called security-scanner to the existing kyberforge plugin"
    expected_output: >
      The skill produces two files: plugins/kyberforge/agents/security-scanner.md (Claude Code)
      and plugins/kyberforge/agents/security-scanner.agent.md (Copilot CLI). Both files share
      an identical system prompt body.
    assertions:
      - "Output references plugins/kyberforge/agents/security-scanner.md"
      - "Output references plugins/kyberforge/agents/security-scanner.agent.md"
      - "Skill does not produce only one of the two files"

  - id: claude-frontmatter-required-fields
    name: Claude Code agent file contains required frontmatter fields
    type: deterministic
    prompt: "Write an agent for linting that reads files and reports violations"
    expected_output: >
      The generated .md file contains a YAML frontmatter block with at minimum name: and
      description: fields. The description field contains delegation guidance — when Claude
      should invoke this agent.
    assertions:
      - "Generated frontmatter contains name:"
      - "Generated frontmatter contains description:"
      - "The description value is not empty or a placeholder"

  - id: copilot-handoff-comment
    name: Copilot CLI .agent.md contains handoff comment for Claude-only fields
    type: deterministic
    prompt: "Add a plugin agent to the kyberforge plugin that uses permissionMode: bypassPermissions and isolation: worktree"
    expected_output: >
      The generated .agent.md file contains a comment at the top noting that permissionMode
      and isolation are Claude Code-only fields with no Copilot CLI equivalent.
    assertions:
      - "The .agent.md file contains a HANDOFF comment or equivalent note"
      - "The comment names at least one Claude-only field that was not ported"
      - "The .agent.md frontmatter does not contain permissionMode or isolation fields"

  - id: description-field-quality
    name: LLM rubric — description field names specific tasks not just a role
    type: llm-rubric
    prompt: "Create a standalone subagent called api-tester that tests REST API endpoints for correctness and schema compliance"
    expected_output: >
      The description field in the generated agent file should specify delegation conditions
      precisely enough that an orchestrator model can decide whether to invoke this agent
      without ambiguity. It should name the task domain, the conditions for delegation,
      and ideally one or more negative cases. A description that only says "Tests APIs" or
      restates the agent name is insufficient.
    assertions:
      - "The description field mentions what kind of task triggers delegation to this agent"
      - "The description field does not simply restate the agent name"
      - "The description is specific enough to distinguish this agent from a general-purpose testing agent"