holocron/plugins/kyberforge/skills/plugin-create/evals/eval.yaml

skill_name: plugin-create

trigger_tests:
  - id: explicit-create-named-plugin
    name: "Explicit trigger — create named plugin"
    query: "create a new plugin called security-tools"
    should_trigger: true

  - id: explicit-scaffold-plugin
    name: "Explicit trigger — scaffold plugin"
    query: "scaffold a plugin called developer-tools"
    should_trigger: true

  - id: implicit-add-marketplace-plugin
    name: "Implicit trigger — add plugin to marketplace"
    query: "add a new plugin to the marketplace for developer tools"
    should_trigger: true

  - id: implicit-package-skills-into-plugin
    name: "Implicit trigger — package skills into a plugin"
    query: "I want to package up my skills into a distributable plugin"
    should_trigger: true

  - id: negative-validate-marketplace
    name: "Negative trigger — validate marketplace.json (routes to /marketplace-architect)"
    query: "validate my marketplace.json"
    should_trigger: false

  - id: negative-write-skill
    name: "Negative trigger — write a skill (routes to /write-skill)"
    query: "write a skill for code review"
    should_trigger: false

  - id: negative-adopt-external-plugin
    name: "Negative trigger — adopt external plugin (routes to /marketplace-architect)"
    query: "adopt this external plugin into my marketplace"
    should_trigger: false

output_tests:
  - id: gate-a-shows-plan
    name: "Gate A presents file list and marketplace entry before writing"
    type: deterministic
    prompt: >
      Create a new plugin called marketplace-tools, description: 'Tools for managing the
      holocron marketplace', author: Jane Doe, email: jane@example.com,
      URL: https://github.com/jane
    expected_output: >
      The skill presents a plan listing all files that will be written under
      plugins/marketplace-tools/, shows the new marketplace.json entry, and
      explicitly asks for approval before writing any file.
    assertions:
      - "Output lists files to be written under plugins/marketplace-tools/"
      - "Output includes a marketplace.json entry with name 'marketplace-tools'"
      - "Output explicitly asks for approval and does not proceed without it"
      - "Output does not contain any written file confirmation before approval is given"

  - id: reserved-name-rejected
    name: "Reserved plugin name is rejected before Gate A"
    type: deterministic
    prompt: "Create a new plugin called claude-tools"
    expected_output: >
      The skill halts before presenting Gate A, reports that 'claude-tools' matches
      the reserved name pattern 'claude-*', and asks the user to provide a different name.
    assertions:
      - "Output does not present Gate A or a file list"
      - "Output identifies 'claude-tools' as a reserved name"
      - "Output asks the user to provide a replacement name before continuing"

  - id: no-skill-stub-generated
    name: "No skill stub generated — skills/ is empty with README only"
    type: deterministic
    prompt: >
      Create a new plugin called marketplace-tools, description: 'Tools for managing the
      holocron marketplace', author: Jane Doe, email: jane@example.com,
      URL: https://github.com/jane. Approve Gate A and Gate B.
    expected_output: >
      The plugin scaffold does not include any SKILL.md file. The skills/ directory
      is present with a README only. The skill directs the user to /write-skill
      to add skill content.
    assertions:
      - "Output does not include creation of any SKILL.md file"
      - "Output includes a skills/ directory entry with README.md only"
      - "Output mentions /write-skill for adding skill content"

  - id: handoff-message-present
    name: "Hand-off message directs user to /marketplace-architect"
    type: deterministic
    prompt: >
      Create a new plugin called marketplace-tools, description: 'Tools for managing the
      holocron marketplace', author: Jane Doe, email: jane@example.com,
      URL: https://github.com/jane. Approve Gate A and Gate B.
    expected_output: >
      After writing files and running validation, the skill prints a hand-off message
      that names the created plugin and instructs the user to run /marketplace-architect.
    assertions:
      - "Output contains the string '/marketplace-architect'"
      - "Output includes the plugin name 'marketplace-tools' in the hand-off message"
      - "Output does not auto-invoke /marketplace-architect itself"

  - id: full-flow-quality
    name: "Full flow — correct sequencing, substitution, and hand-off"
    type: llm-rubric
    prompt: >
      Create a new plugin called developer-tools with description 'Developer productivity
      toolkit', author: Alex Smith, email: alex@example.com, URL: https://github.com/alex.
      Approve Gate A and Gate B.
    expected_output: >
      The skill executes the full create-plugin flow: collects all five inputs one at a
      time, presents Gate A with a file plan, copies and substitutes the template, presents
      Gate B with full file contents, writes files after Gate B approval, updates
      marketplace.json, runs validation, and prints a hand-off message.
    assertions:
      - "All five inputs (plugin name, description, author name, email, URL) were collected individually before Gate A was presented"
      - "Gate A clearly listed all files to be written and the new marketplace.json entry, and waited for explicit approval"
      - "Gate B showed the full substituted content of every file before writing"
      - "None of the strings PLUGIN_NAME, PLUGIN_DESCRIPTION, AUTHOR_NAME, AUTHOR_EMAIL, or AUTHOR_URL appear in the written file output"
      - "The validation step either ran claude plugin validate . and reported output, or explicitly stated it was unavailable"
      - "The final message directed the user to run /marketplace-architect and included the plugin name 'developer-tools'"