holocron/plugins/bin/evals/research/research/eval.yaml

skill_name: research

trigger_tests:
  - id: explicit-basic
    name: "Explicit trigger — basic invocation with topic and path"
    query: "/research stripe .agents/skills/stripe/references/"
    should_trigger: true

  - id: explicit-with-urls
    name: "Explicit trigger — topic, path, and starting URLs provided"
    query: "Research the FastMCP library and put refs in .agents/skills/mcp/references/ — start here: https://fastmcp.dev/llms.txt"
    should_trigger: true

  - id: implicit-pre-skill
    name: "Implicit trigger — looking up docs before writing a skill"
    query: "Before we write the GitHub Actions skill, can you look up the API and put the docs somewhere useful?"
    should_trigger: true

  - id: negative-write-docs
    name: "Negative — documenting existing code goes to write-docs"
    query: "Document the auth module for me"
    should_trigger: false

  - id: negative-diagnose
    name: "Negative — debugging investigation goes to diagnose"
    query: "Research why these integration tests are failing"
    should_trigger: false

  - id: negative-context7-direct-question
    name: "Negative — direct doc question goes to context7-mcp, not research"
    query: "What are the Next.js middleware options?"
    should_trigger: false

  - id: negative-starting-urls-skips-context7
    name: "Explicit starting URLs — Context7 step skipped, user source respected"
    query: "Research fastapi at docs/refs/fastapi/ — start here: https://fastapi.tiangolo.com/llms.txt"
    should_trigger: true

  - id: implicit-library-context7-attempted
    name: "Implicit library research — skill triggers and should attempt Context7 before websearch"
    query: "Before we write the Redis skill, grab the docs and put them in .agents/skills/redis/references/"
    should_trigger: true

  - id: concept-topic-still-triggers-research
    name: "Concept topic — research triggers but Context7 step is skipped (not a named library)"
    query: "/research 'JWT authentication concepts' docs/refs/jwt/"
    should_trigger: true

output_tests:
  - id: sources-always-written
    name: "sources.md is always produced at the output path"
    type: deterministic
    prompt: "/research redis .agents/skills/redis/references/"
    expected_output: >-
      A sources.md file written to .agents/skills/redis/references/sources.md
      containing at least one source section with a URL and status field.
    assertions:
      - "Output path .agents/skills/redis/references/sources.md is created"
      - "sources.md contains at least one ## <slug> section"
      - "Each section in sources.md includes a URL: line and a Status: line"

  - id: topic-frontmatter
    name: "Every topic file has required frontmatter fields"
    type: deterministic
    prompt: "/research stripe .agents/skills/stripe/references/"
    expected_output: >-
      One or more topic .md files written to .agents/skills/stripe/references/,
      each with YAML frontmatter containing topic: and source_keys: fields.
    assertions:
      - "Every .md file except sources.md contains a YAML frontmatter block delimited by ---"
      - "Every topic file frontmatter contains a topic: field"
      - "Every topic file frontmatter contains a source_keys: field with at least one entry"

  - id: no-empty-files
    name: "No topic files written when no content is found"
    type: deterministic
    prompt: "/research xyzzy-nonexistent-tool-404 /tmp/refs-test/"
    expected_output: >-
      No topic .md files written. Skill reports what was searched and asks user
      to provide starting URLs. sources.md may still be written if sources were attempted.
    assertions:
      - "Skill does not write any topic .md files to the output path"
      - "Skill reports the search terms or URLs it tried"
      - "Skill asks the user to provide starting URLs or confirm the topic name"

  - id: output-path-respected
    name: "All files written inside the provided output path only"
    type: deterministic
    prompt: "/research openai .agents/skills/openai/references/"
    expected_output: >-
      All written files are under .agents/skills/openai/references/. No files
      created outside that directory.
    assertions:
      - "Every file path in the write operations starts with .agents/skills/openai/references/"
      - "No files are written to the current working directory root or any other path"

  - id: context7-resolves-source-recorded
    name: "Context7 source recorded in sources.md with context7: URL when library resolves"
    type: deterministic
    prompt: "/research next.js docs/refs/nextjs/"
    expected_output: >-
      sources.md contains at least one entry with a URL starting with context7:
      (e.g. context7:/vercel/next.js) and a slug matching context7-*.
    assertions:
      - "sources.md contains a ## context7-* section"
      - "The URL: line for that section starts with context7:"
      - "At least one topic file lists the context7-* slug in its source_keys frontmatter"

  - id: context7-fallback-to-web
    name: "Fallback to websearch when Context7 does not resolve the library"
    type: deterministic
    prompt: "/research xyzzy-internal-tool-abc docs/refs/xyzzy/"
    expected_output: >-
      No context7-* source in sources.md. Skill falls back to websearch.
      If no web results found, skill reports failure and does not write topic files.
    assertions:
      - "sources.md does not contain a ## context7-* section"
      - "Skill either writes topic files sourced from web URLs or reports no sources found"

  - id: context7-skipped-for-non-library
    name: "Context7 step skipped when topic is not a library/framework/API"
    type: deterministic
    prompt: "/research 'JWT authentication concepts' docs/refs/jwt/"
    expected_output: >-
      No resolve-library-id call made. Skill proceeds directly to websearch.
      sources.md contains only web-sourced slugs.
    assertions:
      - "sources.md does not contain a ## context7-* section"
      - "Topic files are sourced from web URLs, not Context7"

  - id: content-quality
    name: "Extracted content is relevant, concise, and well-organised"
    type: llm-rubric
    prompt: "/research fastapi .agents/skills/fastapi/references/"
    expected_output: >-
      A set of topic files covering FastAPI concepts relevant to skill authoring:
      installation, API reference, configuration, and examples at minimum. Content
      is concise summaries, not verbatim page dumps. sources.md accurately maps
      each source to the files it contributed to.
    assertions:
      - "Topic files cover concepts relevant to FastAPI (routing, request handling, dependency injection, or similar) — not generic web framework content"
      - "Body prose in each topic file is a synthesis or extraction, not a verbatim copy of the source page"
      - "source_keys in each topic file match slugs present in sources.md"
      - "sources.md contributing files list accurately reflects which topic files each source contributed to"