skill_name: promptfoo

trigger_tests:
  - id: explicit-setup
    name: "Explicit trigger — project setup"
    query: "set up promptfoo in my project"
    should_trigger: true

  - id: explicit-run-eval
    name: "Explicit trigger — run eval"
    query: "help me run a promptfoo eval"
    should_trigger: true

  - id: implicit-model-comparison
    name: "Implicit trigger — model comparison"
    query: "I want to compare GPT-4 and Claude on the same test cases"
    should_trigger: true

  - id: implicit-ci-llm-testing
    name: "Implicit trigger — CI LLM testing"
    query: "how do I add LLM testing to my pull request pipeline?"
    should_trigger: true

  - id: negative-express-tests
    name: "Negative trigger — non-LLM integration tests"
    query: "How do I write integration tests for my Express API?"
    should_trigger: false

  - id: negative-ragas
    name: "Negative trigger — different eval framework"
    query: "Set up RAGAS for evaluating my RAG pipeline"
    should_trigger: false

  - id: negative-general-llm-eval
    name: "Negative trigger — general LLM quality question"
    query: "What's the best way to evaluate LLM output quality?"
    should_trigger: false

output_tests:
  - id: output-pinned-version
    name: "Deterministic — pinned version used, @latest absent"
    type: deterministic
    prompt: "install promptfoo"
    expected_output: >
      Shell commands that install or invoke Promptfoo using the pinned version
      0.121.17. No @latest tag appears anywhere in the output.
    assertions:
      - "Output contains the string '0.121.17'"
      - "Output does NOT contain '@latest'"
      - "Output does NOT contain 'promptfoo@latest'"

  - id: output-config-structure
    name: "Deterministic — valid promptfooconfig.yaml structure"
    type: deterministic
    prompt: "create a promptfooconfig.yaml for comparing two models on my customer support prompts"
    expected_output: >
      A YAML configuration file with all required top-level keys for a working
      promptfoo evaluation: prompts, providers (at least two entries), tests with
      at least one test case containing vars and assert.
    assertions:
      - "Output contains 'prompts:'"
      - "Output contains 'providers:'"
      - "Output contains 'tests:'"
      - "Output contains 'assert:'"

  - id: output-no-hardcoded-keys
    name: "Deterministic — no hardcoded API keys in output"
    type: deterministic
    prompt: "set up promptfoo with OpenAI and show me the config"
    expected_output: >
      A setup guide and config that references API keys only via environment
      variables. No literal API key values, no placeholder strings like sk-abc123
      or sk-ant-abc123.
    assertions:
      - "Output does NOT contain 'sk-' followed by alphanumeric characters"
      - "Output references OPENAI_API_KEY or ANTHROPIC_API_KEY as an environment variable"
      - "Output does NOT contain a hardcoded token or credential value"

  - id: output-acquisition-warning
    name: "Deterministic — acquisition warning for non-OpenAI workloads"
    type: deterministic
    prompt: "evaluate my Anthropic-only pipeline with promptfoo"
    expected_output: >
      The response includes a note about Promptfoo's OpenAI acquisition (March 2026)
      and either mentions version pinning or names at least one documented fallback
      (DeepEval or Arize Phoenix).
    assertions:
      - "Output contains 'acquisition' OR output contains 'OpenAI' in a warning context"
      - "Output contains 'DeepEval' OR output contains 'Arize' OR output contains 'Phoenix' OR output contains 'pin'"

  - id: output-quality-rubric
    name: "LLM-rubric — complete setup is actionable and compliant"
    type: llm-rubric
    prompt: "set up a complete promptfoo project for testing my customer support chatbot"
    expected_output: >
      A complete, actionable guide that: scaffolds a project with the pinned version,
      produces a valid promptfooconfig.yaml with realistic prompts, at least one
      provider, and test cases with assertions, references environment variables for
      API keys, and provides the commands needed to run the first eval.
    assertions:
      - >
        The output is actionable: a developer could follow the steps without additional
        research and end up with a working promptfoo project
      - >
        The config example is structurally valid promptfoo YAML: it has prompts,
        providers, and tests sections with realistic placeholder content
      - >
        API keys are handled correctly: all examples use environment variable references,
        not literal key values
      - >
        The pinned version (0.121.17) appears in at least one command — the output does
        not instruct the user to install an unpinned version