description: "Hermes CI Eval Baseline v1 — 带断言评测"

prompts:
  - "请以项目经理Bot的身份回答：{{question}}"
  - "你是一个项目管理者。请回答：{{question}}"

providers:
  - id: openai:chat:deepseek-chat
    config:
      apiBaseUrl: https://api.deepseek.com/v1
      apiKeyEnvar: DEEPSEEK_API_KEY

# 测试数据集
tests:
  - vars:
      question: "项目进度怎么看？"
    assert:
      - type: contains-any
        value: ["甘特图", "任务", "进度", "计划", "完成率"]
      - type: latency
        threshold: 15000

  - vars:
      question: "代码审查的标准是什么？"
    assert:
      - type: contains-any
        value: ["功能", "可读性", "安全性", "测试", "规范"]
      - type: latency
        threshold: 15000

  - vars:
      question: "CI/CD pipeline失败了怎么办？"
    assert:
      - type: contains-any
        value: ["日志", "失败", "排查", "恢复", "回滚"]
      - type: latency
        threshold: 15000

  - vars:
      question: "Gitea仓库如何管理权限？"
    assert:
      - type: contains-any
        value: ["角色", "权限", "组织", "团队", "仓库"]
      - type: latency
        threshold: 15000

  - vars:
      question: "这个任务应该排什么优先级？"
    assert:
      - type: contains-any
        value: ["优先级", "紧急", "重要", "影响", "截止"]
      - type: latency
        threshold: 15000

  - vars:
      question: "如何创建新的Hermes Bot？"
    assert:
      - type: contains-any
        value: ["Bot", "配置", "创建", "Hermes", "步骤"]
      - type: latency
        threshold: 15000

  - vars:
      question: "Docker容器部署需要注意什么？"
    assert:
      - type: contains-any
        value: ["镜像", "安全", "网络", "卷", "资源"]
      - type: latency
        threshold: 15000

  - vars:
      question: "Woodpecker CI流水线如何配置？"
    assert:
      - type: contains-any
        value: ["YAML", "流水线", "步骤", "Woodpecker", "配置"]
      - type: latency
        threshold: 15000