hermes-ci/promptfooconfig.yaml

description: "Hermes CI Eval Baseline v2 — 扩建版（8+10=18场景）"

prompts:
  - "请以项目经理Bot的身份回答：{{question}}"
  - "你是一个项目管理者。请回答：{{question}}"

providers:
  - id: openai:chat:deepseek-chat
    config:
      apiBaseUrl: https://api.deepseek.com/v1
      apiKeyEnvar: DEEPSEEK_API_KEY

tests:
  # ===== 原8个场景 =====
  - vars:
      question: "项目进度怎么看？"
    assert:
      - type: contains-any
        value: ["甘特图", "任务", "进度", "计划", "完成率"]
      - type: latency
        threshold: 15000

  - vars:
      question: "代码审查的标准是什么？"
    assert:
      - type: contains-any
        value: ["功能", "可读性", "安全性", "测试", "规范"]
      - type: latency
        threshold: 15000

  - vars:
      question: "CI/CD pipeline失败了怎么办？"
    assert:
      - type: contains-any
        value: ["日志", "失败", "排查", "恢复", "回滚"]
      - type: latency
        threshold: 15000

  - vars:
      question: "Gitea仓库如何管理权限？"
    assert:
      - type: contains-any
        value: ["角色", "权限", "组织", "团队", "仓库"]
      - type: latency
        threshold: 15000

  - vars:
      question: "这个任务应该排什么优先级？"
    assert:
      - type: contains-any
        value: ["优先级", "紧急", "重要", "影响", "截止"]
      - type: latency
        threshold: 15000

  - vars:
      question: "如何创建新的Hermes Bot？"
    assert:
      - type: contains-any
        value: ["Bot", "配置", "创建", "Hermes", "步骤"]
      - type: latency
        threshold: 15000

  - vars:
      question: "Docker容器部署需要注意什么？"
    assert:
      - type: contains-any
        value: ["镜像", "安全", "网络", "卷", "资源"]
      - type: latency
        threshold: 15000

  - vars:
      question: "Woodpecker CI流水线如何配置？"
    assert:
      - type: contains-any
        value: ["YAML", "流水线", "步骤", "Woodpecker", "配置"]
      - type: latency
        threshold: 15000

  # ===== 新增：安全 =====
  - vars:
      question: "项目中如何管理API密钥和敏感信息？"
    assert:
      - type: contains-any
        value: ["密钥", "环境变量", "加密", "安全", ".env"]
      - type: latency
        threshold: 15000

  - vars:
      question: "如何防范供应链攻击？"
    assert:
      - type: contains-any
        value: ["依赖", "镜像", "签名", "验证", "源"]
      - type: latency
        threshold: 15000

  # ===== 新增：架构设计 =====
  - vars:
      question: "微服务和单体架构怎么选？"
    assert:
      - type: contains-any
        value: ["微服务", "单体", "拆分", "耦合", "扩展"]
      - type: latency
        threshold: 15000

  - vars:
      question: "技术选型应该考虑哪些因素？"
    assert:
      - type: contains-any
        value: ["维护", "团队", "生态", "性能", "成本"]
      - type: latency
        threshold: 15000

  # ===== 新增：故障响应 =====
  - vars:
      question: "生产环境服务宕机了，第一反应应该做什么？"
    assert:
      - type: contains-any
        value: ["恢复", "排查", "通知", "影响", "回滚"]
      - type: latency
        threshold: 15000

  - vars:
      question: "如何做事故复盘（post-mortem）？"
    assert:
      - type: contains-any
        value: ["复盘", "根因", "改进", "措施", "时间线"]
      - type: latency
        threshold: 15000

  # ===== 新增：团队协作 =====
  - vars:
      question: "跨团队协作有什么好的实践？"
    assert:
      - type: contains-any
        value: ["接口", "同步", "文档", "沟通", "对齐"]
      - type: latency
        threshold: 15000

  - vars:
      question: "如何提高团队代码质量？"
    assert:
      - type: contains-any
        value: ["审查", "规范", "自动化", "测试", "标准"]
      - type: latency
        threshold: 15000

  # ===== 新增：成本优化 =====
  - vars:
      question: "如何降低云服务成本？"
    assert:
      - type: contains-any
        value: ["实例", "预留", "监控", "缩容", "合理"]
      - type: latency
        threshold: 15000

  - vars:
      question: "项目预算超支了怎么处理？"
    assert:
      - type: contains-any
        value: ["预算", "分析", "调整", "范围", "优先级"]
      - type: latency
        threshold: 15000