feat: initial Hermes CI eval baseline

- promptfoo eval config with 8 test scenarios - Assertions: contains-any + latency checks - Woodpecker pipeline: eval-baseline + report steps - Evaluates Hermes Bot's project management responses
2026-07-09 20:14:15 +08:00
commit 5ea2710ed2
2 changed files with 109 additions and 0 deletions
@@ -0,0 +1,77 @@
+description: "Hermes CI Eval Baseline v1 — 带断言评测"
+
+prompts:
+  - "请以项目经理Bot的身份回答：{{question}}"
+  - "你是一个项目管理者。请回答：{{question}}"
+
+providers:
+  - id: openai:chat:deepseek-chat
+    config:
+      apiBaseUrl: https://api.deepseek.com/v1
+      apiKeyEnvar: DEEPSEEK_API_KEY
+
+# 测试数据集
+tests:
+  - vars:
+      question: "项目进度怎么看？"
+    assert:
+      - type: contains-any
+        value: ["甘特图", "任务", "进度", "计划", "完成率"]
+      - type: latency
+        threshold: 15000
+
+  - vars:
+      question: "代码审查的标准是什么？"
+    assert:
+      - type: contains-any
+        value: ["功能", "可读性", "安全性", "测试", "规范"]
+      - type: latency
+        threshold: 15000
+
+  - vars:
+      question: "CI/CD pipeline失败了怎么办？"
+    assert:
+      - type: contains-any
+        value: ["日志", "失败", "排查", "恢复", "回滚"]
+      - type: latency
+        threshold: 15000
+
+  - vars:
+      question: "Gitea仓库如何管理权限？"
+    assert:
+      - type: contains-any
+        value: ["角色", "权限", "组织", "团队", "仓库"]
+      - type: latency
+        threshold: 15000
+
+  - vars:
+      question: "这个任务应该排什么优先级？"
+    assert:
+      - type: contains-any
+        value: ["优先级", "紧急", "重要", "影响", "截止"]
+      - type: latency
+        threshold: 15000
+
+  - vars:
+      question: "如何创建新的Hermes Bot？"
+    assert:
+      - type: contains-any
+        value: ["Bot", "配置", "创建", "Hermes", "步骤"]
+      - type: latency
+        threshold: 15000
+
+  - vars:
+      question: "Docker容器部署需要注意什么？"
+    assert:
+      - type: contains-any
+        value: ["镜像", "安全", "网络", "卷", "资源"]
+      - type: latency
+        threshold: 15000
+
+  - vars:
+      question: "Woodpecker CI流水线如何配置？"
+    assert:
+      - type: contains-any
+        value: ["YAML", "流水线", "步骤", "Woodpecker", "配置"]
+      - type: latency
+        threshold: 15000