feat: initial Hermes CI eval baseline

- promptfoo eval config with 8 test scenarios
- Assertions: contains-any + latency checks
- Woodpecker pipeline: eval-baseline + report steps
- Evaluates Hermes Bot's project management responses
This commit is contained in:
Hermes CI Bot
2026-07-09 20:14:15 +08:00
commit 5ea2710ed2
2 changed files with 109 additions and 0 deletions
+32
View File
@@ -0,0 +1,32 @@
when:
branch: main
event: [push, pull_request]
steps:
eval-baseline:
image: node:23-alpine
environment:
DEEPSEEK_API_KEY:
from_secret: DEEPSEEK_API_KEY
commands:
- apk add --no-cache git
- npm install -g promptfoo
- cd promptfoo-eval
- promptfoo eval --max-concurrency 2 --no-cache
- promptfoo results --format json > /tmp/eval-results.json
- |
FAILURES=$(cat /tmp/eval-results.json | grep -o '"failures":[0-9]*' | grep -o '[0-9]*')
if [ "$FAILURES" != "0" ] && [ -n "$FAILURES" ]; then
echo "❌ Eval FAILED: $FAILURES test failures"
exit 1
fi
- echo "✅ All eval tests passed!"
report:
image: alpine:latest
commands:
- echo "Hermes CI Eval Baseline - $(date -u '+%Y-%m-%d %H:%M UTC')"
- echo "Status: ✅ PASS"
- echo "Next steps: Review results and update eval dataset"
when:
status: success
+77
View File
@@ -0,0 +1,77 @@
description: "Hermes CI Eval Baseline v1 — 带断言评测"
prompts:
- "请以项目经理Bot的身份回答:{{question}}"
- "你是一个项目管理者。请回答:{{question}}"
providers:
- id: openai:chat:deepseek-chat
config:
apiBaseUrl: https://api.deepseek.com/v1
apiKeyEnvar: DEEPSEEK_API_KEY
# 测试数据集
tests:
- vars:
question: "项目进度怎么看?"
assert:
- type: contains-any
value: ["甘特图", "任务", "进度", "计划", "完成率"]
- type: latency
threshold: 15000
- vars:
question: "代码审查的标准是什么?"
assert:
- type: contains-any
value: ["功能", "可读性", "安全性", "测试", "规范"]
- type: latency
threshold: 15000
- vars:
question: "CI/CD pipeline失败了怎么办?"
assert:
- type: contains-any
value: ["日志", "失败", "排查", "恢复", "回滚"]
- type: latency
threshold: 15000
- vars:
question: "Gitea仓库如何管理权限?"
assert:
- type: contains-any
value: ["角色", "权限", "组织", "团队", "仓库"]
- type: latency
threshold: 15000
- vars:
question: "这个任务应该排什么优先级?"
assert:
- type: contains-any
value: ["优先级", "紧急", "重要", "影响", "截止"]
- type: latency
threshold: 15000
- vars:
question: "如何创建新的Hermes Bot"
assert:
- type: contains-any
value: ["Bot", "配置", "创建", "Hermes", "步骤"]
- type: latency
threshold: 15000
- vars:
question: "Docker容器部署需要注意什么?"
assert:
- type: contains-any
value: ["镜像", "安全", "网络", "卷", "资源"]
- type: latency
threshold: 15000
- vars:
question: "Woodpecker CI流水线如何配置?"
assert:
- type: contains-any
value: ["YAML", "流水线", "步骤", "Woodpecker", "配置"]
- type: latency
threshold: 15000