feat: initial Hermes CI eval baseline
- promptfoo eval config with 8 test scenarios - Assertions: contains-any + latency checks - Woodpecker pipeline: eval-baseline + report steps - Evaluates Hermes Bot's project management responses
This commit is contained in:
@@ -0,0 +1,32 @@
|
||||
when:
|
||||
branch: main
|
||||
event: [push, pull_request]
|
||||
|
||||
steps:
|
||||
eval-baseline:
|
||||
image: node:23-alpine
|
||||
environment:
|
||||
DEEPSEEK_API_KEY:
|
||||
from_secret: DEEPSEEK_API_KEY
|
||||
commands:
|
||||
- apk add --no-cache git
|
||||
- npm install -g promptfoo
|
||||
- cd promptfoo-eval
|
||||
- promptfoo eval --max-concurrency 2 --no-cache
|
||||
- promptfoo results --format json > /tmp/eval-results.json
|
||||
- |
|
||||
FAILURES=$(cat /tmp/eval-results.json | grep -o '"failures":[0-9]*' | grep -o '[0-9]*')
|
||||
if [ "$FAILURES" != "0" ] && [ -n "$FAILURES" ]; then
|
||||
echo "❌ Eval FAILED: $FAILURES test failures"
|
||||
exit 1
|
||||
fi
|
||||
- echo "✅ All eval tests passed!"
|
||||
|
||||
report:
|
||||
image: alpine:latest
|
||||
commands:
|
||||
- echo "Hermes CI Eval Baseline - $(date -u '+%Y-%m-%d %H:%M UTC')"
|
||||
- echo "Status: ✅ PASS"
|
||||
- echo "Next steps: Review results and update eval dataset"
|
||||
when:
|
||||
status: success
|
||||
@@ -0,0 +1,77 @@
|
||||
description: "Hermes CI Eval Baseline v1 — 带断言评测"
|
||||
|
||||
prompts:
|
||||
- "请以项目经理Bot的身份回答:{{question}}"
|
||||
- "你是一个项目管理者。请回答:{{question}}"
|
||||
|
||||
providers:
|
||||
- id: openai:chat:deepseek-chat
|
||||
config:
|
||||
apiBaseUrl: https://api.deepseek.com/v1
|
||||
apiKeyEnvar: DEEPSEEK_API_KEY
|
||||
|
||||
# 测试数据集
|
||||
tests:
|
||||
- vars:
|
||||
question: "项目进度怎么看?"
|
||||
assert:
|
||||
- type: contains-any
|
||||
value: ["甘特图", "任务", "进度", "计划", "完成率"]
|
||||
- type: latency
|
||||
threshold: 15000
|
||||
|
||||
- vars:
|
||||
question: "代码审查的标准是什么?"
|
||||
assert:
|
||||
- type: contains-any
|
||||
value: ["功能", "可读性", "安全性", "测试", "规范"]
|
||||
- type: latency
|
||||
threshold: 15000
|
||||
|
||||
- vars:
|
||||
question: "CI/CD pipeline失败了怎么办?"
|
||||
assert:
|
||||
- type: contains-any
|
||||
value: ["日志", "失败", "排查", "恢复", "回滚"]
|
||||
- type: latency
|
||||
threshold: 15000
|
||||
|
||||
- vars:
|
||||
question: "Gitea仓库如何管理权限?"
|
||||
assert:
|
||||
- type: contains-any
|
||||
value: ["角色", "权限", "组织", "团队", "仓库"]
|
||||
- type: latency
|
||||
threshold: 15000
|
||||
|
||||
- vars:
|
||||
question: "这个任务应该排什么优先级?"
|
||||
assert:
|
||||
- type: contains-any
|
||||
value: ["优先级", "紧急", "重要", "影响", "截止"]
|
||||
- type: latency
|
||||
threshold: 15000
|
||||
|
||||
- vars:
|
||||
question: "如何创建新的Hermes Bot?"
|
||||
assert:
|
||||
- type: contains-any
|
||||
value: ["Bot", "配置", "创建", "Hermes", "步骤"]
|
||||
- type: latency
|
||||
threshold: 15000
|
||||
|
||||
- vars:
|
||||
question: "Docker容器部署需要注意什么?"
|
||||
assert:
|
||||
- type: contains-any
|
||||
value: ["镜像", "安全", "网络", "卷", "资源"]
|
||||
- type: latency
|
||||
threshold: 15000
|
||||
|
||||
- vars:
|
||||
question: "Woodpecker CI流水线如何配置?"
|
||||
assert:
|
||||
- type: contains-any
|
||||
value: ["YAML", "流水线", "步骤", "Woodpecker", "配置"]
|
||||
- type: latency
|
||||
threshold: 15000
|
||||
Reference in New Issue
Block a user