feat: initial Hermes CI eval baseline

- promptfoo eval config with 8 test scenarios
- Assertions: contains-any + latency checks
- Woodpecker pipeline: eval-baseline + report steps
- Evaluates Hermes Bot's project management responses
This commit is contained in:
Hermes CI Bot
2026-07-09 20:14:15 +08:00
commit 5ea2710ed2
2 changed files with 109 additions and 0 deletions
+32
View File
@@ -0,0 +1,32 @@
when:
branch: main
event: [push, pull_request]
steps:
eval-baseline:
image: node:23-alpine
environment:
DEEPSEEK_API_KEY:
from_secret: DEEPSEEK_API_KEY
commands:
- apk add --no-cache git
- npm install -g promptfoo
- cd promptfoo-eval
- promptfoo eval --max-concurrency 2 --no-cache
- promptfoo results --format json > /tmp/eval-results.json
- |
FAILURES=$(cat /tmp/eval-results.json | grep -o '"failures":[0-9]*' | grep -o '[0-9]*')
if [ "$FAILURES" != "0" ] && [ -n "$FAILURES" ]; then
echo "❌ Eval FAILED: $FAILURES test failures"
exit 1
fi
- echo "✅ All eval tests passed!"
report:
image: alpine:latest
commands:
- echo "Hermes CI Eval Baseline - $(date -u '+%Y-%m-%d %H:%M UTC')"
- echo "Status: ✅ PASS"
- echo "Next steps: Review results and update eval dataset"
when:
status: success