feat: initial Hermes CI eval baseline
- promptfoo eval config with 8 test scenarios - Assertions: contains-any + latency checks - Woodpecker pipeline: eval-baseline + report steps - Evaluates Hermes Bot's project management responses
This commit is contained in:
@@ -0,0 +1,32 @@
|
||||
when:
|
||||
branch: main
|
||||
event: [push, pull_request]
|
||||
|
||||
steps:
|
||||
eval-baseline:
|
||||
image: node:23-alpine
|
||||
environment:
|
||||
DEEPSEEK_API_KEY:
|
||||
from_secret: DEEPSEEK_API_KEY
|
||||
commands:
|
||||
- apk add --no-cache git
|
||||
- npm install -g promptfoo
|
||||
- cd promptfoo-eval
|
||||
- promptfoo eval --max-concurrency 2 --no-cache
|
||||
- promptfoo results --format json > /tmp/eval-results.json
|
||||
- |
|
||||
FAILURES=$(cat /tmp/eval-results.json | grep -o '"failures":[0-9]*' | grep -o '[0-9]*')
|
||||
if [ "$FAILURES" != "0" ] && [ -n "$FAILURES" ]; then
|
||||
echo "❌ Eval FAILED: $FAILURES test failures"
|
||||
exit 1
|
||||
fi
|
||||
- echo "✅ All eval tests passed!"
|
||||
|
||||
report:
|
||||
image: alpine:latest
|
||||
commands:
|
||||
- echo "Hermes CI Eval Baseline - $(date -u '+%Y-%m-%d %H:%M UTC')"
|
||||
- echo "Status: ✅ PASS"
|
||||
- echo "Next steps: Review results and update eval dataset"
|
||||
when:
|
||||
status: success
|
||||
Reference in New Issue
Block a user