Hermes CI Bot
5ea2710ed2
- promptfoo eval config with 8 test scenarios - Assertions: contains-any + latency checks - Woodpecker pipeline: eval-baseline + report steps - Evaluates Hermes Bot's project management responses
33 lines
954 B
YAML
33 lines
954 B
YAML
when:
|
|
branch: main
|
|
event: [push, pull_request]
|
|
|
|
steps:
|
|
eval-baseline:
|
|
image: node:23-alpine
|
|
environment:
|
|
DEEPSEEK_API_KEY:
|
|
from_secret: DEEPSEEK_API_KEY
|
|
commands:
|
|
- apk add --no-cache git
|
|
- npm install -g promptfoo
|
|
- cd promptfoo-eval
|
|
- promptfoo eval --max-concurrency 2 --no-cache
|
|
- promptfoo results --format json > /tmp/eval-results.json
|
|
- |
|
|
FAILURES=$(cat /tmp/eval-results.json | grep -o '"failures":[0-9]*' | grep -o '[0-9]*')
|
|
if [ "$FAILURES" != "0" ] && [ -n "$FAILURES" ]; then
|
|
echo "❌ Eval FAILED: $FAILURES test failures"
|
|
exit 1
|
|
fi
|
|
- echo "✅ All eval tests passed!"
|
|
|
|
report:
|
|
image: alpine:latest
|
|
commands:
|
|
- echo "Hermes CI Eval Baseline - $(date -u '+%Y-%m-%d %H:%M UTC')"
|
|
- echo "Status: ✅ PASS"
|
|
- echo "Next steps: Review results and update eval dataset"
|
|
when:
|
|
status: success
|