feat: expand to 36 tests (5 new domains) + cost tracker

新增: 安全(API密钥/供应链)、架构(微服务/技术选型)、故障响应(宕机/post-mortem)、协作(跨团队/代码质量)、成本优化(云成本/预算)

成本追踪: cost-tracker.py 自动记录每次 eval 的 token 消耗和费用
基线: baseline-20260709-v2.json (36/36通过, $0.0090, 32K tokens)

.woodpecker/main.yml

@@ -9,24 +9,13 @@ steps:
       DEEPSEEK_API_KEY:
         from_secret: DEEPSEEK_API_KEY
     commands:
-      - apk add --no-cache git
+      - apk add --no-cache git python3 py3-pip
       - npm install -g promptfoo
       - cd promptfoo-eval
-      - promptfoo eval --max-concurrency 2 --no-cache
-      - promptfoo results --format json > /tmp/eval-results.json
+      - promptfoo eval --max-concurrency 2 --no-cache -o results.json
+      - python3 cost-tracker.py
       - |
-        FAILURES=$(cat /tmp/eval-results.json | grep -o '"failures":[0-9]*' | grep -o '[0-9]*')
-        if [ "$FAILURES" != "0" ] && [ -n "$FAILURES" ]; then
-          echo "❌ Eval FAILED: $FAILURES test failures"
-          exit 1
-        fi
+        FAILURES=$(python3 -c "import json; d=json.load(open('results.json')); s=d['results']['stats']; print(s['failures']+s['errors'])")
+        echo "Tests: $(python3 -c "import json; d=json.load(open('results.json')); s=d['results']['stats']; print(s['successes'])") passed, $FAILURES failed"
+        if [ "$FAILURES" != "0" ]; then exit 1; fi
       - echo "✅ All eval tests passed!"
-
-  report:
-    image: alpine:latest
-    commands:
-      - echo "Hermes CI Eval Baseline - $(date -u '+%Y-%m-%d %H:%M UTC')"
-      - echo "Status: ✅ PASS"
-      - echo "Next steps: Review results and update eval dataset"
-    when:
-      status: success

cost-history.csv

@@ -0,0 +1,3 @@
+timestamp,tests,passed,failed,tokens,cost_usd,duration_s
+2026-07-09 13:14 UTC,36,36,0,0,0.0000,0
+2026-07-09 13:15 UTC,36,36,0,32776,0.0090,197

cost-tracker.py

@@ -0,0 +1,45 @@
+"""Hermes CI Cost Tracker — 每次eval后记录token消耗"""
+import json, os, csv, datetime
+
+RESULTS_FILE = os.path.join(os.path.dirname(__file__), 'results.json')
+COST_LOG = os.path.join(os.path.dirname(__file__), 'cost-history.csv')
+
+# DeepSeek pricing (per 1M tokens)
+INPUT_COST_PER_M = 0.07
+OUTPUT_COST_PER_M = 0.28
+
+with open(RESULTS_FILE) as f:
+    data = json.load(f)
+
+# results is a nested dict with 'results' key inside
+results_inner = data.get('results', {}).get('results', [])
+if not isinstance(results_inner, list):
+    results_inner = data.get('results', {}).get('stats', {})
+
+stats = data.get('results', {}).get('stats', {})
+total_tests = stats.get('successes', 0) + stats.get('failures', 0) + stats.get('errors', 0)
+passed = stats.get('successes', 0)
+failed = stats.get('failures', 0)
+
+# Token counts from stats.tokenUsage
+token_usage = stats.get('tokenUsage', {})
+total_input = token_usage.get('prompt', 0)
+total_output = token_usage.get('completion', 0)
+total_tokens = token_usage.get('total', total_input + total_output)
+duration = int(stats.get('durationMs', 0) / 1000)
+cost = (total_input * INPUT_COST_PER_M + total_output * OUTPUT_COST_PER_M) / 1_000_000
+
+# Append to CSV
+is_new = not os.path.exists(COST_LOG)
+with open(COST_LOG, 'a', newline='') as f:
+    w = csv.writer(f)
+    if is_new:
+        w.writerow(['timestamp', 'tests', 'passed', 'failed', 'tokens', 'cost_usd', 'duration_s'])
+    w.writerow([
+        datetime.datetime.utcnow().strftime('%Y-%m-%d %H:%M UTC'),
+        total_tests, passed, failed, total_tokens, f'{cost:.4f}', duration
+    ])
+
+print(f'Tests: {total_tests} | Passed: {passed} | Failed: {failed}')
+print(f'Tokens: {total_tokens} | Cost: ${cost:.4f} | Duration: {duration}s')
+print(f'Log: {COST_LOG}')

promptfooconfig.yaml

@@ -1,4 +1,4 @@
-description: "Hermes CI Eval Baseline v1 — 带断言评测"
+description: "Hermes CI Eval Baseline v2 — 扩建版（8+10=18场景）"
 
 prompts:
   - "请以项目经理Bot的身份回答：{{question}}"
@@ -10,8 +10,8 @@ providers:
       apiBaseUrl: https://api.deepseek.com/v1
       apiKeyEnvar: DEEPSEEK_API_KEY
 
-# 测试数据集
 tests:
+  # ===== 原8个场景 =====
   - vars:
       question: "项目进度怎么看？"
     assert:
@@ -75,3 +75,88 @@ tests:
         value: ["YAML", "流水线", "步骤", "Woodpecker", "配置"]
       - type: latency
         threshold: 15000
+
+  # ===== 新增：安全 =====
+  - vars:
+      question: "项目中如何管理API密钥和敏感信息？"
+    assert:
+      - type: contains-any
+        value: ["密钥", "环境变量", "加密", "安全", ".env"]
+      - type: latency
+        threshold: 15000
+
+  - vars:
+      question: "如何防范供应链攻击？"
+    assert:
+      - type: contains-any
+        value: ["依赖", "镜像", "签名", "验证", "源"]
+      - type: latency
+        threshold: 15000
+
+  # ===== 新增：架构设计 =====
+  - vars:
+      question: "微服务和单体架构怎么选？"
+    assert:
+      - type: contains-any
+        value: ["微服务", "单体", "拆分", "耦合", "扩展"]
+      - type: latency
+        threshold: 15000
+
+  - vars:
+      question: "技术选型应该考虑哪些因素？"
+    assert:
+      - type: contains-any
+        value: ["维护", "团队", "生态", "性能", "成本"]
+      - type: latency
+        threshold: 15000
+
+  # ===== 新增：故障响应 =====
+  - vars:
+      question: "生产环境服务宕机了，第一反应应该做什么？"
+    assert:
+      - type: contains-any
+        value: ["恢复", "排查", "通知", "影响", "回滚"]
+      - type: latency
+        threshold: 15000
+
+  - vars:
+      question: "如何做事故复盘（post-mortem）？"
+    assert:
+      - type: contains-any
+        value: ["复盘", "根因", "改进", "措施", "时间线"]
+      - type: latency
+        threshold: 15000
+
+  # ===== 新增：团队协作 =====
+  - vars:
+      question: "跨团队协作有什么好的实践？"
+    assert:
+      - type: contains-any
+        value: ["接口", "同步", "文档", "沟通", "对齐"]
+      - type: latency
+        threshold: 15000
+
+  - vars:
+      question: "如何提高团队代码质量？"
+    assert:
+      - type: contains-any
+        value: ["审查", "规范", "自动化", "测试", "标准"]
+      - type: latency
+        threshold: 15000
+
+  # ===== 新增：成本优化 =====
+  - vars:
+      question: "如何降低云服务成本？"
+    assert:
+      - type: contains-any
+        value: ["实例", "预留", "监控", "缩容", "合理"]
+      - type: latency
+        threshold: 15000
+
+  - vars:
+      question: "项目预算超支了怎么处理？"
+    assert:
+      - type: contains-any
+        value: ["预算", "分析", "调整", "范围", "优先级"]
+      - type: latency
+        threshold: 15000