feat: expand to 36 tests (5 new domains) + cost tracker
新增: 安全(API密钥/供应链)、架构(微服务/技术选型)、故障响应(宕机/post-mortem)、协作(跨团队/代码质量)、成本优化(云成本/预算) 成本追踪: cost-tracker.py 自动记录每次 eval 的 token 消耗和费用 基线: baseline-20260709-v2.json (36/36通过, $0.0090, 32K tokens)
This commit is contained in:
@@ -0,0 +1,45 @@
|
||||
"""Hermes CI Cost Tracker — 每次eval后记录token消耗"""
|
||||
import json, os, csv, datetime
|
||||
|
||||
RESULTS_FILE = os.path.join(os.path.dirname(__file__), 'results.json')
|
||||
COST_LOG = os.path.join(os.path.dirname(__file__), 'cost-history.csv')
|
||||
|
||||
# DeepSeek pricing (per 1M tokens)
|
||||
INPUT_COST_PER_M = 0.07
|
||||
OUTPUT_COST_PER_M = 0.28
|
||||
|
||||
with open(RESULTS_FILE) as f:
|
||||
data = json.load(f)
|
||||
|
||||
# results is a nested dict with 'results' key inside
|
||||
results_inner = data.get('results', {}).get('results', [])
|
||||
if not isinstance(results_inner, list):
|
||||
results_inner = data.get('results', {}).get('stats', {})
|
||||
|
||||
stats = data.get('results', {}).get('stats', {})
|
||||
total_tests = stats.get('successes', 0) + stats.get('failures', 0) + stats.get('errors', 0)
|
||||
passed = stats.get('successes', 0)
|
||||
failed = stats.get('failures', 0)
|
||||
|
||||
# Token counts from stats.tokenUsage
|
||||
token_usage = stats.get('tokenUsage', {})
|
||||
total_input = token_usage.get('prompt', 0)
|
||||
total_output = token_usage.get('completion', 0)
|
||||
total_tokens = token_usage.get('total', total_input + total_output)
|
||||
duration = int(stats.get('durationMs', 0) / 1000)
|
||||
cost = (total_input * INPUT_COST_PER_M + total_output * OUTPUT_COST_PER_M) / 1_000_000
|
||||
|
||||
# Append to CSV
|
||||
is_new = not os.path.exists(COST_LOG)
|
||||
with open(COST_LOG, 'a', newline='') as f:
|
||||
w = csv.writer(f)
|
||||
if is_new:
|
||||
w.writerow(['timestamp', 'tests', 'passed', 'failed', 'tokens', 'cost_usd', 'duration_s'])
|
||||
w.writerow([
|
||||
datetime.datetime.utcnow().strftime('%Y-%m-%d %H:%M UTC'),
|
||||
total_tests, passed, failed, total_tokens, f'{cost:.4f}', duration
|
||||
])
|
||||
|
||||
print(f'Tests: {total_tests} | Passed: {passed} | Failed: {failed}')
|
||||
print(f'Tokens: {total_tokens} | Cost: ${cost:.4f} | Duration: {duration}s')
|
||||
print(f'Log: {COST_LOG}')
|
||||
Reference in New Issue
Block a user