Hermes CI Bot
fd9aa7cbc1
新增: 安全(API密钥/供应链)、架构(微服务/技术选型)、故障响应(宕机/post-mortem)、协作(跨团队/代码质量)、成本优化(云成本/预算) 成本追踪: cost-tracker.py 自动记录每次 eval 的 token 消耗和费用 基线: baseline-20260709-v2.json (36/36通过, $0.0090, 32K tokens)
46 lines
1.7 KiB
Python
46 lines
1.7 KiB
Python
"""Hermes CI Cost Tracker — 每次eval后记录token消耗"""
|
|
import json, os, csv, datetime
|
|
|
|
RESULTS_FILE = os.path.join(os.path.dirname(__file__), 'results.json')
|
|
COST_LOG = os.path.join(os.path.dirname(__file__), 'cost-history.csv')
|
|
|
|
# DeepSeek pricing (per 1M tokens)
|
|
INPUT_COST_PER_M = 0.07
|
|
OUTPUT_COST_PER_M = 0.28
|
|
|
|
with open(RESULTS_FILE) as f:
|
|
data = json.load(f)
|
|
|
|
# results is a nested dict with 'results' key inside
|
|
results_inner = data.get('results', {}).get('results', [])
|
|
if not isinstance(results_inner, list):
|
|
results_inner = data.get('results', {}).get('stats', {})
|
|
|
|
stats = data.get('results', {}).get('stats', {})
|
|
total_tests = stats.get('successes', 0) + stats.get('failures', 0) + stats.get('errors', 0)
|
|
passed = stats.get('successes', 0)
|
|
failed = stats.get('failures', 0)
|
|
|
|
# Token counts from stats.tokenUsage
|
|
token_usage = stats.get('tokenUsage', {})
|
|
total_input = token_usage.get('prompt', 0)
|
|
total_output = token_usage.get('completion', 0)
|
|
total_tokens = token_usage.get('total', total_input + total_output)
|
|
duration = int(stats.get('durationMs', 0) / 1000)
|
|
cost = (total_input * INPUT_COST_PER_M + total_output * OUTPUT_COST_PER_M) / 1_000_000
|
|
|
|
# Append to CSV
|
|
is_new = not os.path.exists(COST_LOG)
|
|
with open(COST_LOG, 'a', newline='') as f:
|
|
w = csv.writer(f)
|
|
if is_new:
|
|
w.writerow(['timestamp', 'tests', 'passed', 'failed', 'tokens', 'cost_usd', 'duration_s'])
|
|
w.writerow([
|
|
datetime.datetime.utcnow().strftime('%Y-%m-%d %H:%M UTC'),
|
|
total_tests, passed, failed, total_tokens, f'{cost:.4f}', duration
|
|
])
|
|
|
|
print(f'Tests: {total_tests} | Passed: {passed} | Failed: {failed}')
|
|
print(f'Tokens: {total_tokens} | Cost: ${cost:.4f} | Duration: {duration}s')
|
|
print(f'Log: {COST_LOG}')
|