Hermes CI Fix
8c5853f64b
deviation_engine: link kpi_value_id on deviation/trend alert rows; skip when period not attributable or kpi_values row missing; dedupe key now (entity_id,kpi_id,kpi_value_id,alert_type) deviation_push: same-source R1/R5 guards, no-period request writes nothing tests: add test_alert_caliber_r5_residual_20260919 (13 cases incl. negative controls); adapt r1r4/r5 fixtures to the supplemental ruling
477 lines
19 KiB
Python
477 lines
19 KiB
Python
"""差异预警引擎 — 管理会计OS
|
||
实际 vs 预算/目标对比，超阈值自动推送预警
|
||
|
||
功能:
|
||
1. 实际 vs 预算差异计算（差异额/差异率）
|
||
2. 同比/环比差异计算
|
||
3. 趋势异常检测（连续N期下滑/上升）
|
||
4. 差异预警触发（集成到现有预警系统）
|
||
"""
|
||
import logging
|
||
import json
|
||
from datetime import datetime
|
||
from typing import Optional
|
||
|
||
from app.database import get_session_local
|
||
from app.models import KPIDefinition, KPIValue, KPIAlert, BudgetPlan, SystemConfig
|
||
# 裁定 R1/R2/R3（2026-09-19 财务口径）：demo 账套排除 / 期间窗口 / 规整月期间
|
||
from app.utils.alert_entity_scope import (
|
||
excluded_entity_ids, kpi_excluded, period_in_window, window_lower_period,
|
||
guard_alert_period, # R5-补1（2026-09-19）：期间可归期 + 关联 kpi_values
|
||
)
|
||
|
||
logger = logging.getLogger("cma.deviation")
|
||
|
||
|
||
# 越高越好型KPI默认列表（P2-⑤ 2026-08-28: 提为 system_configs 可配置）
|
||
DEFAULT_HIGHER_BETTER = [
|
||
"SALES_TOTAL", "CUSTOMER_COUNT", "SALES_PROFIT_RATE",
|
||
"RECEIVABLE_TURNOVER", "TURNOVER_RATE",
|
||
"CUSTOMER_SATISFACTION", "ORDER_DELIVERY_RATE",
|
||
]
|
||
CONFIG_KEY_HIGHER_BETTER = "kpi_alert_higher_better"
|
||
|
||
|
||
def get_higher_better_codes(db, entity_id: int = None) -> list:
|
||
"""读取越高越好型KPI编码列表（system_configs 可维护，无配置回落默认）"""
|
||
cfg = db.query(SystemConfig).filter(
|
||
SystemConfig.config_key == CONFIG_KEY_HIGHER_BETTER
|
||
).first()
|
||
if cfg and cfg.config_value:
|
||
try:
|
||
codes = json.loads(cfg.config_value)
|
||
if isinstance(codes, list):
|
||
return [str(c) for c in codes]
|
||
except Exception:
|
||
logger.warning("system_configs[%s] 解析失败, 回落默认", CONFIG_KEY_HIGHER_BETTER)
|
||
return list(DEFAULT_HIGHER_BETTER)
|
||
|
||
|
||
# ============================================================
|
||
# 差异计算
|
||
# ============================================================
|
||
|
||
def calc_deviation(actual: float, budget: float) -> dict:
|
||
"""计算差异额和差异率"""
|
||
if budget is None or budget == 0:
|
||
return {
|
||
"deviation_amount": None,
|
||
"deviation_rate": None,
|
||
"is_over_budget": None,
|
||
}
|
||
amount = round(actual - budget, 2)
|
||
rate = round(amount / budget * 100, 2)
|
||
return {
|
||
"deviation_amount": amount,
|
||
"deviation_rate": rate,
|
||
"is_over_budget": amount > 0,
|
||
}
|
||
|
||
|
||
def get_budget_for_kpi(db, kpi_id: int, period: str, version: str = None) -> Optional[float]:
|
||
"""获取指定KPI在指定期间的预算值"""
|
||
query = db.query(BudgetPlan).filter(
|
||
BudgetPlan.kpi_id == kpi_id,
|
||
BudgetPlan.period == period,
|
||
BudgetPlan.status == "active",
|
||
)
|
||
if version:
|
||
query = query.filter(BudgetPlan.version == version)
|
||
plan = query.order_by(BudgetPlan.updated_at.desc()).first()
|
||
return plan.budget_value if plan else None
|
||
|
||
|
||
def get_kpi_value_row(db, kpi_id: int, period: str):
|
||
"""取 (kpi_id, period) 的 KPIValue 行（取行口径唯一来源）。
|
||
|
||
与 get_actual_for_kpi 同源：预警行关联的 kpi_values 行 = 数值来源那一行。
|
||
"""
|
||
return db.query(KPIValue).filter(
|
||
KPIValue.kpi_id == kpi_id,
|
||
KPIValue.period == period,
|
||
).order_by(KPIValue.calculated_at.desc()).first()
|
||
|
||
|
||
def get_actual_for_kpi(db, kpi_id: int, period: str) -> Optional[float]:
|
||
"""获取指定KPI在指定期间的实际值"""
|
||
row = get_kpi_value_row(db, kpi_id, period)
|
||
return row.actual_value if row else None
|
||
|
||
|
||
def resolve_alert_kpi_value(db, kpi, period: str):
|
||
"""R5-补1（财务裁定 2026-09-19）：解析差异/趋势预警行必须关联的 `kpi_values` 行。
|
||
|
||
裁定原文：「此类预警行必须在落库时写入 `kpi_value_id`，指向 **同 `entity_id` +
|
||
同 `kpi_id` + 文案所示规整月期间** 的 `kpi_values` 行。文案期间不可解析为 `YYYY-MM`，
|
||
或该 (kpi_id, period) 无 `kpi_values` 行 → 按 R5 `guard_alert_period` 跳过不入库
|
||
（不得以 `kpi_value_id=NULL` 落行——『文案里有期间』不构成可归期）。」
|
||
|
||
- 账套约束由 `kpi_id` 传递：KPIDefinition 归属单一账套，故 kpi_id 已唯一确定 entity_id；
|
||
取行口径与 `get_kpi_value_row`（= `get_actual_for_kpi` 的来源行）完全一致，
|
||
保证「预警关联的那一行」就是该预警数值的来源行，不会关联到别的账套/别的期间。
|
||
- 返回 `(kpi_value_row | None, skip_reason | None)`；跳过原因供调用点计数与日志。
|
||
"""
|
||
if not guard_alert_period(period, source="deviation_engine.resolve_alert_kpi_value",
|
||
detail="kpi=%s" % getattr(kpi, "kpi_code", "")):
|
||
return None, "R5_non_attributable_period"
|
||
row = get_kpi_value_row(db, kpi.id, period)
|
||
if row is None:
|
||
logger.info(
|
||
"R5-补1 期间关联：跳过不入库（该 (kpi, period) 无 kpi_values 行，期间不可派生）"
|
||
" kpi=%s period=%s", getattr(kpi, "kpi_code", ""), period)
|
||
return None, "no_kpi_value_row"
|
||
return row, None
|
||
|
||
|
||
def calc_period_deviation(db, kpi_id: int, period: str) -> dict:
|
||
"""单KPI单期的差异计算"""
|
||
actual = get_actual_for_kpi(db, kpi_id, period)
|
||
budget = get_budget_for_kpi(db, kpi_id, period)
|
||
|
||
kpi = db.query(KPIDefinition).filter(KPIDefinition.id == kpi_id).first()
|
||
kpi_code = kpi.kpi_code if kpi else "unknown"
|
||
|
||
# 如果没预算值，用 target_value 作为替代
|
||
if budget is None and kpi:
|
||
# 尝试把年度目标按月均分
|
||
month = int(period.split("-")[1])
|
||
target = kpi.target_value
|
||
if target and target > 0 and kpi.frequency == "monthly":
|
||
budget = round(target / 12, 2)
|
||
|
||
deviation = calc_deviation(actual, budget) if actual is not None else None
|
||
|
||
result = {
|
||
"kpi_id": kpi_id,
|
||
"kpi_code": kpi_code,
|
||
"period": period,
|
||
"actual_value": actual,
|
||
"budget_value": budget,
|
||
}
|
||
if deviation:
|
||
result.update(deviation)
|
||
return result
|
||
|
||
|
||
# ============================================================
|
||
# 同比/环比差异
|
||
# ============================================================
|
||
|
||
def calc_period_diff(db, kpi_id: int, current_period: str, diff_type: str = "yoy") -> dict:
|
||
"""计算同比（上年同期）或环比（上期）差异"""
|
||
year, month = current_period.split("-")
|
||
y, m = int(year), int(month)
|
||
|
||
if diff_type == "yoy":
|
||
# 同比：上年同期
|
||
prev_period = f"{y-1}-{m:02d}"
|
||
label = "同比"
|
||
elif diff_type == "mom":
|
||
# 环比：上个月
|
||
prev_m = m - 1
|
||
prev_y = y
|
||
if prev_m <= 0:
|
||
prev_m += 12
|
||
prev_y -= 1
|
||
prev_period = f"{prev_y}-{prev_m:02d}"
|
||
label = "环比"
|
||
else:
|
||
return {"error": f"未知比较类型: {diff_type}"}
|
||
|
||
current = get_actual_for_kpi(db, kpi_id, current_period)
|
||
previous = get_actual_for_kpi(db, kpi_id, prev_period)
|
||
|
||
kpi = db.query(KPIDefinition).filter(KPIDefinition.id == kpi_id).first()
|
||
kpi_code = kpi.kpi_code if kpi else "unknown"
|
||
|
||
if current is None or previous is None or previous == 0:
|
||
return {
|
||
"kpi_id": kpi_id,
|
||
"kpi_code": kpi_code,
|
||
"type": diff_type,
|
||
"label": label,
|
||
"current_period": current_period,
|
||
"prev_period": prev_period,
|
||
"current_value": current,
|
||
"prev_value": previous,
|
||
"diff_amount": None,
|
||
"diff_rate": None,
|
||
}
|
||
|
||
diff_amount = round(current - previous, 2)
|
||
diff_rate = round(diff_amount / previous * 100, 2)
|
||
return {
|
||
"kpi_id": kpi_id,
|
||
"kpi_code": kpi_code,
|
||
"type": diff_type,
|
||
"label": label,
|
||
"current_period": current_period,
|
||
"prev_period": prev_period,
|
||
"current_value": current,
|
||
"prev_value": previous,
|
||
"diff_amount": diff_amount,
|
||
"diff_rate": diff_rate,
|
||
}
|
||
|
||
|
||
# ============================================================
|
||
# 趋势检测
|
||
# ============================================================
|
||
|
||
def check_trend_anomaly(db, kpi_id: int, period: str, consecutive: int = 3) -> dict:
|
||
"""检测连续N期下滑或上升的趋势异常"""
|
||
kpi = db.query(KPIDefinition).filter(KPIDefinition.id == kpi_id).first()
|
||
if not kpi:
|
||
return {"anomaly": False}
|
||
|
||
# 获取包括当前期在内的近期数据
|
||
year, month = period.split("-")
|
||
y, m = int(year), int(month)
|
||
|
||
values = []
|
||
for i in range(consecutive + 2): # 多取2期做参考
|
||
p = f"{y}-{m:02d}"
|
||
v = get_actual_for_kpi(db, kpi_id, p)
|
||
if v is not None:
|
||
values.append({"period": p, "value": v})
|
||
m -= 1
|
||
if m <= 0:
|
||
m += 12
|
||
y -= 1
|
||
|
||
values.reverse() # 按时间正序
|
||
if len(values) < consecutive:
|
||
return {"anomaly": False, "reason": "数据不足"}
|
||
|
||
last_n = values[-consecutive:]
|
||
all_decreasing = all(last_n[i]["value"] > last_n[i + 1]["value"] for i in range(len(last_n) - 1))
|
||
all_increasing = all(last_n[i]["value"] < last_n[i + 1]["value"] for i in range(len(last_n) - 1))
|
||
|
||
if all_decreasing:
|
||
return {
|
||
"anomaly": True,
|
||
"type": "continuous_decline",
|
||
"level": "yellow" if consecutive >= 3 else "green",
|
||
"periods": [v["period"] for v in last_n],
|
||
"values": [v["value"] for v in last_n],
|
||
"message": f"{kpi.kpi_name} 连续{consecutive}期下滑",
|
||
}
|
||
if all_increasing:
|
||
return {
|
||
"anomaly": True,
|
||
"type": "continuous_rise",
|
||
"level": "yellow" if consecutive >= 3 else "green",
|
||
"periods": [v["period"] for v in last_n],
|
||
"values": [v["value"] for v in last_n],
|
||
"message": f"{kpi.kpi_name} 连续{consecutive}期上升（可能过热）",
|
||
}
|
||
|
||
return {"anomaly": False}
|
||
|
||
|
||
# ============================================================
|
||
# 差异预警触发
|
||
# ============================================================
|
||
|
||
def build_deviation_alert(db, kpi, period: str, entity_id: int = 1, min_rate: float = 10.0) -> dict:
|
||
"""统一告警构建 — 双出口共享一套逻辑 (P2-⑤ 2026-08-28)
|
||
|
||
预算告警(budget_deviation_alerts) 与 KPIAlert 都调用本函数，差异仅级别映射：
|
||
- budget 出口: warning/critical @ 20/50
|
||
- KPIAlert 出口: yellow/red @ 10/30
|
||
归因(P1-③): attribution 拆解 + 场景建议 由 alert_attribution 组装。
|
||
|
||
返回:
|
||
triggered: bool 是否触发
|
||
level: budget出口级别 warning/critical
|
||
kpi_alert_level: KPIAlert出口级别 yellow/red
|
||
deviation: calc_period_deviation 结果
|
||
suggestion: 模板建议文案
|
||
alert_type: 归因场景类型
|
||
attribution: 归因JSON dict
|
||
scenario_id: 场景建议ID
|
||
"""
|
||
from app.utils.alert_attribution import build_attribution, match_scenario
|
||
|
||
deviation = calc_period_deviation(db, kpi.id, period)
|
||
if deviation.get("deviation_rate") is None:
|
||
return {"triggered": False}
|
||
|
||
rate = abs(deviation["deviation_rate"])
|
||
actual = deviation.get("actual_value")
|
||
budget = deviation.get("budget_value")
|
||
|
||
# 方向性：越高越好型（配置化，system_configs.kpi_alert_higher_better）
|
||
higher_better = kpi.kpi_code in get_higher_better_codes(db, entity_id)
|
||
|
||
if higher_better:
|
||
# 实际低于预算才是问题
|
||
if not (actual is not None and budget is not None and actual < budget and rate >= min_rate):
|
||
return {"triggered": False}
|
||
suggestion = (
|
||
f"实际值低于预算 {rate}%，建议分析业务量未达预期的原因（子KPI拆解见归因），"
|
||
f"制定增量获客或转化提升计划"
|
||
)
|
||
else:
|
||
# 实际高于预算才是问题（成本型）
|
||
if not (actual is not None and budget is not None and actual > budget and rate >= min_rate):
|
||
return {"triggered": False}
|
||
suggestion = (
|
||
f"实际值超出预算 {rate}%，建议核查超支原因（科目明细拆解见归因）并采取控制措施"
|
||
)
|
||
|
||
# 级别映射（双出口）
|
||
budget_level = "critical" if rate > 50 else "warning"
|
||
kpi_alert_level = "red" if rate >= 30 else "yellow"
|
||
|
||
# 归因组装 (P1-③)
|
||
alert_type = None
|
||
attribution = None
|
||
scenario_id = None
|
||
try:
|
||
attribution, alert_type = build_attribution(db, kpi.id, period)
|
||
scenario = match_scenario(db, alert_type)
|
||
if scenario:
|
||
scenario_id = scenario["scenario_id"]
|
||
except Exception as e: # 归因失败不阻断告警主流程
|
||
logger.warning("归因组装失败 kpi=%s: %s", kpi.kpi_code, e)
|
||
|
||
return {
|
||
"triggered": True,
|
||
"level": budget_level,
|
||
"kpi_alert_level": kpi_alert_level,
|
||
"deviation": deviation,
|
||
"suggestion": suggestion,
|
||
"alert_type": alert_type,
|
||
"attribution": attribution,
|
||
"scenario_id": scenario_id,
|
||
}
|
||
|
||
|
||
def run_deviation_check(db_session, period: str = None) -> int:
|
||
"""运行差异预警检查，返回新增预警数（统一走 build_deviation_alert，P2-⑤）
|
||
|
||
2026-09-19 财务口径裁定 R1/R2/R3（`alert_entity_scope`）：
|
||
· R1 —— `status='demo'` 的账套不参与遍历（demo 仅演示，不产生业务预警）；
|
||
· R2/R3 —— 期间须为规整月 `YYYY-MM` 且落在窗口（当月往前 6 个自然月，含当月）内。
|
||
· R5-补1 —— 差异/趋势分支**必须落 `kpi_value_id` 关联**（同 kpi 同期间的 `kpi_values` 行）；
|
||
文案期间不可解析为 `YYYY-MM` 或该 (kpi, period) 无 `kpi_values` 行 → 跳过不入库；
|
||
趋势类以「计算基准期（趋势末端当期）」为关联期间；差异类去重键
|
||
= `(entity_id, kpi_id, kpi_value_id, alert_type)`。
|
||
"""
|
||
if period is None:
|
||
period = datetime.now().strftime("%Y-%m")
|
||
if not period_in_window(period):
|
||
logger.warning(" 差异预警跳过（R2/R3）: 期间 %s 非规整月或早于窗口下限 %s",
|
||
period, window_lower_period())
|
||
return 0
|
||
|
||
excluded = excluded_entity_ids(db_session)
|
||
kpis = db_session.query(KPIDefinition).filter(
|
||
KPIDefinition.status == "active"
|
||
).all()
|
||
|
||
new_count = 0
|
||
skipped_demo = 0
|
||
skipped_period_linkage = 0
|
||
skipped_linkage_kpis = []
|
||
for kpi in kpis:
|
||
# R1（裁定 2026-09-19）：demo 演示账套不参与预警遍历
|
||
if kpi_excluded(db_session, kpi, excluded):
|
||
skipped_demo += 1
|
||
continue
|
||
# 1. 差异预警：实际 vs 预算（统一逻辑）
|
||
result = build_deviation_alert(db_session, kpi, period)
|
||
if result["triggered"]:
|
||
deviation = result["deviation"]
|
||
alert_msg = (
|
||
f"{kpi.kpi_name}[{period}] 差异预警: 实际{deviation['actual_value']} "
|
||
f"vs 预算{deviation['budget_value']}，"
|
||
f"差异率{deviation['deviation_rate']}%"
|
||
)
|
||
alert_type = result["alert_type"] or "actual"
|
||
|
||
# R5-补1（裁定 2026-09-19）：期间可归期 + 必须落 kpi_value_id 关联
|
||
#（文案里的期间不构成可归期；无 kpi_values 行 → 跳过不入库）
|
||
linked_kv, _skip_reason = resolve_alert_kpi_value(db_session, kpi, period)
|
||
if linked_kv is None:
|
||
skipped_period_linkage += 1
|
||
skipped_linkage_kpis.append("%s[%s]" % (kpi.kpi_code, period))
|
||
else:
|
||
# 去重键（R5-补1 变更）：(entity_id, kpi_id, kpi_value_id, alert_type)
|
||
#（原按 `alert_message contains period` 判断，改关联后按可归期目标去重）
|
||
existing = db_session.query(KPIAlert).filter(
|
||
KPIAlert.entity_id == (kpi.entity_id or 1),
|
||
KPIAlert.kpi_id == kpi.id,
|
||
KPIAlert.kpi_value_id == linked_kv.id,
|
||
KPIAlert.alert_type == alert_type,
|
||
KPIAlert.status.in_(["pending", "processing"]),
|
||
).first()
|
||
|
||
if not existing:
|
||
alert = KPIAlert(
|
||
entity_id=kpi.entity_id or 1, # 2026-09-13 F1: 显式写归属（原缺省=1 串账套）
|
||
kpi_id=kpi.id,
|
||
kpi_value_id=linked_kv.id, # R5-补1: 期间经该行可派生（可归期）
|
||
alert_level=result["kpi_alert_level"],
|
||
alert_message=f"[差异预警] {alert_msg}",
|
||
alert_type=alert_type,
|
||
suggestion=result["suggestion"],
|
||
status="pending",
|
||
)
|
||
db_session.add(alert)
|
||
new_count += 1
|
||
logger.info(f" 新增差异预警 [{result['kpi_alert_level']}] {kpi.kpi_name}: 差异率{deviation['deviation_rate']}%")
|
||
|
||
# 2. 趋势异常检测（每期检查连续3期）
|
||
trend = check_trend_anomaly(db_session, kpi.id, period, consecutive=3)
|
||
if trend.get("anomaly") and trend.get("level") in ("yellow", "red"):
|
||
trend_alert_msg = trend["message"]
|
||
# R5-补1：趋势类以「计算基准期（趋势末端当期）」为关联期间；该期无行 → 跳过
|
||
trend_periods = trend.get("periods") or []
|
||
base_period = trend_periods[-1] if trend_periods else period
|
||
linked_kv, _skip_reason = resolve_alert_kpi_value(db_session, kpi, base_period)
|
||
if linked_kv is None:
|
||
skipped_period_linkage += 1
|
||
skipped_linkage_kpis.append("%s[%s]" % (kpi.kpi_code, base_period))
|
||
else:
|
||
existing_trend = db_session.query(KPIAlert).filter(
|
||
KPIAlert.kpi_id == kpi.id,
|
||
KPIAlert.alert_message.contains("[趋势预警]"),
|
||
KPIAlert.status.in_(["pending", "processing"]),
|
||
).first()
|
||
|
||
if not existing_trend:
|
||
alert = KPIAlert(
|
||
entity_id=kpi.entity_id or 1, # 2026-09-13 F1: 显式写归属
|
||
kpi_id=kpi.id,
|
||
kpi_value_id=linked_kv.id, # R5-补1: 趋势末端当期关联
|
||
alert_level=trend["level"],
|
||
alert_message=f"[趋势预警] {trend_alert_msg}",
|
||
status="pending",
|
||
)
|
||
db_session.add(alert)
|
||
new_count += 1
|
||
logger.info(f" 新增趋势预警 [{trend['level']}] {trend_alert_msg}")
|
||
|
||
if skipped_demo:
|
||
logger.info(" 差异预警跳过 demo 账套 KPI %s 项（R1，裁定 2026-09-19）", skipped_demo)
|
||
if skipped_period_linkage:
|
||
logger.info(" 差异/趋势预警跳过（R5-补1 期间不可归期或无 kpi_values 行）KPI %s 项: %s",
|
||
skipped_period_linkage, skipped_linkage_kpis)
|
||
db_session.commit()
|
||
return new_count
|
||
|
||
|
||
if __name__ == "__main__":
|
||
logging.basicConfig(
|
||
level=logging.INFO,
|
||
format="%(asctime)s [%(levelname)s] %(name)s: %(message)s",
|
||
)
|
||
db = get_session_local()()
|
||
try:
|
||
n = run_deviation_check(db)
|
||
print(f"差异预警检查完成: 新增 {n} 条")
|
||
finally:
|
||
db.close()
|