fix(alerts): kpi_alerts 去重键改生成侧语义键 dedupe_key(A2)— 文案改动不再重复落库
ci/woodpecker/push/main Pipeline was successful

- 列 dedupe_key VARCHAR(160) NULL + 唯一索引 (entity_id,kpi_id,alert_type,dedupe_key)
- write_alerts 三级判重:dedupe_key 命中 / 历史 NULL 行反解兼容 / 无命中插入
- 并发兜底:savepoint 捕获 IntegrityError 计跳过
- dry-run would_insert 改用 count_would_insert(修复 len(hits) 虚高)
- DDL 幂等执行器 scripts/ddl_kpi_alerts_dedupe.py(check/apply/rollback)
- tests/test_dedupe_key.py 7 passed(T1-T7b);alert/overdue 回归 173 passed
This commit is contained in:
wecom-fullstack
2026-10-22 11:29:54 +08:00
parent adcd118498
commit 9bbe3001bf
5 changed files with 441 additions and 17 deletions
+161 -15
View File
@@ -4,9 +4,10 @@
* **阈值/责任人/排除名单全部来自 `alert_rules.params`(DB 行)**,本模块不出现任何
业务阈值数字;缺键即该判据关闭(显式失效,不回落隐式默认值)。
* **只 INSERT**:命中写 `kpi_alerts(alert_type='overdue')`,同
`(entity_id, kpi_id, alert_type, alert_message)` 已存在则跳过(`alert_message`
含 period+对象 ⇒ 天然去重键)。**零 UPDATE / 零 DELETE**。
* **只 INSERT**:命中写 `kpi_alerts(alert_type='overdue')`,去重键为**生成侧语义键**
`dedupe_key = period|scope|ident`(单户 `cust|<unit_code>`、聚合 `agg|<族名>`),
由唯一索引兜底;文案口径变化不再产生重复行。历史行 `dedupe_key IS NULL` 走只读反解兼容。
**零 UPDATE / 零 DELETE**。
* `assignee` 必填(`assignee_by_unit[unit_code]` → `assignee_default`),两者皆空
⇒ `AssigneeMissing`(调用方转 400,fail-closed)——禁落空 assignee 的行。
* 口径:只读 `借方应收户口径`(`app.utils.ar_caliber`),负档位不抹平。
@@ -15,10 +16,12 @@ from __future__ import annotations
import calendar
import logging
import re
from decimal import Decimal, ROUND_HALF_UP
from typing import Any, Dict, List, Optional, Tuple
from sqlalchemy import text
from sqlalchemy.exc import IntegrityError
from sqlalchemy.orm import Session
from app.models import KPIAlert
@@ -738,35 +741,178 @@ def hits(result: Dict[str, Any]) -> List[Dict[str, Any]]:
return [h for h in all_hits if str(h.get("assignee") or "").strip()]
def write_alerts(db: Session, entity_id: int, kpi_id: int, result: Dict[str, Any]) -> Dict[str, int]:
"""把命中集合写成 `kpi_alerts(alert_type='overdue')`;历史行零触碰。"""
inserted = 0
skipped = 0
# ============================================================
# 去重键(生成侧语义键):period|scope|ident
# ============================================================
_AGG_KEY_FAMILY = {
"share_90plus": "share_90plus",
"count_mom": "count_mom",
"structure_90_150": "structure", # 族名,不跟随判据代次
"structure_150p": "structure",
}
_CUST_MSG_RE = re.compile(r"(([A-Za-z0-9_\-]+))(\d{4}-\d{2})")
_PERIOD_RE = re.compile(r"(\d{4}-\d{2})")
def _agg_family(key: str) -> str:
if key in _AGG_KEY_FAMILY:
return _AGG_KEY_FAMILY[key]
return str(key).split("_")[0] or "unknown"
def _item_dedupe_key(item: Dict[str, Any], period: str) -> Optional[str]:
"""生成侧幂等键;不解析文案。
- 单户:`<period>|cust|<unit_code>`
- 聚合:`<period>|agg|<族名>`(structure_90_150 / structure_150p 同归 structure)
- 无法判定 → None(调用方退回文案逐字比较,方向安全)
"""
p = str(item.get("period") or period or "").strip()
if not p:
return None
unit_code = str(item.get("unit_code") or "").strip()
if unit_code:
return "%s|cust|%s" % (p, unit_code)
key = str(item.get("key") or "").strip()
if key:
return "%s|agg|%s" % (p, _agg_family(key))
return None
def _key_from_message(msg: str) -> Optional[str]:
"""历史兼容:从旧文案只读反解 dedupe_key(仅用于 dedupe_key IS NULL 的行)。
反解失败返回 None ⇒ 视为未命中 ⇒ 照常插入(fail-open,多插一条而非吞真实告警)。
"""
if not msg:
return None
m = _CUST_MSG_RE.search(msg)
if m:
return "%s|cust|%s" % (m.group(2), m.group(1))
pm = _PERIOD_RE.search(msg)
if not pm:
return None
period = pm.group(1)
if "90 天以上" in msg and "占应收总额" in msg:
return "%s|agg|share_90plus" % period
if "逾期户数" in msg and "环比" in msg:
return "%s|agg|count_mom" % period
if "账龄" in msg and ("结构" in msg or "90–150" in msg or "150 天以上" in msg):
return "%s|agg|structure" % period
return None
def _legacy_keys(db: Session, entity_id: int, kpi_id: int) -> List[str]:
"""取历史行(dedupe_key IS NULL)反解出的键集合(边界层,量级=历史行数)。"""
rows = db.query(KPIAlert.alert_message).filter(
KPIAlert.entity_id == entity_id,
KPIAlert.kpi_id == kpi_id,
KPIAlert.alert_type == ALERT_TYPE,
KPIAlert.dedupe_key.is_(None),
).all()
keys = []
for (m,) in rows:
k = _key_from_message(m or "")
if k:
keys.append(k)
return keys
def count_would_insert(db: Session, entity_id: int, kpi_id: Optional[int],
result: Dict[str, Any]) -> int:
"""dry-run 用:按与 write_alerts 完全相同的判重口径,算出真正会插入的条数。"""
if kpi_id is None:
return 0
period = str(result.get("period") or "")
existing = {
k for (k,) in db.query(KPIAlert.dedupe_key).filter(
KPIAlert.entity_id == entity_id,
KPIAlert.kpi_id == kpi_id,
KPIAlert.alert_type == ALERT_TYPE,
KPIAlert.dedupe_key.isnot(None),
).all() if k
}
existing.update(_legacy_keys(db, entity_id, kpi_id))
seen = set()
n = 0
for h in hits(result):
msg = h.get("alert_message") or ""
if not msg:
continue
exists = db.query(KPIAlert.id).filter(
dk = _item_dedupe_key(h, period)
if dk is None:
# 无键 ⇒ 退回文案逐字(与旧行为一致)
n += 1
continue
if dk in existing or dk in seen:
continue
seen.add(dk)
n += 1
return n
def write_alerts(db: Session, entity_id: int, kpi_id: int, result: Dict[str, Any]) -> Dict[str, int]:
"""把命中集合写成 `kpi_alerts(alert_type='overdue')`;历史行零触碰。
判重三级:① dedupe_key 命中 ② 历史 NULL 行反解键命中 ③ 无命中 → INSERT。
并发兜底:唯一索引 + savepoint 捕获 IntegrityError 计跳过。
"""
inserted = 0
skipped = 0
period = str(result.get("period") or "")
existing = {
k for (k,) in db.query(KPIAlert.dedupe_key).filter(
KPIAlert.entity_id == entity_id,
KPIAlert.kpi_id == kpi_id,
KPIAlert.alert_type == ALERT_TYPE,
KPIAlert.alert_message == msg,
).first()
if exists:
skipped += 1
KPIAlert.dedupe_key.isnot(None),
).all() if k
}
existing.update(_legacy_keys(db, entity_id, kpi_id))
for h in hits(result):
msg = h.get("alert_message") or ""
if not msg:
continue
db.add(KPIAlert(
dk = _item_dedupe_key(h, period)
if dk is not None:
if dk in existing:
skipped += 1
continue
else:
# 无键(理论上不会发生):退回逐字判重(旧行为)
dup = db.query(KPIAlert.id).filter(
KPIAlert.entity_id == entity_id,
KPIAlert.kpi_id == kpi_id,
KPIAlert.alert_type == ALERT_TYPE,
KPIAlert.alert_message == msg,
).first()
if dup:
skipped += 1
continue
row = KPIAlert(
entity_id=entity_id,
kpi_id=kpi_id,
kpi_value_id=None,
alert_level=h.get("level") or "yellow",
alert_message=msg,
alert_type=ALERT_TYPE,
dedupe_key=dk,
suggestion=h.get("suggestion") or None,
status="pending",
assignee=h.get("assignee") or None,
))
inserted += 1
)
sp = db.begin_nested()
try:
db.add(row)
db.flush()
sp.commit()
inserted += 1
if dk:
existing.add(dk)
except IntegrityError:
sp.rollback()
skipped += 1
if inserted:
db.commit()
return {"inserted": inserted, "skipped_duplicate": skipped}