feat: 数据治理 — 入库约束+元数据卡片+编码清洗+审计看板
This commit is contained in:
@@ -260,7 +260,7 @@ def run_all_alert_checks(db: Session = Depends(get_db)):
|
||||
|
||||
value = latest_value.actual_value
|
||||
period = latest_value.period
|
||||
params = rule.params or {}
|
||||
import json; params = json.loads(rule.params) if isinstance(rule.params, str) else (rule.params or {})
|
||||
|
||||
alert_level = None
|
||||
alert_message = None
|
||||
@@ -534,7 +534,7 @@ def _check_forecast_alerts(db: Session) -> int:
|
||||
continue
|
||||
|
||||
# 检查预测值是否超限
|
||||
params = rule.params or {}
|
||||
import json; params = json.loads(rule.params) if isinstance(rule.params, str) else (rule.params or {})
|
||||
params["kpi"] = kpi
|
||||
for forecast in latest_forecasts:
|
||||
value = forecast.predicted_cash
|
||||
|
||||
@@ -253,10 +253,72 @@ def quality_stats(db: Session = Depends(get_db)):
|
||||
if cnt:
|
||||
type_counts[t] = cnt
|
||||
|
||||
# ── 数据审计看板统计 ──
|
||||
# KPI完整度评分
|
||||
all_kpis = db.query(KPIDefinition).filter(KPIDefinition.status == "active").all()
|
||||
total = len(all_kpis)
|
||||
complete_kpis = 0
|
||||
missing_metadata_count = 0
|
||||
missing_data_count = 0
|
||||
stale_data_count = 0
|
||||
|
||||
from datetime import datetime, timedelta
|
||||
six_months_ago = datetime.now() - timedelta(days=180)
|
||||
|
||||
for kpi in all_kpis:
|
||||
# 元数据完整度检查
|
||||
has_meta = all([
|
||||
kpi.formula and kpi.formula.strip(),
|
||||
kpi.data_source and kpi.data_source.strip(),
|
||||
kpi.data_owner and kpi.data_owner.strip(),
|
||||
kpi.unit and kpi.unit.strip(),
|
||||
kpi.target_value is not None,
|
||||
])
|
||||
if has_meta:
|
||||
complete_kpis += 1
|
||||
else:
|
||||
missing_metadata_count += 1
|
||||
|
||||
# 数据缺失检查(是否有实际值)
|
||||
val = db.query(KPIValue).filter(
|
||||
KPIValue.kpi_id == kpi.id,
|
||||
KPIValue.actual_value.isnot(None),
|
||||
).first()
|
||||
if not val:
|
||||
missing_data_count += 1
|
||||
|
||||
# 超30天未更新预警
|
||||
latest_val = db.query(KPIValue).filter(
|
||||
KPIValue.kpi_id == kpi.id,
|
||||
KPIValue.actual_value.isnot(None),
|
||||
).order_by(KPIValue.period.desc()).first()
|
||||
if latest_val and latest_val.calculated_at:
|
||||
if latest_val.calculated_at < six_months_ago:
|
||||
stale_data_count += 1
|
||||
|
||||
completeness_score = round(complete_kpis / total * 100, 1) if total > 0 else 0
|
||||
missing_rate = round(missing_data_count / total * 100, 1) if total > 0 else 0
|
||||
|
||||
return {
|
||||
"total_kpis": total_kpis,
|
||||
"total_logs": total_logs,
|
||||
"open_logs": open_logs,
|
||||
"severity_counts": severity_counts,
|
||||
"type_counts": type_counts,
|
||||
# 数据审计看板
|
||||
"completeness": {
|
||||
"score": completeness_score,
|
||||
"complete": complete_kpis,
|
||||
"total": total,
|
||||
"missing_metadata": missing_metadata_count,
|
||||
},
|
||||
"data_missing": {
|
||||
"count": missing_data_count,
|
||||
"rate": missing_rate,
|
||||
"total": total,
|
||||
},
|
||||
"stale_data": {
|
||||
"count": stale_data_count,
|
||||
"threshold_days": 180,
|
||||
},
|
||||
}
|
||||
|
||||
@@ -406,12 +406,43 @@ def get_kpi(kpi_id: int, db: Session = Depends(get_db)):
|
||||
return kpi_to_dict(kpi)
|
||||
|
||||
|
||||
def _validate_kpi_data(data: dict, is_update: bool = False):
|
||||
"""数据治理:入库必检 + 元数据校验"""
|
||||
errors = []
|
||||
|
||||
# 规则1: target_value 必填
|
||||
tv = data.get("target_value")
|
||||
if tv is None or (isinstance(tv, (int, float)) and tv < 0 and not is_update):
|
||||
if not is_update or "target_value" in data:
|
||||
if tv is None:
|
||||
errors.append("目标值(target_value)不能为空")
|
||||
|
||||
# 规则1: unit 必填
|
||||
unit = data.get("unit")
|
||||
if not unit or (isinstance(unit, str) and unit.strip() == ""):
|
||||
if not is_update or "unit" in data:
|
||||
errors.append("单位(unit)不能为空")
|
||||
|
||||
# 规则2: 元数据必填 — formula/data_source/data_owner
|
||||
for field, label in [("formula", "计算公式"), ("data_source", "数据来源"), ("data_owner", "数据责任人")]:
|
||||
val = data.get(field)
|
||||
if not val or (isinstance(val, str) and val.strip() == ""):
|
||||
if not is_update or field in data:
|
||||
errors.append(f"元数据字段'{label}'({field})不能为空")
|
||||
|
||||
return errors
|
||||
|
||||
|
||||
@router.post("")
|
||||
def create_kpi(data: dict, db: Session = Depends(get_db), user=WRITE_ROLES):
|
||||
# 检查编码唯一性
|
||||
existing = db.query(KPIDefinition).filter(KPIDefinition.kpi_code == data.get("kpi_code", "")).first()
|
||||
if existing:
|
||||
raise HTTPException(400, f"KPI编码 {data['kpi_code']} 已存在")
|
||||
# 数据治理校验
|
||||
errs = _validate_kpi_data(data, is_update=False)
|
||||
if errs:
|
||||
raise HTTPException(422, detail={"message": "数据校验不通过", "errors": errs})
|
||||
kpi = KPIDefinition(**data)
|
||||
db.add(kpi)
|
||||
db.commit()
|
||||
@@ -425,6 +456,10 @@ def update_kpi(kpi_id: int, data: dict, db: Session = Depends(get_db), user=WRIT
|
||||
kpi = db.query(KPIDefinition).filter(KPIDefinition.id == kpi_id).first()
|
||||
if not kpi:
|
||||
raise HTTPException(404, "KPI不存在")
|
||||
# 数据治理校验(更新时只检查传了但为空的字段)
|
||||
errs = _validate_kpi_data(data, is_update=True)
|
||||
if errs:
|
||||
raise HTTPException(422, detail={"message": "数据校验不通过", "errors": errs})
|
||||
for k, v in data.items():
|
||||
if hasattr(kpi, k) and v is not None:
|
||||
setattr(kpi, k, v)
|
||||
|
||||
Reference in New Issue
Block a user