feat: 数据治理 — 入库约束+元数据卡片+编码清洗+审计看板

This commit is contained in:
Hermes CI Fix
2026-07-22 12:06:52 +08:00
parent ec6af751a5
commit 748c2da43f
8 changed files with 452 additions and 2 deletions
+2 -2
View File
@@ -260,7 +260,7 @@ def run_all_alert_checks(db: Session = Depends(get_db)):
value = latest_value.actual_value
period = latest_value.period
params = rule.params or {}
import json; params = json.loads(rule.params) if isinstance(rule.params, str) else (rule.params or {})
alert_level = None
alert_message = None
@@ -534,7 +534,7 @@ def _check_forecast_alerts(db: Session) -> int:
continue
# 检查预测值是否超限
params = rule.params or {}
import json; params = json.loads(rule.params) if isinstance(rule.params, str) else (rule.params or {})
params["kpi"] = kpi
for forecast in latest_forecasts:
value = forecast.predicted_cash
+62
View File
@@ -253,10 +253,72 @@ def quality_stats(db: Session = Depends(get_db)):
if cnt:
type_counts[t] = cnt
# ── 数据审计看板统计 ──
# KPI完整度评分
all_kpis = db.query(KPIDefinition).filter(KPIDefinition.status == "active").all()
total = len(all_kpis)
complete_kpis = 0
missing_metadata_count = 0
missing_data_count = 0
stale_data_count = 0
from datetime import datetime, timedelta
six_months_ago = datetime.now() - timedelta(days=180)
for kpi in all_kpis:
# 元数据完整度检查
has_meta = all([
kpi.formula and kpi.formula.strip(),
kpi.data_source and kpi.data_source.strip(),
kpi.data_owner and kpi.data_owner.strip(),
kpi.unit and kpi.unit.strip(),
kpi.target_value is not None,
])
if has_meta:
complete_kpis += 1
else:
missing_metadata_count += 1
# 数据缺失检查(是否有实际值)
val = db.query(KPIValue).filter(
KPIValue.kpi_id == kpi.id,
KPIValue.actual_value.isnot(None),
).first()
if not val:
missing_data_count += 1
# 超30天未更新预警
latest_val = db.query(KPIValue).filter(
KPIValue.kpi_id == kpi.id,
KPIValue.actual_value.isnot(None),
).order_by(KPIValue.period.desc()).first()
if latest_val and latest_val.calculated_at:
if latest_val.calculated_at < six_months_ago:
stale_data_count += 1
completeness_score = round(complete_kpis / total * 100, 1) if total > 0 else 0
missing_rate = round(missing_data_count / total * 100, 1) if total > 0 else 0
return {
"total_kpis": total_kpis,
"total_logs": total_logs,
"open_logs": open_logs,
"severity_counts": severity_counts,
"type_counts": type_counts,
# 数据审计看板
"completeness": {
"score": completeness_score,
"complete": complete_kpis,
"total": total,
"missing_metadata": missing_metadata_count,
},
"data_missing": {
"count": missing_data_count,
"rate": missing_rate,
"total": total,
},
"stale_data": {
"count": stale_data_count,
"threshold_days": 180,
},
}
+35
View File
@@ -406,12 +406,43 @@ def get_kpi(kpi_id: int, db: Session = Depends(get_db)):
return kpi_to_dict(kpi)
def _validate_kpi_data(data: dict, is_update: bool = False):
"""数据治理:入库必检 + 元数据校验"""
errors = []
# 规则1: target_value 必填
tv = data.get("target_value")
if tv is None or (isinstance(tv, (int, float)) and tv < 0 and not is_update):
if not is_update or "target_value" in data:
if tv is None:
errors.append("目标值(target_value)不能为空")
# 规则1: unit 必填
unit = data.get("unit")
if not unit or (isinstance(unit, str) and unit.strip() == ""):
if not is_update or "unit" in data:
errors.append("单位(unit)不能为空")
# 规则2: 元数据必填 — formula/data_source/data_owner
for field, label in [("formula", "计算公式"), ("data_source", "数据来源"), ("data_owner", "数据责任人")]:
val = data.get(field)
if not val or (isinstance(val, str) and val.strip() == ""):
if not is_update or field in data:
errors.append(f"元数据字段'{label}'({field})不能为空")
return errors
@router.post("")
def create_kpi(data: dict, db: Session = Depends(get_db), user=WRITE_ROLES):
# 检查编码唯一性
existing = db.query(KPIDefinition).filter(KPIDefinition.kpi_code == data.get("kpi_code", "")).first()
if existing:
raise HTTPException(400, f"KPI编码 {data['kpi_code']} 已存在")
# 数据治理校验
errs = _validate_kpi_data(data, is_update=False)
if errs:
raise HTTPException(422, detail={"message": "数据校验不通过", "errors": errs})
kpi = KPIDefinition(**data)
db.add(kpi)
db.commit()
@@ -425,6 +456,10 @@ def update_kpi(kpi_id: int, data: dict, db: Session = Depends(get_db), user=WRIT
kpi = db.query(KPIDefinition).filter(KPIDefinition.id == kpi_id).first()
if not kpi:
raise HTTPException(404, "KPI不存在")
# 数据治理校验(更新时只检查传了但为空的字段)
errs = _validate_kpi_data(data, is_update=True)
if errs:
raise HTTPException(422, detail={"message": "数据校验不通过", "errors": errs})
for k, v in data.items():
if hasattr(kpi, k) and v is not None:
setattr(kpi, k, v)
+2
View File
@@ -59,6 +59,8 @@ class KPIDefinition(Base):
formula_desc = Column(String(500), nullable=True, comment="公式说明")
data_source_type = Column(String(20), default="manual", comment="erp/business/excel/manual")
data_source_config = Column(JSON, nullable=True, comment="数据源配置")
data_source = Column(String(500), default="待补充", comment="数据来源")
data_owner = Column(String(100), default="待指定", comment="数据责任人")
frequency = Column(String(20), default="monthly", comment="daily/weekly/monthly/quarterly/yearly")
unit = Column(String(50), default="%", comment="单位")
target_value = Column(Float, nullable=True, comment="目标值")