任总 2026-09-28 13:38 放行「4 和 5」→ 落笔最小实现: - 新表 quality_rule_config(39 项:37 规则级 + 2 全局)= 唯一配置源 - 三层解析:账套行 → 全账套模板(entity_id=0) → 代码内置默认(=清单 §7 建议值,零行为差异) - 端点 /api/cma/data-quality/rule-config(GET/PUT)+ /history + /meta:写入仅 ceo/it, platform 级仅 it,演示账套(functional)写禁,变更落 operation_logs 同事务 - 数据质量页新增「规则配置」只读 Tab(懒加载) - 唯一行为变更:污染判定去掉 sync —— 酣客 zxt-sync 系统迁入数据不再被误判(口径可配置) - 出厂默认落库脚本 scripts/seed_dq_rule_config.py(--check/--apply/--rollback,幂等 39/0) - 回归:新套件 35 例 + 存量 1508 passed(唯一 FAILED 为他人未完成 migrate_subjects_entity_id_20260923.py 改动所致,HEAD 干净树同例通过) 生产写(DDL/seed)与 push 均走审批门:DDL=54434656 push=3bd00774
This commit is contained in:
+505
-153
@@ -4,7 +4,7 @@
|
||||
from fastapi import APIRouter, Depends, HTTPException, Query, Request
|
||||
from sqlalchemy.orm import Session
|
||||
from sqlalchemy import func, and_, text
|
||||
from typing import Optional
|
||||
from typing import Optional, Any, Tuple
|
||||
from datetime import datetime, timedelta
|
||||
import json
|
||||
import logging
|
||||
@@ -12,7 +12,11 @@ import logging
|
||||
from app.database import get_db
|
||||
from app.deps import assert_entity_scope, get_entity_id
|
||||
from app.auth_middleware import require_auth, require_role
|
||||
from app.models import KPIDefinition, KPIValue, KpiDataQualityLog, OperationLog
|
||||
from app.models import (KPIDefinition, KPIValue, KpiDataQualityLog, OperationLog,
|
||||
QualityRuleConfig, User)
|
||||
from app.utils.oplog import log_op
|
||||
from app import dq_rule_config as dqcfg
|
||||
from pydantic import BaseModel, Field
|
||||
from app.api.kpis import kpi_to_dict
|
||||
from app.utils.alert_entity_scope import month_period_clause
|
||||
|
||||
@@ -485,6 +489,100 @@ RULE_ACTIONS = {
|
||||
}
|
||||
|
||||
|
||||
# ============================================================
|
||||
# 规则配置取值口(ASK-20260928-05)—— 引擎侧**唯一**配置读取入口
|
||||
# 取值经 `app/dq_rule_config.py` 三层解析(账套行 → 模板行(entity_id=0) → 内置默认;
|
||||
# 禁止跨账套回退)。本文件**不得**再出现可配置项的裸露常量(阈值 / 关键词 / 条数下限 /
|
||||
# 差异阈值与方向 / 扣分上限与系数 / 明细条数)—— 否则「改了配置不生效」。
|
||||
# 锁死面(级别、判定表达式本体、评分函数、作用域、规则集…)仍在本文件硬编码,见清单 §4。
|
||||
# ============================================================
|
||||
RECON_DIFF_EXPR = ("ROUND((COALESCE(SUM(b.budget_value),0) - d.target_yearly) "
|
||||
"/ d.target_yearly * 100, 1)")
|
||||
RECON_BASE_SQL = ("FROM kpi_definitions d JOIN budget_plans b ON b.kpi_id = d.id "
|
||||
"WHERE d.status='active' AND d.target_yearly > 0 AND b.status='active'")
|
||||
|
||||
|
||||
def _load_cfg(db: Session, eid: int) -> dict:
|
||||
"""完整解析结果(含每项来源与 config_version);任何异常 → 内置默认形态(读路径绝不 500)。"""
|
||||
try:
|
||||
return dqcfg.resolve(db, eid)
|
||||
except Exception: # noqa: BLE001
|
||||
logger.warning("DQ_RULE_CONFIG_FALLBACK_BUILTIN entity=%s", eid, exc_info=True)
|
||||
return dqcfg.builtin_resolution(eid)
|
||||
|
||||
|
||||
def _cfg_values(db: Session, eid: int) -> dict:
|
||||
"""引擎取值口:{rule_key: {param: value}, ..., 'global': {...}}"""
|
||||
return _load_cfg(db, eid)["values"]
|
||||
|
||||
|
||||
def _rule_cfg(cfg: dict, rule: str) -> dict:
|
||||
return (cfg or {}).get(rule) or {}
|
||||
|
||||
|
||||
def _detail_limit(cfg: dict, rule: str) -> int:
|
||||
return int(_rule_cfg(cfg, rule).get("detail_limit") or dqcfg.DEFAULT_DETAIL_LIMIT)
|
||||
|
||||
|
||||
def _unit_threshold(cfg: dict) -> int:
|
||||
return int(_rule_cfg(cfg, "unit_check").get("amount_threshold", 10000))
|
||||
|
||||
|
||||
def _dup_min_count(cfg: dict) -> int:
|
||||
return int(_rule_cfg(cfg, "dup_alert").get("min_count", 2))
|
||||
|
||||
|
||||
def _manual_source(cfg: dict) -> str:
|
||||
return str(_rule_cfg(cfg, "virtual_pollution").get("manual_source_value") or "manual")
|
||||
|
||||
|
||||
def _recon_pct(cfg: dict) -> float:
|
||||
return float(_rule_cfg(cfg, "reconciliation").get("diff_threshold_pct", 20))
|
||||
|
||||
|
||||
def _recon_having(cfg: dict) -> str:
|
||||
"""规则7 比较方向:仅三种枚举(abs/upper_only/lower_only),不接受自由表达式。"""
|
||||
mode = str(_rule_cfg(cfg, "reconciliation").get("compare_mode") or "abs")
|
||||
if mode == "upper_only":
|
||||
return "diff_pct > :pct"
|
||||
if mode == "lower_only":
|
||||
return "diff_pct < (0 - :pct)"
|
||||
return "ABS(diff_pct) > :pct"
|
||||
|
||||
|
||||
def _entity_clause(eid: int, col: str) -> Tuple[str, dict]:
|
||||
"""账套过滤子句 + 绑定参数(eid=0 → 全账套,无子句)。"""
|
||||
if eid:
|
||||
return f" AND {col} = :eid", {"eid": eid}
|
||||
return "", {}
|
||||
|
||||
|
||||
def _vp_predicate(cfg: dict) -> Tuple[str, dict]:
|
||||
"""规则4 污染主体谓词:嫌疑关键词(大小写不敏感包含式)**减去**来源白名单(豁免优先)。
|
||||
|
||||
关键词/白名单均来自配置;匹配方式固定为包含式(禁正则与 SQL 片段 —— 锁死面第 6 条);
|
||||
关键词为空 ⇒ 恒假谓词(不误报任何行)。绑定参数化,无字面量拼接(注入面 = 0)。
|
||||
"""
|
||||
vp = _rule_cfg(cfg, "virtual_pollution")
|
||||
keywords = [str(k) for k in (vp.get("suspect_keywords") or []) if str(k).strip()]
|
||||
whitelist = [str(w) for w in (vp.get("source_whitelist") or []) if str(w).strip()]
|
||||
params: dict = {}
|
||||
parts = []
|
||||
for i, kw in enumerate(keywords):
|
||||
key = f"vpkw{i}"
|
||||
parts.append(f"LOWER(cp.source) LIKE :{key}")
|
||||
params[key] = f"%{kw.lower()}%"
|
||||
pred = "(" + " OR ".join(parts) + ")" if parts else "(1 = 0)"
|
||||
if whitelist:
|
||||
names = []
|
||||
for i, w in enumerate(whitelist):
|
||||
key = f"vpwl{i}"
|
||||
names.append(f":{key}")
|
||||
params[key] = w.lower()
|
||||
pred += f" AND LOWER(cp.source) NOT IN ({', '.join(names)})"
|
||||
return pred, params
|
||||
|
||||
|
||||
def _plan_id_set(db: Session, eid: int):
|
||||
"""已存在的 cash_plans id 集合(规则3 判孤儿用;eid 非 0 时只看该实体)"""
|
||||
sql = "SELECT id FROM cash_plans" + (" WHERE entity_id = :eid" if eid else "")
|
||||
@@ -512,64 +610,79 @@ def _pending_alert_plan_links(db: Session, eid: int):
|
||||
return parsed, rows
|
||||
|
||||
|
||||
def _dup_groups(parsed):
|
||||
"""规则2 判据:同 plan_id 多条 pending 预警 → [(plan_id, [(alert_id, kpi_id, ent_id), ...])],按条数降序"""
|
||||
def _dup_groups(parsed, min_count: int = 2):
|
||||
"""规则2 判据:同 plan_id 的 pending 预警条数 **>= min_count**(默认 2 == 原 `len(ids) > 1`)
|
||||
|
||||
min_count 来自配置(`dup_alert.min_count`,清单 §3.2,边界 [2,20] 越界拒绝)。
|
||||
"""
|
||||
threshold = max(2, int(min_count or 2))
|
||||
group_map = {}
|
||||
for alert_id, kpi_id, pid, ent_id in parsed:
|
||||
group_map.setdefault(pid, []).append((alert_id, kpi_id, ent_id))
|
||||
groups = [(pid, ids) for pid, ids in group_map.items() if len(ids) > 1]
|
||||
groups = [(pid, ids) for pid, ids in group_map.items() if len(ids) >= threshold]
|
||||
groups.sort(key=lambda x: -len(x[1]))
|
||||
return groups
|
||||
|
||||
|
||||
def _detail_count(db: Session, rule: str, eid: int, manual: bool = False) -> int:
|
||||
def _detail_count(db: Session, rule: str, eid: int, manual: bool = False,
|
||||
cfg: Optional[dict] = None) -> int:
|
||||
"""本规则真实总数:一律 COUNT(*)(dup/orphan 由预警分组得出,同样不受 LIMIT 影响)"""
|
||||
cfg = cfg or _cfg_values(db, eid)
|
||||
|
||||
if rule == "dup_alert":
|
||||
parsed, _ = _pending_alert_plan_links(db, eid)
|
||||
return len(_dup_groups(parsed))
|
||||
return len(_dup_groups(parsed, _dup_min_count(cfg)))
|
||||
if rule == "orphan_check":
|
||||
parsed, _ = _pending_alert_plan_links(db, eid)
|
||||
pset = _plan_id_set(db, eid)
|
||||
return len([1 for _a, _k, pid, _e in parsed if pid not in pset])
|
||||
|
||||
ent = " AND cp.entity_id = :eid" if eid else ""
|
||||
ent, params = _entity_clause(eid, "cp.entity_id")
|
||||
if rule == "unit_check":
|
||||
sql = "SELECT COUNT(*) FROM cash_plans cp WHERE cp.amount > 10000" + ent
|
||||
sql = "SELECT COUNT(*) FROM cash_plans cp WHERE cp.amount > :amt" + ent
|
||||
params["amt"] = _unit_threshold(cfg)
|
||||
elif rule == "virtual_pollution":
|
||||
if manual:
|
||||
sql = "SELECT COUNT(*) FROM cash_plans cp WHERE cp.source = 'manual'" + ent
|
||||
sql = "SELECT COUNT(*) FROM cash_plans cp WHERE cp.source = :msrc" + ent
|
||||
params["msrc"] = _manual_source(cfg)
|
||||
else:
|
||||
sql = ("SELECT COUNT(*) FROM cash_plans cp WHERE (cp.source LIKE '%test%' "
|
||||
"OR cp.source LIKE '%sync%' OR cp.source LIKE '%虚拟%' OR cp.source LIKE '%demo%')" + ent)
|
||||
pred, vp_params = _vp_predicate(cfg)
|
||||
params.update(vp_params)
|
||||
sql = "SELECT COUNT(*) FROM cash_plans cp WHERE " + pred + ent
|
||||
elif rule == "entity_check":
|
||||
sql = ("SELECT COUNT(*) FROM kpi_values v JOIN kpi_definitions d ON v.kpi_id = d.id "
|
||||
"WHERE v.entity_id IS NOT NULL AND v.entity_id != d.entity_id")
|
||||
if eid:
|
||||
sql += " AND v.entity_id = :eid"
|
||||
params["eid"] = eid
|
||||
elif rule == "kpi_completeness":
|
||||
sql = ("SELECT COUNT(*) FROM kpi_definitions d WHERE d.status='active' "
|
||||
"AND NOT EXISTS (SELECT 1 FROM kpi_values v WHERE v.kpi_id = d.id)")
|
||||
if eid:
|
||||
sql += " AND d.entity_id = :eid"
|
||||
params["eid"] = eid
|
||||
elif rule == "reconciliation":
|
||||
sql = ("SELECT COUNT(*) FROM (SELECT d.id, "
|
||||
"ROUND((COALESCE(SUM(b.budget_value),0) - d.target_yearly) / d.target_yearly * 100, 1) AS diff_pct "
|
||||
"FROM kpi_definitions d JOIN budget_plans b ON b.kpi_id = d.id "
|
||||
"WHERE d.status='active' AND d.target_yearly > 0 AND b.status='active'")
|
||||
sql = ("SELECT COUNT(*) FROM (SELECT d.id, " + RECON_DIFF_EXPR + " AS diff_pct "
|
||||
+ RECON_BASE_SQL)
|
||||
if eid:
|
||||
sql += " AND d.entity_id = :eid"
|
||||
sql += " GROUP BY d.id HAVING ABS(diff_pct) > 20) t"
|
||||
params["eid"] = eid
|
||||
sql += " GROUP BY d.id HAVING " + _recon_having(cfg) + ") t"
|
||||
params["pct"] = _recon_pct(cfg)
|
||||
else:
|
||||
return 0
|
||||
return int(db.execute(text(sql), {"eid": eid}).scalar() or 0)
|
||||
return int(db.execute(text(sql), params).scalar() or 0)
|
||||
|
||||
|
||||
def _detail_items(db: Session, rule: str, eid: int, limit: int, offset: int, manual: bool = False):
|
||||
def _detail_items(db: Session, rule: str, eid: int, limit: int, offset: int,
|
||||
manual: bool = False, cfg: Optional[dict] = None):
|
||||
"""结构化明细条目(服务端 LIMIT/OFFSET 分页;字段:record_id/entity_id/source/key_fields/summary/suggested_action)"""
|
||||
cfg = cfg or _cfg_values(db, eid)
|
||||
|
||||
if rule == "dup_alert":
|
||||
parsed, _ = _pending_alert_plan_links(db, eid)
|
||||
items = []
|
||||
for pid, alerts in _dup_groups(parsed):
|
||||
for pid, alerts in _dup_groups(parsed, _dup_min_count(cfg)):
|
||||
items.append({
|
||||
"record_id": pid,
|
||||
"entity_id": alerts[0][2],
|
||||
@@ -598,41 +711,43 @@ def _detail_items(db: Session, rule: str, eid: int, limit: int, offset: int, man
|
||||
})
|
||||
return items[offset:offset + limit]
|
||||
|
||||
params = {"eid": eid, "lim": limit, "off": offset}
|
||||
ent = " AND cp.entity_id = :eid" if eid else ""
|
||||
ent, params = _entity_clause(eid, "cp.entity_id")
|
||||
params.update({"lim": limit, "off": offset})
|
||||
|
||||
if rule == "unit_check":
|
||||
threshold = _unit_threshold(cfg)
|
||||
rows = db.execute(text(
|
||||
"SELECT cp.id, cp.entity_id, cp.amount, cp.source, cp.description FROM cash_plans cp "
|
||||
"WHERE cp.amount > 10000" + ent + " ORDER BY cp.amount DESC LIMIT :lim OFFSET :off"
|
||||
), params).fetchall()
|
||||
"WHERE cp.amount > :amt" + ent + " ORDER BY cp.amount DESC LIMIT :lim OFFSET :off"
|
||||
), {**params, "amt": threshold}).fetchall()
|
||||
return [{
|
||||
"record_id": r.id,
|
||||
"entity_id": r.entity_id,
|
||||
"source": RULE_SOURCES[rule],
|
||||
"key_fields": {"amount": float(r.amount) if r.amount is not None else None,
|
||||
"source": r.source, "description": r.description,
|
||||
"threshold": ">10000(万元口径可疑)"},
|
||||
"summary": "计划#%d 金额 %s(超 10000 阈值,疑似单位错乱)" % (r.id, r.amount),
|
||||
"threshold": ">%s(万元口径可疑)" % threshold},
|
||||
"summary": "计划#%d 金额 %s(超 %s 阈值,疑似单位错乱)" % (r.id, r.amount, threshold),
|
||||
"suggested_action": RULE_ACTIONS[rule],
|
||||
} for r in rows]
|
||||
|
||||
if rule == "virtual_pollution":
|
||||
if manual:
|
||||
where = "cp.source = 'manual'"
|
||||
where, kw_params = "cp.source = :msrc", {"msrc": _manual_source(cfg)}
|
||||
scope_text = "%s(待人工确认)" % _manual_source(cfg)
|
||||
else:
|
||||
where = ("(cp.source LIKE '%test%' OR cp.source LIKE '%sync%' "
|
||||
"OR cp.source LIKE '%虚拟%' OR cp.source LIKE '%demo%')")
|
||||
where, kw_params = _vp_predicate(cfg)
|
||||
scope_text = "污染源(%s)" % "/".join(cfg["virtual_pollution"]["suspect_keywords"])
|
||||
rows = db.execute(text(
|
||||
"SELECT cp.id, cp.entity_id, cp.source, cp.description FROM cash_plans cp "
|
||||
"WHERE " + where + ent + " ORDER BY cp.id LIMIT :lim OFFSET :off"
|
||||
), params).fetchall()
|
||||
), {**params, **kw_params}).fetchall()
|
||||
return [{
|
||||
"record_id": r.id,
|
||||
"entity_id": r.entity_id,
|
||||
"source": RULE_SOURCES[rule],
|
||||
"key_fields": {"source": r.source, "description": r.description,
|
||||
"scope": "manual(待人工确认)" if manual else "污染源(test/sync/虚拟/demo)"},
|
||||
"scope": scope_text},
|
||||
"summary": "计划#%d source=%s%s" % (r.id, r.source, "" if not manual else "(待人工确认)"),
|
||||
"suggested_action": RULE_ACTIONS[rule],
|
||||
} for r in rows]
|
||||
@@ -644,6 +759,8 @@ def _detail_items(db: Session, rule: str, eid: int, limit: int, offset: int, man
|
||||
if eid:
|
||||
sql += " AND v.entity_id = :eid"
|
||||
sql += " ORDER BY v.id LIMIT :lim OFFSET :off"
|
||||
if not eid:
|
||||
params.pop("eid", None)
|
||||
rows = db.execute(text(sql), params).fetchall()
|
||||
return [{
|
||||
"record_id": r.id,
|
||||
@@ -661,6 +778,8 @@ def _detail_items(db: Session, rule: str, eid: int, limit: int, offset: int, man
|
||||
if eid:
|
||||
sql += " AND d.entity_id = :eid"
|
||||
sql += " ORDER BY d.id LIMIT :lim OFFSET :off"
|
||||
if not eid:
|
||||
params.pop("eid", None)
|
||||
rows = db.execute(text(sql), params).fetchall()
|
||||
return [{
|
||||
"record_id": r.id,
|
||||
@@ -674,13 +793,15 @@ def _detail_items(db: Session, rule: str, eid: int, limit: int, offset: int, man
|
||||
if rule == "reconciliation":
|
||||
sql = ("SELECT d.id, d.entity_id, d.kpi_code, d.kpi_name, d.target_yearly, "
|
||||
"COALESCE(SUM(b.budget_value),0) AS monthly_sum, "
|
||||
"ROUND((COALESCE(SUM(b.budget_value),0) - d.target_yearly) / d.target_yearly * 100, 1) AS diff_pct "
|
||||
"FROM kpi_definitions d JOIN budget_plans b ON b.kpi_id = d.id "
|
||||
"WHERE d.status='active' AND d.target_yearly > 0 AND b.status='active'")
|
||||
+ RECON_DIFF_EXPR + " AS diff_pct " + RECON_BASE_SQL)
|
||||
if eid:
|
||||
sql += " AND d.entity_id = :eid"
|
||||
sql += " GROUP BY d.id HAVING ABS(diff_pct) > 20 ORDER BY ABS(diff_pct) DESC LIMIT :lim OFFSET :off"
|
||||
rows = db.execute(text(sql), params).fetchall()
|
||||
sql += (" GROUP BY d.id HAVING " + _recon_having(cfg)
|
||||
+ " ORDER BY ABS(diff_pct) DESC LIMIT :lim OFFSET :off")
|
||||
recon_params = {**params, "pct": _recon_pct(cfg)}
|
||||
if not eid:
|
||||
recon_params.pop("eid", None)
|
||||
rows = db.execute(text(sql), recon_params).fetchall()
|
||||
return [{
|
||||
"record_id": r.id,
|
||||
"entity_id": r.entity_id,
|
||||
@@ -695,16 +816,16 @@ def _detail_items(db: Session, rule: str, eid: int, limit: int, offset: int, man
|
||||
return []
|
||||
|
||||
|
||||
|
||||
# ============================================================
|
||||
# 财务七规则检查(governance-check)— 2026-08-30 P1 最终方案
|
||||
# 端点收敛(cma-residual-cross-entity-cleanup-20260914):
|
||||
# /check-governance 已下线 —— 全仓无调用方(前端仅残留 api 定义、无 .vue 调用);
|
||||
# 其 entity_id 参数无会话鉴权(0=全部/1/2 任取)→ 跨租户;与 governance-check 功能重复。
|
||||
# 本端点评分口径:
|
||||
# * 评分规则:error 扣 min(15, count*3),warning 扣 min(10, count*1)
|
||||
# * 评分规则:error 扣 min(error 上限, count×系数),warning 扣 min(warning 上限, count×系数)
|
||||
# —— 上限/系数自 2026-09-28(ASK-20260928-05)起取自 `dq_rule_config`(默认 = 原 15×3 / 10×1)
|
||||
# * 规则2/3 用 Python 解析 suggestion JSON(不依赖 MySQL JSON 函数)
|
||||
# * 规则4 区分 error(test/sync/虚拟) 与 manual(待人工确认 warning)
|
||||
# * 规则4 区分 污染关键词(error)与 manual(待人工确认 warning)
|
||||
# * 规则7 按 status='active' 口径(与 budget.py 一致,避免多版本叠加失真)
|
||||
# 只读幂等:不写库、不创建 KpiDataQualityLog
|
||||
# ============================================================
|
||||
@@ -721,142 +842,204 @@ def _extract_plan_id(suggestion: str):
|
||||
return None
|
||||
|
||||
|
||||
def _run_governance_checks(db: Session, entity_id: int = 0):
|
||||
"""执行财务七规则,返回 issues 列表(含 deducted 扣分)。entity_id=0 表示全部实体。"""
|
||||
eid = entity_id or 0
|
||||
ent = " AND cp.entity_id = :eid" if eid else ""
|
||||
issues = []
|
||||
def _disabled_issue(rule: str, cfg: dict) -> dict:
|
||||
"""被配置关闭的规则:零扣分、零明细,但**仍占位**(规则集锁死 7 条,响应形状稳定)。"""
|
||||
return {"rule": rule, "level": RULES_META[rule]["level"], "enabled": False,
|
||||
"count": 0, "detail": [], "deducted": 0}
|
||||
|
||||
# ── 规则1 单位校验:amount > 10000(万元口径可疑)──
|
||||
rows = db.execute(text(
|
||||
"SELECT cp.id, cp.entity_id, cp.amount, cp.source, cp.description "
|
||||
"FROM cash_plans cp WHERE cp.amount > 10000" + ent + " ORDER BY cp.amount DESC LIMIT 200"
|
||||
), {"eid": eid}).fetchall()
|
||||
unit_total = _detail_count(db, "unit_check", eid) # 真实总数(不再受明细 LIMIT 200 截断)
|
||||
issues.append({
|
||||
"rule": "unit_check", "level": "error",
|
||||
"count": unit_total,
|
||||
"detail": [f"plan#{r.id} 金额{r.amount}(疑似元)" for r in rows[:DETAIL_LIMIT]]
|
||||
+ (["…等%d条" % unit_total] if unit_total > DETAIL_LIMIT else []),
|
||||
})
|
||||
|
||||
def _run_governance_checks(db: Session, entity_id: int = 0):
|
||||
"""执行财务七规则,返回 issues 列表(含 deducted 扣分)。entity_id=0 表示全部实体。
|
||||
|
||||
配置化(ASK-20260928-05):阈值/关键词/条数下限/差异阈值与方向/扣分上限与系数/明细条数
|
||||
一律经 `_cfg_values()`(三层解析)取值;级别与判定表达式本体**锁死**(见 dq_rule_config)。
|
||||
"""
|
||||
eid = entity_id or 0
|
||||
cfg = _cfg_values(db, eid)
|
||||
resolution = _load_cfg(db, eid)
|
||||
issues = []
|
||||
disabled = []
|
||||
|
||||
# ── 规则1 单位校验:amount > 阈值(默认 10000 万元口径可疑)──
|
||||
if cfg["unit_check"].get("enabled", True):
|
||||
threshold = _unit_threshold(cfg)
|
||||
limit = _detail_limit(cfg, "unit_check")
|
||||
ent, params = _entity_clause(eid, "cp.entity_id")
|
||||
rows = db.execute(text(
|
||||
"SELECT cp.id, cp.entity_id, cp.amount, cp.source, cp.description "
|
||||
"FROM cash_plans cp WHERE cp.amount > :amt" + ent + " ORDER BY cp.amount DESC LIMIT 200"
|
||||
), {**params, "amt": threshold}).fetchall()
|
||||
unit_total = _detail_count(db, "unit_check", eid, cfg=cfg) # 真实总数(不受明细 LIMIT 截断)
|
||||
issues.append({
|
||||
"rule": "unit_check", "level": RULES_META["unit_check"]["level"],
|
||||
"count": unit_total,
|
||||
"detail": [f"plan#{r.id} 金额{r.amount}(疑似元)" for r in rows[:limit]]
|
||||
+ (["…等%d条" % unit_total] if unit_total > limit else []),
|
||||
})
|
||||
else:
|
||||
disabled.append("unit_check")
|
||||
issues.append(_disabled_issue("unit_check", cfg))
|
||||
|
||||
# ── 规则2/3 共用数据:pending cash_plan 预警(实体筛选经 kpi_definitions)──
|
||||
parsed, alert_rows = _pending_alert_plan_links(db, eid)
|
||||
parsed, _alert_rows = _pending_alert_plan_links(db, eid)
|
||||
|
||||
# 已存在的 cash_plans id 集合(规则3 判断孤儿用;实体筛选时仅看该实体下 plan)
|
||||
plan_id_set = _plan_id_set(db, eid)
|
||||
|
||||
# ── 规则2 重复预警:同 plan_id 多条 pending 预警 ──
|
||||
dup_groups = _dup_groups(parsed)
|
||||
issues.append({
|
||||
"rule": "dup_alert", "level": "error",
|
||||
"count": len(dup_groups),
|
||||
"detail": [f"plan#{pid} 重复预警×{len(ids)}" for pid, ids in dup_groups[:DETAIL_LIMIT]]
|
||||
+ (["…等%d个plan" % len(dup_groups)] if len(dup_groups) > DETAIL_LIMIT else []),
|
||||
})
|
||||
# ── 规则2 重复预警:同 plan_id 的 pending 预警 ≥ min_count(默认 2)──
|
||||
if cfg["dup_alert"].get("enabled", True):
|
||||
limit = _detail_limit(cfg, "dup_alert")
|
||||
dup_groups = _dup_groups(parsed, _dup_min_count(cfg))
|
||||
issues.append({
|
||||
"rule": "dup_alert", "level": RULES_META["dup_alert"]["level"],
|
||||
"count": len(dup_groups),
|
||||
"detail": [f"plan#{pid} 重复预警×{len(ids)}" for pid, ids in dup_groups[:limit]]
|
||||
+ (["…等%d个plan" % len(dup_groups)] if len(dup_groups) > limit else []),
|
||||
})
|
||||
else:
|
||||
disabled.append("dup_alert")
|
||||
issues.append(_disabled_issue("dup_alert", cfg))
|
||||
|
||||
# ── 规则3 孤儿预警:plan_id 指向不存在的 cash_plans ──
|
||||
orphan_rows = [(aid, kid, pid) for aid, kid, pid, _ent in parsed if pid not in plan_id_set]
|
||||
issues.append({
|
||||
"rule": "orphan_check", "level": "error",
|
||||
"count": len(orphan_rows),
|
||||
"detail": [f"预警#{aid}(kpi#{kid}) → plan#{pid} 不存在" for aid, kid, pid in orphan_rows[:DETAIL_LIMIT]]
|
||||
+ (["…等%d条" % len(orphan_rows)] if len(orphan_rows) > DETAIL_LIMIT else []),
|
||||
})
|
||||
if cfg["orphan_check"].get("enabled", True):
|
||||
limit = _detail_limit(cfg, "orphan_check")
|
||||
orphan_rows = [(aid, kid, pid) for aid, kid, pid, _ent in parsed if pid not in plan_id_set]
|
||||
issues.append({
|
||||
"rule": "orphan_check", "level": RULES_META["orphan_check"]["level"],
|
||||
"count": len(orphan_rows),
|
||||
"detail": [f"预警#{aid}(kpi#{kid}) → plan#{pid} 不存在" for aid, kid, pid in orphan_rows[:limit]]
|
||||
+ (["…等%d条" % len(orphan_rows)] if len(orphan_rows) > limit else []),
|
||||
})
|
||||
else:
|
||||
disabled.append("orphan_check")
|
||||
issues.append(_disabled_issue("orphan_check", cfg))
|
||||
|
||||
# ── 规则4 虚拟污染:source 含 test/sync/虚拟 → error;source='manual' → 待人工确认 warning ──
|
||||
rows = db.execute(text(
|
||||
"SELECT cp.id, cp.entity_id, cp.source, cp.description FROM cash_plans cp "
|
||||
"WHERE (cp.source LIKE '%test%' OR cp.source LIKE '%sync%' OR cp.source LIKE '%虚拟%' OR cp.source LIKE '%demo%')"
|
||||
+ ent + " ORDER BY cp.id LIMIT 200"
|
||||
), {"eid": eid}).fetchall()
|
||||
manual_rows = db.execute(text(
|
||||
"SELECT cp.id, cp.entity_id, cp.source, cp.description FROM cash_plans cp "
|
||||
"WHERE cp.source = 'manual'" + ent + " ORDER BY cp.id LIMIT 200"
|
||||
), {"eid": eid}).fetchall()
|
||||
poll_total = _detail_count(db, "virtual_pollution", eid) # 真实总数(不再受 LIMIT 200 截断)
|
||||
manual_total = _detail_count(db, "virtual_pollution", eid, manual=True)
|
||||
issues.append({
|
||||
"rule": "virtual_pollution", "level": "error",
|
||||
"count": poll_total,
|
||||
"detail": [f"plan#{r.id} source={r.source} desc={r.description or ''}" for r in rows[:DETAIL_LIMIT]]
|
||||
+ (["…等%d条" % poll_total] if poll_total > DETAIL_LIMIT else []),
|
||||
"manual_count": manual_total,
|
||||
"manual_detail": [f"plan#{r.id} source=manual(待人工确认)" for r in manual_rows[:DETAIL_LIMIT]]
|
||||
+ (["…等%d条" % manual_total] if manual_total > DETAIL_LIMIT else []),
|
||||
})
|
||||
# ── 规则4 虚拟污染:source 命中嫌疑关键词(去白名单)→ error;manual 值 → 待人工确认 warning ──
|
||||
if cfg["virtual_pollution"].get("enabled", True):
|
||||
limit = _detail_limit(cfg, "virtual_pollution")
|
||||
ent, params = _entity_clause(eid, "cp.entity_id")
|
||||
pred, vp_params = _vp_predicate(cfg)
|
||||
rows = db.execute(text(
|
||||
"SELECT cp.id, cp.entity_id, cp.source, cp.description FROM cash_plans cp "
|
||||
"WHERE " + pred + ent + " ORDER BY cp.id LIMIT 200"
|
||||
), {**params, **vp_params}).fetchall()
|
||||
manual_rows = db.execute(text(
|
||||
"SELECT cp.id, cp.entity_id, cp.source, cp.description FROM cash_plans cp "
|
||||
"WHERE cp.source = :msrc" + ent + " ORDER BY cp.id LIMIT 200"
|
||||
), {**params, "msrc": _manual_source(cfg)}).fetchall()
|
||||
poll_total = _detail_count(db, "virtual_pollution", eid, cfg=cfg)
|
||||
manual_total = _detail_count(db, "virtual_pollution", eid, manual=True, cfg=cfg)
|
||||
issues.append({
|
||||
"rule": "virtual_pollution", "level": RULES_META["virtual_pollution"]["level"],
|
||||
"count": poll_total,
|
||||
"detail": [f"plan#{r.id} source={r.source} desc={r.description or ''}" for r in rows[:limit]]
|
||||
+ (["…等%d条" % poll_total] if poll_total > limit else []),
|
||||
"manual_count": manual_total,
|
||||
"manual_detail": [f"plan#{r.id} source=manual(待人工确认)" for r in manual_rows[:limit]]
|
||||
+ (["…等%d条" % manual_total] if manual_total > limit else []),
|
||||
})
|
||||
else:
|
||||
disabled.append("virtual_pollution")
|
||||
issues.append(_disabled_issue("virtual_pollution", cfg))
|
||||
|
||||
# ── 规则5 实体归属:kpi_values.entity_id IS NOT NULL 且 != kpi_definitions.entity_id ──
|
||||
ent_sql = (
|
||||
"SELECT v.id, v.kpi_id, d.kpi_code, v.entity_id AS v_eid, d.entity_id AS d_eid "
|
||||
"FROM kpi_values v JOIN kpi_definitions d ON v.kpi_id = d.id "
|
||||
"WHERE v.entity_id IS NOT NULL AND v.entity_id != d.entity_id"
|
||||
)
|
||||
if eid:
|
||||
ent_sql += " AND v.entity_id = :eid"
|
||||
ent_sql += " ORDER BY v.id LIMIT 200"
|
||||
ent_rows = db.execute(text(ent_sql), {"eid": eid}).fetchall()
|
||||
ent_total = _detail_count(db, "entity_check", eid) # 真实总数
|
||||
issues.append({
|
||||
"rule": "entity_check", "level": "error",
|
||||
"count": ent_total,
|
||||
"detail": [f"值#{r.id} {r.kpi_code} 实体{r.v_eid}≠定义实体{r.d_eid}" for r in ent_rows[:DETAIL_LIMIT]]
|
||||
+ (["…等%d条" % ent_total] if ent_total > DETAIL_LIMIT else []),
|
||||
})
|
||||
if cfg["entity_check"].get("enabled", True):
|
||||
limit = _detail_limit(cfg, "entity_check")
|
||||
ent_sql = (
|
||||
"SELECT v.id, v.kpi_id, d.kpi_code, v.entity_id AS v_eid, d.entity_id AS d_eid "
|
||||
"FROM kpi_values v JOIN kpi_definitions d ON v.kpi_id = d.id "
|
||||
"WHERE v.entity_id IS NOT NULL AND v.entity_id != d.entity_id"
|
||||
)
|
||||
if eid:
|
||||
ent_sql += " AND v.entity_id = :eid"
|
||||
ent_sql += " ORDER BY v.id LIMIT 200"
|
||||
ent_rows = db.execute(text(ent_sql), {"eid": eid}).fetchall()
|
||||
ent_total = _detail_count(db, "entity_check", eid, cfg=cfg) # 真实总数
|
||||
issues.append({
|
||||
"rule": "entity_check", "level": RULES_META["entity_check"]["level"],
|
||||
"count": ent_total,
|
||||
"detail": [f"值#{r.id} {r.kpi_code} 实体{r.v_eid}≠定义实体{r.d_eid}" for r in ent_rows[:limit]]
|
||||
+ (["…等%d条" % ent_total] if ent_total > limit else []),
|
||||
})
|
||||
else:
|
||||
disabled.append("entity_check")
|
||||
issues.append(_disabled_issue("entity_check", cfg))
|
||||
|
||||
# ── 规则6 KPI完整性:active KPI 无任何实际值 ──
|
||||
comp_sql = (
|
||||
"SELECT d.id, d.kpi_code, d.kpi_name FROM kpi_definitions d "
|
||||
"WHERE d.status='active' AND NOT EXISTS (SELECT 1 FROM kpi_values v WHERE v.kpi_id = d.id)"
|
||||
)
|
||||
if eid:
|
||||
comp_sql += " AND d.entity_id = :eid"
|
||||
comp_sql += " ORDER BY d.id LIMIT 300"
|
||||
comp_rows = db.execute(text(comp_sql), {"eid": eid}).fetchall()
|
||||
comp_total = _detail_count(db, "kpi_completeness", eid) # 真实总数(不再受 LIMIT 300 截断)
|
||||
issues.append({
|
||||
"rule": "kpi_completeness", "level": "warning",
|
||||
"count": comp_total,
|
||||
"detail": [f"{r.kpi_code} {r.kpi_name}(无值)" for r in comp_rows[:DETAIL_LIMIT]]
|
||||
+ (["…等%d个KPI" % comp_total] if comp_total > DETAIL_LIMIT else []),
|
||||
})
|
||||
if cfg["kpi_completeness"].get("enabled", True):
|
||||
limit = _detail_limit(cfg, "kpi_completeness")
|
||||
comp_sql = (
|
||||
"SELECT d.id, d.kpi_code, d.kpi_name FROM kpi_definitions d "
|
||||
"WHERE d.status='active' AND NOT EXISTS (SELECT 1 FROM kpi_values v WHERE v.kpi_id = d.id)"
|
||||
)
|
||||
if eid:
|
||||
comp_sql += " AND d.entity_id = :eid"
|
||||
comp_sql += " ORDER BY d.id LIMIT 300"
|
||||
comp_rows = db.execute(text(comp_sql), {"eid": eid}).fetchall()
|
||||
comp_total = _detail_count(db, "kpi_completeness", eid, cfg=cfg)
|
||||
issues.append({
|
||||
"rule": "kpi_completeness", "level": RULES_META["kpi_completeness"]["level"],
|
||||
"count": comp_total,
|
||||
"detail": [f"{r.kpi_code} {r.kpi_name}(无值)" for r in comp_rows[:limit]]
|
||||
+ (["…等%d个KPI" % comp_total] if comp_total > limit else []),
|
||||
})
|
||||
else:
|
||||
disabled.append("kpi_completeness")
|
||||
issues.append(_disabled_issue("kpi_completeness", cfg))
|
||||
|
||||
# ── 规则7 勾稽验证:预算(active口径)年度合计 vs 年度目标 差异>20% ──
|
||||
recon_sql = (
|
||||
"SELECT d.kpi_code, d.kpi_name, d.target_yearly, "
|
||||
"COALESCE(SUM(b.budget_value),0) AS monthly_sum, "
|
||||
"ROUND((COALESCE(SUM(b.budget_value),0) - d.target_yearly) / d.target_yearly * 100, 1) AS diff_pct "
|
||||
"FROM kpi_definitions d JOIN budget_plans b ON b.kpi_id = d.id "
|
||||
"WHERE d.status='active' AND d.target_yearly > 0 AND b.status='active'"
|
||||
)
|
||||
if eid:
|
||||
recon_sql += " AND d.entity_id = :eid"
|
||||
recon_sql += " GROUP BY d.id HAVING ABS(diff_pct) > 20 ORDER BY ABS(diff_pct) DESC LIMIT 200"
|
||||
recon_rows = db.execute(text(recon_sql), {"eid": eid}).fetchall()
|
||||
recon_total = _detail_count(db, "reconciliation", eid) # 真实总数
|
||||
issues.append({
|
||||
"rule": "reconciliation", "level": "warning",
|
||||
"count": recon_total,
|
||||
"detail": [f"{r.kpi_code} 预算合计{round(r.monthly_sum, 1)} vs 年度目标{r.target_yearly} 差异{r.diff_pct}%" for r in recon_rows[:DETAIL_LIMIT]]
|
||||
+ (["…等%d个KPI" % recon_total] if recon_total > DETAIL_LIMIT else []),
|
||||
})
|
||||
# ── 规则7 勾稽验证:预算(active口径)合计 vs 年度目标,差异按配置阈值/方向判定 ──
|
||||
if cfg["reconciliation"].get("enabled", True):
|
||||
limit = _detail_limit(cfg, "reconciliation")
|
||||
recon_sql = (
|
||||
"SELECT d.kpi_code, d.kpi_name, d.target_yearly, "
|
||||
"COALESCE(SUM(b.budget_value),0) AS monthly_sum, "
|
||||
+ RECON_DIFF_EXPR + " AS diff_pct " + RECON_BASE_SQL
|
||||
)
|
||||
if eid:
|
||||
recon_sql += " AND d.entity_id = :eid"
|
||||
recon_sql += (" GROUP BY d.id HAVING " + _recon_having(cfg)
|
||||
+ " ORDER BY ABS(diff_pct) DESC LIMIT 200")
|
||||
recon_rows = db.execute(text(recon_sql),
|
||||
{"eid": eid, "pct": _recon_pct(cfg)}).fetchall()
|
||||
recon_total = _detail_count(db, "reconciliation", eid, cfg=cfg)
|
||||
issues.append({
|
||||
"rule": "reconciliation", "level": RULES_META["reconciliation"]["level"],
|
||||
"count": recon_total,
|
||||
"detail": [f"{r.kpi_code} 预算合计{round(r.monthly_sum, 1)} vs 年度目标{r.target_yearly} 差异{r.diff_pct}%"
|
||||
for r in recon_rows[:limit]]
|
||||
+ (["…等%d个KPI" % recon_total] if recon_total > limit else []),
|
||||
})
|
||||
else:
|
||||
disabled.append("reconciliation")
|
||||
issues.append(_disabled_issue("reconciliation", cfg))
|
||||
|
||||
# ── 评分:error 扣 min(15, count*3),warning 扣 min(10, count*1);规则4 manual 按 warning 附加扣 ──
|
||||
# ── 评分:各规则的「上限 × 系数」取自配置(默认 error 15×3 / warning 10×1;规则4 manual 附加)──
|
||||
total_deduct = 0
|
||||
for item in issues:
|
||||
rule = item["rule"]
|
||||
rule_cfg = cfg.get(rule) or {}
|
||||
if not rule_cfg.get("enabled", True):
|
||||
item["deducted"] = 0
|
||||
continue
|
||||
ded = 0
|
||||
if item["count"] > 0:
|
||||
ded += min(15, item["count"] * 3) if item["level"] == "error" else min(10, item["count"] * 1)
|
||||
cap = int(rule_cfg.get("deduct_cap",
|
||||
15 if item["level"] == "error" else 10))
|
||||
factor = int(rule_cfg.get("deduct_factor",
|
||||
3 if item["level"] == "error" else 1))
|
||||
ded += min(cap, item["count"] * factor)
|
||||
# 规则4 附加:manual 待人工确认(warning 性质)
|
||||
manual_cnt = item.get("manual_count") or 0
|
||||
if item["rule"] == "virtual_pollution" and manual_cnt > 0:
|
||||
ded += min(10, manual_cnt * 1)
|
||||
if rule == "virtual_pollution" and manual_cnt > 0:
|
||||
ded += min(int(rule_cfg.get("manual_cap", 10)),
|
||||
manual_cnt * int(rule_cfg.get("manual_factor", 1)))
|
||||
item["deducted"] = ded
|
||||
total_deduct += ded
|
||||
score = max(0, 100 - total_deduct)
|
||||
|
||||
passed = [item["rule"] for item in issues if item["count"] == 0]
|
||||
passed = [item["rule"] for item in issues if item["count"] == 0 and item["rule"] not in disabled]
|
||||
|
||||
excluded = list(resolution.get("scope", {}).get("excluded_entities") or [])
|
||||
return {
|
||||
"checked_at": datetime.now().isoformat(timespec="seconds"),
|
||||
"entity_id": eid,
|
||||
@@ -865,7 +1048,15 @@ def _run_governance_checks(db: Session, entity_id: int = 0):
|
||||
"total_rules": len(issues),
|
||||
"issues": issues,
|
||||
"passed": passed,
|
||||
"disabled": disabled,
|
||||
"rules_meta": RULES_META,
|
||||
# 配置化(ASK-20260928-05):口径快照指纹 —— 事后可解释「当时的分数是哪一版配置算出来的」
|
||||
"config_version": resolution.get("config_version"),
|
||||
"config_source": {"table_ready": resolution.get("table_ready"),
|
||||
"entity_config_enabled": resolution.get("entity_config_enabled")},
|
||||
# 演示/测试账套排除(F6):默认空集 ⇒ 不改变评分语义,只做标注
|
||||
"scope_excluded_entities": excluded,
|
||||
"excluded_from_business_caliber": bool(eid and eid in excluded),
|
||||
}
|
||||
|
||||
|
||||
@@ -878,7 +1069,8 @@ def governance_check(
|
||||
"""财务七规则检查(DAMA数据治理)— 只读幂等,不写库。
|
||||
|
||||
7条规则: unit_check/dup_alert/orphan_check/virtual_pollution/entity_check/kpi_completeness/reconciliation
|
||||
评分: 满分100,error 扣 min(15, count*3),warning 扣 min(10, count*1),score=max(0, 100-总扣分)
|
||||
评分: 满分100,各规则扣分 = min(配置上限, 命中数×配置系数)(默认 error 15×3、warning 10×1),
|
||||
score = max(0, 100 − 总扣分);阈值/关键词/权重自 2026-09-28 起由 quality_rule_config 配置(ASK-20260928-05)。
|
||||
实体筛选: entity_id 参数(0=全部,仅无会话token的Bot通道)。
|
||||
|
||||
多租户(cma-cross-entity-endpoints-auth-20260914):
|
||||
@@ -899,7 +1091,7 @@ def rule_detail(
|
||||
rule: Optional[str] = Query(None, description="七规则 key 之一;缺省/空 → 空集(不 500)"),
|
||||
page: int = Query(1, ge=1, description="页码,从 1 开始"),
|
||||
page_size: int = Query(50, ge=1, le=200, description="每页条数,默认 50,上限 200"),
|
||||
scope: Optional[str] = Query(None, description="virtual_pollution 专用:manual=source='manual' 待人工确认组"),
|
||||
scope: Optional[str] = Query(None, description="virtual_pollution 专用:manual=待人工确认组"),
|
||||
db: Session = Depends(get_db),
|
||||
entity_id: int = Depends(get_entity_id),
|
||||
):
|
||||
@@ -909,7 +1101,8 @@ def rule_detail(
|
||||
- `items` 结构化字段:record_id / entity_id / source / key_fields / summary / suggested_action;
|
||||
- 服务端分页(LIMIT/OFFSET),单次响应不塞全量;
|
||||
- 账套由会话 token 解析(Depends(get_entity_id)),**不新增账套入参**;
|
||||
- 兼容:rule 缺省/空 → 空集(200,不 500);未知 rule → 400 + 明确文案。
|
||||
- 兼容:rule 缺省/空 → 空集(200,不 500);未知 rule → 400 + 明确文案;
|
||||
- 判据与 `governance-check` 共用同一配置解析(2026-09-28 起),两处口径不会漂移。
|
||||
"""
|
||||
key = (rule or "").strip()
|
||||
if not key:
|
||||
@@ -920,18 +1113,177 @@ def rule_detail(
|
||||
|
||||
manual = key == "virtual_pollution" and (scope or "").strip().lower() == "manual"
|
||||
eid = entity_id or 0
|
||||
total = _detail_count(db, key, eid, manual=manual)
|
||||
cfg = _cfg_values(db, eid)
|
||||
total = _detail_count(db, key, eid, manual=manual, cfg=cfg)
|
||||
offset = (page - 1) * page_size
|
||||
items = _detail_items(db, key, eid, page_size, offset, manual=manual)
|
||||
items = _detail_items(db, key, eid, page_size, offset, manual=manual, cfg=cfg)
|
||||
return {
|
||||
"rule": key,
|
||||
"level": RULES_META[key]["level"],
|
||||
"name": RULES_META[key]["name"],
|
||||
"desc": RULES_META[key]["desc"],
|
||||
"scope": "manual" if manual else None,
|
||||
"enabled": bool((cfg.get(key) or {}).get("enabled", True)),
|
||||
"total": total,
|
||||
"page": page,
|
||||
"page_size": page_size,
|
||||
"items": items,
|
||||
}
|
||||
|
||||
|
||||
# ============================================================
|
||||
# 规则配置端点(ASK-20260928-05 最小实现:新表 + 端点 + 只读 UI)
|
||||
# 读:GET /rule-config(四角色,账套由 token 绑定,不新增账套入参)
|
||||
# 写:PUT /rule-config(ceo/it;平台级参数仅 it;Bot 通道禁写;变更与审计同事务)
|
||||
# 史:GET /rule-config/history(ceo/it)
|
||||
# 边界(清单 §4/§5):级别、判定表达式、评分函数、作用域等 10 类锁死 —— 不在 SPEC 即拒绝。
|
||||
# ============================================================
|
||||
|
||||
|
||||
class DqRuleConfigUpdate(BaseModel):
|
||||
"""单参数配置变更(一次只改一档:清单 §5.1「一次只改一档,改完复跑七规则对账」)。"""
|
||||
rule_key: str = Field(..., description="七规则 key 之一,或 global(全局项)")
|
||||
param_key: str = Field(..., description="参数名(见 GET /rule-config 的 params)")
|
||||
value: Any = Field(None, description="新值(类型见参数规格:bool/int/float/str/数组/枚举)")
|
||||
change_reason: str = Field(..., description=f"变更理由(≥{dqcfg.MIN_REASON_LEN} 字,落审计)")
|
||||
|
||||
|
||||
@router.get("/rule-config")
|
||||
def get_rule_config(
|
||||
db: Session = Depends(get_db),
|
||||
entity_id: int = Depends(get_entity_id),
|
||||
):
|
||||
"""财务七规则配置(**只读**):三层解析后的生效值 + 每项来源 + 锁死项清单。
|
||||
|
||||
来源语义:`entity`=本账套覆盖行 / `template`=全账套模板行(entity_id=0) / `builtin`=代码内置默认。
|
||||
账套由会话 token 解析(`Depends(get_entity_id)`),**不新增账套入参**;演示/测试账套只读不可写。
|
||||
"""
|
||||
res = _load_cfg(db, entity_id)
|
||||
return {
|
||||
"entity_id": entity_id,
|
||||
"config_entity_id": res["config_entity_id"],
|
||||
"entity_config_enabled": res["entity_config_enabled"],
|
||||
"config_entities": list(dqcfg.CONFIG_ENTITIES),
|
||||
"table_ready": res["table_ready"],
|
||||
"config_version": res["config_version"],
|
||||
"rule_param_count": dqcfg.count_rule_params(),
|
||||
"rules": {
|
||||
key: {
|
||||
"name": RULES_META[key]["name"],
|
||||
"level": RULES_META[key]["level"],
|
||||
"desc": RULES_META[key]["desc"],
|
||||
"params": res["rules"][key],
|
||||
} for key in dqcfg.RULE_KEYS
|
||||
},
|
||||
"global_params": res["global"],
|
||||
"scope": res["scope"],
|
||||
"scope_note": "演示/测试账套排除(F6)当前只做标注、不改评分语义;默认空集 = 不排除",
|
||||
"locked": [dict(it) for it in dqcfg.LOCKED_ITEMS],
|
||||
"readonly": True,
|
||||
"generated_at": datetime.now().isoformat(timespec="seconds"),
|
||||
}
|
||||
|
||||
|
||||
@router.get("/rule-config/history")
|
||||
def get_rule_config_history(
|
||||
limit: int = Query(20, ge=1, le=200, description="返回条数上限"),
|
||||
db: Session = Depends(get_db),
|
||||
entity_id: int = Depends(get_entity_id),
|
||||
_user: User = Depends(require_role("ceo", "it")),
|
||||
):
|
||||
"""本账套配置变更历史(ceo/it)——读 `operation_logs`(action=update_dq_rule_config)。
|
||||
|
||||
`operation_logs` 无 entity_id 列 ⇒ 按 `detail.entity_id` 过滤(清单 §5.2 既定口径)。
|
||||
"""
|
||||
return {"entity_id": entity_id, "items": dqcfg.history(db, entity_id, limit)}
|
||||
|
||||
|
||||
@router.put("/rule-config")
|
||||
def update_rule_config(
|
||||
payload: DqRuleConfigUpdate,
|
||||
request: Request,
|
||||
db: Session = Depends(get_db),
|
||||
entity_id: int = Depends(get_entity_id),
|
||||
current_user: User = Depends(require_role("ceo", "it")),
|
||||
):
|
||||
"""修改单个配置项(ceo/it;平台级参数仅 it)。
|
||||
|
||||
校验顺序:账套可写 → Bot 通道禁写 → 参数在开放面(锁死/未知一律 400)
|
||||
→ 平台级仅 it → change_reason → 取值边界(拒绝或钳制)→ 写行 + 审计同事务。
|
||||
"""
|
||||
if entity_id not in dqcfg.CONFIG_ENTITIES:
|
||||
raise HTTPException(
|
||||
400, f"账套 entity_id={entity_id} 不开放规则配置(仅 {list(dqcfg.CONFIG_ENTITIES)};"
|
||||
f"演示/测试账套只读,ASK-20260928-05 口径②)")
|
||||
if str(request.headers.get("X-BOT-KEY") or "").strip():
|
||||
# 锁死面第 9 条:Bot 通道无 users 行 → 无操作人可留痕 = 审计断链
|
||||
raise HTTPException(403, "Bot 通道禁止写规则配置(无操作人可留痕,审计会断链)")
|
||||
|
||||
rule_key = (payload.rule_key or "").strip()
|
||||
param_key = (payload.param_key or "").strip()
|
||||
if dqcfg.param_spec(rule_key, param_key) is None:
|
||||
# 复用参数校验器的文案:锁死项 / 未知 key / 该规则无开放参数
|
||||
probe = dqcfg.validate_param(rule_key, param_key, payload.value)
|
||||
raise HTTPException(400, probe["error"])
|
||||
|
||||
level = dqcfg.open_level_of(rule_key, param_key)
|
||||
if level == dqcfg.OPEN_LEVEL_PLATFORM and current_user.role != "it":
|
||||
raise HTTPException(403, f"参数 {rule_key}.{param_key} 为平台级配置(仅 it 可写)")
|
||||
|
||||
reason = (payload.change_reason or "").strip()
|
||||
if len(reason) < dqcfg.MIN_REASON_LEN:
|
||||
raise HTTPException(400, f"change_reason 必填且不少于 {dqcfg.MIN_REASON_LEN} 字"
|
||||
f"(防无痕调参刷分;清单 §5.1)")
|
||||
|
||||
checked = dqcfg.validate_param(rule_key, param_key, payload.value)
|
||||
if not checked["ok"]:
|
||||
raise HTTPException(400, checked["error"])
|
||||
|
||||
rule_name = RULES_META.get(rule_key, {}).get("name") or rule_key
|
||||
try:
|
||||
result = dqcfg.upsert_param(db, entity_id, rule_key, param_key, checked["value"],
|
||||
username=getattr(current_user, "username", None))
|
||||
log_op(
|
||||
db, dqcfg.ACTION_UPDATE, target_type=dqcfg.TARGET_TYPE, target_id=result["id"],
|
||||
user_id=current_user.id, entity_id=entity_id,
|
||||
detail={
|
||||
"entity_id": entity_id,
|
||||
"rule_key": rule_key,
|
||||
"rule_name": rule_name,
|
||||
"param_key": param_key,
|
||||
"before": result["before"],
|
||||
"after": result["after"],
|
||||
"reason": reason,
|
||||
"open_level": level,
|
||||
"client_ip": (request.client.host if request.client else None),
|
||||
"operator": {"user_id": current_user.id,
|
||||
"username": getattr(current_user, "username", None),
|
||||
"role": current_user.role},
|
||||
},
|
||||
)
|
||||
# 变更与审计同一事务提交:审计写不进 ⇒ 配置变更一并回滚
|
||||
# (绝不留「改了但没审计」的窗口;形态照抄 tenant.py:148 先例)
|
||||
db.commit()
|
||||
except Exception:
|
||||
db.rollback()
|
||||
logger.error("DQ_RULE_CONFIG_AUDIT_WRITE_FAILED entity=%s %s.%s user=%s → 变更已回滚",
|
||||
entity_id, rule_key, param_key, getattr(current_user, "id", None),
|
||||
exc_info=True)
|
||||
raise HTTPException(500, "审计写入失败,配置变更已回滚")
|
||||
|
||||
after = _load_cfg(db, entity_id)
|
||||
return {
|
||||
"success": True,
|
||||
"entity_id": entity_id,
|
||||
"rule_key": rule_key,
|
||||
"param_key": param_key,
|
||||
"before": result["before"],
|
||||
"before_source": result["before_source"],
|
||||
"after": checked["value"],
|
||||
"warning": checked["warning"],
|
||||
"config_version": after["config_version"],
|
||||
"changed_by": getattr(current_user, "username", None),
|
||||
"audit": {"action": dqcfg.ACTION_UPDATE, "target_type": dqcfg.TARGET_TYPE,
|
||||
"target_id": result["id"]},
|
||||
"note": "配置即生效(下次 governance-check 按新口径计算);历史评分不重算",
|
||||
}
|
||||
|
||||
@@ -0,0 +1,618 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""DAMA 数据质量治理规则可配置项(财务七规则)—— 单一事实源 + 三层解析
|
||||
|
||||
任务:ASK-20260928-05(任总 2026-09-28 13:38「放行 4 和 5」,本项目 Bot 转呈)
|
||||
来源清单:wecom-dama《财务七规则可配置化 — 可配置项清单 + 建议默认值 + 权限/留痕建议》
|
||||
(`wecom-dama/pending-tasks/_reports/project-20260923-quality-rules-config-inventory-dama.md`)
|
||||
|
||||
本模块承担三件事(唯一来源,勿在别处复制常量)
|
||||
----------------------------------------------
|
||||
1. **开放面**:37 个规则级参数(`SPEC`)+ 2 个全局项(`GLOBAL_SPEC`)。清单 §3 逐条落成代码;
|
||||
测试断言项数 = 37 / 2,防后续会话漏项或私自扩面。
|
||||
2. **锁死面**:清单 §4 的 10 类(判定表达式本体 / 级别归属 / 评分函数 / 实体作用域 /
|
||||
规则集与数量 / 匹配方式自由化 / kpi_completeness 口径源 / `entity_id IS NULL` 处置 /
|
||||
Bot 通道写配置 / 越界权重)——不在 SPEC 里的键一律 400(写端点),
|
||||
`LOCKED_PARAM_KEYS` 仅用于把「锁死」讲清楚(可读文案),不是白名单。
|
||||
3. **三层解析**:账套行(`entity_id`)→ 模板行(`entity_id=0`)→ 代码内置默认;
|
||||
**禁止跨账套回退**(历史事故:`cash.current_balance` 全局键被全账套共用)。
|
||||
|
||||
口径(任总 2026-09-28 放行原文:「口径用清单默认值」)
|
||||
----------------------------------------------------
|
||||
* 默认值 = 清单 §7「建议默认值」总表,即「现行值,上线零行为差异」,
|
||||
**唯一例外**:`virtual_pollution.suspect_keywords` 去掉 `sync`
|
||||
(`zxt-sync` 是酣客生产同步来源,现行 134 行 100% 假阳性)——本单唯一的行为变更;
|
||||
* 演示/测试账套(3/4/5/6)**不开配置**(`CONFIG_ENTITIES = (1, 2)`,清单 §6.4 + ASK-05 口径②);
|
||||
读路径仍按三层解析返回生效值(模板/内置默认),只是写端点拒绝。
|
||||
|
||||
配置值一律以 JSON 字符串落库(`param_value`),`value_type` 记录类型以便前端/审计呈现。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import logging
|
||||
from typing import Any, Dict, List, Optional, Tuple
|
||||
|
||||
from sqlalchemy import inspect
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
logger = logging.getLogger("cma.dq.rule_config")
|
||||
|
||||
#: 表名(DDL 走部署链特权迁移步 `backend/scripts/db_migrate.py --apply`,属生产写审批范围)
|
||||
TABLE_NAME = "quality_rule_config"
|
||||
#: 全账套模板行的 entity_id(0 = 模板,不参与具体账套取数)
|
||||
TEMPLATE_ENTITY_ID = 0
|
||||
#: 开放配置写入的真实账套(口径②:演示/测试账套不开配置)
|
||||
CONFIG_ENTITIES: Tuple[int, ...] = (1, 2)
|
||||
#: 内置默认明细展示条数(清单 §3.9 全局项 `detail_limit` 的出厂值)
|
||||
DEFAULT_DETAIL_LIMIT = 10
|
||||
#: 写配置必须带变更理由,最短长度(清单 §5.1「反刷分约束」第 1 条)
|
||||
MIN_REASON_LEN = 10
|
||||
|
||||
RULE_KEYS: Tuple[str, ...] = (
|
||||
"unit_check", "dup_alert", "orphan_check", "virtual_pollution",
|
||||
"entity_check", "kpi_completeness", "reconciliation",
|
||||
)
|
||||
|
||||
# 全局项(清单 §3.9):platform 级,仅 it 可写
|
||||
GLOBAL_RULE_KEY = "global"
|
||||
|
||||
# 规则级参数统一三件套(每条规则都有;`detail_limit` 为 platform 级)
|
||||
_CAPS = {
|
||||
"deduct_cap": dict(type="int", default=15, open_level="entity",
|
||||
bounds=(0, 50), overflow="clamp",
|
||||
desc="本规则扣分上限(0 = 只报不计分)"),
|
||||
"deduct_factor": dict(type="int", default=3, open_level="entity",
|
||||
bounds=(0, 10), overflow="clamp",
|
||||
desc="本规则每命中一条的扣分系数"),
|
||||
}
|
||||
_WARN_CAPS = {
|
||||
"deduct_cap": dict(type="int", default=10, open_level="entity",
|
||||
bounds=(0, 50), overflow="clamp",
|
||||
desc="本规则扣分上限(0 = 只报不计分)"),
|
||||
"deduct_factor": dict(type="int", default=1, open_level="entity",
|
||||
bounds=(0, 10), overflow="clamp",
|
||||
desc="本规则每命中一条的扣分系数"),
|
||||
}
|
||||
_DETAIL_LIMIT = dict(type="int", default=DEFAULT_DETAIL_LIMIT, open_level="platform",
|
||||
bounds=(1, 100), overflow="clamp",
|
||||
desc="本规则明细展示条数(平台级,仅 it 可写)")
|
||||
_ENABLED = dict(type="bool", default=True, open_level="entity",
|
||||
desc="启用开关(关闭后该规则不参与评分,明细不再展示)")
|
||||
|
||||
#: 7 条规则的开放参数(37 项;清单 §3.1–§3.7 逐条落成)
|
||||
SPEC: Dict[str, Dict[str, dict]] = {
|
||||
"unit_check": {
|
||||
"enabled": dict(_ENABLED),
|
||||
"amount_threshold": dict(type="int", default=10000, open_level="entity",
|
||||
bounds=(100, 1000000), overflow="reject",
|
||||
desc="金额阈值(万元):cash_plans.amount > 阈值 即命中(越界拒绝)"),
|
||||
**_CAPS,
|
||||
"detail_limit": dict(_DETAIL_LIMIT),
|
||||
},
|
||||
"dup_alert": {
|
||||
"enabled": dict(_ENABLED),
|
||||
"min_count": dict(type="int", default=2, open_level="entity",
|
||||
bounds=(2, 20), overflow="reject",
|
||||
desc="同一 plan_id 的 pending 预警条数下限(≥ 该值即重复;越界拒绝)"),
|
||||
**_CAPS,
|
||||
"detail_limit": dict(_DETAIL_LIMIT),
|
||||
},
|
||||
"orphan_check": {
|
||||
"enabled": dict(_ENABLED),
|
||||
**_CAPS,
|
||||
"detail_limit": dict(_DETAIL_LIMIT),
|
||||
},
|
||||
"virtual_pollution": {
|
||||
"enabled": dict(_ENABLED),
|
||||
"suspect_keywords": dict(
|
||||
type="json", default=["test", "虚拟", "demo"], open_level="entity",
|
||||
max_items=20, item_max_len=32, forbidden_chars="%_",
|
||||
desc="嫌疑关键词表(大小写不敏感包含式匹配;不含 sync —— 本单唯一行为变更)"),
|
||||
"source_whitelist": dict(
|
||||
type="json",
|
||||
default=["zxt-sync", "bohai_ar", "bohai_ar_20260630", "bohai_prepay_20260630",
|
||||
"bohai_ap_20260630", "receivables_migrate"],
|
||||
open_level="entity", max_items=20, item_max_len=32, forbidden_chars="%_",
|
||||
desc="来源白名单(豁免优先于关键词:命中即不算污染)"),
|
||||
"manual_source_value": dict(type="str", default="manual", open_level="entity",
|
||||
max_len=50,
|
||||
desc="「待人工确认」的来源值(非污染,按 warning 附加扣分)"),
|
||||
**_CAPS,
|
||||
"manual_cap": dict(type="int", default=10, open_level="entity",
|
||||
bounds=(0, 30), overflow="clamp",
|
||||
desc="待人工确认(manual)附加上限"),
|
||||
"manual_factor": dict(type="int", default=1, open_level="entity",
|
||||
bounds=(0, 5), overflow="clamp",
|
||||
desc="待人工确认(manual)附加系数"),
|
||||
"detail_limit": dict(_DETAIL_LIMIT),
|
||||
},
|
||||
"entity_check": {
|
||||
"enabled": dict(_ENABLED),
|
||||
**_CAPS,
|
||||
"detail_limit": dict(_DETAIL_LIMIT),
|
||||
},
|
||||
"kpi_completeness": {
|
||||
"enabled": dict(_ENABLED),
|
||||
**_WARN_CAPS,
|
||||
"detail_limit": dict(_DETAIL_LIMIT),
|
||||
},
|
||||
"reconciliation": {
|
||||
"enabled": dict(_ENABLED),
|
||||
"diff_threshold_pct": dict(type="float", default=20, open_level="entity",
|
||||
bounds=(1, 200), overflow="reject",
|
||||
desc="预算月度合计 vs 年度目标差异阈值(%);越界拒绝"),
|
||||
"compare_mode": dict(type="enum", default="abs", open_level="entity",
|
||||
enum=("abs", "upper_only", "lower_only"),
|
||||
desc="比较方向:abs=双侧 / upper_only=仅超目标 / lower_only=仅低于目标"),
|
||||
**_WARN_CAPS,
|
||||
"detail_limit": dict(_DETAIL_LIMIT),
|
||||
},
|
||||
}
|
||||
|
||||
#: 全局项(2 项;清单 §3.9)
|
||||
GLOBAL_SPEC: Dict[str, dict] = {
|
||||
"detail_limit": dict(_DETAIL_LIMIT),
|
||||
"scope.excluded_entities": dict(
|
||||
type="json", default=[], open_level="platform", max_items=20, item_max_len=6,
|
||||
desc="排除出「经营口径」的账套(演示/测试;默认空集 = 不排除)。"
|
||||
"⚠️ 本项当前**只做标注不改评分语义**(F6 属业务口径,待任总/财务裁定后接线)"),
|
||||
}
|
||||
|
||||
# ── 锁死面(清单 §4;仅用于给出可读文案,非白名单)─────────────────────────
|
||||
LOCKED_ITEMS: Tuple[dict, ...] = (
|
||||
dict(key="expression", item="判定表达式本体(字段/算子/SQL 片段/正则)",
|
||||
reason="表达式可配 = 治理规则可被一次配置改成永久绿灯"),
|
||||
dict(key="level", item="级别归属(error/warning)",
|
||||
reason="级别直接决定扣分上限,降级是最廉价的绕分路径;确有差异请用已开放的扣分权重表达"),
|
||||
dict(key="score_fn", item="评分函数(max(0,100−Σ)、7 条固定、各规则独立封顶后累加、无总封顶)",
|
||||
reason="评分是跨规则/跨期可比的口径基线"),
|
||||
dict(key="entity_scope", item="实体作用域语义(规则 1 用 cash_plans.entity_id;2/3/6/7 用 kpi_definitions.entity_id;5 用 kpi_values.entity_id)",
|
||||
reason="作用域 = 安全边界,开放会引入跨账套借用通路"),
|
||||
dict(key="rule_set", item="规则 key 集合与规则数量(固定 7 条)",
|
||||
reason="规则集是治理骨架,新增规则必须走代码评审"),
|
||||
dict(key="match_mode", item="匹配方式自由化(关键词升级为正则/SQL 片段)",
|
||||
reason="注入面 + 不可验证;匹配方式固定为大小写不敏感包含式"),
|
||||
dict(key="completeness_caliber", item="kpi_completeness 口径源 kpi_definitions.status='active'(含豁免名单 exempt_kpi_codes)",
|
||||
reason="KPI 存废开关已是 status,再加豁免名单 = 双开关口径分裂"),
|
||||
dict(key="null_entity_policy", item="规则5 kpi_values.entity_id IS NULL 行的处置",
|
||||
reason="NULL 语义未裁定前保持 ignore"),
|
||||
dict(key="bot_write", item="Bot 通道(X-BOT-KEY)写配置",
|
||||
reason="Bot 通道无 users 行 → 无操作人可留痕 = 审计断链"),
|
||||
dict(key="weight_range", item="负系数 / 超大上限 / 非整数权重",
|
||||
reason="负系数会变成加分(可刷分);一律钳制到开放区间"),
|
||||
)
|
||||
#: 键名 → 锁死文案(写端点遇到这些 param_key 时给「锁死」提示而非笼统的未知参数)
|
||||
LOCKED_PARAM_KEYS = {it["key"]: it for it in LOCKED_ITEMS}
|
||||
|
||||
# 常见别名(历史/前端习惯叫法)→ 锁死项 key
|
||||
_LOCKED_ALIASES = {
|
||||
"expr": "expression", "predicate": "expression", "sql": "expression",
|
||||
"rule_level": "level", "severity": "level",
|
||||
"score": "score_fn", "score_func": "score_fn", "total_deduct_cap": "score_fn",
|
||||
"exempt_kpi_codes": "completeness_caliber", "status_source": "completeness_caliber",
|
||||
"entity_id_source": "entity_scope", "scope_field": "entity_scope",
|
||||
"rules": "rule_set", "rule_keys": "rule_set", "rule_count": "rule_set",
|
||||
"match_regex": "match_mode", "regex": "match_mode", "pattern": "match_mode",
|
||||
"null_policy": "null_entity_policy",
|
||||
}
|
||||
|
||||
OPEN_LEVEL_ENTITY = "entity"
|
||||
OPEN_LEVEL_PLATFORM = "platform"
|
||||
|
||||
|
||||
# ══════════════════════════════════════════════════════════════════
|
||||
# 参数查找 / 校验
|
||||
# ══════════════════════════════════════════════════════════════════
|
||||
def is_global_param(rule_key: str) -> bool:
|
||||
return rule_key == GLOBAL_RULE_KEY
|
||||
|
||||
|
||||
def spec_of(rule_key: str) -> Dict[str, dict]:
|
||||
"""返回某 rule_key 的参数规格表(全局项用 rule_key='global')。"""
|
||||
if is_global_param(rule_key):
|
||||
return GLOBAL_SPEC
|
||||
return SPEC.get(rule_key) or {}
|
||||
|
||||
|
||||
def param_spec(rule_key: str, param_key: str) -> Optional[dict]:
|
||||
return spec_of(rule_key).get(param_key)
|
||||
|
||||
|
||||
def all_param_pairs() -> List[Tuple[str, str]]:
|
||||
"""(rule_key, param_key) 全量清单——37 规则级 + 2 全局项。"""
|
||||
pairs = [(r, p) for r in RULE_KEYS for p in SPEC[r]]
|
||||
pairs += [(GLOBAL_RULE_KEY, p) for p in GLOBAL_SPEC]
|
||||
return pairs
|
||||
|
||||
|
||||
def count_rule_params() -> int:
|
||||
"""规则级开放参数项数(断言用,清单 §10.2 = 37)。"""
|
||||
return sum(len(SPEC[r]) for r in RULE_KEYS)
|
||||
|
||||
|
||||
def open_level_of(rule_key: str, param_key: str) -> str:
|
||||
spec = param_spec(rule_key, param_key) or {}
|
||||
return spec.get("open_level") or OPEN_LEVEL_ENTITY
|
||||
|
||||
|
||||
def value_type_of(rule_key: str, param_key: str) -> str:
|
||||
spec = param_spec(rule_key, param_key) or {}
|
||||
return spec.get("type") or "json"
|
||||
|
||||
|
||||
def dumps_value(value: Any) -> str:
|
||||
"""统一序列化(落库格式;bool/int/float/str/数组一律 JSON)。"""
|
||||
return json.dumps(value, ensure_ascii=False)
|
||||
|
||||
|
||||
def loads_value(raw: Any, value_type: str = "") -> Any:
|
||||
"""反序列化;非 JSON 的历史/手写行容错(返回 None 表示不可用 → 上层回落)。"""
|
||||
if raw is None:
|
||||
return None
|
||||
if isinstance(raw, (int, float, bool, list, dict)):
|
||||
return raw
|
||||
try:
|
||||
return json.loads(str(raw))
|
||||
except (ValueError, TypeError):
|
||||
text = str(raw)
|
||||
if value_type == "int":
|
||||
try:
|
||||
return int(float(text))
|
||||
except (ValueError, TypeError):
|
||||
return None
|
||||
if value_type == "float":
|
||||
try:
|
||||
return float(text)
|
||||
except (ValueError, TypeError):
|
||||
return None
|
||||
if value_type == "bool":
|
||||
if text.strip().lower() in ("1", "true", "yes"):
|
||||
return True
|
||||
if text.strip().lower() in ("0", "false", "no"):
|
||||
return False
|
||||
return None
|
||||
if value_type == "str":
|
||||
return text
|
||||
return None
|
||||
|
||||
|
||||
def _coerce_type(value: Any, spec: dict) -> Tuple[bool, Any]:
|
||||
"""类型强转(拒绝型:类型不符 → 不可用)。bool 与 int 严格区分(Python 里 bool 是 int 子类)。"""
|
||||
t = spec.get("type") or "json"
|
||||
if t == "bool":
|
||||
if isinstance(value, bool):
|
||||
return True, value
|
||||
if isinstance(value, str) and value.strip().lower() in ("true", "false"):
|
||||
return True, value.strip().lower() == "true"
|
||||
if isinstance(value, int) and value in (0, 1):
|
||||
return True, bool(value)
|
||||
return False, None
|
||||
if t == "int":
|
||||
if isinstance(value, bool):
|
||||
return False, None
|
||||
if isinstance(value, int):
|
||||
return True, value
|
||||
if isinstance(value, float) and float(value).is_integer():
|
||||
return True, int(value)
|
||||
if isinstance(value, str):
|
||||
s = value.strip()
|
||||
try:
|
||||
return True, int(s)
|
||||
except ValueError:
|
||||
return False, None
|
||||
return False, None
|
||||
if t == "float":
|
||||
if isinstance(value, bool):
|
||||
return False, None
|
||||
if isinstance(value, (int, float)):
|
||||
return True, float(value)
|
||||
if isinstance(value, str):
|
||||
try:
|
||||
return True, float(value.strip())
|
||||
except ValueError:
|
||||
return False, None
|
||||
return False, None
|
||||
if t == "str":
|
||||
if not isinstance(value, str):
|
||||
return False, None
|
||||
return True, value
|
||||
if t == "enum":
|
||||
if not isinstance(value, str):
|
||||
return False, None
|
||||
return True, value.strip()
|
||||
if t == "json":
|
||||
if not isinstance(value, list):
|
||||
return False, None
|
||||
return True, value
|
||||
return False, None
|
||||
|
||||
|
||||
def validate_param(rule_key: str, param_key: str, value: Any) -> dict:
|
||||
"""参数级校验(清单 §3 的「取值边界 / 越界处理」逐条落地)。
|
||||
|
||||
返回 ``{"ok": bool, "value": <规范化值>, "error": str|None, "warning": str|None}``;
|
||||
越界处理两类:``reject``(400,附区间文案)/ ``clamp``(钳制到端点 + 响应告警)。
|
||||
"""
|
||||
spec = param_spec(rule_key, param_key)
|
||||
if spec is None:
|
||||
locked = LOCKED_PARAM_KEYS.get(
|
||||
param_key, LOCKED_PARAM_KEYS.get(_LOCKED_ALIASES.get(param_key, ""))
|
||||
)
|
||||
if locked:
|
||||
return dict(ok=False, value=None,
|
||||
error=f"参数 {param_key} 属锁死项「{locked['item']}」,不允许配置:{locked['reason']}",
|
||||
warning=None)
|
||||
return dict(ok=False, value=None,
|
||||
error=f"未知参数 rule={rule_key} param={param_key};"
|
||||
f"可选: {', '.join(spec_of(rule_key).keys()) or '(该规则无开放参数)'}",
|
||||
warning=None)
|
||||
|
||||
passed, value = _coerce_type(value, spec)
|
||||
if not passed:
|
||||
return dict(ok=False, value=None,
|
||||
error=f"{rule_key}.{param_key} 类型非法:要求 {spec.get('type')}", warning=None)
|
||||
|
||||
caution = None
|
||||
if spec.get("type") == "enum":
|
||||
if value not in tuple(spec.get("enum") or ()):
|
||||
return dict(ok=False, value=None,
|
||||
error=f"{rule_key}.{param_key} 取值非法:仅支持 {', '.join(spec['enum'])}",
|
||||
warning=None)
|
||||
return dict(ok=True, value=value, error=None, warning=None)
|
||||
|
||||
if spec.get("type") == "json":
|
||||
max_items = int(spec.get("max_items") or 0)
|
||||
item_max_len = int(spec.get("item_max_len") or 0)
|
||||
forbidden = spec.get("forbidden_chars") or ""
|
||||
if max_items and len(value) > max_items:
|
||||
return dict(ok=False, value=None,
|
||||
error=f"{rule_key}.{param_key} 元素过多:上限 {max_items} 项(当前 {len(value)})",
|
||||
warning=None)
|
||||
cleaned: List[str] = []
|
||||
for it in value:
|
||||
if not isinstance(it, str):
|
||||
if spec.get("item_max_len") == 6 and isinstance(it, int) and not isinstance(it, bool):
|
||||
it = str(it) # scope.excluded_entities:账套号允许传 int
|
||||
else:
|
||||
return dict(ok=False, value=None,
|
||||
error=f"{rule_key}.{param_key} 仅允许字符串数组元素", warning=None)
|
||||
s = it.strip()
|
||||
if not s:
|
||||
return dict(ok=False, value=None,
|
||||
error=f"{rule_key}.{param_key} 不允许空串元素", warning=None)
|
||||
if item_max_len and len(s) > item_max_len:
|
||||
return dict(ok=False, value=None,
|
||||
error=f"{rule_key}.{param_key} 元素过长:单元素上限 {item_max_len} 字符({s})",
|
||||
warning=None)
|
||||
if any(ch in s for ch in forbidden):
|
||||
return dict(ok=False, value=None,
|
||||
error=f"{rule_key}.{param_key} 元素含非法字符 {forbidden}(不支持通配符/自由模式)",
|
||||
warning=None)
|
||||
cleaned.append(s)
|
||||
# 去重保序(重复项无意义且会让 LIKE 子句膨胀)
|
||||
seen, uniq = set(), []
|
||||
for s in cleaned:
|
||||
if s not in seen:
|
||||
seen.add(s)
|
||||
uniq.append(s)
|
||||
if len(uniq) != len(cleaned):
|
||||
caution = f"{rule_key}.{param_key} 已去重:{len(cleaned)} → {len(uniq)} 项"
|
||||
return dict(ok=True, value=uniq, error=None, warning=caution)
|
||||
|
||||
if spec.get("type") == "str":
|
||||
max_len = int(spec.get("max_len") or 0)
|
||||
s = value
|
||||
if not s.strip():
|
||||
return dict(ok=False, value=None, error=f"{rule_key}.{param_key} 不允许空串", warning=None)
|
||||
if max_len and len(s) > max_len:
|
||||
return dict(ok=False, value=None,
|
||||
error=f"{rule_key}.{param_key} 过长:上限 {max_len} 字符", warning=None)
|
||||
return dict(ok=True, value=s, error=None, warning=None)
|
||||
|
||||
bounds = spec.get("bounds")
|
||||
if bounds:
|
||||
lo, hi = bounds
|
||||
if value < lo or value > hi:
|
||||
if spec.get("overflow") == "clamp":
|
||||
clamped = min(max(value, lo), hi)
|
||||
logger.warning("DQ_RULE_CONFIG_CLAMPED %s.%s %s → %s(区间 [%s, %s])",
|
||||
rule_key, param_key, value, clamped, lo, hi)
|
||||
return dict(ok=True, value=clamped, error=None,
|
||||
warning=f"{rule_key}.{param_key} 越界已钳制到区间端点:{value} → {clamped}"
|
||||
f"(允许区间 [{lo}, {hi}])")
|
||||
return dict(ok=False, value=None,
|
||||
error=f"{rule_key}.{param_key} 越界:允许区间 [{lo}, {hi}],收到 {value}",
|
||||
warning=None)
|
||||
return dict(ok=True, value=value, error=None, warning=None)
|
||||
|
||||
|
||||
# ══════════════════════════════════════════════════════════════════
|
||||
# 读路径:三层解析
|
||||
# ══════════════════════════════════════════════════════════════════
|
||||
def table_ready(db: Session) -> bool:
|
||||
"""配置表是否就绪(DDL 未落地时**不得**让读路径 500:回落内置默认)。"""
|
||||
try:
|
||||
return bool(inspect(db.get_bind()).has_table(TABLE_NAME))
|
||||
except Exception as exc: # noqa: BLE001
|
||||
logger.warning("DQ_RULE_CONFIG_TABLE_PROBE_FAILED %s: %s", TABLE_NAME, exc)
|
||||
return False
|
||||
|
||||
|
||||
def _load_rows(db: Session, entity_id: int) -> Dict[Tuple[str, str], dict]:
|
||||
"""读模板行 + 本账套行(一次查询);返回 {(rule_key, param_key): {entity_id, value, raw, row}}。"""
|
||||
from app.models import QualityRuleConfig
|
||||
|
||||
eids = [TEMPLATE_ENTITY_ID]
|
||||
if entity_id != TEMPLATE_ENTITY_ID:
|
||||
eids.append(entity_id)
|
||||
rows = db.query(QualityRuleConfig).filter(QualityRuleConfig.entity_id.in_(eids)).all()
|
||||
out: Dict[Tuple[str, str], dict] = {}
|
||||
for row in rows:
|
||||
vt = value_type_of(row.rule_key, row.param_key)
|
||||
val = loads_value(row.param_value, vt)
|
||||
if val is None:
|
||||
logger.warning("DQ_RULE_CONFIG_ROW_UNPARSABLE id=%s %s.%s value=%r → 忽略该行(回落下一层)",
|
||||
row.id, row.rule_key, row.param_key, row.param_value)
|
||||
continue
|
||||
out[(row.rule_key, row.param_key)] = {
|
||||
"entity_id": row.entity_id, "value": val, "row": row,
|
||||
}
|
||||
return out
|
||||
|
||||
|
||||
def resolve(db: Session, entity_id: int) -> dict:
|
||||
"""三层解析当前生效配置(账套行 → 模板行 → 内置默认;**不跨账套回退**)。
|
||||
|
||||
返回结构(前端/审计/引擎共用)::
|
||||
|
||||
{"entity_id": 1, "config_version": "…", "table_ready": True,
|
||||
"rules": {rule_key: {param_key: {"value":…, "source": "entity|template|builtin",
|
||||
"open_level":…, "desc":…, "updated_by":…, "updated_at":…}}},
|
||||
"global": {...同 rules 的形状…},
|
||||
"values": {rule_key: {param_key: <裸值>}}, # 引擎侧取值口
|
||||
"scope": {"excluded_entities": [...]}}
|
||||
"""
|
||||
eid = int(entity_id or 0)
|
||||
ready = table_ready(db)
|
||||
rows = _load_rows(db, eid) if ready else {}
|
||||
return _assemble(eid, ready, rows)
|
||||
|
||||
|
||||
def builtin_resolution(entity_id: int = 0) -> dict:
|
||||
"""纯内置默认的解析结果(配置表缺失/读异常时的回落形态,字段与 `resolve()` 逐字一致)。"""
|
||||
return _assemble(int(entity_id or 0), False, {})
|
||||
|
||||
|
||||
def _assemble(eid: int, ready: bool, rows: Dict[Tuple[str, str], dict]) -> dict:
|
||||
"""把「行 + 内置默认」组装成对外结构(resolve / builtin_resolution 共用)。"""
|
||||
|
||||
def _one(rule_key: str, param_key: str, spec: dict) -> dict:
|
||||
val, source, row = spec.get("default"), "builtin", None
|
||||
tmpl = rows.get((rule_key, param_key))
|
||||
if tmpl and tmpl["entity_id"] == TEMPLATE_ENTITY_ID:
|
||||
val, source, row = tmpl["value"], "template", tmpl["row"]
|
||||
elif eid != TEMPLATE_ENTITY_ID:
|
||||
own = rows.get((rule_key, param_key))
|
||||
if own and own["entity_id"] == eid:
|
||||
val, source, row = own["value"], "entity", own["row"]
|
||||
entry = {
|
||||
"value": val, "source": source, "open_level": spec.get("open_level"),
|
||||
"desc": spec.get("desc"), "updated_by": None, "updated_at": None,
|
||||
}
|
||||
if row is not None:
|
||||
entry["updated_by"] = getattr(row, "updated_by", None)
|
||||
ua = getattr(row, "updated_at", None)
|
||||
entry["updated_at"] = ua.isoformat(timespec="seconds") if ua else None
|
||||
entry["row_id"] = row.id
|
||||
return entry
|
||||
|
||||
rules: Dict[str, Dict[str, dict]] = {}
|
||||
values: Dict[str, Dict[str, Any]] = {}
|
||||
for rule_key in RULE_KEYS:
|
||||
rules[rule_key] = {p: _one(rule_key, p, s) for p, s in SPEC[rule_key].items()}
|
||||
values[rule_key] = {p: e["value"] for p, e in rules[rule_key].items()}
|
||||
glob: Dict[str, dict] = {p: _one(GLOBAL_RULE_KEY, p, s) for p, s in GLOBAL_SPEC.items()}
|
||||
values[GLOBAL_RULE_KEY] = {p: e["value"] for p, e in glob.items()}
|
||||
|
||||
# 规则级 detail_limit 未配置时,回落全局 detail_limit(清单 §3.9)
|
||||
global_limit = values[GLOBAL_RULE_KEY].get("detail_limit") or DEFAULT_DETAIL_LIMIT
|
||||
for rule_key in RULE_KEYS:
|
||||
entry = rules[rule_key]["detail_limit"]
|
||||
if entry["source"] == "builtin":
|
||||
entry["value"] = global_limit
|
||||
entry["source"] = "builtin(global)"
|
||||
values[rule_key]["detail_limit"] = entry["value"]
|
||||
|
||||
version = hashlib.sha1(
|
||||
json.dumps(values, sort_keys=True, ensure_ascii=False).encode("utf-8")
|
||||
).hexdigest()[:12]
|
||||
return {
|
||||
"entity_id": eid,
|
||||
"config_entity_id": eid if eid in CONFIG_ENTITIES else TEMPLATE_ENTITY_ID,
|
||||
"entity_config_enabled": eid in CONFIG_ENTITIES,
|
||||
"table_ready": ready,
|
||||
"config_version": version,
|
||||
"rules": rules,
|
||||
"global": glob,
|
||||
"values": values,
|
||||
"scope": {"excluded_entities": values[GLOBAL_RULE_KEY].get("scope.excluded_entities") or []},
|
||||
}
|
||||
|
||||
|
||||
def engine_values(db: Session, entity_id: int) -> dict:
|
||||
"""引擎取值口(data_quality 规则引擎唯一的配置读取入口)。"""
|
||||
return resolve(db, entity_id)["values"]
|
||||
|
||||
|
||||
# ══════════════════════════════════════════════════════════════════
|
||||
# 写路径:单参数 upsert(逐项「一次只改一档」)
|
||||
# ══════════════════════════════════════════════════════════════════
|
||||
def upsert_param(db: Session, entity_id: int, rule_key: str, param_key: str, value: Any,
|
||||
*, username: Optional[str] = None) -> dict:
|
||||
"""写一行配置(调用方负责权限/校验/审计/事务边界);返回 {id, before, after, source_before}。"""
|
||||
from app.models import QualityRuleConfig
|
||||
|
||||
before = resolve(db, entity_id)
|
||||
scope = before.get("global") if is_global_param(rule_key) else before.get("rules")
|
||||
before_entry = (scope or {}).get(rule_key, {}).get(param_key, {}) or {}
|
||||
value_type = value_type_of(rule_key, param_key)
|
||||
payload = dumps_value(value)
|
||||
|
||||
row = (db.query(QualityRuleConfig)
|
||||
.filter(QualityRuleConfig.entity_id == int(entity_id),
|
||||
QualityRuleConfig.rule_key == rule_key,
|
||||
QualityRuleConfig.param_key == param_key)
|
||||
.first())
|
||||
if row is None:
|
||||
row = QualityRuleConfig(
|
||||
entity_id=int(entity_id), rule_key=rule_key, param_key=param_key,
|
||||
param_value=payload, value_type=value_type,
|
||||
open_level=open_level_of(rule_key, param_key), is_default=False,
|
||||
description=(param_spec(rule_key, param_key) or {}).get("desc"),
|
||||
updated_by=username,
|
||||
)
|
||||
db.add(row)
|
||||
else:
|
||||
row.param_value = payload
|
||||
row.value_type = value_type
|
||||
row.is_default = False
|
||||
row.updated_by = username
|
||||
db.flush() # 拿到 id,且撞唯一键时当场暴露
|
||||
return {
|
||||
"id": row.id,
|
||||
"before": before_entry.get("value"),
|
||||
"before_source": before_entry.get("source"),
|
||||
"after": value,
|
||||
}
|
||||
|
||||
|
||||
def history(db: Session, entity_id: int, limit: int = 20) -> List[dict]:
|
||||
"""本账套的配置变更历史(读 `operation_logs`;该表无 entity_id 列 → 按 detail JSON 过滤)。"""
|
||||
from app.models import OperationLog
|
||||
|
||||
rows = (db.query(OperationLog)
|
||||
.filter(OperationLog.action == ACTION_UPDATE)
|
||||
.order_by(OperationLog.id.desc())
|
||||
.limit(max(limit * 5, 50))
|
||||
.all())
|
||||
out: List[dict] = []
|
||||
for r in rows:
|
||||
detail = r.detail if isinstance(r.detail, dict) else {}
|
||||
if detail.get("entity_id") != int(entity_id):
|
||||
continue
|
||||
out.append({
|
||||
"id": r.id, "at": r.created_at.isoformat(timespec="seconds") if r.created_at else None,
|
||||
"user_id": r.user_id,
|
||||
"rule_key": detail.get("rule_key"), "param_key": detail.get("param_key"),
|
||||
"before": detail.get("before"), "after": detail.get("after"),
|
||||
"reason": detail.get("reason"), "target_id": r.target_id,
|
||||
})
|
||||
if len(out) >= limit:
|
||||
break
|
||||
return out
|
||||
|
||||
|
||||
#: 审计动作名(清单 §5.2 逐字)
|
||||
ACTION_UPDATE = "update_dq_rule_config"
|
||||
TARGET_TYPE = "dq_rule_config"
|
||||
@@ -181,6 +181,36 @@ class OperationLog(Base):
|
||||
created_at = Column(DateTime, server_default=func.now())
|
||||
|
||||
|
||||
class QualityRuleConfig(Base):
|
||||
"""DAMA 数据质量治理规则可配置项(财务七规则)— ASK-20260928-05
|
||||
|
||||
逐 param 一行;`entity_id=0` = **全账套模板行**(不参与具体账套取数)。
|
||||
读取三层:账套行 → 模板行(0) → 代码内置默认(禁止跨账套回退);
|
||||
规格/默认值/边界/校验的**单一来源** = `app/dq_rule_config.py`(本模型只存行)。
|
||||
DDL 走部署链特权迁移步 `backend/scripts/db_migrate.py --apply`(生产写审批范围)。
|
||||
"""
|
||||
__tablename__ = "quality_rule_config"
|
||||
id = Column(Integer, primary_key=True, index=True)
|
||||
entity_id = Column(Integer, nullable=False, default=0, index=True,
|
||||
comment="账套;0=全账套默认模板")
|
||||
rule_key = Column(String(50), nullable=False, comment="七规则 key 或 global(全局项)")
|
||||
param_key = Column(String(50), nullable=False, comment="参数名(见 dq_rule_config.SPEC)")
|
||||
param_value = Column(String(500), nullable=False, comment="值(JSON 序列化字符串)")
|
||||
value_type = Column(String(20), nullable=False, default="int",
|
||||
comment="int/float/bool/str/json/enum")
|
||||
open_level = Column(String(10), nullable=False, default="entity",
|
||||
comment="entity=账套级(ceo/it) / platform=平台级(仅it)")
|
||||
is_default = Column(Boolean, nullable=False, default=False,
|
||||
comment="1=出厂默认行(用于「恢复默认」与审计对照)")
|
||||
description = Column(String(500), nullable=True, comment="参数说明(面向使用者)")
|
||||
updated_by = Column(String(50), nullable=True, comment="最后修改人 users.username")
|
||||
updated_at = Column(DateTime, server_default=func.now(), onupdate=func.now())
|
||||
created_at = Column(DateTime, server_default=func.now())
|
||||
__table_args__ = (
|
||||
UniqueConstraint("entity_id", "rule_key", "param_key", name="uq_rule_param"),
|
||||
)
|
||||
|
||||
|
||||
class NotificationChannel(Base):
|
||||
"""通知渠道配置"""
|
||||
__tablename__ = "notification_channels"
|
||||
|
||||
Reference in New Issue
Block a user