wecom-fullstack
36f4f54799
ci/woodpecker/push/main Pipeline failed
任总 2026-09-28 13:38 放行「4 和 5」→ 落笔最小实现： - 新表 quality_rule_config（39 项：37 规则级 + 2 全局）= 唯一配置源 - 三层解析：账套行 → 全账套模板(entity_id=0) → 代码内置默认（=清单 §7 建议值，零行为差异） - 端点 /api/cma/data-quality/rule-config（GET/PUT）+ /history + /meta：写入仅 ceo/it， platform 级仅 it，演示账套(functional)写禁，变更落 operation_logs 同事务 - 数据质量页新增「规则配置」只读 Tab（懒加载） - 唯一行为变更：污染判定去掉 sync —— 酣客 zxt-sync 系统迁入数据不再被误判（口径可配置） - 出厂默认落库脚本 scripts/seed_dq_rule_config.py（--check/--apply/--rollback，幂等 39/0） - 回归：新套件 35 例 + 存量 1508 passed（唯一 FAILED 为他人未完成 migrate_subjects_entity_id_20260923.py 改动所致，HEAD 干净树同例通过） 生产写（DDL/seed）与 push 均走审批门：DDL=54434656 push=3bd00774
619 lines
30 KiB
Python
619 lines
30 KiB
Python
# -*- coding: utf-8 -*-
|
||
"""DAMA 数据质量治理规则可配置项（财务七规则）—— 单一事实源 + 三层解析
|
||
|
||
任务：ASK-20260928-05（任总 2026-09-28 13:38「放行 4 和 5」，本项目 Bot 转呈）
|
||
来源清单：wecom-dama《财务七规则可配置化 — 可配置项清单 + 建议默认值 + 权限/留痕建议》
|
||
（`wecom-dama/pending-tasks/_reports/project-20260923-quality-rules-config-inventory-dama.md`）
|
||
|
||
本模块承担三件事（唯一来源，勿在别处复制常量）
|
||
----------------------------------------------
|
||
1. **开放面**：37 个规则级参数（`SPEC`）+ 2 个全局项（`GLOBAL_SPEC`）。清单 §3 逐条落成代码；
|
||
测试断言项数 = 37 / 2，防后续会话漏项或私自扩面。
|
||
2. **锁死面**：清单 §4 的 10 类（判定表达式本体 / 级别归属 / 评分函数 / 实体作用域 /
|
||
规则集与数量 / 匹配方式自由化 / kpi_completeness 口径源 / `entity_id IS NULL` 处置 /
|
||
Bot 通道写配置 / 越界权重）——不在 SPEC 里的键一律 400（写端点），
|
||
`LOCKED_PARAM_KEYS` 仅用于把「锁死」讲清楚（可读文案），不是白名单。
|
||
3. **三层解析**：账套行（`entity_id`）→ 模板行（`entity_id=0`）→ 代码内置默认；
|
||
**禁止跨账套回退**（历史事故：`cash.current_balance` 全局键被全账套共用）。
|
||
|
||
口径（任总 2026-09-28 放行原文：「口径用清单默认值」）
|
||
----------------------------------------------------
|
||
* 默认值 = 清单 §7「建议默认值」总表，即「现行值，上线零行为差异」，
|
||
**唯一例外**：`virtual_pollution.suspect_keywords` 去掉 `sync`
|
||
（`zxt-sync` 是酣客生产同步来源，现行 134 行 100% 假阳性）——本单唯一的行为变更；
|
||
* 演示/测试账套（3/4/5/6）**不开配置**（`CONFIG_ENTITIES = (1, 2)`，清单 §6.4 + ASK-05 口径②）；
|
||
读路径仍按三层解析返回生效值（模板/内置默认），只是写端点拒绝。
|
||
|
||
配置值一律以 JSON 字符串落库（`param_value`），`value_type` 记录类型以便前端/审计呈现。
|
||
"""
|
||
from __future__ import annotations
|
||
|
||
import hashlib
|
||
import json
|
||
import logging
|
||
from typing import Any, Dict, List, Optional, Tuple
|
||
|
||
from sqlalchemy import inspect
|
||
from sqlalchemy.orm import Session
|
||
|
||
logger = logging.getLogger("cma.dq.rule_config")
|
||
|
||
#: 表名（DDL 走部署链特权迁移步 `backend/scripts/db_migrate.py --apply`，属生产写审批范围）
|
||
TABLE_NAME = "quality_rule_config"
|
||
#: 全账套模板行的 entity_id（0 = 模板，不参与具体账套取数）
|
||
TEMPLATE_ENTITY_ID = 0
|
||
#: 开放配置写入的真实账套（口径②：演示/测试账套不开配置）
|
||
CONFIG_ENTITIES: Tuple[int, ...] = (1, 2)
|
||
#: 内置默认明细展示条数（清单 §3.9 全局项 `detail_limit` 的出厂值）
|
||
DEFAULT_DETAIL_LIMIT = 10
|
||
#: 写配置必须带变更理由，最短长度（清单 §5.1「反刷分约束」第 1 条）
|
||
MIN_REASON_LEN = 10
|
||
|
||
RULE_KEYS: Tuple[str, ...] = (
|
||
"unit_check", "dup_alert", "orphan_check", "virtual_pollution",
|
||
"entity_check", "kpi_completeness", "reconciliation",
|
||
)
|
||
|
||
# 全局项（清单 §3.9）：platform 级，仅 it 可写
|
||
GLOBAL_RULE_KEY = "global"
|
||
|
||
# 规则级参数统一三件套（每条规则都有；`detail_limit` 为 platform 级）
|
||
_CAPS = {
|
||
"deduct_cap": dict(type="int", default=15, open_level="entity",
|
||
bounds=(0, 50), overflow="clamp",
|
||
desc="本规则扣分上限（0 = 只报不计分）"),
|
||
"deduct_factor": dict(type="int", default=3, open_level="entity",
|
||
bounds=(0, 10), overflow="clamp",
|
||
desc="本规则每命中一条的扣分系数"),
|
||
}
|
||
_WARN_CAPS = {
|
||
"deduct_cap": dict(type="int", default=10, open_level="entity",
|
||
bounds=(0, 50), overflow="clamp",
|
||
desc="本规则扣分上限（0 = 只报不计分）"),
|
||
"deduct_factor": dict(type="int", default=1, open_level="entity",
|
||
bounds=(0, 10), overflow="clamp",
|
||
desc="本规则每命中一条的扣分系数"),
|
||
}
|
||
_DETAIL_LIMIT = dict(type="int", default=DEFAULT_DETAIL_LIMIT, open_level="platform",
|
||
bounds=(1, 100), overflow="clamp",
|
||
desc="本规则明细展示条数（平台级，仅 it 可写）")
|
||
_ENABLED = dict(type="bool", default=True, open_level="entity",
|
||
desc="启用开关（关闭后该规则不参与评分，明细不再展示）")
|
||
|
||
#: 7 条规则的开放参数（37 项；清单 §3.1–§3.7 逐条落成）
|
||
SPEC: Dict[str, Dict[str, dict]] = {
|
||
"unit_check": {
|
||
"enabled": dict(_ENABLED),
|
||
"amount_threshold": dict(type="int", default=10000, open_level="entity",
|
||
bounds=(100, 1000000), overflow="reject",
|
||
desc="金额阈值（万元）：cash_plans.amount > 阈值 即命中（越界拒绝）"),
|
||
**_CAPS,
|
||
"detail_limit": dict(_DETAIL_LIMIT),
|
||
},
|
||
"dup_alert": {
|
||
"enabled": dict(_ENABLED),
|
||
"min_count": dict(type="int", default=2, open_level="entity",
|
||
bounds=(2, 20), overflow="reject",
|
||
desc="同一 plan_id 的 pending 预警条数下限（≥ 该值即重复；越界拒绝）"),
|
||
**_CAPS,
|
||
"detail_limit": dict(_DETAIL_LIMIT),
|
||
},
|
||
"orphan_check": {
|
||
"enabled": dict(_ENABLED),
|
||
**_CAPS,
|
||
"detail_limit": dict(_DETAIL_LIMIT),
|
||
},
|
||
"virtual_pollution": {
|
||
"enabled": dict(_ENABLED),
|
||
"suspect_keywords": dict(
|
||
type="json", default=["test", "虚拟", "demo"], open_level="entity",
|
||
max_items=20, item_max_len=32, forbidden_chars="%_",
|
||
desc="嫌疑关键词表（大小写不敏感包含式匹配；不含 sync —— 本单唯一行为变更）"),
|
||
"source_whitelist": dict(
|
||
type="json",
|
||
default=["zxt-sync", "bohai_ar", "bohai_ar_20260630", "bohai_prepay_20260630",
|
||
"bohai_ap_20260630", "receivables_migrate"],
|
||
open_level="entity", max_items=20, item_max_len=32, forbidden_chars="%_",
|
||
desc="来源白名单（豁免优先于关键词：命中即不算污染）"),
|
||
"manual_source_value": dict(type="str", default="manual", open_level="entity",
|
||
max_len=50,
|
||
desc="「待人工确认」的来源值（非污染，按 warning 附加扣分）"),
|
||
**_CAPS,
|
||
"manual_cap": dict(type="int", default=10, open_level="entity",
|
||
bounds=(0, 30), overflow="clamp",
|
||
desc="待人工确认（manual）附加上限"),
|
||
"manual_factor": dict(type="int", default=1, open_level="entity",
|
||
bounds=(0, 5), overflow="clamp",
|
||
desc="待人工确认（manual）附加系数"),
|
||
"detail_limit": dict(_DETAIL_LIMIT),
|
||
},
|
||
"entity_check": {
|
||
"enabled": dict(_ENABLED),
|
||
**_CAPS,
|
||
"detail_limit": dict(_DETAIL_LIMIT),
|
||
},
|
||
"kpi_completeness": {
|
||
"enabled": dict(_ENABLED),
|
||
**_WARN_CAPS,
|
||
"detail_limit": dict(_DETAIL_LIMIT),
|
||
},
|
||
"reconciliation": {
|
||
"enabled": dict(_ENABLED),
|
||
"diff_threshold_pct": dict(type="float", default=20, open_level="entity",
|
||
bounds=(1, 200), overflow="reject",
|
||
desc="预算月度合计 vs 年度目标差异阈值（%）；越界拒绝"),
|
||
"compare_mode": dict(type="enum", default="abs", open_level="entity",
|
||
enum=("abs", "upper_only", "lower_only"),
|
||
desc="比较方向：abs=双侧 / upper_only=仅超目标 / lower_only=仅低于目标"),
|
||
**_WARN_CAPS,
|
||
"detail_limit": dict(_DETAIL_LIMIT),
|
||
},
|
||
}
|
||
|
||
#: 全局项（2 项；清单 §3.9）
|
||
GLOBAL_SPEC: Dict[str, dict] = {
|
||
"detail_limit": dict(_DETAIL_LIMIT),
|
||
"scope.excluded_entities": dict(
|
||
type="json", default=[], open_level="platform", max_items=20, item_max_len=6,
|
||
desc="排除出「经营口径」的账套（演示/测试；默认空集 = 不排除）。"
|
||
"⚠️ 本项当前**只做标注不改评分语义**（F6 属业务口径，待任总/财务裁定后接线）"),
|
||
}
|
||
|
||
# ── 锁死面（清单 §4；仅用于给出可读文案，非白名单）─────────────────────────
|
||
LOCKED_ITEMS: Tuple[dict, ...] = (
|
||
dict(key="expression", item="判定表达式本体（字段/算子/SQL 片段/正则）",
|
||
reason="表达式可配 = 治理规则可被一次配置改成永久绿灯"),
|
||
dict(key="level", item="级别归属（error/warning）",
|
||
reason="级别直接决定扣分上限，降级是最廉价的绕分路径；确有差异请用已开放的扣分权重表达"),
|
||
dict(key="score_fn", item="评分函数（max(0,100−Σ)、7 条固定、各规则独立封顶后累加、无总封顶）",
|
||
reason="评分是跨规则/跨期可比的口径基线"),
|
||
dict(key="entity_scope", item="实体作用域语义（规则 1 用 cash_plans.entity_id；2/3/6/7 用 kpi_definitions.entity_id；5 用 kpi_values.entity_id）",
|
||
reason="作用域 = 安全边界，开放会引入跨账套借用通路"),
|
||
dict(key="rule_set", item="规则 key 集合与规则数量（固定 7 条）",
|
||
reason="规则集是治理骨架，新增规则必须走代码评审"),
|
||
dict(key="match_mode", item="匹配方式自由化（关键词升级为正则/SQL 片段）",
|
||
reason="注入面 + 不可验证；匹配方式固定为大小写不敏感包含式"),
|
||
dict(key="completeness_caliber", item="kpi_completeness 口径源 kpi_definitions.status='active'（含豁免名单 exempt_kpi_codes）",
|
||
reason="KPI 存废开关已是 status，再加豁免名单 = 双开关口径分裂"),
|
||
dict(key="null_entity_policy", item="规则5 kpi_values.entity_id IS NULL 行的处置",
|
||
reason="NULL 语义未裁定前保持 ignore"),
|
||
dict(key="bot_write", item="Bot 通道（X-BOT-KEY）写配置",
|
||
reason="Bot 通道无 users 行 → 无操作人可留痕 = 审计断链"),
|
||
dict(key="weight_range", item="负系数 / 超大上限 / 非整数权重",
|
||
reason="负系数会变成加分（可刷分）；一律钳制到开放区间"),
|
||
)
|
||
#: 键名 → 锁死文案（写端点遇到这些 param_key 时给「锁死」提示而非笼统的未知参数）
|
||
LOCKED_PARAM_KEYS = {it["key"]: it for it in LOCKED_ITEMS}
|
||
|
||
# 常见别名（历史/前端习惯叫法）→ 锁死项 key
|
||
_LOCKED_ALIASES = {
|
||
"expr": "expression", "predicate": "expression", "sql": "expression",
|
||
"rule_level": "level", "severity": "level",
|
||
"score": "score_fn", "score_func": "score_fn", "total_deduct_cap": "score_fn",
|
||
"exempt_kpi_codes": "completeness_caliber", "status_source": "completeness_caliber",
|
||
"entity_id_source": "entity_scope", "scope_field": "entity_scope",
|
||
"rules": "rule_set", "rule_keys": "rule_set", "rule_count": "rule_set",
|
||
"match_regex": "match_mode", "regex": "match_mode", "pattern": "match_mode",
|
||
"null_policy": "null_entity_policy",
|
||
}
|
||
|
||
OPEN_LEVEL_ENTITY = "entity"
|
||
OPEN_LEVEL_PLATFORM = "platform"
|
||
|
||
|
||
# ══════════════════════════════════════════════════════════════════
|
||
# 参数查找 / 校验
|
||
# ══════════════════════════════════════════════════════════════════
|
||
def is_global_param(rule_key: str) -> bool:
|
||
return rule_key == GLOBAL_RULE_KEY
|
||
|
||
|
||
def spec_of(rule_key: str) -> Dict[str, dict]:
|
||
"""返回某 rule_key 的参数规格表（全局项用 rule_key='global'）。"""
|
||
if is_global_param(rule_key):
|
||
return GLOBAL_SPEC
|
||
return SPEC.get(rule_key) or {}
|
||
|
||
|
||
def param_spec(rule_key: str, param_key: str) -> Optional[dict]:
|
||
return spec_of(rule_key).get(param_key)
|
||
|
||
|
||
def all_param_pairs() -> List[Tuple[str, str]]:
|
||
"""(rule_key, param_key) 全量清单——37 规则级 + 2 全局项。"""
|
||
pairs = [(r, p) for r in RULE_KEYS for p in SPEC[r]]
|
||
pairs += [(GLOBAL_RULE_KEY, p) for p in GLOBAL_SPEC]
|
||
return pairs
|
||
|
||
|
||
def count_rule_params() -> int:
|
||
"""规则级开放参数项数（断言用，清单 §10.2 = 37）。"""
|
||
return sum(len(SPEC[r]) for r in RULE_KEYS)
|
||
|
||
|
||
def open_level_of(rule_key: str, param_key: str) -> str:
|
||
spec = param_spec(rule_key, param_key) or {}
|
||
return spec.get("open_level") or OPEN_LEVEL_ENTITY
|
||
|
||
|
||
def value_type_of(rule_key: str, param_key: str) -> str:
|
||
spec = param_spec(rule_key, param_key) or {}
|
||
return spec.get("type") or "json"
|
||
|
||
|
||
def dumps_value(value: Any) -> str:
|
||
"""统一序列化（落库格式；bool/int/float/str/数组一律 JSON）。"""
|
||
return json.dumps(value, ensure_ascii=False)
|
||
|
||
|
||
def loads_value(raw: Any, value_type: str = "") -> Any:
|
||
"""反序列化；非 JSON 的历史/手写行容错（返回 None 表示不可用 → 上层回落）。"""
|
||
if raw is None:
|
||
return None
|
||
if isinstance(raw, (int, float, bool, list, dict)):
|
||
return raw
|
||
try:
|
||
return json.loads(str(raw))
|
||
except (ValueError, TypeError):
|
||
text = str(raw)
|
||
if value_type == "int":
|
||
try:
|
||
return int(float(text))
|
||
except (ValueError, TypeError):
|
||
return None
|
||
if value_type == "float":
|
||
try:
|
||
return float(text)
|
||
except (ValueError, TypeError):
|
||
return None
|
||
if value_type == "bool":
|
||
if text.strip().lower() in ("1", "true", "yes"):
|
||
return True
|
||
if text.strip().lower() in ("0", "false", "no"):
|
||
return False
|
||
return None
|
||
if value_type == "str":
|
||
return text
|
||
return None
|
||
|
||
|
||
def _coerce_type(value: Any, spec: dict) -> Tuple[bool, Any]:
|
||
"""类型强转（拒绝型：类型不符 → 不可用）。bool 与 int 严格区分（Python 里 bool 是 int 子类）。"""
|
||
t = spec.get("type") or "json"
|
||
if t == "bool":
|
||
if isinstance(value, bool):
|
||
return True, value
|
||
if isinstance(value, str) and value.strip().lower() in ("true", "false"):
|
||
return True, value.strip().lower() == "true"
|
||
if isinstance(value, int) and value in (0, 1):
|
||
return True, bool(value)
|
||
return False, None
|
||
if t == "int":
|
||
if isinstance(value, bool):
|
||
return False, None
|
||
if isinstance(value, int):
|
||
return True, value
|
||
if isinstance(value, float) and float(value).is_integer():
|
||
return True, int(value)
|
||
if isinstance(value, str):
|
||
s = value.strip()
|
||
try:
|
||
return True, int(s)
|
||
except ValueError:
|
||
return False, None
|
||
return False, None
|
||
if t == "float":
|
||
if isinstance(value, bool):
|
||
return False, None
|
||
if isinstance(value, (int, float)):
|
||
return True, float(value)
|
||
if isinstance(value, str):
|
||
try:
|
||
return True, float(value.strip())
|
||
except ValueError:
|
||
return False, None
|
||
return False, None
|
||
if t == "str":
|
||
if not isinstance(value, str):
|
||
return False, None
|
||
return True, value
|
||
if t == "enum":
|
||
if not isinstance(value, str):
|
||
return False, None
|
||
return True, value.strip()
|
||
if t == "json":
|
||
if not isinstance(value, list):
|
||
return False, None
|
||
return True, value
|
||
return False, None
|
||
|
||
|
||
def validate_param(rule_key: str, param_key: str, value: Any) -> dict:
|
||
"""参数级校验（清单 §3 的「取值边界 / 越界处理」逐条落地）。
|
||
|
||
返回 ``{"ok": bool, "value": <规范化值>, "error": str|None, "warning": str|None}``；
|
||
越界处理两类：``reject``（400，附区间文案）/ ``clamp``（钳制到端点 + 响应告警）。
|
||
"""
|
||
spec = param_spec(rule_key, param_key)
|
||
if spec is None:
|
||
locked = LOCKED_PARAM_KEYS.get(
|
||
param_key, LOCKED_PARAM_KEYS.get(_LOCKED_ALIASES.get(param_key, ""))
|
||
)
|
||
if locked:
|
||
return dict(ok=False, value=None,
|
||
error=f"参数 {param_key} 属锁死项「{locked['item']}」，不允许配置：{locked['reason']}",
|
||
warning=None)
|
||
return dict(ok=False, value=None,
|
||
error=f"未知参数 rule={rule_key} param={param_key}；"
|
||
f"可选: {', '.join(spec_of(rule_key).keys()) or '（该规则无开放参数）'}",
|
||
warning=None)
|
||
|
||
passed, value = _coerce_type(value, spec)
|
||
if not passed:
|
||
return dict(ok=False, value=None,
|
||
error=f"{rule_key}.{param_key} 类型非法：要求 {spec.get('type')}", warning=None)
|
||
|
||
caution = None
|
||
if spec.get("type") == "enum":
|
||
if value not in tuple(spec.get("enum") or ()):
|
||
return dict(ok=False, value=None,
|
||
error=f"{rule_key}.{param_key} 取值非法：仅支持 {', '.join(spec['enum'])}",
|
||
warning=None)
|
||
return dict(ok=True, value=value, error=None, warning=None)
|
||
|
||
if spec.get("type") == "json":
|
||
max_items = int(spec.get("max_items") or 0)
|
||
item_max_len = int(spec.get("item_max_len") or 0)
|
||
forbidden = spec.get("forbidden_chars") or ""
|
||
if max_items and len(value) > max_items:
|
||
return dict(ok=False, value=None,
|
||
error=f"{rule_key}.{param_key} 元素过多：上限 {max_items} 项（当前 {len(value)}）",
|
||
warning=None)
|
||
cleaned: List[str] = []
|
||
for it in value:
|
||
if not isinstance(it, str):
|
||
if spec.get("item_max_len") == 6 and isinstance(it, int) and not isinstance(it, bool):
|
||
it = str(it) # scope.excluded_entities：账套号允许传 int
|
||
else:
|
||
return dict(ok=False, value=None,
|
||
error=f"{rule_key}.{param_key} 仅允许字符串数组元素", warning=None)
|
||
s = it.strip()
|
||
if not s:
|
||
return dict(ok=False, value=None,
|
||
error=f"{rule_key}.{param_key} 不允许空串元素", warning=None)
|
||
if item_max_len and len(s) > item_max_len:
|
||
return dict(ok=False, value=None,
|
||
error=f"{rule_key}.{param_key} 元素过长：单元素上限 {item_max_len} 字符（{s}）",
|
||
warning=None)
|
||
if any(ch in s for ch in forbidden):
|
||
return dict(ok=False, value=None,
|
||
error=f"{rule_key}.{param_key} 元素含非法字符 {forbidden}（不支持通配符/自由模式）",
|
||
warning=None)
|
||
cleaned.append(s)
|
||
# 去重保序（重复项无意义且会让 LIKE 子句膨胀）
|
||
seen, uniq = set(), []
|
||
for s in cleaned:
|
||
if s not in seen:
|
||
seen.add(s)
|
||
uniq.append(s)
|
||
if len(uniq) != len(cleaned):
|
||
caution = f"{rule_key}.{param_key} 已去重：{len(cleaned)} → {len(uniq)} 项"
|
||
return dict(ok=True, value=uniq, error=None, warning=caution)
|
||
|
||
if spec.get("type") == "str":
|
||
max_len = int(spec.get("max_len") or 0)
|
||
s = value
|
||
if not s.strip():
|
||
return dict(ok=False, value=None, error=f"{rule_key}.{param_key} 不允许空串", warning=None)
|
||
if max_len and len(s) > max_len:
|
||
return dict(ok=False, value=None,
|
||
error=f"{rule_key}.{param_key} 过长：上限 {max_len} 字符", warning=None)
|
||
return dict(ok=True, value=s, error=None, warning=None)
|
||
|
||
bounds = spec.get("bounds")
|
||
if bounds:
|
||
lo, hi = bounds
|
||
if value < lo or value > hi:
|
||
if spec.get("overflow") == "clamp":
|
||
clamped = min(max(value, lo), hi)
|
||
logger.warning("DQ_RULE_CONFIG_CLAMPED %s.%s %s → %s（区间 [%s, %s]）",
|
||
rule_key, param_key, value, clamped, lo, hi)
|
||
return dict(ok=True, value=clamped, error=None,
|
||
warning=f"{rule_key}.{param_key} 越界已钳制到区间端点：{value} → {clamped}"
|
||
f"（允许区间 [{lo}, {hi}]）")
|
||
return dict(ok=False, value=None,
|
||
error=f"{rule_key}.{param_key} 越界：允许区间 [{lo}, {hi}]，收到 {value}",
|
||
warning=None)
|
||
return dict(ok=True, value=value, error=None, warning=None)
|
||
|
||
|
||
# ══════════════════════════════════════════════════════════════════
|
||
# 读路径：三层解析
|
||
# ══════════════════════════════════════════════════════════════════
|
||
def table_ready(db: Session) -> bool:
|
||
"""配置表是否就绪（DDL 未落地时**不得**让读路径 500：回落内置默认）。"""
|
||
try:
|
||
return bool(inspect(db.get_bind()).has_table(TABLE_NAME))
|
||
except Exception as exc: # noqa: BLE001
|
||
logger.warning("DQ_RULE_CONFIG_TABLE_PROBE_FAILED %s: %s", TABLE_NAME, exc)
|
||
return False
|
||
|
||
|
||
def _load_rows(db: Session, entity_id: int) -> Dict[Tuple[str, str], dict]:
|
||
"""读模板行 + 本账套行（一次查询）；返回 {(rule_key, param_key): {entity_id, value, raw, row}}。"""
|
||
from app.models import QualityRuleConfig
|
||
|
||
eids = [TEMPLATE_ENTITY_ID]
|
||
if entity_id != TEMPLATE_ENTITY_ID:
|
||
eids.append(entity_id)
|
||
rows = db.query(QualityRuleConfig).filter(QualityRuleConfig.entity_id.in_(eids)).all()
|
||
out: Dict[Tuple[str, str], dict] = {}
|
||
for row in rows:
|
||
vt = value_type_of(row.rule_key, row.param_key)
|
||
val = loads_value(row.param_value, vt)
|
||
if val is None:
|
||
logger.warning("DQ_RULE_CONFIG_ROW_UNPARSABLE id=%s %s.%s value=%r → 忽略该行（回落下一层）",
|
||
row.id, row.rule_key, row.param_key, row.param_value)
|
||
continue
|
||
out[(row.rule_key, row.param_key)] = {
|
||
"entity_id": row.entity_id, "value": val, "row": row,
|
||
}
|
||
return out
|
||
|
||
|
||
def resolve(db: Session, entity_id: int) -> dict:
|
||
"""三层解析当前生效配置（账套行 → 模板行 → 内置默认；**不跨账套回退**）。
|
||
|
||
返回结构（前端/审计/引擎共用）::
|
||
|
||
{"entity_id": 1, "config_version": "…", "table_ready": True,
|
||
"rules": {rule_key: {param_key: {"value":…, "source": "entity|template|builtin",
|
||
"open_level":…, "desc":…, "updated_by":…, "updated_at":…}}},
|
||
"global": {...同 rules 的形状…},
|
||
"values": {rule_key: {param_key: <裸值>}}, # 引擎侧取值口
|
||
"scope": {"excluded_entities": [...]}}
|
||
"""
|
||
eid = int(entity_id or 0)
|
||
ready = table_ready(db)
|
||
rows = _load_rows(db, eid) if ready else {}
|
||
return _assemble(eid, ready, rows)
|
||
|
||
|
||
def builtin_resolution(entity_id: int = 0) -> dict:
|
||
"""纯内置默认的解析结果（配置表缺失/读异常时的回落形态，字段与 `resolve()` 逐字一致）。"""
|
||
return _assemble(int(entity_id or 0), False, {})
|
||
|
||
|
||
def _assemble(eid: int, ready: bool, rows: Dict[Tuple[str, str], dict]) -> dict:
|
||
"""把「行 + 内置默认」组装成对外结构（resolve / builtin_resolution 共用）。"""
|
||
|
||
def _one(rule_key: str, param_key: str, spec: dict) -> dict:
|
||
val, source, row = spec.get("default"), "builtin", None
|
||
tmpl = rows.get((rule_key, param_key))
|
||
if tmpl and tmpl["entity_id"] == TEMPLATE_ENTITY_ID:
|
||
val, source, row = tmpl["value"], "template", tmpl["row"]
|
||
elif eid != TEMPLATE_ENTITY_ID:
|
||
own = rows.get((rule_key, param_key))
|
||
if own and own["entity_id"] == eid:
|
||
val, source, row = own["value"], "entity", own["row"]
|
||
entry = {
|
||
"value": val, "source": source, "open_level": spec.get("open_level"),
|
||
"desc": spec.get("desc"), "updated_by": None, "updated_at": None,
|
||
}
|
||
if row is not None:
|
||
entry["updated_by"] = getattr(row, "updated_by", None)
|
||
ua = getattr(row, "updated_at", None)
|
||
entry["updated_at"] = ua.isoformat(timespec="seconds") if ua else None
|
||
entry["row_id"] = row.id
|
||
return entry
|
||
|
||
rules: Dict[str, Dict[str, dict]] = {}
|
||
values: Dict[str, Dict[str, Any]] = {}
|
||
for rule_key in RULE_KEYS:
|
||
rules[rule_key] = {p: _one(rule_key, p, s) for p, s in SPEC[rule_key].items()}
|
||
values[rule_key] = {p: e["value"] for p, e in rules[rule_key].items()}
|
||
glob: Dict[str, dict] = {p: _one(GLOBAL_RULE_KEY, p, s) for p, s in GLOBAL_SPEC.items()}
|
||
values[GLOBAL_RULE_KEY] = {p: e["value"] for p, e in glob.items()}
|
||
|
||
# 规则级 detail_limit 未配置时，回落全局 detail_limit（清单 §3.9）
|
||
global_limit = values[GLOBAL_RULE_KEY].get("detail_limit") or DEFAULT_DETAIL_LIMIT
|
||
for rule_key in RULE_KEYS:
|
||
entry = rules[rule_key]["detail_limit"]
|
||
if entry["source"] == "builtin":
|
||
entry["value"] = global_limit
|
||
entry["source"] = "builtin(global)"
|
||
values[rule_key]["detail_limit"] = entry["value"]
|
||
|
||
version = hashlib.sha1(
|
||
json.dumps(values, sort_keys=True, ensure_ascii=False).encode("utf-8")
|
||
).hexdigest()[:12]
|
||
return {
|
||
"entity_id": eid,
|
||
"config_entity_id": eid if eid in CONFIG_ENTITIES else TEMPLATE_ENTITY_ID,
|
||
"entity_config_enabled": eid in CONFIG_ENTITIES,
|
||
"table_ready": ready,
|
||
"config_version": version,
|
||
"rules": rules,
|
||
"global": glob,
|
||
"values": values,
|
||
"scope": {"excluded_entities": values[GLOBAL_RULE_KEY].get("scope.excluded_entities") or []},
|
||
}
|
||
|
||
|
||
def engine_values(db: Session, entity_id: int) -> dict:
|
||
"""引擎取值口（data_quality 规则引擎唯一的配置读取入口）。"""
|
||
return resolve(db, entity_id)["values"]
|
||
|
||
|
||
# ══════════════════════════════════════════════════════════════════
|
||
# 写路径：单参数 upsert（逐项「一次只改一档」）
|
||
# ══════════════════════════════════════════════════════════════════
|
||
def upsert_param(db: Session, entity_id: int, rule_key: str, param_key: str, value: Any,
|
||
*, username: Optional[str] = None) -> dict:
|
||
"""写一行配置（调用方负责权限/校验/审计/事务边界）；返回 {id, before, after, source_before}。"""
|
||
from app.models import QualityRuleConfig
|
||
|
||
before = resolve(db, entity_id)
|
||
scope = before.get("global") if is_global_param(rule_key) else before.get("rules")
|
||
before_entry = (scope or {}).get(rule_key, {}).get(param_key, {}) or {}
|
||
value_type = value_type_of(rule_key, param_key)
|
||
payload = dumps_value(value)
|
||
|
||
row = (db.query(QualityRuleConfig)
|
||
.filter(QualityRuleConfig.entity_id == int(entity_id),
|
||
QualityRuleConfig.rule_key == rule_key,
|
||
QualityRuleConfig.param_key == param_key)
|
||
.first())
|
||
if row is None:
|
||
row = QualityRuleConfig(
|
||
entity_id=int(entity_id), rule_key=rule_key, param_key=param_key,
|
||
param_value=payload, value_type=value_type,
|
||
open_level=open_level_of(rule_key, param_key), is_default=False,
|
||
description=(param_spec(rule_key, param_key) or {}).get("desc"),
|
||
updated_by=username,
|
||
)
|
||
db.add(row)
|
||
else:
|
||
row.param_value = payload
|
||
row.value_type = value_type
|
||
row.is_default = False
|
||
row.updated_by = username
|
||
db.flush() # 拿到 id，且撞唯一键时当场暴露
|
||
return {
|
||
"id": row.id,
|
||
"before": before_entry.get("value"),
|
||
"before_source": before_entry.get("source"),
|
||
"after": value,
|
||
}
|
||
|
||
|
||
def history(db: Session, entity_id: int, limit: int = 20) -> List[dict]:
|
||
"""本账套的配置变更历史（读 `operation_logs`；该表无 entity_id 列 → 按 detail JSON 过滤）。"""
|
||
from app.models import OperationLog
|
||
|
||
rows = (db.query(OperationLog)
|
||
.filter(OperationLog.action == ACTION_UPDATE)
|
||
.order_by(OperationLog.id.desc())
|
||
.limit(max(limit * 5, 50))
|
||
.all())
|
||
out: List[dict] = []
|
||
for r in rows:
|
||
detail = r.detail if isinstance(r.detail, dict) else {}
|
||
if detail.get("entity_id") != int(entity_id):
|
||
continue
|
||
out.append({
|
||
"id": r.id, "at": r.created_at.isoformat(timespec="seconds") if r.created_at else None,
|
||
"user_id": r.user_id,
|
||
"rule_key": detail.get("rule_key"), "param_key": detail.get("param_key"),
|
||
"before": detail.get("before"), "after": detail.get("after"),
|
||
"reason": detail.get("reason"), "target_id": r.target_id,
|
||
})
|
||
if len(out) >= limit:
|
||
break
|
||
return out
|
||
|
||
|
||
#: 审计动作名（清单 §5.2 逐字）
|
||
ACTION_UPDATE = "update_dq_rule_config"
|
||
TARGET_TYPE = "dq_rule_config"
|