feat(data-quality): 财务七规则可配置化最小实现(ASK-20260928-05)
ci/woodpecker/push/main Pipeline failed

任总 2026-09-28 13:38 放行「4 和 5」→ 落笔最小实现:
- 新表 quality_rule_config(39 项:37 规则级 + 2 全局)= 唯一配置源
- 三层解析:账套行 → 全账套模板(entity_id=0) → 代码内置默认(=清单 §7 建议值,零行为差异)
- 端点 /api/cma/data-quality/rule-config(GET/PUT)+ /history + /meta:写入仅 ceo/it,
  platform 级仅 it,演示账套(functional)写禁,变更落 operation_logs 同事务
- 数据质量页新增「规则配置」只读 Tab(懒加载)
- 唯一行为变更:污染判定去掉 sync —— 酣客 zxt-sync 系统迁入数据不再被误判(口径可配置)
- 出厂默认落库脚本 scripts/seed_dq_rule_config.py(--check/--apply/--rollback,幂等 39/0)
- 回归:新套件 35 例 + 存量 1508 passed(唯一 FAILED 为他人未完成
  migrate_subjects_entity_id_20260923.py 改动所致,HEAD 干净树同例通过)

生产写(DDL/seed)与 push 均走审批门:DDL=54434656 push=3bd00774
This commit is contained in:
wecom-fullstack
2026-09-28 13:56:16 +08:00
parent 242f3cf5ec
commit 36f4f54799
6 changed files with 2093 additions and 153 deletions
+505 -153
View File
@@ -4,7 +4,7 @@
from fastapi import APIRouter, Depends, HTTPException, Query, Request
from sqlalchemy.orm import Session
from sqlalchemy import func, and_, text
from typing import Optional
from typing import Optional, Any, Tuple
from datetime import datetime, timedelta
import json
import logging
@@ -12,7 +12,11 @@ import logging
from app.database import get_db
from app.deps import assert_entity_scope, get_entity_id
from app.auth_middleware import require_auth, require_role
from app.models import KPIDefinition, KPIValue, KpiDataQualityLog, OperationLog
from app.models import (KPIDefinition, KPIValue, KpiDataQualityLog, OperationLog,
QualityRuleConfig, User)
from app.utils.oplog import log_op
from app import dq_rule_config as dqcfg
from pydantic import BaseModel, Field
from app.api.kpis import kpi_to_dict
from app.utils.alert_entity_scope import month_period_clause
@@ -485,6 +489,100 @@ RULE_ACTIONS = {
}
# ============================================================
# 规则配置取值口(ASK-20260928-05)—— 引擎侧**唯一**配置读取入口
# 取值经 `app/dq_rule_config.py` 三层解析(账套行 → 模板行(entity_id=0) → 内置默认;
# 禁止跨账套回退)。本文件**不得**再出现可配置项的裸露常量(阈值 / 关键词 / 条数下限 /
# 差异阈值与方向 / 扣分上限与系数 / 明细条数)—— 否则「改了配置不生效」。
# 锁死面(级别、判定表达式本体、评分函数、作用域、规则集…)仍在本文件硬编码,见清单 §4。
# ============================================================
RECON_DIFF_EXPR = ("ROUND((COALESCE(SUM(b.budget_value),0) - d.target_yearly) "
"/ d.target_yearly * 100, 1)")
RECON_BASE_SQL = ("FROM kpi_definitions d JOIN budget_plans b ON b.kpi_id = d.id "
"WHERE d.status='active' AND d.target_yearly > 0 AND b.status='active'")
def _load_cfg(db: Session, eid: int) -> dict:
"""完整解析结果(含每项来源与 config_version);任何异常 → 内置默认形态(读路径绝不 500)。"""
try:
return dqcfg.resolve(db, eid)
except Exception: # noqa: BLE001
logger.warning("DQ_RULE_CONFIG_FALLBACK_BUILTIN entity=%s", eid, exc_info=True)
return dqcfg.builtin_resolution(eid)
def _cfg_values(db: Session, eid: int) -> dict:
"""引擎取值口:{rule_key: {param: value}, ..., 'global': {...}}"""
return _load_cfg(db, eid)["values"]
def _rule_cfg(cfg: dict, rule: str) -> dict:
return (cfg or {}).get(rule) or {}
def _detail_limit(cfg: dict, rule: str) -> int:
return int(_rule_cfg(cfg, rule).get("detail_limit") or dqcfg.DEFAULT_DETAIL_LIMIT)
def _unit_threshold(cfg: dict) -> int:
return int(_rule_cfg(cfg, "unit_check").get("amount_threshold", 10000))
def _dup_min_count(cfg: dict) -> int:
return int(_rule_cfg(cfg, "dup_alert").get("min_count", 2))
def _manual_source(cfg: dict) -> str:
return str(_rule_cfg(cfg, "virtual_pollution").get("manual_source_value") or "manual")
def _recon_pct(cfg: dict) -> float:
return float(_rule_cfg(cfg, "reconciliation").get("diff_threshold_pct", 20))
def _recon_having(cfg: dict) -> str:
"""规则7 比较方向:仅三种枚举(abs/upper_only/lower_only),不接受自由表达式。"""
mode = str(_rule_cfg(cfg, "reconciliation").get("compare_mode") or "abs")
if mode == "upper_only":
return "diff_pct > :pct"
if mode == "lower_only":
return "diff_pct < (0 - :pct)"
return "ABS(diff_pct) > :pct"
def _entity_clause(eid: int, col: str) -> Tuple[str, dict]:
"""账套过滤子句 + 绑定参数(eid=0 → 全账套,无子句)。"""
if eid:
return f" AND {col} = :eid", {"eid": eid}
return "", {}
def _vp_predicate(cfg: dict) -> Tuple[str, dict]:
"""规则4 污染主体谓词:嫌疑关键词(大小写不敏感包含式)**减去**来源白名单(豁免优先)。
关键词/白名单均来自配置;匹配方式固定为包含式(禁正则与 SQL 片段 —— 锁死面第 6 条);
关键词为空 ⇒ 恒假谓词(不误报任何行)。绑定参数化,无字面量拼接(注入面 = 0)。
"""
vp = _rule_cfg(cfg, "virtual_pollution")
keywords = [str(k) for k in (vp.get("suspect_keywords") or []) if str(k).strip()]
whitelist = [str(w) for w in (vp.get("source_whitelist") or []) if str(w).strip()]
params: dict = {}
parts = []
for i, kw in enumerate(keywords):
key = f"vpkw{i}"
parts.append(f"LOWER(cp.source) LIKE :{key}")
params[key] = f"%{kw.lower()}%"
pred = "(" + " OR ".join(parts) + ")" if parts else "(1 = 0)"
if whitelist:
names = []
for i, w in enumerate(whitelist):
key = f"vpwl{i}"
names.append(f":{key}")
params[key] = w.lower()
pred += f" AND LOWER(cp.source) NOT IN ({', '.join(names)})"
return pred, params
def _plan_id_set(db: Session, eid: int):
"""已存在的 cash_plans id 集合(规则3 判孤儿用;eid 非 0 时只看该实体)"""
sql = "SELECT id FROM cash_plans" + (" WHERE entity_id = :eid" if eid else "")
@@ -512,64 +610,79 @@ def _pending_alert_plan_links(db: Session, eid: int):
return parsed, rows
def _dup_groups(parsed):
"""规则2 判据:同 plan_id 多条 pending 预警 → [(plan_id, [(alert_id, kpi_id, ent_id), ...])],按条数降序"""
def _dup_groups(parsed, min_count: int = 2):
"""规则2 判据:同 plan_id 的 pending 预警条数 **>= min_count**(默认 2 == 原 `len(ids) > 1`)
min_count 来自配置(`dup_alert.min_count`,清单 §3.2,边界 [2,20] 越界拒绝)。
"""
threshold = max(2, int(min_count or 2))
group_map = {}
for alert_id, kpi_id, pid, ent_id in parsed:
group_map.setdefault(pid, []).append((alert_id, kpi_id, ent_id))
groups = [(pid, ids) for pid, ids in group_map.items() if len(ids) > 1]
groups = [(pid, ids) for pid, ids in group_map.items() if len(ids) >= threshold]
groups.sort(key=lambda x: -len(x[1]))
return groups
def _detail_count(db: Session, rule: str, eid: int, manual: bool = False) -> int:
def _detail_count(db: Session, rule: str, eid: int, manual: bool = False,
cfg: Optional[dict] = None) -> int:
"""本规则真实总数:一律 COUNT(*)(dup/orphan 由预警分组得出,同样不受 LIMIT 影响)"""
cfg = cfg or _cfg_values(db, eid)
if rule == "dup_alert":
parsed, _ = _pending_alert_plan_links(db, eid)
return len(_dup_groups(parsed))
return len(_dup_groups(parsed, _dup_min_count(cfg)))
if rule == "orphan_check":
parsed, _ = _pending_alert_plan_links(db, eid)
pset = _plan_id_set(db, eid)
return len([1 for _a, _k, pid, _e in parsed if pid not in pset])
ent = " AND cp.entity_id = :eid" if eid else ""
ent, params = _entity_clause(eid, "cp.entity_id")
if rule == "unit_check":
sql = "SELECT COUNT(*) FROM cash_plans cp WHERE cp.amount > 10000" + ent
sql = "SELECT COUNT(*) FROM cash_plans cp WHERE cp.amount > :amt" + ent
params["amt"] = _unit_threshold(cfg)
elif rule == "virtual_pollution":
if manual:
sql = "SELECT COUNT(*) FROM cash_plans cp WHERE cp.source = 'manual'" + ent
sql = "SELECT COUNT(*) FROM cash_plans cp WHERE cp.source = :msrc" + ent
params["msrc"] = _manual_source(cfg)
else:
sql = ("SELECT COUNT(*) FROM cash_plans cp WHERE (cp.source LIKE '%test%' "
"OR cp.source LIKE '%sync%' OR cp.source LIKE '%虚拟%' OR cp.source LIKE '%demo%')" + ent)
pred, vp_params = _vp_predicate(cfg)
params.update(vp_params)
sql = "SELECT COUNT(*) FROM cash_plans cp WHERE " + pred + ent
elif rule == "entity_check":
sql = ("SELECT COUNT(*) FROM kpi_values v JOIN kpi_definitions d ON v.kpi_id = d.id "
"WHERE v.entity_id IS NOT NULL AND v.entity_id != d.entity_id")
if eid:
sql += " AND v.entity_id = :eid"
params["eid"] = eid
elif rule == "kpi_completeness":
sql = ("SELECT COUNT(*) FROM kpi_definitions d WHERE d.status='active' "
"AND NOT EXISTS (SELECT 1 FROM kpi_values v WHERE v.kpi_id = d.id)")
if eid:
sql += " AND d.entity_id = :eid"
params["eid"] = eid
elif rule == "reconciliation":
sql = ("SELECT COUNT(*) FROM (SELECT d.id, "
"ROUND((COALESCE(SUM(b.budget_value),0) - d.target_yearly) / d.target_yearly * 100, 1) AS diff_pct "
"FROM kpi_definitions d JOIN budget_plans b ON b.kpi_id = d.id "
"WHERE d.status='active' AND d.target_yearly > 0 AND b.status='active'")
sql = ("SELECT COUNT(*) FROM (SELECT d.id, " + RECON_DIFF_EXPR + " AS diff_pct "
+ RECON_BASE_SQL)
if eid:
sql += " AND d.entity_id = :eid"
sql += " GROUP BY d.id HAVING ABS(diff_pct) > 20) t"
params["eid"] = eid
sql += " GROUP BY d.id HAVING " + _recon_having(cfg) + ") t"
params["pct"] = _recon_pct(cfg)
else:
return 0
return int(db.execute(text(sql), {"eid": eid}).scalar() or 0)
return int(db.execute(text(sql), params).scalar() or 0)
def _detail_items(db: Session, rule: str, eid: int, limit: int, offset: int, manual: bool = False):
def _detail_items(db: Session, rule: str, eid: int, limit: int, offset: int,
manual: bool = False, cfg: Optional[dict] = None):
"""结构化明细条目(服务端 LIMIT/OFFSET 分页;字段:record_id/entity_id/source/key_fields/summary/suggested_action)"""
cfg = cfg or _cfg_values(db, eid)
if rule == "dup_alert":
parsed, _ = _pending_alert_plan_links(db, eid)
items = []
for pid, alerts in _dup_groups(parsed):
for pid, alerts in _dup_groups(parsed, _dup_min_count(cfg)):
items.append({
"record_id": pid,
"entity_id": alerts[0][2],
@@ -598,41 +711,43 @@ def _detail_items(db: Session, rule: str, eid: int, limit: int, offset: int, man
})
return items[offset:offset + limit]
params = {"eid": eid, "lim": limit, "off": offset}
ent = " AND cp.entity_id = :eid" if eid else ""
ent, params = _entity_clause(eid, "cp.entity_id")
params.update({"lim": limit, "off": offset})
if rule == "unit_check":
threshold = _unit_threshold(cfg)
rows = db.execute(text(
"SELECT cp.id, cp.entity_id, cp.amount, cp.source, cp.description FROM cash_plans cp "
"WHERE cp.amount > 10000" + ent + " ORDER BY cp.amount DESC LIMIT :lim OFFSET :off"
), params).fetchall()
"WHERE cp.amount > :amt" + ent + " ORDER BY cp.amount DESC LIMIT :lim OFFSET :off"
), {**params, "amt": threshold}).fetchall()
return [{
"record_id": r.id,
"entity_id": r.entity_id,
"source": RULE_SOURCES[rule],
"key_fields": {"amount": float(r.amount) if r.amount is not None else None,
"source": r.source, "description": r.description,
"threshold": ">10000(万元口径可疑)"},
"summary": "计划#%d 金额 %s(超 10000 阈值,疑似单位错乱)" % (r.id, r.amount),
"threshold": ">%s(万元口径可疑)" % threshold},
"summary": "计划#%d 金额 %s(超 %s 阈值,疑似单位错乱)" % (r.id, r.amount, threshold),
"suggested_action": RULE_ACTIONS[rule],
} for r in rows]
if rule == "virtual_pollution":
if manual:
where = "cp.source = 'manual'"
where, kw_params = "cp.source = :msrc", {"msrc": _manual_source(cfg)}
scope_text = "%s(待人工确认)" % _manual_source(cfg)
else:
where = ("(cp.source LIKE '%test%' OR cp.source LIKE '%sync%' "
"OR cp.source LIKE '%虚拟%' OR cp.source LIKE '%demo%')")
where, kw_params = _vp_predicate(cfg)
scope_text = "污染源(%s)" % "/".join(cfg["virtual_pollution"]["suspect_keywords"])
rows = db.execute(text(
"SELECT cp.id, cp.entity_id, cp.source, cp.description FROM cash_plans cp "
"WHERE " + where + ent + " ORDER BY cp.id LIMIT :lim OFFSET :off"
), params).fetchall()
), {**params, **kw_params}).fetchall()
return [{
"record_id": r.id,
"entity_id": r.entity_id,
"source": RULE_SOURCES[rule],
"key_fields": {"source": r.source, "description": r.description,
"scope": "manual(待人工确认)" if manual else "污染源(test/sync/虚拟/demo)"},
"scope": scope_text},
"summary": "计划#%d source=%s%s" % (r.id, r.source, "" if not manual else "(待人工确认)"),
"suggested_action": RULE_ACTIONS[rule],
} for r in rows]
@@ -644,6 +759,8 @@ def _detail_items(db: Session, rule: str, eid: int, limit: int, offset: int, man
if eid:
sql += " AND v.entity_id = :eid"
sql += " ORDER BY v.id LIMIT :lim OFFSET :off"
if not eid:
params.pop("eid", None)
rows = db.execute(text(sql), params).fetchall()
return [{
"record_id": r.id,
@@ -661,6 +778,8 @@ def _detail_items(db: Session, rule: str, eid: int, limit: int, offset: int, man
if eid:
sql += " AND d.entity_id = :eid"
sql += " ORDER BY d.id LIMIT :lim OFFSET :off"
if not eid:
params.pop("eid", None)
rows = db.execute(text(sql), params).fetchall()
return [{
"record_id": r.id,
@@ -674,13 +793,15 @@ def _detail_items(db: Session, rule: str, eid: int, limit: int, offset: int, man
if rule == "reconciliation":
sql = ("SELECT d.id, d.entity_id, d.kpi_code, d.kpi_name, d.target_yearly, "
"COALESCE(SUM(b.budget_value),0) AS monthly_sum, "
"ROUND((COALESCE(SUM(b.budget_value),0) - d.target_yearly) / d.target_yearly * 100, 1) AS diff_pct "
"FROM kpi_definitions d JOIN budget_plans b ON b.kpi_id = d.id "
"WHERE d.status='active' AND d.target_yearly > 0 AND b.status='active'")
+ RECON_DIFF_EXPR + " AS diff_pct " + RECON_BASE_SQL)
if eid:
sql += " AND d.entity_id = :eid"
sql += " GROUP BY d.id HAVING ABS(diff_pct) > 20 ORDER BY ABS(diff_pct) DESC LIMIT :lim OFFSET :off"
rows = db.execute(text(sql), params).fetchall()
sql += (" GROUP BY d.id HAVING " + _recon_having(cfg)
+ " ORDER BY ABS(diff_pct) DESC LIMIT :lim OFFSET :off")
recon_params = {**params, "pct": _recon_pct(cfg)}
if not eid:
recon_params.pop("eid", None)
rows = db.execute(text(sql), recon_params).fetchall()
return [{
"record_id": r.id,
"entity_id": r.entity_id,
@@ -695,16 +816,16 @@ def _detail_items(db: Session, rule: str, eid: int, limit: int, offset: int, man
return []
# ============================================================
# 财务七规则检查(governance-check)— 2026-08-30 P1 最终方案
# 端点收敛(cma-residual-cross-entity-cleanup-20260914):
# /check-governance 已下线 —— 全仓无调用方(前端仅残留 api 定义、无 .vue 调用);
# 其 entity_id 参数无会话鉴权(0=全部/1/2 任取)→ 跨租户;与 governance-check 功能重复。
# 本端点评分口径:
# * 评分规则:error 扣 min(15, count*3),warning 扣 min(10, count*1)
# * 评分规则:error 扣 min(error 上限, count×系数),warning 扣 min(warning 上限, count×系数)
# —— 上限/系数自 2026-09-28(ASK-20260928-05)起取自 `dq_rule_config`(默认 = 原 15×3 / 10×1)
# * 规则2/3 用 Python 解析 suggestion JSON(不依赖 MySQL JSON 函数)
# * 规则4 区分 error(test/sync/虚拟) 与 manual(待人工确认 warning)
# * 规则4 区分 污染关键词(error)与 manual(待人工确认 warning)
# * 规则7 按 status='active' 口径(与 budget.py 一致,避免多版本叠加失真)
# 只读幂等:不写库、不创建 KpiDataQualityLog
# ============================================================
@@ -721,142 +842,204 @@ def _extract_plan_id(suggestion: str):
return None
def _run_governance_checks(db: Session, entity_id: int = 0):
"""执行财务七规则,返回 issues 列表(含 deducted 扣分)。entity_id=0 表示全部实体。"""
eid = entity_id or 0
ent = " AND cp.entity_id = :eid" if eid else ""
issues = []
def _disabled_issue(rule: str, cfg: dict) -> dict:
"""被配置关闭的规则:零扣分、零明细,但**仍占位**(规则集锁死 7 条,响应形状稳定)。"""
return {"rule": rule, "level": RULES_META[rule]["level"], "enabled": False,
"count": 0, "detail": [], "deducted": 0}
# ── 规则1 单位校验:amount > 10000(万元口径可疑)──
rows = db.execute(text(
"SELECT cp.id, cp.entity_id, cp.amount, cp.source, cp.description "
"FROM cash_plans cp WHERE cp.amount > 10000" + ent + " ORDER BY cp.amount DESC LIMIT 200"
), {"eid": eid}).fetchall()
unit_total = _detail_count(db, "unit_check", eid) # 真实总数(不再受明细 LIMIT 200 截断)
issues.append({
"rule": "unit_check", "level": "error",
"count": unit_total,
"detail": [f"plan#{r.id} 金额{r.amount}(疑似元)" for r in rows[:DETAIL_LIMIT]]
+ (["…等%d条" % unit_total] if unit_total > DETAIL_LIMIT else []),
})
def _run_governance_checks(db: Session, entity_id: int = 0):
"""执行财务七规则,返回 issues 列表(含 deducted 扣分)。entity_id=0 表示全部实体。
配置化(ASK-20260928-05):阈值/关键词/条数下限/差异阈值与方向/扣分上限与系数/明细条数
一律经 `_cfg_values()`(三层解析)取值;级别与判定表达式本体**锁死**(见 dq_rule_config)。
"""
eid = entity_id or 0
cfg = _cfg_values(db, eid)
resolution = _load_cfg(db, eid)
issues = []
disabled = []
# ── 规则1 单位校验:amount > 阈值(默认 10000 万元口径可疑)──
if cfg["unit_check"].get("enabled", True):
threshold = _unit_threshold(cfg)
limit = _detail_limit(cfg, "unit_check")
ent, params = _entity_clause(eid, "cp.entity_id")
rows = db.execute(text(
"SELECT cp.id, cp.entity_id, cp.amount, cp.source, cp.description "
"FROM cash_plans cp WHERE cp.amount > :amt" + ent + " ORDER BY cp.amount DESC LIMIT 200"
), {**params, "amt": threshold}).fetchall()
unit_total = _detail_count(db, "unit_check", eid, cfg=cfg) # 真实总数(不受明细 LIMIT 截断)
issues.append({
"rule": "unit_check", "level": RULES_META["unit_check"]["level"],
"count": unit_total,
"detail": [f"plan#{r.id} 金额{r.amount}(疑似元)" for r in rows[:limit]]
+ (["…等%d条" % unit_total] if unit_total > limit else []),
})
else:
disabled.append("unit_check")
issues.append(_disabled_issue("unit_check", cfg))
# ── 规则2/3 共用数据:pending cash_plan 预警(实体筛选经 kpi_definitions)──
parsed, alert_rows = _pending_alert_plan_links(db, eid)
parsed, _alert_rows = _pending_alert_plan_links(db, eid)
# 已存在的 cash_plans id 集合(规则3 判断孤儿用;实体筛选时仅看该实体下 plan)
plan_id_set = _plan_id_set(db, eid)
# ── 规则2 重复预警:同 plan_id 多条 pending 预警 ──
dup_groups = _dup_groups(parsed)
issues.append({
"rule": "dup_alert", "level": "error",
"count": len(dup_groups),
"detail": [f"plan#{pid} 重复预警×{len(ids)}" for pid, ids in dup_groups[:DETAIL_LIMIT]]
+ (["…等%d个plan" % len(dup_groups)] if len(dup_groups) > DETAIL_LIMIT else []),
})
# ── 规则2 重复预警:同 plan_id 的 pending 预警 ≥ min_count(默认 2)──
if cfg["dup_alert"].get("enabled", True):
limit = _detail_limit(cfg, "dup_alert")
dup_groups = _dup_groups(parsed, _dup_min_count(cfg))
issues.append({
"rule": "dup_alert", "level": RULES_META["dup_alert"]["level"],
"count": len(dup_groups),
"detail": [f"plan#{pid} 重复预警×{len(ids)}" for pid, ids in dup_groups[:limit]]
+ (["…等%d个plan" % len(dup_groups)] if len(dup_groups) > limit else []),
})
else:
disabled.append("dup_alert")
issues.append(_disabled_issue("dup_alert", cfg))
# ── 规则3 孤儿预警:plan_id 指向不存在的 cash_plans ──
orphan_rows = [(aid, kid, pid) for aid, kid, pid, _ent in parsed if pid not in plan_id_set]
issues.append({
"rule": "orphan_check", "level": "error",
"count": len(orphan_rows),
"detail": [f"预警#{aid}(kpi#{kid}) → plan#{pid} 不存在" for aid, kid, pid in orphan_rows[:DETAIL_LIMIT]]
+ (["…等%d条" % len(orphan_rows)] if len(orphan_rows) > DETAIL_LIMIT else []),
})
if cfg["orphan_check"].get("enabled", True):
limit = _detail_limit(cfg, "orphan_check")
orphan_rows = [(aid, kid, pid) for aid, kid, pid, _ent in parsed if pid not in plan_id_set]
issues.append({
"rule": "orphan_check", "level": RULES_META["orphan_check"]["level"],
"count": len(orphan_rows),
"detail": [f"预警#{aid}(kpi#{kid}) → plan#{pid} 不存在" for aid, kid, pid in orphan_rows[:limit]]
+ (["…等%d条" % len(orphan_rows)] if len(orphan_rows) > limit else []),
})
else:
disabled.append("orphan_check")
issues.append(_disabled_issue("orphan_check", cfg))
# ── 规则4 虚拟污染:source 含 test/sync/虚拟 → error;source='manual' → 待人工确认 warning ──
rows = db.execute(text(
"SELECT cp.id, cp.entity_id, cp.source, cp.description FROM cash_plans cp "
"WHERE (cp.source LIKE '%test%' OR cp.source LIKE '%sync%' OR cp.source LIKE '%虚拟%' OR cp.source LIKE '%demo%')"
+ ent + " ORDER BY cp.id LIMIT 200"
), {"eid": eid}).fetchall()
manual_rows = db.execute(text(
"SELECT cp.id, cp.entity_id, cp.source, cp.description FROM cash_plans cp "
"WHERE cp.source = 'manual'" + ent + " ORDER BY cp.id LIMIT 200"
), {"eid": eid}).fetchall()
poll_total = _detail_count(db, "virtual_pollution", eid) # 真实总数(不再受 LIMIT 200 截断)
manual_total = _detail_count(db, "virtual_pollution", eid, manual=True)
issues.append({
"rule": "virtual_pollution", "level": "error",
"count": poll_total,
"detail": [f"plan#{r.id} source={r.source} desc={r.description or ''}" for r in rows[:DETAIL_LIMIT]]
+ (["…等%d条" % poll_total] if poll_total > DETAIL_LIMIT else []),
"manual_count": manual_total,
"manual_detail": [f"plan#{r.id} source=manual(待人工确认)" for r in manual_rows[:DETAIL_LIMIT]]
+ (["…等%d条" % manual_total] if manual_total > DETAIL_LIMIT else []),
})
# ── 规则4 虚拟污染:source 命中嫌疑关键词(去白名单)→ error;manual 值 → 待人工确认 warning ──
if cfg["virtual_pollution"].get("enabled", True):
limit = _detail_limit(cfg, "virtual_pollution")
ent, params = _entity_clause(eid, "cp.entity_id")
pred, vp_params = _vp_predicate(cfg)
rows = db.execute(text(
"SELECT cp.id, cp.entity_id, cp.source, cp.description FROM cash_plans cp "
"WHERE " + pred + ent + " ORDER BY cp.id LIMIT 200"
), {**params, **vp_params}).fetchall()
manual_rows = db.execute(text(
"SELECT cp.id, cp.entity_id, cp.source, cp.description FROM cash_plans cp "
"WHERE cp.source = :msrc" + ent + " ORDER BY cp.id LIMIT 200"
), {**params, "msrc": _manual_source(cfg)}).fetchall()
poll_total = _detail_count(db, "virtual_pollution", eid, cfg=cfg)
manual_total = _detail_count(db, "virtual_pollution", eid, manual=True, cfg=cfg)
issues.append({
"rule": "virtual_pollution", "level": RULES_META["virtual_pollution"]["level"],
"count": poll_total,
"detail": [f"plan#{r.id} source={r.source} desc={r.description or ''}" for r in rows[:limit]]
+ (["…等%d条" % poll_total] if poll_total > limit else []),
"manual_count": manual_total,
"manual_detail": [f"plan#{r.id} source=manual(待人工确认)" for r in manual_rows[:limit]]
+ (["…等%d条" % manual_total] if manual_total > limit else []),
})
else:
disabled.append("virtual_pollution")
issues.append(_disabled_issue("virtual_pollution", cfg))
# ── 规则5 实体归属:kpi_values.entity_id IS NOT NULL 且 != kpi_definitions.entity_id ──
ent_sql = (
"SELECT v.id, v.kpi_id, d.kpi_code, v.entity_id AS v_eid, d.entity_id AS d_eid "
"FROM kpi_values v JOIN kpi_definitions d ON v.kpi_id = d.id "
"WHERE v.entity_id IS NOT NULL AND v.entity_id != d.entity_id"
)
if eid:
ent_sql += " AND v.entity_id = :eid"
ent_sql += " ORDER BY v.id LIMIT 200"
ent_rows = db.execute(text(ent_sql), {"eid": eid}).fetchall()
ent_total = _detail_count(db, "entity_check", eid) # 真实总数
issues.append({
"rule": "entity_check", "level": "error",
"count": ent_total,
"detail": [f"值#{r.id} {r.kpi_code} 实体{r.v_eid}≠定义实体{r.d_eid}" for r in ent_rows[:DETAIL_LIMIT]]
+ (["…等%d条" % ent_total] if ent_total > DETAIL_LIMIT else []),
})
if cfg["entity_check"].get("enabled", True):
limit = _detail_limit(cfg, "entity_check")
ent_sql = (
"SELECT v.id, v.kpi_id, d.kpi_code, v.entity_id AS v_eid, d.entity_id AS d_eid "
"FROM kpi_values v JOIN kpi_definitions d ON v.kpi_id = d.id "
"WHERE v.entity_id IS NOT NULL AND v.entity_id != d.entity_id"
)
if eid:
ent_sql += " AND v.entity_id = :eid"
ent_sql += " ORDER BY v.id LIMIT 200"
ent_rows = db.execute(text(ent_sql), {"eid": eid}).fetchall()
ent_total = _detail_count(db, "entity_check", eid, cfg=cfg) # 真实总数
issues.append({
"rule": "entity_check", "level": RULES_META["entity_check"]["level"],
"count": ent_total,
"detail": [f"值#{r.id} {r.kpi_code} 实体{r.v_eid}≠定义实体{r.d_eid}" for r in ent_rows[:limit]]
+ (["…等%d条" % ent_total] if ent_total > limit else []),
})
else:
disabled.append("entity_check")
issues.append(_disabled_issue("entity_check", cfg))
# ── 规则6 KPI完整性:active KPI 无任何实际值 ──
comp_sql = (
"SELECT d.id, d.kpi_code, d.kpi_name FROM kpi_definitions d "
"WHERE d.status='active' AND NOT EXISTS (SELECT 1 FROM kpi_values v WHERE v.kpi_id = d.id)"
)
if eid:
comp_sql += " AND d.entity_id = :eid"
comp_sql += " ORDER BY d.id LIMIT 300"
comp_rows = db.execute(text(comp_sql), {"eid": eid}).fetchall()
comp_total = _detail_count(db, "kpi_completeness", eid) # 真实总数(不再受 LIMIT 300 截断)
issues.append({
"rule": "kpi_completeness", "level": "warning",
"count": comp_total,
"detail": [f"{r.kpi_code} {r.kpi_name}(无值)" for r in comp_rows[:DETAIL_LIMIT]]
+ (["…等%d个KPI" % comp_total] if comp_total > DETAIL_LIMIT else []),
})
if cfg["kpi_completeness"].get("enabled", True):
limit = _detail_limit(cfg, "kpi_completeness")
comp_sql = (
"SELECT d.id, d.kpi_code, d.kpi_name FROM kpi_definitions d "
"WHERE d.status='active' AND NOT EXISTS (SELECT 1 FROM kpi_values v WHERE v.kpi_id = d.id)"
)
if eid:
comp_sql += " AND d.entity_id = :eid"
comp_sql += " ORDER BY d.id LIMIT 300"
comp_rows = db.execute(text(comp_sql), {"eid": eid}).fetchall()
comp_total = _detail_count(db, "kpi_completeness", eid, cfg=cfg)
issues.append({
"rule": "kpi_completeness", "level": RULES_META["kpi_completeness"]["level"],
"count": comp_total,
"detail": [f"{r.kpi_code} {r.kpi_name}(无值)" for r in comp_rows[:limit]]
+ (["…等%d个KPI" % comp_total] if comp_total > limit else []),
})
else:
disabled.append("kpi_completeness")
issues.append(_disabled_issue("kpi_completeness", cfg))
# ── 规则7 勾稽验证:预算(active口径)年度合计 vs 年度目标 差异>20% ──
recon_sql = (
"SELECT d.kpi_code, d.kpi_name, d.target_yearly, "
"COALESCE(SUM(b.budget_value),0) AS monthly_sum, "
"ROUND((COALESCE(SUM(b.budget_value),0) - d.target_yearly) / d.target_yearly * 100, 1) AS diff_pct "
"FROM kpi_definitions d JOIN budget_plans b ON b.kpi_id = d.id "
"WHERE d.status='active' AND d.target_yearly > 0 AND b.status='active'"
)
if eid:
recon_sql += " AND d.entity_id = :eid"
recon_sql += " GROUP BY d.id HAVING ABS(diff_pct) > 20 ORDER BY ABS(diff_pct) DESC LIMIT 200"
recon_rows = db.execute(text(recon_sql), {"eid": eid}).fetchall()
recon_total = _detail_count(db, "reconciliation", eid) # 真实总数
issues.append({
"rule": "reconciliation", "level": "warning",
"count": recon_total,
"detail": [f"{r.kpi_code} 预算合计{round(r.monthly_sum, 1)} vs 年度目标{r.target_yearly} 差异{r.diff_pct}%" for r in recon_rows[:DETAIL_LIMIT]]
+ (["…等%d个KPI" % recon_total] if recon_total > DETAIL_LIMIT else []),
})
# ── 规则7 勾稽验证:预算(active口径)合计 vs 年度目标,差异按配置阈值/方向判定 ──
if cfg["reconciliation"].get("enabled", True):
limit = _detail_limit(cfg, "reconciliation")
recon_sql = (
"SELECT d.kpi_code, d.kpi_name, d.target_yearly, "
"COALESCE(SUM(b.budget_value),0) AS monthly_sum, "
+ RECON_DIFF_EXPR + " AS diff_pct " + RECON_BASE_SQL
)
if eid:
recon_sql += " AND d.entity_id = :eid"
recon_sql += (" GROUP BY d.id HAVING " + _recon_having(cfg)
+ " ORDER BY ABS(diff_pct) DESC LIMIT 200")
recon_rows = db.execute(text(recon_sql),
{"eid": eid, "pct": _recon_pct(cfg)}).fetchall()
recon_total = _detail_count(db, "reconciliation", eid, cfg=cfg)
issues.append({
"rule": "reconciliation", "level": RULES_META["reconciliation"]["level"],
"count": recon_total,
"detail": [f"{r.kpi_code} 预算合计{round(r.monthly_sum, 1)} vs 年度目标{r.target_yearly} 差异{r.diff_pct}%"
for r in recon_rows[:limit]]
+ (["…等%d个KPI" % recon_total] if recon_total > limit else []),
})
else:
disabled.append("reconciliation")
issues.append(_disabled_issue("reconciliation", cfg))
# ── 评分:error 扣 min(15, count*3),warning 扣 min(10, count*1);规则4 manual 按 warning 附加扣 ──
# ── 评分:各规则的「上限 × 系数」取自配置(默认 error 15×3 / warning 10×1;规则4 manual 附加)──
total_deduct = 0
for item in issues:
rule = item["rule"]
rule_cfg = cfg.get(rule) or {}
if not rule_cfg.get("enabled", True):
item["deducted"] = 0
continue
ded = 0
if item["count"] > 0:
ded += min(15, item["count"] * 3) if item["level"] == "error" else min(10, item["count"] * 1)
cap = int(rule_cfg.get("deduct_cap",
15 if item["level"] == "error" else 10))
factor = int(rule_cfg.get("deduct_factor",
3 if item["level"] == "error" else 1))
ded += min(cap, item["count"] * factor)
# 规则4 附加:manual 待人工确认(warning 性质)
manual_cnt = item.get("manual_count") or 0
if item["rule"] == "virtual_pollution" and manual_cnt > 0:
ded += min(10, manual_cnt * 1)
if rule == "virtual_pollution" and manual_cnt > 0:
ded += min(int(rule_cfg.get("manual_cap", 10)),
manual_cnt * int(rule_cfg.get("manual_factor", 1)))
item["deducted"] = ded
total_deduct += ded
score = max(0, 100 - total_deduct)
passed = [item["rule"] for item in issues if item["count"] == 0]
passed = [item["rule"] for item in issues if item["count"] == 0 and item["rule"] not in disabled]
excluded = list(resolution.get("scope", {}).get("excluded_entities") or [])
return {
"checked_at": datetime.now().isoformat(timespec="seconds"),
"entity_id": eid,
@@ -865,7 +1048,15 @@ def _run_governance_checks(db: Session, entity_id: int = 0):
"total_rules": len(issues),
"issues": issues,
"passed": passed,
"disabled": disabled,
"rules_meta": RULES_META,
# 配置化(ASK-20260928-05):口径快照指纹 —— 事后可解释「当时的分数是哪一版配置算出来的」
"config_version": resolution.get("config_version"),
"config_source": {"table_ready": resolution.get("table_ready"),
"entity_config_enabled": resolution.get("entity_config_enabled")},
# 演示/测试账套排除(F6):默认空集 ⇒ 不改变评分语义,只做标注
"scope_excluded_entities": excluded,
"excluded_from_business_caliber": bool(eid and eid in excluded),
}
@@ -878,7 +1069,8 @@ def governance_check(
"""财务七规则检查(DAMA数据治理)— 只读幂等,不写库。
7条规则: unit_check/dup_alert/orphan_check/virtual_pollution/entity_check/kpi_completeness/reconciliation
评分: 满分100,error 扣 min(15, count*3),warning 扣 min(10, count*1),score=max(0, 100-总扣分)
评分: 满分100,各规则扣分 = min(配置上限, 命中数×配置系数)(默认 error 15×3、warning 10×1),
score = max(0, 100 − 总扣分);阈值/关键词/权重自 2026-09-28 起由 quality_rule_config 配置(ASK-20260928-05)。
实体筛选: entity_id 参数(0=全部,仅无会话token的Bot通道)。
多租户(cma-cross-entity-endpoints-auth-20260914):
@@ -899,7 +1091,7 @@ def rule_detail(
rule: Optional[str] = Query(None, description="七规则 key 之一;缺省/空 → 空集(不 500)"),
page: int = Query(1, ge=1, description="页码,从 1 开始"),
page_size: int = Query(50, ge=1, le=200, description="每页条数,默认 50,上限 200"),
scope: Optional[str] = Query(None, description="virtual_pollution 专用:manual=source='manual' 待人工确认组"),
scope: Optional[str] = Query(None, description="virtual_pollution 专用:manual=待人工确认组"),
db: Session = Depends(get_db),
entity_id: int = Depends(get_entity_id),
):
@@ -909,7 +1101,8 @@ def rule_detail(
- `items` 结构化字段:record_id / entity_id / source / key_fields / summary / suggested_action;
- 服务端分页(LIMIT/OFFSET),单次响应不塞全量;
- 账套由会话 token 解析(Depends(get_entity_id)),**不新增账套入参**;
- 兼容:rule 缺省/空 → 空集(200,不 500);未知 rule → 400 + 明确文案。
- 兼容:rule 缺省/空 → 空集(200,不 500);未知 rule → 400 + 明确文案;
- 判据与 `governance-check` 共用同一配置解析(2026-09-28 起),两处口径不会漂移。
"""
key = (rule or "").strip()
if not key:
@@ -920,18 +1113,177 @@ def rule_detail(
manual = key == "virtual_pollution" and (scope or "").strip().lower() == "manual"
eid = entity_id or 0
total = _detail_count(db, key, eid, manual=manual)
cfg = _cfg_values(db, eid)
total = _detail_count(db, key, eid, manual=manual, cfg=cfg)
offset = (page - 1) * page_size
items = _detail_items(db, key, eid, page_size, offset, manual=manual)
items = _detail_items(db, key, eid, page_size, offset, manual=manual, cfg=cfg)
return {
"rule": key,
"level": RULES_META[key]["level"],
"name": RULES_META[key]["name"],
"desc": RULES_META[key]["desc"],
"scope": "manual" if manual else None,
"enabled": bool((cfg.get(key) or {}).get("enabled", True)),
"total": total,
"page": page,
"page_size": page_size,
"items": items,
}
# ============================================================
# 规则配置端点(ASK-20260928-05 最小实现:新表 + 端点 + 只读 UI)
# 读:GET /rule-config(四角色,账套由 token 绑定,不新增账套入参)
# 写:PUT /rule-config(ceo/it;平台级参数仅 it;Bot 通道禁写;变更与审计同事务)
# 史:GET /rule-config/history(ceo/it)
# 边界(清单 §4/§5):级别、判定表达式、评分函数、作用域等 10 类锁死 —— 不在 SPEC 即拒绝。
# ============================================================
class DqRuleConfigUpdate(BaseModel):
"""单参数配置变更(一次只改一档:清单 §5.1「一次只改一档,改完复跑七规则对账」)。"""
rule_key: str = Field(..., description="七规则 key 之一,或 global(全局项)")
param_key: str = Field(..., description="参数名(见 GET /rule-config 的 params)")
value: Any = Field(None, description="新值(类型见参数规格:bool/int/float/str/数组/枚举)")
change_reason: str = Field(..., description=f"变更理由(≥{dqcfg.MIN_REASON_LEN} 字,落审计)")
@router.get("/rule-config")
def get_rule_config(
db: Session = Depends(get_db),
entity_id: int = Depends(get_entity_id),
):
"""财务七规则配置(**只读**):三层解析后的生效值 + 每项来源 + 锁死项清单。
来源语义:`entity`=本账套覆盖行 / `template`=全账套模板行(entity_id=0) / `builtin`=代码内置默认。
账套由会话 token 解析(`Depends(get_entity_id)`),**不新增账套入参**;演示/测试账套只读不可写。
"""
res = _load_cfg(db, entity_id)
return {
"entity_id": entity_id,
"config_entity_id": res["config_entity_id"],
"entity_config_enabled": res["entity_config_enabled"],
"config_entities": list(dqcfg.CONFIG_ENTITIES),
"table_ready": res["table_ready"],
"config_version": res["config_version"],
"rule_param_count": dqcfg.count_rule_params(),
"rules": {
key: {
"name": RULES_META[key]["name"],
"level": RULES_META[key]["level"],
"desc": RULES_META[key]["desc"],
"params": res["rules"][key],
} for key in dqcfg.RULE_KEYS
},
"global_params": res["global"],
"scope": res["scope"],
"scope_note": "演示/测试账套排除(F6)当前只做标注、不改评分语义;默认空集 = 不排除",
"locked": [dict(it) for it in dqcfg.LOCKED_ITEMS],
"readonly": True,
"generated_at": datetime.now().isoformat(timespec="seconds"),
}
@router.get("/rule-config/history")
def get_rule_config_history(
limit: int = Query(20, ge=1, le=200, description="返回条数上限"),
db: Session = Depends(get_db),
entity_id: int = Depends(get_entity_id),
_user: User = Depends(require_role("ceo", "it")),
):
"""本账套配置变更历史(ceo/it)——读 `operation_logs`(action=update_dq_rule_config)。
`operation_logs` 无 entity_id 列 ⇒ 按 `detail.entity_id` 过滤(清单 §5.2 既定口径)。
"""
return {"entity_id": entity_id, "items": dqcfg.history(db, entity_id, limit)}
@router.put("/rule-config")
def update_rule_config(
payload: DqRuleConfigUpdate,
request: Request,
db: Session = Depends(get_db),
entity_id: int = Depends(get_entity_id),
current_user: User = Depends(require_role("ceo", "it")),
):
"""修改单个配置项(ceo/it;平台级参数仅 it)。
校验顺序:账套可写 → Bot 通道禁写 → 参数在开放面(锁死/未知一律 400)
→ 平台级仅 it → change_reason → 取值边界(拒绝或钳制)→ 写行 + 审计同事务。
"""
if entity_id not in dqcfg.CONFIG_ENTITIES:
raise HTTPException(
400, f"账套 entity_id={entity_id} 不开放规则配置(仅 {list(dqcfg.CONFIG_ENTITIES)};"
f"演示/测试账套只读,ASK-20260928-05 口径②)")
if str(request.headers.get("X-BOT-KEY") or "").strip():
# 锁死面第 9 条:Bot 通道无 users 行 → 无操作人可留痕 = 审计断链
raise HTTPException(403, "Bot 通道禁止写规则配置(无操作人可留痕,审计会断链)")
rule_key = (payload.rule_key or "").strip()
param_key = (payload.param_key or "").strip()
if dqcfg.param_spec(rule_key, param_key) is None:
# 复用参数校验器的文案:锁死项 / 未知 key / 该规则无开放参数
probe = dqcfg.validate_param(rule_key, param_key, payload.value)
raise HTTPException(400, probe["error"])
level = dqcfg.open_level_of(rule_key, param_key)
if level == dqcfg.OPEN_LEVEL_PLATFORM and current_user.role != "it":
raise HTTPException(403, f"参数 {rule_key}.{param_key} 为平台级配置(仅 it 可写)")
reason = (payload.change_reason or "").strip()
if len(reason) < dqcfg.MIN_REASON_LEN:
raise HTTPException(400, f"change_reason 必填且不少于 {dqcfg.MIN_REASON_LEN} 字"
f"(防无痕调参刷分;清单 §5.1)")
checked = dqcfg.validate_param(rule_key, param_key, payload.value)
if not checked["ok"]:
raise HTTPException(400, checked["error"])
rule_name = RULES_META.get(rule_key, {}).get("name") or rule_key
try:
result = dqcfg.upsert_param(db, entity_id, rule_key, param_key, checked["value"],
username=getattr(current_user, "username", None))
log_op(
db, dqcfg.ACTION_UPDATE, target_type=dqcfg.TARGET_TYPE, target_id=result["id"],
user_id=current_user.id, entity_id=entity_id,
detail={
"entity_id": entity_id,
"rule_key": rule_key,
"rule_name": rule_name,
"param_key": param_key,
"before": result["before"],
"after": result["after"],
"reason": reason,
"open_level": level,
"client_ip": (request.client.host if request.client else None),
"operator": {"user_id": current_user.id,
"username": getattr(current_user, "username", None),
"role": current_user.role},
},
)
# 变更与审计同一事务提交:审计写不进 ⇒ 配置变更一并回滚
# (绝不留「改了但没审计」的窗口;形态照抄 tenant.py:148 先例)
db.commit()
except Exception:
db.rollback()
logger.error("DQ_RULE_CONFIG_AUDIT_WRITE_FAILED entity=%s %s.%s user=%s → 变更已回滚",
entity_id, rule_key, param_key, getattr(current_user, "id", None),
exc_info=True)
raise HTTPException(500, "审计写入失败,配置变更已回滚")
after = _load_cfg(db, entity_id)
return {
"success": True,
"entity_id": entity_id,
"rule_key": rule_key,
"param_key": param_key,
"before": result["before"],
"before_source": result["before_source"],
"after": checked["value"],
"warning": checked["warning"],
"config_version": after["config_version"],
"changed_by": getattr(current_user, "username", None),
"audit": {"action": dqcfg.ACTION_UPDATE, "target_type": dqcfg.TARGET_TYPE,
"target_id": result["id"]},
"note": "配置即生效(下次 governance-check 按新口径计算);历史评分不重算",
}