feat(cma): audit write-side entity closure + entity-dimension data dictionary

task dama-entity-isolation-sweep-20260923 (sections 2-2 4/5, merged by projectBot 2026-09-23 18:2x)

4) operation_logs write side: every audit write carries the account (no DDL, no column added)

   - new app/utils/oplog.py::log_op() single wrapper: sets detail.entity_id; scope=global for

     by-design non-entity writes (platform dict / role-level); scope=batch + OPLOG_BATCH_NO_ENTITY

     sentinel for cross-account batches; no context -> OPLOG_NO_ENTITY sentinel (never guess)

   - 16 remaining construction points converted (data/alert_rules/bi_reports/kpi_causality/

     bot_bridge x2/budget_generate/driver_budget/kpis _log x3/templates _log x2/erp_sync);

     pre-4 state had 20 sites without entity_id, now 0 outside the wrapper

   - read side untouched (knowledge_service still user_id fallback); history not backfilled

5) docs/entity-dimension-data-dictionary.md: 8-table three-state registry

   (kpi_templates/driver_factor_templates/scenario_suggestions/bot_bridge_config = non-entity;

    kpi_subject_map = inherited via kpi_id; analysis_results/subjects = account dimension,

    legacy rows still NULL -> other task; operation_logs = non-entity column + detail.entity_id)

Evidence: RED 2 failed/7 passed (reverted call sites) -> GREEN full 1428 passed; 9 new tests

including two read-back rows; app md5 restore verified identical; no cross-account writes

LOCAL commit only - push is gated (ahead contains the P1 entity-dim batch commit fb4609b)
This commit is contained in:
wecom-fullstack
2026-09-23 23:08:26 +08:00
parent fb4609be14
commit 467472311d
18 changed files with 525 additions and 71 deletions
@@ -28,6 +28,8 @@ import os
import re
import sys
import ast
import pytest
from fastapi.testclient import TestClient
from sqlalchemy import create_engine, inspect, text
@@ -35,6 +37,8 @@ from sqlalchemy.orm import Session
from app.models import Entity, OperationLog, RolePermission, SystemConfig
from app.database import Base
from app.utils.oplog import (BATCH_NO_ENTITY_TAG, NO_ENTITY_TAG, SCOPE_BATCH,
SCOPE_GLOBAL, log_op)
from tests.conftest import TEST_USER_PASSWORD, auth_header, create_test_user
BUDGET = "/api/cma/budget"
@@ -378,3 +382,157 @@ class TestMigrationScript:
with pytest.raises(SystemExit) as e:
self._run(mod, ["--apply", "--allow-db", "sqlite", "--target-entity", "99", "--json"])
assert e.value.code == 2
# ══════════════════════════════════════════════
# ④ operation_logs 写侧统一带账套(`detail.entity_id`,零 DDL)
# 任务单 §二之二④(项目Bot 2026-09-23 18:2x 并入)
# ══════════════════════════════════════════════
BACKEND_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
OPLOG_REL = os.path.join("app", "utils", "oplog.py")
def _operation_log_construction_sites():
"""全仓 `OperationLog(` 构造点 → [(相对路径, 行号, 所在语句源码)]
语句源码取「包含该构造调用的完整 Assign/Expr 语句」(多行 dict 字面量整段在内),
用于断言「写入时带账套」这件事落在**语句级**,而不是碰巧命中了别处的 entity_id。
"""
out = []
for rel in ("app", "scripts"):
for dp, _dn, fns in os.walk(os.path.join(BACKEND_ROOT, rel)):
if "__pycache__" in dp:
continue
for fn in fns:
if not fn.endswith(".py"):
continue
path = os.path.join(dp, fn)
src = open(path, encoding="utf-8").read()
tree = ast.parse(src)
lines = src.splitlines()
stmts = [(n.lineno, n.end_lineno) for n in ast.walk(tree)
if isinstance(n, (ast.Assign, ast.Expr, ast.Return))
and getattr(n, "lineno", None) and getattr(n, "end_lineno", None)]
for node in ast.walk(tree):
if not isinstance(node, ast.Call):
continue
fn_node = node.func
name = getattr(fn_node, "id", None) or getattr(fn_node, "attr", None)
if name != "OperationLog":
continue
cand = [s for s in stmts if s[0] <= node.lineno and s[1] >= node.end_lineno]
s, e = min(cand, key=lambda x: x[1] - x[0]) if cand else (node.lineno, node.end_lineno)
out.append((os.path.relpath(path, BACKEND_ROOT).replace("\\", "/"),
node.lineno, "\n".join(lines[s - 1:e])))
return out
class TestAuditWriteSideEntityId:
"""④ 写侧收口:所有审计行带账套,且集中封装(不加列、不回填历史)"""
def test_no_new_raw_OperationLog_sites_lack_entity(self):
"""源级断言:除集中封装件外,每个构造点的语句里必须出现 entity_id"""
sites = _operation_log_construction_sites()
assert len(sites) >= 13, f"构造点数量异常(疑似收集失败): {len(sites)}"
missing = [(p, l) for p, l, seg in sites
if p != OPLOG_REL and "entity_id" not in seg]
assert missing == [], f"以下构造点未带账套(禁止猜账套,必须显式): {missing}"
def test_wrapper_is_single_place_constructing_rows(self):
"""集中封装:裸构造 OperationLog 只允许出现在 app/utils/oplog.py(防第 28 处再漏)"""
wrappers = [(p, l) for p, l, _seg in _operation_log_construction_sites() if p == OPLOG_REL]
assert len(wrappers) == 1, f"oplog.py 应恰好 1 处裸构造(log_op 本体): {wrappers}"
def test_log_op_callers_pass_entity_or_scope(self):
"""所有 `log_op(` 调用点必须传 entity_id= 或 scope=(否则等于留覆盖缺口)"""
bad = []
for rel in ("app", "scripts"):
for dp, _dn, fns in os.walk(os.path.join(BACKEND_ROOT, rel)):
if "__pycache__" in dp:
continue
for fn in fns:
if not fn.endswith(".py"):
continue
path = os.path.join(dp, fn)
tree = ast.parse(open(path, encoding="utf-8").read())
for node in ast.walk(tree):
if isinstance(node, ast.Call) and getattr(node.func, "id", None) == "log_op":
if not ({"entity_id", "scope"} & {k.arg for k in node.keywords}):
bad.append((os.path.relpath(path, BACKEND_ROOT), node.lineno))
assert bad == [], f"log_op 调用点缺账套参数: {bad}"
# ── 落库实证 ────────────────────────────────
def test_readback_entity_id_persisted(self, db: Session):
"""落库实证 ①:log_op 写入 → **raw SELECT 回查数据库**,detail.entity_id 在表里"""
log_op(db, "unit_probe_entity", target_type="kpi", target_id=123,
detail={"kpi_code": "PROBE_KPI"}, entity_id=2, commit=True)
row = db.execute(text("SELECT action, detail FROM operation_logs "
"WHERE action = 'unit_probe_entity'")).fetchone()
assert row is not None, "审计行未落库"
detail = json.loads(row[1]) if isinstance(row[1], str) else row[1]
assert detail.get("entity_id") == 2, f"落库 detail 未带账套: {detail}"
assert detail.get("kpi_code") == "PROBE_KPI", f"原 detail 内容被破坏: {detail}"
def test_readback_kpi_create_endpoint_audit_entity(self, client: TestClient, db: Session):
"""落库实证 ②:真实写端点(POST /kpis,会话账套 e2)→ 回查审计行 detail.entity_id == 2"""
_seed_two_entities(db)
create_test_user(db)
token = _login_as(client, "testadmin", 2) # 博海 e2
resp = client.post("/api/cma/kpis", headers=auth_header(token), json={
"kpi_code": "F_AUDIT_PROBE_E2", "kpi_name": "审计落库探针",
"dimension": "finance", "target_value": 100.0, "unit": "%",
"formula": "测试公式", "data_source": "测试系统", "data_owner": "测试管理员",
})
assert resp.status_code == 200, resp.text[:300]
row = db.execute(text("SELECT detail FROM operation_logs WHERE action = 'create' "
"AND target_type = 'kpi' ORDER BY id DESC LIMIT 1")).scalar()
assert row is not None, "端点未写审计行"
detail = json.loads(row) if isinstance(row, str) else row
assert detail.get("entity_id") == 2, f"审计 detail 未带会话账套: {detail}"
# ── 形态与哨兵 ──────────────────────────────
def test_legacy_json_string_detail_is_not_nested(self, db: Session):
"""历史写法(detail=json.dumps({...}))归一后不嵌套:entity_id 与原键同一层"""
log_op(db, "unit_probe_legacy", target_type="budget", target_id=1,
detail=json.dumps({"year": 2026, "mode": "rolling"}, ensure_ascii=False),
entity_id=1, commit=True)
detail = db.execute(text("SELECT detail FROM operation_logs "
"WHERE action = 'unit_probe_legacy'")).scalar()
detail = json.loads(detail) if isinstance(detail, str) else detail
assert detail == {"year": 2026, "mode": "rolling", "entity_id": 1}, detail
def test_global_scope_no_gap_warning(self, db: Session, caplog):
"""非账套维度(平台/角色级):写 entity_scope=global,且不产生缺口告警"""
with caplog.at_level("WARNING", logger="cma.oplog"):
log_op(db, "unit_probe_global", target_type="template", target_id=1,
detail={"kpi_code": "T1"}, scope=SCOPE_GLOBAL, commit=True)
detail = db.execute(text("SELECT detail FROM operation_logs "
"WHERE action = 'unit_probe_global'")).scalar()
detail = json.loads(detail) if isinstance(detail, str) else detail
assert detail.get("entity_scope") == "global" and "entity_id" not in detail, detail
assert NO_ENTITY_TAG not in caplog.text
def test_missing_context_warns_sentinel_never_guesses(self, db: Session, caplog):
"""无账套上下文:留空 + 可 grep 哨兵(绝不猜账套)"""
with caplog.at_level("WARNING", logger="cma.oplog"):
log_op(db, "unit_probe_unknown", target_type="kpi", target_id=1,
detail={"note": "no context"}, commit=True)
assert NO_ENTITY_TAG in caplog.text
detail = db.execute(text("SELECT detail FROM operation_logs "
"WHERE action = 'unit_probe_unknown'")).scalar()
detail = json.loads(detail) if isinstance(detail, str) else detail
assert "entity_id" not in detail, detail
def test_batch_scope_warns_own_sentinel(self, db: Session, caplog):
"""跨账套批量:entity_scope=batch + 独立哨兵(不猜单账套)"""
with caplog.at_level("WARNING", logger="cma.oplog"):
log_op(db, "unit_probe_batch", target_type="kpi", detail={"total": 3},
scope=SCOPE_BATCH, commit=True)
assert BATCH_NO_ENTITY_TAG in caplog.text
detail = db.execute(text("SELECT detail FROM operation_logs "
"WHERE action = 'unit_probe_batch'")).scalar()
detail = json.loads(detail) if isinstance(detail, str) else detail
assert detail.get("entity_scope") == "batch", detail