#!/usr/bin/env python3 # -*- coding: utf-8 -*- """`subjects` 账套分层 DDL + 存量回填 — 2026-09-23 任务：`0-P1-project-20260923-entity-dimension-convergence-fullstack` §三③ （裁定件 `project-entity-dimension-ruling-20260923` §3.1「方案 A：分层」） 背景 ---- `subjects` 207 行混装且 `subject_code` **全局唯一**（`uk_subject_code`）： · 43 行标准准则科目（1001 库存现金 …，`remark IS NULL`）= 天然全局 · 164 行 `remark='zxt-sync'` 酣客（智享通）企业明细 = 账套私有 双 token 实测两账套拿到**同一份 207 行** ⇒ 博海页面出现酣客的 `100106 酒窖备用金` / `100107 现金备用卡（唐）` / `100108 友赞商户`（结构性串配置）。 分层口径（`knowledge_articles` 同款已验模式） ------------------------------------------ `entity_id IS NULL` = 全局标准准则科目（两账套共享） `entity_id = N` = 账套 N 私有明细 唯一键 `(subject_code)` → `(entity_id, subject_code)` ⚠️ MySQL 把 NULL 视为互不相同的索引值 ⇒ 「全局行同码重复」索引拦不住， 由写入口（`POST /api/cma/bot/subjects` UPSERT 判重）兜住，不可依赖索引。 动作（四段可独立跑，互不隐含） ---------------------------- --check 只读预检（默认）：列/索引现状 + 行数分布 + 唯一键重复检查，零写入 --apply DDL：ADD COLUMN entity_id INT NULL + INDEX + UNIQUE(entity_id,subject_code) − DROP INDEX uk_subject_code（幂等：已存在的步骤自动跳过） --backfill 存量回填（**须 --evidence **，与 DDL 分开放行；口径乙 = 2026-09-23 财务裁定）： `remark='zxt-sync'` 的 164 行**剔除排除白名单**后 → 156 行置 `--target-entity` （默认 1）；`remark IS NULL` 的 43 行 ＋ 排除白名单 8 行保持/复位 `NULL`（合计 51） ⇒ 写后 `GROUP BY entity_id` = {1:156, NULL:51} 取证件 `exclude_subject_codes`：逐码命中须 = `expect_hits_per_code` 且总和须 = `total_expect`，否则 rc=3（**不做部分回填**）；**锚点双轨**：`by_remark_anchor` （zxt-sync 164 / (null) 43 / total 207）与 `counts`（e1 156 / global_null 51） 并存校验，任一不符即 fail-closed rc=3 --check --evidence 只读预检附口径乙校验（白名单逐码命中 + 锚点双轨读数），零写入 --down 回滚：重建 `uk_subject_code`（先断言无同码重复，fail-closed）→ 删新唯一键 → 删列 --show-create 打印 SHOW CREATE TABLE subjects 原文（验收证据） 生产保护（生产资产派单纪律） -------------------------- 目标库名 == 'cma' 时 `--apply/--backfill/--down` 一律拒绝；须 `--allow-prod` **且** 环境变量 `CMA_DDL_APPROVED=1`（= 已拿到 ops 审批单）才放行。`--check/--show-create` 只读不受限。 用法 ---- venv/bin/python3 scripts/migrate_subjects_entity_id_20260923.py --check CMA_DDL_APPROVED=1 venv/bin/python3 scripts/migrate_subjects_entity_id_20260923.py \\ --apply --allow-prod CMA_DDL_APPROVED=1 venv/bin/python3 scripts/migrate_subjects_entity_id_20260923.py \\ --backfill --allow-prod --evidence /path/to/evidence.json --target-entity 1 """ from __future__ import annotations import argparse import json import os import sys sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) from sqlalchemy import create_engine, inspect, text # noqa: E402 TABLE = "subjects" COLUMN = "entity_id" NEW_UNIQUE = "uk_entity_subject" OLD_UNIQUE = "uk_subject_code" NEW_INDEX = "ix_subjects_entity_id" PROD_DB_NAMES = ("cma",) DDL_ADD_COLUMN = ( f"ALTER TABLE {TABLE} ADD COLUMN {COLUMN} INT NULL " "COMMENT '企业ID（NULL=全局标准准则科目 / N=账套私有明细）' AFTER id" ) DDL_ADD_UNIQUE = f"ALTER TABLE {TABLE} ADD UNIQUE INDEX {NEW_UNIQUE} ({COLUMN}, subject_code)" DDL_DROP_OLD_UNIQUE = f"ALTER TABLE {TABLE} DROP INDEX {OLD_UNIQUE}" DDL_ADD_OLD_UNIQUE = f"ALTER TABLE {TABLE} ADD UNIQUE INDEX {OLD_UNIQUE} (subject_code)" DDL_DROP_COLUMN = f"ALTER TABLE {TABLE} DROP COLUMN {COLUMN}" def is_sqlite(engine) -> bool: """沙箱演练（SQLite）与生产（MySQL）的 DDL 方言差异：索引/列定义语句形态不同""" return engine.dialect.name == "sqlite" def stmt_add_column(engine) -> str: """生产形态 = MySQL（带 COMMENT / AFTER）；沙箱 = SQLite 精简形态（语法不支持二者）""" if is_sqlite(engine): return f"ALTER TABLE {TABLE} ADD COLUMN {COLUMN} INT NULL" return DDL_ADD_COLUMN def stmt_add_index(engine, name: str, cols: list, unique: bool = False) -> str: cols_s = ", ".join(cols) if is_sqlite(engine): return f"CREATE {'UNIQUE ' if unique else ''}INDEX {name} ON {TABLE} ({cols_s})" return f"ALTER TABLE {TABLE} ADD {'UNIQUE ' if unique else ''}INDEX {name} ({cols_s})" def stmt_drop_index(engine, name: str) -> str: return f"DROP INDEX {name}" if is_sqlite(engine) else f"ALTER TABLE {TABLE} DROP INDEX {name}" def db_name_of(engine) -> str: try: return engine.url.database or "" except Exception: return "" # ============================================================ # 口径乙（2026-09-23 财务裁定，wecom-finance） # · e1（酣客私有）= remark='zxt-sync' 行 **剔除排除白名单** 后的行数 # · global_null = remark IS NULL 行 ＋ 排除白名单行 # · by_remark_anchor 与 counts 两轨并存，须同时与取证件一致（fail-closed） # ============================================================ EXCLUDE_KEY = "exclude_subject_codes" ANCHOR_KEY = "by_remark_anchor" NULL_REMARK = "(null)" class EvidenceError(Exception): """取证件/锚点校验失败 ⇒ fail-closed（rc=3：零写入、不做部分回填）""" def load_evidence(path) -> dict: """读取证件 JSON 并做结构校验（缺件/缺字段 ⇒ EvidenceError）""" if not path or not os.path.exists(path): raise EvidenceError(f"取证件不存在：{path!r}（--backfill 必须带 --evidence ）") try: with open(path, encoding="utf-8") as fh: ev = json.load(fh) except Exception as exc: # noqa: BLE001 raise EvidenceError(f"取证件解析失败：{exc}") from exc missing = [k for k in ("dama_evidence", "finance_confirmation", "counts") if not ev.get(k)] if missing: raise EvidenceError(f"取证件缺字段 {missing}") return ev def exclude_spec(ev: dict) -> dict: """口径乙排除白名单规格（字段名以取证件 JSON 为准）""" spec = ev.get(EXCLUDE_KEY) if not isinstance(spec, dict): raise EvidenceError(f"取证件缺 {EXCLUDE_KEY} 段（无法确定白名单，拒绝执行）") codes = [str(c).strip() for c in (spec.get("codes") or [])] per = spec.get("expect_hits_per_code") total = spec.get("total_expect") if not codes: raise EvidenceError(f"{EXCLUDE_KEY}.codes 为空，拒绝执行") if not spec.get("fail_closed"): raise EvidenceError(f"{EXCLUDE_KEY}.fail_closed 缺失（非 fail-closed 形态，拒绝执行）") if len(set(codes)) != len(codes): raise EvidenceError(f"{EXCLUDE_KEY}.codes 存在重复：{codes}") if per != 1: raise EvidenceError(f"expect_hits_per_code={per!r} != 1，拒绝执行") if total is None or int(total) != len(codes): raise EvidenceError(f"total_expect={total!r} != 白名单码数 {len(codes)}，拒绝执行") return {"codes": codes, "expect_hits_per_code": 1, "total_expect": int(total)} def probe_exclude_hits(engine, codes) -> dict: """逐码命中数（只读）：subject_code 精确匹配行数""" hits = {} with engine.connect() as conn: for code in codes: hits[code] = int(conn.execute( text(f"SELECT COUNT(*) FROM {TABLE} WHERE subject_code = :c"), {"c": code}).scalar()) return hits def caliber_b_readings(engine, codes) -> dict: """现网口径乙读数（只读）：by_remark 实读 ＋ e1/global_null 推算 ＋ 白名单逐码命中""" st = state(engine) by_remark = dict(st["counts"].get("by_remark") or {}) live_remark = {"zxt-sync": int(by_remark.get("zxt-sync", 0)), NULL_REMARK: int(by_remark.get(NULL_REMARK, 0)), "total": int(st["counts"].get("total") or 0)} hits = probe_exclude_hits(engine, codes) hit_total = sum(hits.values()) live_counts = {"e1": live_remark["zxt-sync"] - hit_total, "global_null": live_remark[NULL_REMARK] + hit_total} return {"state": st, "by_remark": live_remark, "counts": live_counts, "exclude_hits": hits, "exclude_total": hit_total} def verify_caliber_b(engine, ev, spec) -> dict: """双轨锚点校验：白名单逐码命中 ＋ by_remark ＋ counts，三者全部须与取证件一致""" rd = caliber_b_readings(engine, spec["codes"]) hits = rd["exclude_hits"] bad = {c: n for c, n in hits.items() if n != spec["expect_hits_per_code"]} if bad: raise EvidenceError(f"排除白名单逐码命中 != {spec['expect_hits_per_code']}：{bad}（拒绝执行）") if rd["exclude_total"] != spec["total_expect"]: raise EvidenceError(f"排除白名单命中总数 {rd['exclude_total']} != total_expect " f"{spec['total_expect']}（拒绝执行，不做部分回填）") exp_anchor = ev.get(ANCHOR_KEY) if not isinstance(exp_anchor, dict): raise EvidenceError(f"取证件缺 {ANCHOR_KEY}（锚点双轨的 remark 轨缺失，拒绝执行）") counts = ev.get("counts") or {} pairs = [("by_remark_anchor.zxt-sync", exp_anchor.get("zxt-sync"), rd["by_remark"]["zxt-sync"]), (f"by_remark_anchor.{NULL_REMARK}", exp_anchor.get(NULL_REMARK), rd["by_remark"][NULL_REMARK]), ("by_remark_anchor.total", exp_anchor.get("total"), rd["by_remark"]["total"]), ("counts.e1", counts.get("e1"), rd["counts"]["e1"]), ("counts.global_null", counts.get("global_null"), rd["counts"]["global_null"])] mismatch = [(k, e, l) for k, e, l in pairs if e is None or int(e) != int(l)] if mismatch: raise EvidenceError("锚点双轨与现网不符：" + "；".join( f"{k}: evidence={e} vs live={l}" for k, e, l in mismatch)) if not rd["state"]["has_column"]: raise EvidenceError("entity_id 列未落地，先跑 --apply（本单不得重跑 DDL）") rd["spec"] = spec rd["evidence_by_remark"] = {k: exp_anchor.get(k) for k in ("zxt-sync", NULL_REMARK, "total")} rd["evidence_counts"] = {k: counts.get(k) for k in ("e1", "global_null")} return rd def print_caliber_b(rd: dict, evidence_path) -> None: spec = rd.get("spec") or {} print(f"== 口径乙校验（evidence: {evidence_path}）==") print(f" 排除白名单逐码命中：{rd['exclude_hits']}（合计 {rd['exclude_total']}；" f"expect_hits_per_code={spec.get('expect_hits_per_code')} / " f"total_expect={spec.get('total_expect')}）") print(f" 锚点轨 A by_remark（现网）：{rd['by_remark']} ｜ 取证件：{rd['evidence_by_remark']}") print(f" 锚点轨 B counts（现网）：{rd['counts']} ｜ 取证件：{rd['evidence_counts']}") def do_precheck_evidence(engine, evidence_path) -> int: """--check 附 --evidence 的只读口径乙校验（零写入）：通过 rc=0，任一不符 rc=3""" try: ev = load_evidence(evidence_path) spec = exclude_spec(ev) rd = verify_caliber_b(engine, ev, spec) except EvidenceError as exc: print(f"!!! {exc}") print("!!! fail-closed：只读预检未通过（零写入）") return 3 print_caliber_b(rd, evidence_path) print(" [OK] 白名单命中 = total_expect 且锚点双轨一致 ⇒ --backfill 前置判据满足") return 0 def state(engine) -> dict: insp = inspect(engine) tables = set(insp.get_table_names()) out = {"table_exists": TABLE in tables, "has_column": False, "indexes": [], "unique_keys": [], "counts": {}} if not out["table_exists"]: return out out["has_column"] = COLUMN in {c["name"] for c in insp.get_columns(TABLE)} out["indexes"] = sorted({i["name"] for i in insp.get_indexes(TABLE)}) out["unique_keys"] = sorted({uc["name"] for uc in insp.get_unique_constraints(TABLE)}) with engine.connect() as conn: out["counts"]["total"] = conn.execute(text(f"SELECT COUNT(*) FROM {TABLE}")).scalar() if out["has_column"]: rows = conn.execute(text( f"SELECT IFNULL({COLUMN},'NULL'), COUNT(*) FROM {TABLE} GROUP BY 1 ORDER BY 1")).fetchall() out["counts"]["by_entity"] = [[str(r[0]), int(r[1])] for r in rows] rows = conn.execute(text( f"SELECT IFNULL(remark,'(null)'), COUNT(*) FROM {TABLE} GROUP BY 1 ORDER BY 2 DESC")).fetchall() out["counts"]["by_remark"] = [[str(r[0]), int(r[1])] for r in rows] dup_code = conn.execute(text( f"SELECT COUNT(*) FROM (SELECT subject_code FROM {TABLE} GROUP BY subject_code " f"HAVING COUNT(*) > 1) t")).scalar() out["duplicate_subject_code_groups"] = int(dup_code) if out["has_column"]: dup_pair = conn.execute(text( f"SELECT COUNT(*) FROM (SELECT {COLUMN}, subject_code FROM {TABLE} " f"GROUP BY {COLUMN}, subject_code HAVING COUNT(*) > 1) t")).scalar() out["duplicate_entity_code_groups"] = int(dup_pair) return out def print_check(st: dict) -> None: print("== 只读预检（inspector / SELECT，零写入）==") print(f" table_exists={st['table_exists']} has_column={st['has_column']}") print(f" indexes={st['indexes']}") print(f" unique_keys={st['unique_keys']}") print(f" counts={st['counts']}") print(f" duplicate_subject_code_groups={st.get('duplicate_subject_code_groups')}") if "duplicate_entity_code_groups" in st: print(f" duplicate_entity_code_groups={st['duplicate_entity_code_groups']}") def do_apply(engine) -> int: st = state(engine) if not st["table_exists"]: print("!!! 目标库没有 subjects 表，拒绝 --apply") return 3 if st.get("duplicate_subject_code_groups"): print("!!! subjects.subject_code 已有重复组，先处置数据再加/换唯一键") return 3 done = [] with engine.begin() as conn: if not st["has_column"]: add_col = stmt_add_column(engine) conn.execute(text(add_col)) done.append(add_col) cur = state(engine) present = set(cur["indexes"]) | set(cur["unique_keys"]) add_idx = stmt_add_index(engine, NEW_INDEX, [COLUMN]) if NEW_INDEX not in present: conn.execute(text(add_idx)) done.append(add_idx) add_uk = stmt_add_index(engine, NEW_UNIQUE, [COLUMN, "subject_code"], unique=True) if NEW_UNIQUE not in present: conn.execute(text(add_uk)) done.append(add_uk) if OLD_UNIQUE in present: drop_old = stmt_drop_index(engine, OLD_UNIQUE) conn.execute(text(drop_old)) done.append(drop_old) print("== APPLY 完成，执行语句：") for s in done: print(" " + s) if not done: print(" （无语句执行：已是目标形态，幂等）") print_check(state(engine)) return 0 def do_backfill(engine, evidence_path: str, target: int) -> int: """口径乙回填：zxt-sync 剔除排除白名单 → target；其余（含白名单码）保持/复位 NULL""" try: ev = load_evidence(evidence_path) spec = exclude_spec(ev) rd = verify_caliber_b(engine, ev, spec) except EvidenceError as exc: print(f"!!! {exc}") print("!!! fail-closed：拒绝执行 --backfill（事务未开启，零写入、不做部分回填）") return 3 print_caliber_b(rd, evidence_path) codes = spec["codes"] ph = {f"c{i}": c for i, c in enumerate(codes)} in_clause = ", ".join(f":c{i}" for i in range(len(codes))) with engine.begin() as conn: n1 = conn.execute(text( f"UPDATE {TABLE} SET {COLUMN} = :e WHERE remark = 'zxt-sync' " f"AND subject_code NOT IN ({in_clause})"), dict(ph, e=target)).rowcount n0 = conn.execute(text( f"UPDATE {TABLE} SET {COLUMN} = NULL WHERE {COLUMN} IS NOT NULL " f"AND (remark IS NULL OR subject_code IN ({in_clause}))"), ph).rowcount print(f"== BACKFILL 完成（口径乙）：remark='zxt-sync' 且非排除白名单 → entity_id={target} " f"命中 {n1} 行；标准准则科目 ＋ 排除白名单码保持/复位 NULL {n0} 行") print_check(state(engine)) return 0 def do_down(engine, yes: bool) -> int: st = state(engine) if not st["has_column"]: print("== DOWN 跳过：entity_id 列不存在（已是回滚态）") return 0 if st.get("duplicate_subject_code_groups"): print(f"!!! 已存在同码多行（{st['duplicate_subject_code_groups']} 组）⇒ 无法重建全局唯一键 " f"uk_subject_code，拒绝 --down（fail-closed：先人工处置同码行）") return 3 if st["counts"]["by_entity"] and len(st["counts"]["by_entity"]) > 1 and not yes: print(f"!!! 列内已有分账套数据 {st['counts']['by_entity']}：回滚会丢失账套归属，须显式 --yes") return 3 with engine.begin() as conn: present = set(st["indexes"]) | set(st["unique_keys"]) if NEW_UNIQUE in present: conn.execute(text(stmt_drop_index(engine, NEW_UNIQUE))) cur = state(engine) present = set(cur["indexes"]) | set(cur["unique_keys"]) if NEW_INDEX in present: conn.execute(text(stmt_drop_index(engine, NEW_INDEX))) if OLD_UNIQUE not in present: conn.execute(text(stmt_add_index(engine, OLD_UNIQUE, ["subject_code"], unique=True))) conn.execute(text(DDL_DROP_COLUMN)) print("== DOWN 完成：重建 uk_subject_code + 删除 entity_id 列") print_check(state(engine)) return 0 def main(): ap = argparse.ArgumentParser(description="subjects 账套分层 DDL/回填") g = ap.add_mutually_exclusive_group() g.add_argument("--check", action="store_true", help="只读预检（默认）") g.add_argument("--apply", action="store_true", help="执行 DDL（幂等）") g.add_argument("--backfill", action="store_true", help="存量回填（须 --evidence）") g.add_argument("--down", action="store_true", help="回滚") ap.add_argument("--show-create", action="store_true", help="打印 SHOW CREATE TABLE") ap.add_argument("--url", default=None, help="目标库 SQLAlchemy URL（默认 backend/.env）") ap.add_argument("--allow-prod", action="store_true", help="生产库放行开关（须 CMA_DDL_APPROVED=1）") ap.add_argument("--evidence", default=None, help="--backfill 的取证件 JSON") ap.add_argument("--target-entity", type=int, default=1, help="--backfill 目标账套（默认 1=酣客）") ap.add_argument("--yes", action="store_true", help="--down 有分账套数据时的确认开关") args = ap.parse_args() if args.url: engine = create_engine(args.url, echo=False) else: from app.database import get_engine engine = get_engine() dbname = db_name_of(engine) print(f"目标库: {dbname or '(未知)'} | dialect={engine.dialect.name} | host={engine.url.host or 'socket'}") is_prod = dbname in PROD_DB_NAMES if (args.apply or args.backfill or args.down) and is_prod: if not args.allow_prod or os.getenv("CMA_DDL_APPROVED") != "1": print(f"!!! 拒绝执行：目标库是生产库 {dbname}。") print("!!! 生产 DDL/回填须 ops 审批后另派执行（本单边界：不得直连生产写入）。") print("!!! 放行方式：--allow-prod 且 export CMA_DDL_APPROVED=1（持有审批单）。") return 3 if args.apply: return do_apply(engine) if args.backfill: return do_backfill(engine, args.evidence, args.target_entity) if args.down: return do_down(engine, args.yes) st = state(engine) print_check(st) if args.evidence: rc = do_precheck_evidence(engine, args.evidence) if rc != 0: return rc if is_prod: print(" [注意] 目标是生产库：以上仅为只读预检，未做任何写入。") if args.show_create: if engine.dialect.name != "mysql": print("== SHOW CREATE TABLE 跳过（非 MySQL 目标）==") else: with engine.connect() as conn: print(conn.execute(text(f"SHOW CREATE TABLE {TABLE}")).fetchone()[1]) return 0 if __name__ == "__main__": sys.exit(main())