Files
cma-management/backend/scripts/migrate_subjects_entity_id_20260923.py
T

435 lines
21 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""`subjects` 账套分层 DDL + 存量回填 — 2026-09-23
任务:`0-P1-project-20260923-entity-dimension-convergence-fullstack` §三③
(裁定件 `project-entity-dimension-ruling-20260923` §3.1「方案 A:分层」)
背景
----
`subjects` 207 行混装且 `subject_code` **全局唯一**(`uk_subject_code`):
· 43 行标准准则科目(1001 库存现金 …,`remark IS NULL`)= 天然全局
· 164 行 `remark='zxt-sync'` 酣客(智享通)企业明细 = 账套私有
双 token 实测两账套拿到**同一份 207 行** ⇒ 博海页面出现酣客的
`100106 酒窖备用金` / `100107 现金备用卡(唐)` / `100108 友赞商户`(结构性串配置)。
分层口径(`knowledge_articles` 同款已验模式)
------------------------------------------
`entity_id IS NULL` = 全局标准准则科目(两账套共享)
`entity_id = N` = 账套 N 私有明细
唯一键 `(subject_code)` → `(entity_id, subject_code)`
⚠️ MySQL 把 NULL 视为互不相同的索引值 ⇒ 「全局行同码重复」索引拦不住,
由写入口(`POST /api/cma/bot/subjects` UPSERT 判重)兜住,不可依赖索引。
动作(四段可独立跑,互不隐含)
----------------------------
--check 只读预检(默认):列/索引现状 + 行数分布 + 唯一键重复检查,零写入
--apply DDL:ADD COLUMN entity_id INT NULL + INDEX + UNIQUE(entity_id,subject_code)
− DROP INDEX uk_subject_code(幂等:已存在的步骤自动跳过)
--backfill 存量回填(**须 --evidence <json>**,与 DDL 分开放行;口径乙 = 2026-09-23 财务裁定):
`remark='zxt-sync'` 的 164 行**剔除排除白名单**后 → 156 行置 `--target-entity`
(默认 1);`remark IS NULL` 的 43 行 + 排除白名单 8 行保持/复位 `NULL`(合计 51)
⇒ 写后 `GROUP BY entity_id` = {1:156, NULL:51}
取证件 `exclude_subject_codes`:逐码命中须 = `expect_hits_per_code` 且总和须 =
`total_expect`,否则 rc=3(**不做部分回填**);**锚点双轨**:`by_remark_anchor`
(zxt-sync 164 / (null) 43 / total 207)与 `counts`(e1 156 / global_null 51)
并存校验,任一不符即 fail-closed rc=3
--check --evidence <json> 只读预检附口径乙校验(白名单逐码命中 + 锚点双轨读数),零写入
--down 回滚:重建 `uk_subject_code`(先断言无同码重复,fail-closed)→ 删新唯一键 → 删列
--show-create 打印 SHOW CREATE TABLE subjects 原文(验收证据)
生产保护(生产资产派单纪律)
--------------------------
目标库名 == 'cma' 时 `--apply/--backfill/--down` 一律拒绝;须 `--allow-prod` **且**
环境变量 `CMA_DDL_APPROVED=1`(= 已拿到 ops 审批单)才放行。`--check/--show-create` 只读不受限。
用法
----
venv/bin/python3 scripts/migrate_subjects_entity_id_20260923.py --check
CMA_DDL_APPROVED=1 venv/bin/python3 scripts/migrate_subjects_entity_id_20260923.py \\
--apply --allow-prod
CMA_DDL_APPROVED=1 venv/bin/python3 scripts/migrate_subjects_entity_id_20260923.py \\
--backfill --allow-prod --evidence /path/to/evidence.json --target-entity 1
"""
from __future__ import annotations
import argparse
import json
import os
import sys
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
from sqlalchemy import create_engine, inspect, text # noqa: E402
TABLE = "subjects"
COLUMN = "entity_id"
NEW_UNIQUE = "uk_entity_subject"
OLD_UNIQUE = "uk_subject_code"
NEW_INDEX = "ix_subjects_entity_id"
PROD_DB_NAMES = ("cma",)
DDL_ADD_COLUMN = (
f"ALTER TABLE {TABLE} ADD COLUMN {COLUMN} INT NULL "
"COMMENT '企业ID(NULL=全局标准准则科目 / N=账套私有明细)' AFTER id"
)
DDL_ADD_UNIQUE = f"ALTER TABLE {TABLE} ADD UNIQUE INDEX {NEW_UNIQUE} ({COLUMN}, subject_code)"
DDL_DROP_OLD_UNIQUE = f"ALTER TABLE {TABLE} DROP INDEX {OLD_UNIQUE}"
DDL_ADD_OLD_UNIQUE = f"ALTER TABLE {TABLE} ADD UNIQUE INDEX {OLD_UNIQUE} (subject_code)"
DDL_DROP_COLUMN = f"ALTER TABLE {TABLE} DROP COLUMN {COLUMN}"
def is_sqlite(engine) -> bool:
"""沙箱演练(SQLite)与生产(MySQL)的 DDL 方言差异:索引/列定义语句形态不同"""
return engine.dialect.name == "sqlite"
def stmt_add_column(engine) -> str:
"""生产形态 = MySQL(带 COMMENT / AFTER);沙箱 = SQLite 精简形态(语法不支持二者)"""
if is_sqlite(engine):
return f"ALTER TABLE {TABLE} ADD COLUMN {COLUMN} INT NULL"
return DDL_ADD_COLUMN
def stmt_add_index(engine, name: str, cols: list, unique: bool = False) -> str:
cols_s = ", ".join(cols)
if is_sqlite(engine):
return f"CREATE {'UNIQUE ' if unique else ''}INDEX {name} ON {TABLE} ({cols_s})"
return f"ALTER TABLE {TABLE} ADD {'UNIQUE ' if unique else ''}INDEX {name} ({cols_s})"
def stmt_drop_index(engine, name: str) -> str:
return f"DROP INDEX {name}" if is_sqlite(engine) else f"ALTER TABLE {TABLE} DROP INDEX {name}"
def db_name_of(engine) -> str:
try:
return engine.url.database or ""
except Exception:
return ""
# ============================================================
# 口径乙(2026-09-23 财务裁定,wecom-finance)
# · e1(酣客私有)= remark='zxt-sync' 行 **剔除排除白名单** 后的行数
# · global_null = remark IS NULL 行 + 排除白名单行
# · by_remark_anchor 与 counts 两轨并存,须同时与取证件一致(fail-closed)
# ============================================================
EXCLUDE_KEY = "exclude_subject_codes"
ANCHOR_KEY = "by_remark_anchor"
NULL_REMARK = "(null)"
class EvidenceError(Exception):
"""取证件/锚点校验失败 ⇒ fail-closed(rc=3:零写入、不做部分回填)"""
def load_evidence(path) -> dict:
"""读取证件 JSON 并做结构校验(缺件/缺字段 ⇒ EvidenceError)"""
if not path or not os.path.exists(path):
raise EvidenceError(f"取证件不存在:{path!r}(--backfill 必须带 --evidence <json>)")
try:
with open(path, encoding="utf-8") as fh:
ev = json.load(fh)
except Exception as exc: # noqa: BLE001
raise EvidenceError(f"取证件解析失败:{exc}") from exc
missing = [k for k in ("dama_evidence", "finance_confirmation", "counts") if not ev.get(k)]
if missing:
raise EvidenceError(f"取证件缺字段 {missing}")
return ev
def exclude_spec(ev: dict) -> dict:
"""口径乙排除白名单规格(字段名以取证件 JSON 为准)"""
spec = ev.get(EXCLUDE_KEY)
if not isinstance(spec, dict):
raise EvidenceError(f"取证件缺 {EXCLUDE_KEY} 段(无法确定白名单,拒绝执行)")
codes = [str(c).strip() for c in (spec.get("codes") or [])]
per = spec.get("expect_hits_per_code")
total = spec.get("total_expect")
if not codes:
raise EvidenceError(f"{EXCLUDE_KEY}.codes 为空,拒绝执行")
if not spec.get("fail_closed"):
raise EvidenceError(f"{EXCLUDE_KEY}.fail_closed 缺失(非 fail-closed 形态,拒绝执行)")
if len(set(codes)) != len(codes):
raise EvidenceError(f"{EXCLUDE_KEY}.codes 存在重复:{codes}")
if per != 1:
raise EvidenceError(f"expect_hits_per_code={per!r} != 1,拒绝执行")
if total is None or int(total) != len(codes):
raise EvidenceError(f"total_expect={total!r} != 白名单码数 {len(codes)},拒绝执行")
return {"codes": codes, "expect_hits_per_code": 1, "total_expect": int(total)}
def probe_exclude_hits(engine, codes) -> dict:
"""逐码命中数(只读):subject_code 精确匹配行数"""
hits = {}
with engine.connect() as conn:
for code in codes:
hits[code] = int(conn.execute(
text(f"SELECT COUNT(*) FROM {TABLE} WHERE subject_code = :c"), {"c": code}).scalar())
return hits
def caliber_b_readings(engine, codes) -> dict:
"""现网口径乙读数(只读):by_remark 实读 + e1/global_null 推算 + 白名单逐码命中"""
st = state(engine)
by_remark = dict(st["counts"].get("by_remark") or {})
live_remark = {"zxt-sync": int(by_remark.get("zxt-sync", 0)),
NULL_REMARK: int(by_remark.get(NULL_REMARK, 0)),
"total": int(st["counts"].get("total") or 0)}
hits = probe_exclude_hits(engine, codes)
hit_total = sum(hits.values())
live_counts = {"e1": live_remark["zxt-sync"] - hit_total,
"global_null": live_remark[NULL_REMARK] + hit_total}
return {"state": st, "by_remark": live_remark, "counts": live_counts,
"exclude_hits": hits, "exclude_total": hit_total}
def verify_caliber_b(engine, ev, spec) -> dict:
"""双轨锚点校验:白名单逐码命中 + by_remark + counts,三者全部须与取证件一致"""
rd = caliber_b_readings(engine, spec["codes"])
hits = rd["exclude_hits"]
bad = {c: n for c, n in hits.items() if n != spec["expect_hits_per_code"]}
if bad:
raise EvidenceError(f"排除白名单逐码命中 != {spec['expect_hits_per_code']}:{bad}(拒绝执行)")
if rd["exclude_total"] != spec["total_expect"]:
raise EvidenceError(f"排除白名单命中总数 {rd['exclude_total']} != total_expect "
f"{spec['total_expect']}(拒绝执行,不做部分回填)")
exp_anchor = ev.get(ANCHOR_KEY)
if not isinstance(exp_anchor, dict):
raise EvidenceError(f"取证件缺 {ANCHOR_KEY}(锚点双轨的 remark 轨缺失,拒绝执行)")
counts = ev.get("counts") or {}
pairs = [("by_remark_anchor.zxt-sync", exp_anchor.get("zxt-sync"), rd["by_remark"]["zxt-sync"]),
(f"by_remark_anchor.{NULL_REMARK}", exp_anchor.get(NULL_REMARK), rd["by_remark"][NULL_REMARK]),
("by_remark_anchor.total", exp_anchor.get("total"), rd["by_remark"]["total"]),
("counts.e1", counts.get("e1"), rd["counts"]["e1"]),
("counts.global_null", counts.get("global_null"), rd["counts"]["global_null"])]
mismatch = [(k, e, l) for k, e, l in pairs if e is None or int(e) != int(l)]
if mismatch:
raise EvidenceError("锚点双轨与现网不符:" + ";".join(
f"{k}: evidence={e} vs live={l}" for k, e, l in mismatch))
if not rd["state"]["has_column"]:
raise EvidenceError("entity_id 列未落地,先跑 --apply(本单不得重跑 DDL)")
rd["spec"] = spec
rd["evidence_by_remark"] = {k: exp_anchor.get(k) for k in ("zxt-sync", NULL_REMARK, "total")}
rd["evidence_counts"] = {k: counts.get(k) for k in ("e1", "global_null")}
return rd
def print_caliber_b(rd: dict, evidence_path) -> None:
spec = rd.get("spec") or {}
print(f"== 口径乙校验(evidence: {evidence_path})==")
print(f" 排除白名单逐码命中:{rd['exclude_hits']}(合计 {rd['exclude_total']};"
f"expect_hits_per_code={spec.get('expect_hits_per_code')} / "
f"total_expect={spec.get('total_expect')})")
print(f" 锚点轨 A by_remark(现网):{rd['by_remark']} | 取证件:{rd['evidence_by_remark']}")
print(f" 锚点轨 B counts(现网):{rd['counts']} | 取证件:{rd['evidence_counts']}")
def do_precheck_evidence(engine, evidence_path) -> int:
"""--check 附 --evidence 的只读口径乙校验(零写入):通过 rc=0,任一不符 rc=3"""
try:
ev = load_evidence(evidence_path)
spec = exclude_spec(ev)
rd = verify_caliber_b(engine, ev, spec)
except EvidenceError as exc:
print(f"!!! {exc}")
print("!!! fail-closed:只读预检未通过(零写入)")
return 3
print_caliber_b(rd, evidence_path)
print(" [OK] 白名单命中 = total_expect 且锚点双轨一致 ⇒ --backfill 前置判据满足")
return 0
def state(engine) -> dict:
insp = inspect(engine)
tables = set(insp.get_table_names())
out = {"table_exists": TABLE in tables, "has_column": False,
"indexes": [], "unique_keys": [], "counts": {}}
if not out["table_exists"]:
return out
out["has_column"] = COLUMN in {c["name"] for c in insp.get_columns(TABLE)}
out["indexes"] = sorted({i["name"] for i in insp.get_indexes(TABLE)})
out["unique_keys"] = sorted({uc["name"] for uc in insp.get_unique_constraints(TABLE)})
with engine.connect() as conn:
out["counts"]["total"] = conn.execute(text(f"SELECT COUNT(*) FROM {TABLE}")).scalar()
if out["has_column"]:
rows = conn.execute(text(
f"SELECT IFNULL({COLUMN},'NULL'), COUNT(*) FROM {TABLE} GROUP BY 1 ORDER BY 1")).fetchall()
out["counts"]["by_entity"] = [[str(r[0]), int(r[1])] for r in rows]
rows = conn.execute(text(
f"SELECT IFNULL(remark,'(null)'), COUNT(*) FROM {TABLE} GROUP BY 1 ORDER BY 2 DESC")).fetchall()
out["counts"]["by_remark"] = [[str(r[0]), int(r[1])] for r in rows]
dup_code = conn.execute(text(
f"SELECT COUNT(*) FROM (SELECT subject_code FROM {TABLE} GROUP BY subject_code "
f"HAVING COUNT(*) > 1) t")).scalar()
out["duplicate_subject_code_groups"] = int(dup_code)
if out["has_column"]:
dup_pair = conn.execute(text(
f"SELECT COUNT(*) FROM (SELECT {COLUMN}, subject_code FROM {TABLE} "
f"GROUP BY {COLUMN}, subject_code HAVING COUNT(*) > 1) t")).scalar()
out["duplicate_entity_code_groups"] = int(dup_pair)
return out
def print_check(st: dict) -> None:
print("== 只读预检(inspector / SELECT,零写入)==")
print(f" table_exists={st['table_exists']} has_column={st['has_column']}")
print(f" indexes={st['indexes']}")
print(f" unique_keys={st['unique_keys']}")
print(f" counts={st['counts']}")
print(f" duplicate_subject_code_groups={st.get('duplicate_subject_code_groups')}")
if "duplicate_entity_code_groups" in st:
print(f" duplicate_entity_code_groups={st['duplicate_entity_code_groups']}")
def do_apply(engine) -> int:
st = state(engine)
if not st["table_exists"]:
print("!!! 目标库没有 subjects 表,拒绝 --apply")
return 3
if st.get("duplicate_subject_code_groups"):
print("!!! subjects.subject_code 已有重复组,先处置数据再加/换唯一键")
return 3
done = []
with engine.begin() as conn:
if not st["has_column"]:
add_col = stmt_add_column(engine)
conn.execute(text(add_col))
done.append(add_col)
cur = state(engine)
present = set(cur["indexes"]) | set(cur["unique_keys"])
add_idx = stmt_add_index(engine, NEW_INDEX, [COLUMN])
if NEW_INDEX not in present:
conn.execute(text(add_idx))
done.append(add_idx)
add_uk = stmt_add_index(engine, NEW_UNIQUE, [COLUMN, "subject_code"], unique=True)
if NEW_UNIQUE not in present:
conn.execute(text(add_uk))
done.append(add_uk)
if OLD_UNIQUE in present:
drop_old = stmt_drop_index(engine, OLD_UNIQUE)
conn.execute(text(drop_old))
done.append(drop_old)
print("== APPLY 完成,执行语句:")
for s in done:
print(" " + s)
if not done:
print(" (无语句执行:已是目标形态,幂等)")
print_check(state(engine))
return 0
def do_backfill(engine, evidence_path: str, target: int) -> int:
"""口径乙回填:zxt-sync 剔除排除白名单 → target;其余(含白名单码)保持/复位 NULL"""
try:
ev = load_evidence(evidence_path)
spec = exclude_spec(ev)
rd = verify_caliber_b(engine, ev, spec)
except EvidenceError as exc:
print(f"!!! {exc}")
print("!!! fail-closed:拒绝执行 --backfill(事务未开启,零写入、不做部分回填)")
return 3
print_caliber_b(rd, evidence_path)
codes = spec["codes"]
ph = {f"c{i}": c for i, c in enumerate(codes)}
in_clause = ", ".join(f":c{i}" for i in range(len(codes)))
with engine.begin() as conn:
n1 = conn.execute(text(
f"UPDATE {TABLE} SET {COLUMN} = :e WHERE remark = 'zxt-sync' "
f"AND subject_code NOT IN ({in_clause})"), dict(ph, e=target)).rowcount
n0 = conn.execute(text(
f"UPDATE {TABLE} SET {COLUMN} = NULL WHERE {COLUMN} IS NOT NULL "
f"AND (remark IS NULL OR subject_code IN ({in_clause}))"), ph).rowcount
print(f"== BACKFILL 完成(口径乙):remark='zxt-sync' 且非排除白名单 → entity_id={target} "
f"命中 {n1} 行;标准准则科目 + 排除白名单码保持/复位 NULL {n0} 行")
print_check(state(engine))
return 0
def do_down(engine, yes: bool) -> int:
st = state(engine)
if not st["has_column"]:
print("== DOWN 跳过:entity_id 列不存在(已是回滚态)")
return 0
if st.get("duplicate_subject_code_groups"):
print(f"!!! 已存在同码多行({st['duplicate_subject_code_groups']} 组)⇒ 无法重建全局唯一键 "
f"uk_subject_code,拒绝 --down(fail-closed:先人工处置同码行)")
return 3
if st["counts"]["by_entity"] and len(st["counts"]["by_entity"]) > 1 and not yes:
print(f"!!! 列内已有分账套数据 {st['counts']['by_entity']}:回滚会丢失账套归属,须显式 --yes")
return 3
with engine.begin() as conn:
present = set(st["indexes"]) | set(st["unique_keys"])
if NEW_UNIQUE in present:
conn.execute(text(stmt_drop_index(engine, NEW_UNIQUE)))
cur = state(engine)
present = set(cur["indexes"]) | set(cur["unique_keys"])
if NEW_INDEX in present:
conn.execute(text(stmt_drop_index(engine, NEW_INDEX)))
if OLD_UNIQUE not in present:
conn.execute(text(stmt_add_index(engine, OLD_UNIQUE, ["subject_code"], unique=True)))
conn.execute(text(DDL_DROP_COLUMN))
print("== DOWN 完成:重建 uk_subject_code + 删除 entity_id 列")
print_check(state(engine))
return 0
def main():
ap = argparse.ArgumentParser(description="subjects 账套分层 DDL/回填")
g = ap.add_mutually_exclusive_group()
g.add_argument("--check", action="store_true", help="只读预检(默认)")
g.add_argument("--apply", action="store_true", help="执行 DDL(幂等)")
g.add_argument("--backfill", action="store_true", help="存量回填(须 --evidence)")
g.add_argument("--down", action="store_true", help="回滚")
ap.add_argument("--show-create", action="store_true", help="打印 SHOW CREATE TABLE")
ap.add_argument("--url", default=None, help="目标库 SQLAlchemy URL(默认 backend/.env)")
ap.add_argument("--allow-prod", action="store_true", help="生产库放行开关(须 CMA_DDL_APPROVED=1)")
ap.add_argument("--evidence", default=None, help="--backfill 的取证件 JSON")
ap.add_argument("--target-entity", type=int, default=1, help="--backfill 目标账套(默认 1=酣客)")
ap.add_argument("--yes", action="store_true", help="--down 有分账套数据时的确认开关")
args = ap.parse_args()
if args.url:
engine = create_engine(args.url, echo=False)
else:
from app.database import get_engine
engine = get_engine()
dbname = db_name_of(engine)
print(f"目标库: {dbname or '(未知)'} | dialect={engine.dialect.name} | host={engine.url.host or 'socket'}")
is_prod = dbname in PROD_DB_NAMES
if (args.apply or args.backfill or args.down) and is_prod:
if not args.allow_prod or os.getenv("CMA_DDL_APPROVED") != "1":
print(f"!!! 拒绝执行:目标库是生产库 {dbname}。")
print("!!! 生产 DDL/回填须 ops 审批后另派执行(本单边界:不得直连生产写入)。")
print("!!! 放行方式:--allow-prod 且 export CMA_DDL_APPROVED=1(持有审批单)。")
return 3
if args.apply:
return do_apply(engine)
if args.backfill:
return do_backfill(engine, args.evidence, args.target_entity)
if args.down:
return do_down(engine, args.yes)
st = state(engine)
print_check(st)
if args.evidence:
rc = do_precheck_evidence(engine, args.evidence)
if rc != 0:
return rc
if is_prod:
print(" [注意] 目标是生产库:以上仅为只读预检,未做任何写入。")
if args.show_create:
if engine.dialect.name != "mysql":
print("== SHOW CREATE TABLE 跳过(非 MySQL 目标)==")
else:
with engine.connect() as conn:
print(conn.execute(text(f"SHOW CREATE TABLE {TABLE}")).fetchone()[1])
return 0
if __name__ == "__main__":
sys.exit(main())