#!/usr/bin/env python3 # -*- coding: utf-8 -*- """ CMA 数据治理 P0-2 数据清洗脚本（2026-08-15） ================================================ 背景: CMA 系统数据治理 P0 任务, 财务Bot已拍板执行 P0-2 数据清洗。 操作清单: 1. 备份 kpi_values -> kpi_values_bak_20260815（先备份再动数据） 2. 删除异常周期数据: source_batch='986360a37b99'（26条, id 1323-1348, period 列是金额乱码: 2459.2/1021.21/1016.12/71981.55/103833.02/nan 等, 非有效期间） 3. 176条 active 且 target_value=0 的 KPI: formula_desc 追加【目标值待补充】（不删除）。 ⚠️ 任务书 SQL 写 SET remark=..., 但 kpi_definitions 表无 remark 列（已 SHOW COLUMNS 核实）; 实际落点为 formula_desc —— 经查该 176 条 formula_desc 全部非空且为说明文字 （如 EXT_071"待财务提供旧科目表核对归属"），是事实上的备注/说明字段。 4. EXT_071 (id=196, kpi_name=未命名科目(历史导入数据), entity_id=1): 历史Excel导入残留(source_batch=0720ca516636) -> 归档 status='archived', 其 kpi_values 数据保留不动 5. FB_ ×11(process维度) / BH_SALES_ ×10(finance维度) 编码保留不改（只加注释）: - FB_ = 财务Bot自身工作KPI, 涉及前端引用, 改码会破坏财务Bot链路 - BH_SALES_ = 博海销售真实数据, 涉及前端引用 - 本脚本仅输出只读清单确认未改动 约束（铁律）: - 不删除 kpi_definitions 任何行; 不删除 EXT_071 的 kpi_values - 可重复执行（幂等）: 备份表先 DROP IF EXISTS 再重建; 其余操作以 WHERE 条件天然幂等 用法: python3 clean_kpi_data_governance.py """ import pymysql def main(): conn = pymysql.connect(host="127.0.0.1", port=3306, user="cma_user", password="cma_pass_2026", database="cma", charset="utf8mb4") cur = conn.cursor(pymysql.cursors.DictCursor) def q(sql, args=None): cur.execute(sql, args) return cur.fetchall() def c(sql, args=None): return q(sql, args)[0]["c"] print("=" * 64) print("CMA 数据治理 P0-2 数据清洗 开始") print("=" * 64) # ---- 步骤1: 备份 kpi_values（先备份再动数据） ---- print("\n[步骤1] 备份 kpi_values -> kpi_values_bak_20260815") before = c("SELECT COUNT(*) AS c FROM kpi_values") print(f" 备份前 kpi_values 总行数: {before}") cur.execute("DROP TABLE IF EXISTS kpi_values_bak_20260815") cur.execute("CREATE TABLE kpi_values_bak_20260815 AS SELECT * FROM kpi_values") bak = c("SELECT COUNT(*) AS c FROM kpi_values_bak_20260815") print(f" 备份表 kpi_values_bak_20260815 行数: {bak}（应等于 {before}）") assert bak == before, "备份行数不一致, 中止!" conn.commit() # ---- 步骤2: 删除异常周期数据 ---- print("\n[步骤2] 删除异常周期数据 source_batch='986360a37b99'") rows = q("SELECT id, period FROM kpi_values WHERE source_batch='986360a37b99' ORDER BY id") print(f" 删除前命中 {len(rows)} 条", end="") if rows: print(f"（id {rows[0]['id']} ~ {rows[-1]['id']}）") for r in rows: print(f" id={r['id']} period={r['period']!r}") else: print(" —— 该批次已无数据（可能上次执行已删除）, 跳过打印") cur.execute("DELETE FROM kpi_values WHERE source_batch='986360a37b99'") remain = c("SELECT COUNT(*) AS c FROM kpi_values WHERE source_batch='986360a37b99'") total_after = c("SELECT COUNT(*) AS c FROM kpi_values") print(f" 删除后该批次剩余 {remain} 条（应为 0）") print(f" kpi_values 总行数: {before} -> {total_after}（应减少 {len(rows)}）") conn.commit() # ---- 步骤3: active 且 target_value=0 标注【目标值待补充】 ---- print("\n[步骤3] active 且 target_value=0 的 KPI 标注『目标值待补充』（落点 formula_desc）") n = c("SELECT COUNT(*) AS c FROM kpi_definitions WHERE status='active' AND target_value=0") print(f" 命中 {n} 条（任务书预期 176）") assert n == 176, f"与任务书预期 176 不符（实际 {n}）, 中止!" already = c("SELECT COUNT(*) AS c FROM kpi_definitions WHERE status='active' AND target_value=0 AND formula_desc LIKE '%【目标值待补充】%'") print(f" 已带标记 {already} 条（防重复追加）") if already == 0: cur.execute(""" UPDATE kpi_definitions SET formula_desc = CONCAT(IFNULL(NULLIF(formula_desc,''),''), '【目标值待补充】') WHERE status='active' AND target_value=0 """) print(f" UPDATE 影响行数: {cur.rowcount}") else: print(f" 已存在标记 {already} 条, 跳过追加（幂等）") marked = c("SELECT COUNT(*) AS c FROM kpi_definitions WHERE status='active' AND target_value=0 AND formula_desc LIKE '%【目标值待补充】%'") n_after = c("SELECT COUNT(*) AS c FROM kpi_definitions WHERE status='active' AND target_value=0") print(f" 标注后带标记 {marked} 条; active 且 target_value=0 仍为 {n_after} 条（只标注, 不删除）") conn.commit() # ---- 步骤4: EXT_071 归档 ---- print("\n[步骤4] EXT_071（id=196）归档 status='archived'") ext_before = q("SELECT id,kpi_code,kpi_name,entity_id,status FROM kpi_definitions WHERE id=196 AND kpi_code='EXT_071'") print(f" 归档前: {ext_before}") cur.execute("UPDATE kpi_definitions SET status='archived' WHERE id=196 AND kpi_code='EXT_071'") print(f" UPDATE 影响行数: {cur.rowcount}") ext_after = q("SELECT id,kpi_code,kpi_name,entity_id,status FROM kpi_definitions WHERE id=196 AND kpi_code='EXT_071'") print(f" 归档后: {ext_after}") kv = c("SELECT COUNT(*) AS c FROM kpi_values WHERE kpi_id=196") print(f" EXT_071 关联 kpi_values 行数（保留不动）: {kv}") # 只读引用检查（不修改; 引用表不存在则跳过） for label, sql in ( ("kpi_causality", "SELECT COUNT(*) AS c FROM kpi_causality WHERE source_kpi_id=196 OR target_kpi_id=196"), ("kpi_alerts", "SELECT COUNT(*) AS c FROM kpi_alerts WHERE kpi_id=196"), ("alert_rules", "SELECT COUNT(*) AS c FROM alert_rules WHERE kpi_id=196"), ("kpi_hierarchy", "SELECT COUNT(*) AS c FROM kpi_hierarchy WHERE parent_kpi_id=196 OR child_kpi_id=196"), ): try: print(f" 引用表 {label}: {c(sql)} 条（只读检查）") except pymysql.err.ProgrammingError as e: print(f" 引用表 {label}: 表不存在, 跳过（{e}）") conn.commit() # ---- 步骤5: FB_/BH_SALES_ 编码保留（不改） ---- print("\n[步骤5] FB_/BH_SALES_ 编码保留（不改）— 只读确认") print(" 保留原因: FB_=财务Bot自身工作KPI; BH_SALES_=博海销售真实数据; 均涉及前端引用") fb = q("SELECT kpi_code,kpi_name,entity_id,dimension,status FROM kpi_definitions WHERE kpi_code LIKE 'FB\\_%' ORDER BY kpi_code") bh = q("SELECT kpi_code,kpi_name,entity_id,dimension,status FROM kpi_definitions WHERE kpi_code LIKE 'BH\\_%' ORDER BY kpi_code") print(f" FB_ 共 {len(fb)} 条:") for r in fb: print(f" {r['kpi_code']} | {r['kpi_name']} | entity={r['entity_id']} | {r['dimension']} | {r['status']}") print(f" BH_SALES_ 共 {len(bh)} 条:") for r in bh: print(f" {r['kpi_code']} | {r['kpi_name']} | entity={r['entity_id']} | {r['dimension']} | {r['status']}") print(" 本轮未对 FB_/BH_SALES_ 做任何修改") print("\n" + "=" * 64) print("P0-2 数据清洗完成（步骤1-4 已提交; 步骤5 仅只读）") print("=" * 64) conn.close() if __name__ == "__main__": main()