142 lines
7.6 KiB
Python
142 lines
7.6 KiB
Python
#!/usr/bin/env python3
|
||
# -*- coding: utf-8 -*-
|
||
"""
|
||
CMA 数据治理 P0-2 数据清洗脚本（2026-08-15）
|
||
================================================
|
||
背景: CMA 系统数据治理 P0 任务, 财务Bot已拍板执行 P0-2 数据清洗。
|
||
|
||
操作清单:
|
||
1. 备份 kpi_values -> kpi_values_bak_20260815（先备份再动数据）
|
||
2. 删除异常周期数据: source_batch='986360a37b99'（26条, id 1323-1348,
|
||
period 列是金额乱码: 2459.2/1021.21/1016.12/71981.55/103833.02/nan 等, 非有效期间）
|
||
3. 176条 active 且 target_value=0 的 KPI: formula_desc 追加【目标值待补充】（不删除）。
|
||
⚠️ 任务书 SQL 写 SET remark=..., 但 kpi_definitions 表无 remark 列（已 SHOW COLUMNS 核实）;
|
||
实际落点为 formula_desc —— 经查该 176 条 formula_desc 全部非空且为说明文字
|
||
（如 EXT_071"待财务提供旧科目表核对归属"），是事实上的备注/说明字段。
|
||
4. EXT_071 (id=196, kpi_name=未命名科目(历史导入数据), entity_id=1):
|
||
历史Excel导入残留(source_batch=0720ca516636) -> 归档 status='archived',
|
||
其 kpi_values 数据保留不动
|
||
5. FB_ ×11(process维度) / BH_SALES_ ×10(finance维度) 编码保留不改（只加注释）:
|
||
- FB_ = 财务Bot自身工作KPI, 涉及前端引用, 改码会破坏财务Bot链路
|
||
- BH_SALES_ = 博海销售真实数据, 涉及前端引用
|
||
- 本脚本仅输出只读清单确认未改动
|
||
|
||
约束（铁律）:
|
||
- 不删除 kpi_definitions 任何行; 不删除 EXT_071 的 kpi_values
|
||
- 可重复执行（幂等）: 备份表先 DROP IF EXISTS 再重建; 其余操作以 WHERE 条件天然幂等
|
||
|
||
用法: python3 clean_kpi_data_governance.py
|
||
"""
|
||
import pymysql
|
||
|
||
|
||
def main():
|
||
conn = pymysql.connect(host="127.0.0.1", port=3306, user="cma_user",
|
||
password="cma_pass_2026", database="cma", charset="utf8mb4")
|
||
cur = conn.cursor(pymysql.cursors.DictCursor)
|
||
|
||
def q(sql, args=None):
|
||
cur.execute(sql, args)
|
||
return cur.fetchall()
|
||
|
||
def c(sql, args=None):
|
||
return q(sql, args)[0]["c"]
|
||
|
||
print("=" * 64)
|
||
print("CMA 数据治理 P0-2 数据清洗 开始")
|
||
print("=" * 64)
|
||
|
||
# ---- 步骤1: 备份 kpi_values（先备份再动数据） ----
|
||
print("\n[步骤1] 备份 kpi_values -> kpi_values_bak_20260815")
|
||
before = c("SELECT COUNT(*) AS c FROM kpi_values")
|
||
print(f" 备份前 kpi_values 总行数: {before}")
|
||
cur.execute("DROP TABLE IF EXISTS kpi_values_bak_20260815")
|
||
cur.execute("CREATE TABLE kpi_values_bak_20260815 AS SELECT * FROM kpi_values")
|
||
bak = c("SELECT COUNT(*) AS c FROM kpi_values_bak_20260815")
|
||
print(f" 备份表 kpi_values_bak_20260815 行数: {bak}（应等于 {before}）")
|
||
assert bak == before, "备份行数不一致, 中止!"
|
||
conn.commit()
|
||
|
||
# ---- 步骤2: 删除异常周期数据 ----
|
||
print("\n[步骤2] 删除异常周期数据 source_batch='986360a37b99'")
|
||
rows = q("SELECT id, period FROM kpi_values WHERE source_batch='986360a37b99' ORDER BY id")
|
||
print(f" 删除前命中 {len(rows)} 条", end="")
|
||
if rows:
|
||
print(f"（id {rows[0]['id']} ~ {rows[-1]['id']}）")
|
||
for r in rows:
|
||
print(f" id={r['id']} period={r['period']!r}")
|
||
else:
|
||
print(" —— 该批次已无数据（可能上次执行已删除）, 跳过打印")
|
||
cur.execute("DELETE FROM kpi_values WHERE source_batch='986360a37b99'")
|
||
remain = c("SELECT COUNT(*) AS c FROM kpi_values WHERE source_batch='986360a37b99'")
|
||
total_after = c("SELECT COUNT(*) AS c FROM kpi_values")
|
||
print(f" 删除后该批次剩余 {remain} 条（应为 0）")
|
||
print(f" kpi_values 总行数: {before} -> {total_after}（应减少 {len(rows)}）")
|
||
conn.commit()
|
||
|
||
# ---- 步骤3: active 且 target_value=0 标注【目标值待补充】 ----
|
||
print("\n[步骤3] active 且 target_value=0 的 KPI 标注『目标值待补充』（落点 formula_desc）")
|
||
n = c("SELECT COUNT(*) AS c FROM kpi_definitions WHERE status='active' AND target_value=0")
|
||
print(f" 命中 {n} 条（任务书预期 176）")
|
||
assert n == 176, f"与任务书预期 176 不符（实际 {n}）, 中止!"
|
||
already = c("SELECT COUNT(*) AS c FROM kpi_definitions WHERE status='active' AND target_value=0 AND formula_desc LIKE '%【目标值待补充】%'")
|
||
print(f" 已带标记 {already} 条（防重复追加）")
|
||
if already == 0:
|
||
cur.execute("""
|
||
UPDATE kpi_definitions
|
||
SET formula_desc = CONCAT(IFNULL(NULLIF(formula_desc,''),''), '【目标值待补充】')
|
||
WHERE status='active' AND target_value=0
|
||
""")
|
||
print(f" UPDATE 影响行数: {cur.rowcount}")
|
||
else:
|
||
print(f" 已存在标记 {already} 条, 跳过追加（幂等）")
|
||
marked = c("SELECT COUNT(*) AS c FROM kpi_definitions WHERE status='active' AND target_value=0 AND formula_desc LIKE '%【目标值待补充】%'")
|
||
n_after = c("SELECT COUNT(*) AS c FROM kpi_definitions WHERE status='active' AND target_value=0")
|
||
print(f" 标注后带标记 {marked} 条; active 且 target_value=0 仍为 {n_after} 条（只标注, 不删除）")
|
||
conn.commit()
|
||
|
||
# ---- 步骤4: EXT_071 归档 ----
|
||
print("\n[步骤4] EXT_071（id=196）归档 status='archived'")
|
||
ext_before = q("SELECT id,kpi_code,kpi_name,entity_id,status FROM kpi_definitions WHERE id=196 AND kpi_code='EXT_071'")
|
||
print(f" 归档前: {ext_before}")
|
||
cur.execute("UPDATE kpi_definitions SET status='archived' WHERE id=196 AND kpi_code='EXT_071'")
|
||
print(f" UPDATE 影响行数: {cur.rowcount}")
|
||
ext_after = q("SELECT id,kpi_code,kpi_name,entity_id,status FROM kpi_definitions WHERE id=196 AND kpi_code='EXT_071'")
|
||
print(f" 归档后: {ext_after}")
|
||
kv = c("SELECT COUNT(*) AS c FROM kpi_values WHERE kpi_id=196")
|
||
print(f" EXT_071 关联 kpi_values 行数（保留不动）: {kv}")
|
||
# 只读引用检查（不修改; 引用表不存在则跳过）
|
||
for label, sql in (
|
||
("kpi_causality", "SELECT COUNT(*) AS c FROM kpi_causality WHERE source_kpi_id=196 OR target_kpi_id=196"),
|
||
("kpi_alerts", "SELECT COUNT(*) AS c FROM kpi_alerts WHERE kpi_id=196"),
|
||
("alert_rules", "SELECT COUNT(*) AS c FROM alert_rules WHERE kpi_id=196"),
|
||
("kpi_hierarchy", "SELECT COUNT(*) AS c FROM kpi_hierarchy WHERE parent_kpi_id=196 OR child_kpi_id=196"),
|
||
):
|
||
try:
|
||
print(f" 引用表 {label}: {c(sql)} 条（只读检查）")
|
||
except pymysql.err.ProgrammingError as e:
|
||
print(f" 引用表 {label}: 表不存在, 跳过（{e}）")
|
||
conn.commit()
|
||
|
||
# ---- 步骤5: FB_/BH_SALES_ 编码保留（不改） ----
|
||
print("\n[步骤5] FB_/BH_SALES_ 编码保留（不改）— 只读确认")
|
||
print(" 保留原因: FB_=财务Bot自身工作KPI; BH_SALES_=博海销售真实数据; 均涉及前端引用")
|
||
fb = q("SELECT kpi_code,kpi_name,entity_id,dimension,status FROM kpi_definitions WHERE kpi_code LIKE 'FB\\_%' ORDER BY kpi_code")
|
||
bh = q("SELECT kpi_code,kpi_name,entity_id,dimension,status FROM kpi_definitions WHERE kpi_code LIKE 'BH\\_%' ORDER BY kpi_code")
|
||
print(f" FB_ 共 {len(fb)} 条:")
|
||
for r in fb:
|
||
print(f" {r['kpi_code']} | {r['kpi_name']} | entity={r['entity_id']} | {r['dimension']} | {r['status']}")
|
||
print(f" BH_SALES_ 共 {len(bh)} 条:")
|
||
for r in bh:
|
||
print(f" {r['kpi_code']} | {r['kpi_name']} | entity={r['entity_id']} | {r['dimension']} | {r['status']}")
|
||
print(" 本轮未对 FB_/BH_SALES_ 做任何修改")
|
||
|
||
print("\n" + "=" * 64)
|
||
print("P0-2 数据清洗完成（步骤1-4 已提交; 步骤5 仅只读）")
|
||
print("=" * 64)
|
||
conn.close()
|
||
|
||
|
||
if __name__ == "__main__":
|
||
main()
|