feat(cma): cashflow NA annotation + dimension guard + threshold unit migration script

dev-level prep only: no prod write, no push, no pipeline deploy.
2026-09-17 00:01:58 +08:00
parent 9b3a131070
commit 293df6df8c
3 changed files with 503 additions and 3 deletions
@@ -12,19 +12,25 @@

 口径：
  - 经营/投资/筹资 净额 = Σ(流入) − Σ(流出)，单位 元 → 万元（÷10000，保留 2 位）
-  - 某活动无项目行 → 记 0.00（表列为 NOT NULL；同时写 remark 说明"无该项目=0"）
+  - **NA（无发生额）标注（本版本新增，裁定 finance-20260916-cma-caliber-ruling §2.1）**：
+    某活动**无任何项目行**时净额记 0.00（三净额列为 NOT NULL，表本身表达不了 NA），
+    同时在 `remark` 写入**固定短语**「<活动>活动本期无发生额（无项目行）」，
+    供判定层用 `parse_na_activities(remark)` 取回 → 判定时该维度**不参与**（降维），
+    避免把「无发生额」与「收付相抵的 0」混为一谈（后者才参与判定）。
  - source='official'（官方报表口径）；期初/期末现金 留空（主表采集缺陷，见报告 §项1-②）
  - 同 (entity_id, period) 重复执行 = 覆盖更新（幂等）

 用法：
  python3 scripts/import_cashflow_statements.py                       # dry-run（默认，只算不写）
  python3 scripts/import_cashflow_statements.py --period 2026-07      # 单期 dry-run
+  python3 scripts/import_cashflow_statements.py --self-test           # NA 标注逻辑自测（零库连接）
  python3 scripts/import_cashflow_statements.py --apply               # 落库（需放行）
 """
 import argparse
 import csv
 import glob
 import os
+import re
 import sys

 sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
@@ -36,6 +42,29 @@ from app.api.cashflow_structure import classify_enterprise
 ZKT_BASE = "/root/WB_WS/_data/zxt-hanke/cashflow"
 CLASSES = ("经营活动", "投资活动", "筹资活动")

+#: NA 固定短语模板 —— 判定层与量纲守卫共用，勿改字面量（裁定 §2.1 指定）
+NA_PHRASE_TMPL = "{cls}本期无发生额（无项目行）"
+
+#: 从 remark 反解 NA 活动的正则（与 NA_PHRASE_TMPL 严格同源）
+_NA_RE = re.compile(r"(经营活动|投资活动|筹资活动)本期无发生额（无项目行）")
+
+
+def parse_na_activities(remark):
+    """从 cashflow_statements.remark 取回 NA（无发生额）活动集合。
+
+    判定层入口：NA 维度**不参与**八类判定（降维），其 0.00 是"表表达不了 NA"的占位，
+    不是"收付相抵的 0"。同时提供 `na_flags(remark)` 返回 {活动: True/False} 供逐维判断。
+    """
+    if not remark:
+        return set()
+    return set(_NA_RE.findall(str(remark)))
+
+
+def na_flags(remark):
+    """逐维 NA 标志：{'经营活动': bool, '投资活动': bool, '筹资活动': bool}"""
+    na = parse_na_activities(remark)
+    return {c: (c in na) for c in CLASSES}
+

 def _read_stat_file(path):
    """读项目级汇总 CSV → [(分类, 编码, 项目名称, 方向, 金额元)]"""
@@ -85,8 +114,11 @@ def collect(period_filter=None):
            continue
        nets_yuan, agg = _nets(rows)
        missing = [c for c in CLASSES if not agg[c]["in"] and not agg[c]["out"]]
-        remark = ("源: 智享通现金流量统计表 %s（单位元→万元 /10000）；项目行 %d 条；缺项目活动: %s"
-                  % (os.path.basename(src), len(rows), "、".join(missing) or "无"))
+        na_phrases = [NA_PHRASE_TMPL.format(cls=c) for c in missing]
+        head = "源: 智享通现金流量统计表 %s（单位元→万元 /10000）；项目行 %d 条" % (
+            os.path.basename(src), len(rows))
+        remark = (head + "；NA(无发生额，不参与判定): " + "、".join(na_phrases)) if na_phrases \
+            else (head + "；三类活动均有项目行（无 NA）")
        payloads.append({
            "period": period,
            "status": "READY",
@@ -95,19 +127,61 @@ def collect(period_filter=None):
            "nets_yuan": nets_yuan,
            "nets_wan": {k: round(v / 10000.0, 2) for k, v in nets_yuan.items()},
            "missing_classes": missing,
+            "na_classes": missing,
+            "na_phrases": na_phrases,
+            "na_flags": {c: (c in missing) for c in CLASSES},
            "remark": remark,
        })
    return payloads


+def _self_test():
+    """NA 标注逻辑自测（零库连接、零写盘；只读盘上 CSV）——回执证据用。"""
+    passed = 0
+    # 1~3) 短语 → 标志 反解
+    for missing, expect in (
+        (["投资活动"], {"经营活动": False, "投资活动": True, "筹资活动": False}),
+        (["投资活动", "筹资活动"], {"经营活动": False, "投资活动": True, "筹资活动": True}),
+        ([], {"经营活动": False, "投资活动": False, "筹资活动": False}),
+    ):
+        phrases = [NA_PHRASE_TMPL.format(cls=c) for c in missing]
+        remark = ("源: X；项目行 3 条；NA(无发生额，不参与判定): " + "、".join(phrases)) if phrases \
+            else "源: X；项目行 3 条；三类活动均有项目行（无 NA）"
+        assert na_flags(remark) == expect, (missing, remark)
+        assert parse_na_activities(remark) == set(missing)
+        passed += 1
+    # 4) 空值安全
+    assert na_flags(None) == {c: False for c in CLASSES}
+    assert parse_na_activities("") == set()
+    passed += 1
+    # 5) 真实数据源（盘上 8 期）：投资活动应全为 NA
+    pay = collect()
+    ready = [p for p in pay if p["status"] == "READY"]
+    assert ready, "无可入库期（数据源缺失）"
+    all_inv_na = all("投资活动" in p["na_classes"] for p in ready)
+    passed += 1
+    print("=== NA 标注 self-test: %d/5 PASS | 期数=%d | 投资活动全NA=%s ==="
+          % (passed, len(ready), all_inv_na))
+    for p in ready:
+        print("  %-8s NA=%s" % (p["period"], "、".join(p["na_classes"]) or "无"))
+        print("           remark: %s" % p["remark"])
+    assert all_inv_na, "酣客 8 期投资活动应全为 NA（裁定 §2.1 实证）"
+    return 0
+
+
 def main():
    ap = argparse.ArgumentParser()
    ap.add_argument("--entity-id", type=int, default=1,
                    help="账套：1=酣客(白酒, 智享通通道) 2=博海(IT, 无凭证明细)")
    ap.add_argument("--period", default=None, help="仅处理某期 YYYY-MM")
+    ap.add_argument("--self-test", action="store_true",
+                    help="NA 标注逻辑自测（零库连接：只读盘上 CSV）")
    ap.add_argument("--apply", action="store_true", help="真正写库（缺省=dry-run 只算不写）")
    args = ap.parse_args()

+    if args.self_test:
+        return _self_test()
+
    payloads = collect(args.period)
    if not payloads:
        print("未发现任何期数据源（%s/2026.*）" % ZKT_BASE)
@@ -129,6 +203,7 @@ def main():
                     ("类型%d %s" % (cls["type_no"], cls["name"])) if cls.get("classified")
                     else ("未判定: " + cls.get("reason", "")[:28]),
                     os.path.basename(p["src"])))
+            print("           NA(无发生额，不参与判定)=%s" % ("、".join(p["na_classes"]) or "无"))
            if not args.apply:
                continue
            rec = db.query(CashFlowStructureValue).filter(
@@ -0,0 +1,216 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""阈值单位口径修正 + 量纲守卫装配（迁移脚本·准备态）
+
+依据：财务口径裁定 finance-20260916-cma-caliber-ruling.md v1.0 §四（裁定③）+ §3.3/§3.4
+  · 4.1 单位规范：偿债能力比率=**倍**（无量纲）；资产负债率等=%；返利率=%
+  · 4.3 具体阈值裁定（贸易型：白酒经销 / IT 经销）
+  · 3.3 09-09 假红灯 586/587 按「口径错误」作废并重算
+  · 3.4 切换顺序不可颠倒：先单位 → 再期间 → 后调度（本脚本只做第 1 步）
+
+默认 dry-run（只读：列改动 + 影响面复算，不写库）。`--apply` 才写（**需审批**）。
+
+用法：
+  python3 scripts/migrate_kpi_threshold_unit_20260916.py                    # dry-run
+  python3 scripts/migrate_kpi_threshold_unit_20260916.py --apply            # 写阈值/单位（需放行）
+  python3 scripts/migrate_kpi_threshold_unit_20260916.py --apply --resolve-false-reds
+                                                                            # 同时作废假红灯 586/587
+说明：
+  · 影响面口径 = 用阈值复算「最近可用期间(<= 最新已入库月)」的命中条数，
+    分别给「修正前 / 修正后」两组数字（裁定 §4.5 要求：影响面以修正后复算值为准）。
+  · 返利率（F_REBATE_RATE）**不改阈值**（量纲一致、判定有效），仅标注口径待确认（裁定 §4.4）。
+"""
+import argparse
+import sys
+import os
+
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+from sqlalchemy import text  # noqa: E402
+from app.database import get_engine  # noqa: E402
+from app.utils.dimension_guard import check_dimension  # noqa: E402
+from app.utils.threshold_parser import eval_threshold, LEVEL_PRIORITY  # noqa: E402
+
+ENTITY_ID = 1  # 酣客（白酒经销）；博海（IT）无阈值 KPI 待核
+
+#: 迁移计划：kpi_code → 目标 unit / 三档阈值（None = 不改该字段）
+PLAN = {
+    "F_CURRENT_RATIO": {"name": "流动比率", "unit": "倍",
+                        "green": ">=2.0", "yellow": ">=1.0", "red": "<1.0",
+                        "why": "裁定 §4.3：经验值 2:1；<1 才够紧急（旧 <150 为百分数写法）"},
+    "F_QUICK_RATIO": {"name": "速动比率", "unit": "倍",
+                      "green": ">=1.0", "yellow": ">=0.5", "red": "<0.5",
+                      "why": "裁定 §4.3：经验值 1:1（旧 <80 为百分数写法）"},
+    "F_DEBT_RATIO": {"name": "资产负债率", "unit": "%",
+                     "green": "<=60", "yellow": ">60", "red": ">70",
+                     "why": "裁定 §4.3：红 >70 / 黄 60–70 / 绿 ≤60"},
+}
+#: 明确不改、但需标注的项
+UNCHANGED_NOTES = {
+    "F_REBATE_RATE": ("返利率", "%", "裁定 §4.4：量纲一致、判定有效，不改阈值；"
+                                     "48.44% 口径本身待确认 → 预警保留并标注「口径待确认」"),
+}
+
+#: 假红灯（09-09 单位错配产生）—— 裁定 §3.3：按「口径错误」作废
+FALSE_RED_ALERT_IDS = (586, 587)
+VOID_RESOLUTION = "[口径错误·作废] 阈值与实测值跨量级比较（流动/速动比率单位错配），" \
+                  "非财务事实；由 finance-20260916-cma-caliber-ruling §3.3 裁定作废，待单位修正后重算。"
+
+
+def q(sql, params=None):
+    with get_engine().connect() as c:
+        return [dict(r._mapping) for r in c.execute(text(sql), params or {}).fetchall()]
+
+
+def exec_sql(sql, params=None):
+    with get_engine().begin() as c:
+        return c.execute(text(sql), params or {})
+
+
+def fetch_plan_rows():
+    codes = list(PLAN.keys()) + list(UNCHANGED_NOTES.keys())
+    ph = ",".join(":%s" % ("c%d" % i) for i in range(len(codes)))
+    return q("SELECT id, kpi_code, kpi_name, unit, threshold_green, threshold_yellow, threshold_red, status "
+             "FROM kpi_definitions WHERE entity_id=:e AND kpi_code IN (%s) ORDER BY id" % ph,
+             dict({"e": ENTITY_ID}, **{("c%d" % i): c for i, c in enumerate(codes)}))
+
+
+def latest_period():
+    r = q("SELECT MAX(period) AS p FROM kpi_values WHERE period REGEXP '^[0-9]{4}-[0-9]{2}$'")
+    return r[0]["p"] if r else None
+
+
+def impact():
+    """影响面复算：最近可用期间上，各 KPI 在「修正前 / 修正后」的命中条数。"""
+    p = latest_period()
+    rows = q(
+        "SELECT d.id, d.kpi_code, d.kpi_name, d.unit, d.threshold_green AS g, "
+        "d.threshold_yellow AS y, d.threshold_red AS r, v.actual_value AS actual, v.period "
+        "FROM kpi_definitions d JOIN kpi_values v ON v.kpi_id = d.id "
+        "WHERE d.entity_id=:e AND d.status='active' AND v.period = :p AND v.actual_value IS NOT NULL "
+        "AND (d.threshold_red IS NOT NULL OR d.threshold_yellow IS NOT NULL OR d.threshold_green IS NOT NULL) "
+        "ORDER BY d.id", {"e": ENTITY_ID, "p": p})
+
+    def judge(actual, unit, g, y, r):
+        exprs = {"green": g, "yellow": y, "red": r}
+        guard_hit = None
+        for lv in LEVEL_PRIORITY:
+            if exprs[lv]:
+                gd = check_dimension(actual, exprs[lv], unit)
+                if not gd["ok"]:
+                    guard_hit = gd
+                    break
+        if guard_hit:
+            return "口径错误"
+        for lv in LEVEL_PRIORITY:
+            if exprs[lv] and eval_threshold(actual, exprs[lv]):
+                return lv
+        return "info"
+
+    before, after, detail = {"red": 0, "yellow": 0, "green": 0, "info": 0, "口径错误": 0}, \
+                            {"red": 0, "yellow": 0, "green": 0, "info": 0, "口径错误": 0}, []
+    for x in rows:
+        plan = PLAN.get(x["kpi_code"])
+        g2 = plan["green"] if plan else x["g"]
+        y2 = plan["yellow"] if plan else x["y"]
+        r2 = plan["red"] if plan else x["r"]
+        u2 = plan["unit"] if plan else x["unit"]
+        b = judge(x["actual"], x["unit"], x["g"], x["y"], x["r"])
+        a = judge(x["actual"], u2, g2, y2, r2)
+        before[b] = before.get(b, 0) + 1
+        after[a] = after.get(a, 0) + 1
+        if b != a:
+            detail.append({"kpi": "%s/%s" % (x["kpi_code"], x["kpi_name"]), "period": x["period"],
+                           "actual": x["actual"], "unit": x["unit"],
+                           "before": b, "after": a,
+                           "thresholds": "%s|%s|%s → %s|%s|%s" % (x["g"], x["y"], x["r"], g2, y2, r2)})
+    return p, before, after, detail
+
+
+def main():
+    ap = argparse.ArgumentParser()
+    ap.add_argument("--apply", action="store_true", help="真正写库（缺省=dry-run）")
+    ap.add_argument("--resolve-false-reds", action="store_true",
+                    help="同时把假红灯 %s 按口径错误作废（status=resolved）" % (FALSE_RED_ALERT_IDS,))
+    args = ap.parse_args()
+
+    print("=== 阈值单位口径修正 %s | entity_id=%d ===" % ("APPLY" if args.apply else "DRY-RUN", ENTITY_ID))
+    rows = fetch_plan_rows()
+    if not rows:
+        print("!! 未取到目标 KPI 行（kpi_code 不匹配？）")
+        return 2
+
+    print("\n-- 计划改动 --")
+    todo = []
+    for x in rows:
+        plan = PLAN.get(x["kpi_code"])
+        if not plan:
+            note = UNCHANGED_NOTES.get(x["kpi_code"])
+            print("  [不改] id=%-4s %-6s %-8s unit=%s 阈值=%s|%s|%s  ← %s"
+                  % (x["id"], x["kpi_code"], x["kpi_name"], x["unit"],
+                     x["threshold_green"], x["threshold_yellow"], x["threshold_red"],
+                     note[2] if note else ""))
+            continue
+        changed = (x["unit"] != plan["unit"] or x["threshold_green"] != plan["green"]
+                   or x["threshold_yellow"] != plan["yellow"] or x["threshold_red"] != plan["red"])
+        print("  [%s] id=%-4s %-6s %-8s unit %s→%s  阈值 %s|%s|%s → %s|%s|%s"
+              % ("改" if changed else "已符", x["id"], x["kpi_code"], x["kpi_name"],
+                 x["unit"], plan["unit"], x["threshold_green"], x["threshold_yellow"], x["threshold_red"],
+                 plan["green"], plan["yellow"], plan["red"]))
+        print("        理由: %s" % plan["why"])
+        if changed:
+            todo.append((x["id"], plan))
+
+    print("\n-- 影响面复算（最近可用期间） --")
+    p, before, after, detail = impact()
+    print("  期间 P* = %s" % p)
+    print("  修正前: 红 %d / 黄 %d / 绿 %d / 无判定 %d / 口径错误 %d"
+          % (before["red"], before["yellow"], before["green"], before["info"], before["口径错误"]))
+    print("  修正后: 红 %d / 黄 %d / 绿 %d / 无判定 %d / 口径错误 %d"
+          % (after["red"], after["yellow"], after["green"], after["info"], after["口径错误"]))
+    print("  条数变化（仅列变化项）:")
+    for d in detail:
+        print("    %-22s %s actual=%s unit=%s  %s → %s  [%s]"
+              % (d["kpi"], d["period"], d["actual"], d["unit"], d["before"], d["after"], d["thresholds"]))
+    print("  ※ 影响面以「修正后」为准（裁定 §4.5）；此处红/黄为静态阈值复算上界，"
+          "实际落库条数还受期间口径与幂等约束。")
+
+    if args.resolve_false_reds:
+        print("\n-- 假红灯作废 --")
+        ph = ",".join(":a%d" % i for i in range(len(FALSE_RED_ALERT_IDS)))
+        al = q("SELECT id, kpi_id, alert_level, status, alert_message FROM kpi_alerts "
+               "WHERE id IN (%s)" % ph,
+               {("a%d" % i): v for i, v in enumerate(FALSE_RED_ALERT_IDS)})
+        for a in al:
+            print("    id=%s level=%s status=%s msg=%s" % (a["id"], a["alert_level"], a["status"],
+                                                           (a["alert_message"] or "")[:70]))
+        if not al:
+            print("    （未取到该两条预警，可能已清理）")
+
+    if not args.apply:
+        print("\n=== dry-run 结束：未写库（%d 项待改，加 --apply 落库 / 需审批）===" % len(todo))
+        return 0
+
+    for kid, plan in todo:
+        exec_sql("UPDATE kpi_definitions SET unit=:u, threshold_green=:g, threshold_yellow=:y, "
+                 "threshold_red=:r WHERE id=:i AND entity_id=:e",
+                 {"u": plan["unit"], "g": plan["green"], "y": plan["yellow"],
+                  "r": plan["red"], "i": kid, "e": ENTITY_ID})
+    if args.resolve_false_reds:
+        exec_sql("UPDATE kpi_alerts SET status='resolved', resolution=:res, resolved_at=NOW() "
+                 "WHERE id IN (%s)" % ",".join(":a%d" % i for i in range(len(FALSE_RED_ALERT_IDS))),
+                 dict({("a%d" % i): v for i, v in enumerate(FALSE_RED_ALERT_IDS)},
+                      res=VOID_RESOLUTION))
+    print("\n=== 已提交：%d 项阈值/单位%s ===" % (len(todo),
+          "；已作废假红灯 %s" % (FALSE_RED_ALERT_IDS,) if args.resolve_false_reds else ""))
+    print("-- 写入后回查 --")
+    for x in fetch_plan_rows():
+        if x["kpi_code"] in PLAN:
+            print("  id=%-4s %-6s unit=%s 阈值=%s|%s|%s"
+                  % (x["id"], x["kpi_code"], x["unit"], x["threshold_green"],
+                     x["threshold_yellow"], x["threshold_red"]))
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())