feat: 因果链三层验证机制(P2) — 数据验证+人工确认+状态机

- kpi_causality 加列: source_type/verify_status/verified_at/verified_by/entity_id(回填) - 核心服务: app/services/causality_verification.py (Pearson+滞后对齐+状态机) - 数据验证脚本: scripts/correlation-check.py (月度cron, 输出JSON报告) - API: create/update支持source_type, GET /verify-status, PUT /{id}/verify(人工确认) - 全部端点按entity_id账套隔离, kpi/{id}/network/simulate补跨企业校验 - 前端: KPIDetail因果链页显示验证状态徽标(数据证实/存疑/待检) - 测试: test_causality_verification.py 37用例 + 原因果链测试全过(59个) - 50条因果链首轮验证: 1数据证实(#37渠补率到净利润lag1 r=-0.89), 4存疑, 45待检(数据不足)
2026-08-27 15:51:19 +08:00
parent 6b479bfe7d
commit fdc42d443d
10 changed files with 1063 additions and 22 deletions
@@ -0,0 +1,238 @@
+"""因果链验证核心逻辑 — 数据验证（Pearson相关性）+ 状态机 (2026-08-27 P2)
+
+三层验证：
+1. 数据验证（自动化）：kpi_values 历史值 → Pearson 相关系数 + 方向一致性 + 滞后对齐
+2. AI/人工验证：战略回顾会人工打标 human_verified（API PUT /verify）
+3. 状态机流转：
+   pending（初始）→ data_verified / disputed（数据验证 cron）
+   pending/data_verified/disputed → human_verified（人工确认，最终）
+   任何矛盾 → disputed（待检）
+
+判定规则（可解释、可测试）：
+- 对齐后数据点 < MIN_POINTS(4) → pending（数据不足，无法统计验证）
+- |r| >= CORR_THRESHOLD(0.5) 且方向与 direction 一致 → data_verified
+- 否则（点数足够但弱相关/方向矛盾）→ disputed
+- human_verified 为人工最终确认，脚本默认不覆盖（respect_human=True），
+  但若数据矛盾会在报告中给出警示（disputed_note）
+
+脚本入口: scripts/correlation-check.py
+测试入口: tests/test_causality_verification.py
+"""
+from __future__ import annotations
+
+import logging
+import math
+import re
+from typing import Dict, List, Optional, Tuple
+
+logger = logging.getLogger("causality-verification")
+
+# 验证状态
+STATUS_PENDING = "pending"
+STATUS_DATA_VERIFIED = "data_verified"
+STATUS_HUMAN_VERIFIED = "human_verified"
+STATUS_DISPUTED = "disputed"
+ALL_STATUSES = (STATUS_PENDING, STATUS_DATA_VERIFIED, STATUS_HUMAN_VERIFIED, STATUS_DISPUTED)
+
+# 建链来源
+SOURCE_MANUAL = "manual"
+SOURCE_AI = "AI_suggested"
+SOURCE_IMPORTED = "imported"
+ALL_SOURCE_TYPES = (SOURCE_MANUAL, SOURCE_AI, SOURCE_IMPORTED)
+
+# 判定参数
+MIN_POINTS = 4          # 最少对齐数据点（少于则无法统计验证）
+CORR_THRESHOLD = 0.5    # |r| 阈值：达到且方向一致 → 数据证实
+VERIFIER_SCRIPT = "correlation-check"
+
+# period 粒度（对齐时只允许同粒度配对，避免月度/年度量纲混用）
+GRANULARITY_ORDER = ("month", "half", "year")
+
+_PERIOD_RE = {
+    "month": re.compile(r"^(\d{4})-(\d{2})$"),
+    "half": re.compile(r"^(\d{4})-H([12])$"),
+    "year": re.compile(r"^(\d{4})$"),
+}
+
+
+def parse_period(period: str) -> Optional[Tuple[str, int]]:
+    """解析 period 为 (granularity, seq)。
+
+    seq = year*12 + 月序号（0-indexed），可比较/做滞后偏移。
+    - '2026-07' → ('month', 2026*12+6)
+    - '2026-H1' → ('half', 2026*12+5)   (H1≈6月)
+    - '2026-H2' → ('half', 2026*12+11)  (H2≈12月)
+    - '2026'    → ('year', 2026*12+5)   (年中)
+    无法解析 → None
+    """
+    if not period:
+        return None
+    s = str(period).strip()
+    m = _PERIOD_RE["month"].match(s)
+    if m:
+        year, mon = int(m.group(1)), int(m.group(2))
+        if 1 <= mon <= 12:
+            return ("month", year * 12 + (mon - 1))
+    m = _PERIOD_RE["half"].match(s)
+    if m:
+        year, half = int(m.group(1)), int(m.group(2))
+        return ("half", year * 12 + (5 if half == 1 else 11))
+    m = _PERIOD_RE["year"].match(s)
+    if m:
+        return ("year", int(m.group(1)) * 12 + 5)
+    return None
+
+
+def pearson(xs: List[float], ys: List[float]) -> Tuple[Optional[float], int]:
+    """Pearson 相关系数。点数 < 2 返回 (None, n)。"""
+    n = len(xs)
+    if n < 2:
+        return None, n
+    mx = sum(xs) / n
+    my = sum(ys) / n
+    sxy = sum((x - mx) * (y - my) for x, y in zip(xs, ys))
+    sxx = sum((x - mx) ** 2 for x in xs)
+    syy = sum((y - my) ** 2 for y in ys)
+    if sxx <= 0 or syy <= 0:  # 常数列 → 无相关
+        return None, n
+    r = sxy / math.sqrt(sxx * syy)
+    # 数值保护：浮点误差可能略超 [-1,1]
+    return max(-1.0, min(1.0, r)), n
+
+
+def align_series(
+    source_values: List[Tuple[str, float]],
+    target_values: List[Tuple[str, float]],
+    lag_months: int = 0,
+) -> Tuple[Optional[str], List[Tuple[float, float]]]:
+    """按滞后期对齐 source/target 时间序列，返回 (granularity, pairs)。
+
+    语义：source 是因（先发生），target 是果（滞后 lag 月出现）。
+    pair = (source[t], target[t + lag])。
+    只使用同粒度（month/half/year）数据配对，避免量纲混用。
+    按粒度优先级 month > half > year 选取数据点最多的粒度。
+    """
+    parsed_src: Dict[str, Dict[int, float]] = {g: {} for g in GRANULARITY_ORDER}
+    parsed_tgt: Dict[str, Dict[int, float]] = {g: {} for g in GRANULARITY_ORDER}
+    for period, val in source_values:
+        if val is None:
+            continue
+        r = parse_period(period)
+        if r:
+            g, seq = r
+            parsed_src[g][seq] = float(val)
+    for period, val in target_values:
+        if val is None:
+            continue
+        r = parse_period(period)
+        if r:
+            g, seq = r
+            parsed_tgt[g][seq] = float(val)
+
+    best_g, best_pairs = None, []
+    for g in GRANULARITY_ORDER:
+        src_map, tgt_map = parsed_src[g], parsed_tgt[g]
+        pairs = []
+        for seq, sv in sorted(src_map.items()):
+            tv = tgt_map.get(seq + lag_months)
+            if tv is not None:
+                pairs.append((sv, tv))
+        if len(pairs) > len(best_pairs):
+            best_g, best_pairs = g, pairs
+    return best_g, best_pairs
+
+
+def evaluate_chain(
+    source_values: List[Tuple[str, float]],
+    target_values: List[Tuple[str, float]],
+    lag_months: int = 0,
+    direction: str = "positive",
+    min_points: int = MIN_POINTS,
+    corr_threshold: float = CORR_THRESHOLD,
+) -> dict:
+    """对单条因果链做数据验证。
+
+    返回:
+    {
+      granularity, n, r, expected_sign, actual_sign, direction_consistent,
+      status (pending/data_verified/disputed), reason
+    }
+    """
+    granularity, pairs = align_series(source_values, target_values, lag_months)
+    n = len(pairs)
+    r = None
+    if n >= 2:
+        r, _ = pearson([p[0] for p in pairs], [p[1] for p in pairs])
+
+    expected_sign = 1 if direction == "positive" else -1
+    actual_sign = 1 if r is not None and r > 0 else (-1 if r is not None and r < 0 else 0)
+    direction_consistent = r is not None and actual_sign == expected_sign
+
+    if n < min_points or r is None:
+        return {
+            "granularity": granularity, "n": n, "r": r,
+            "expected_sign": expected_sign, "actual_sign": actual_sign,
+            "direction_consistent": direction_consistent,
+            "status": STATUS_PENDING,
+            "reason": f"数据不足（对齐后{n}点，需≥{min_points}点）" if n < min_points else "序列无方差，无法计算相关性",
+        }
+
+    abs_r = abs(r)
+    if abs_r >= corr_threshold and direction_consistent:
+        return {
+            "granularity": granularity, "n": n, "r": r,
+            "expected_sign": expected_sign, "actual_sign": actual_sign,
+            "direction_consistent": True,
+            "status": STATUS_DATA_VERIFIED,
+            "reason": f"|r|={abs_r:.3f}≥{corr_threshold} 且方向一致({direction}) → 数据证实",
+        }
+    if not direction_consistent:
+        return {
+            "granularity": granularity, "n": n, "r": r,
+            "expected_sign": expected_sign, "actual_sign": actual_sign,
+            "direction_consistent": False,
+            "status": STATUS_DISPUTED,
+            "reason": f"方向矛盾: 声明{direction}但实际相关方向{'正' if r > 0 else '负'} (r={r:.3f})",
+        }
+    return {
+        "granularity": granularity, "n": n, "r": r,
+        "expected_sign": expected_sign, "actual_sign": actual_sign,
+        "direction_consistent": True,
+        "status": STATUS_DISPUTED,
+        "reason": f"弱相关: |r|={abs_r:.3f}<{corr_threshold}，数据暂不能证实该因果强度",
+    }
+
+
+def apply_state_machine(
+    current_status: str,
+    eval_status: str,
+    respect_human: bool = True,
+) -> Tuple[str, Optional[str]]:
+    """状态机：根据数据验证结果流转状态。
+
+    规则:
+    - human_verified 是人工最终确认: respect_human=True 时不被脚本覆盖
+      （返回原状态 + disputed_note 警示）
+    - 数据不足(pending) → 保持当前状态（不降级已有结论）
+    - data_verified → 覆盖为非 human_verified 的当前状态
+    - disputed → 覆盖为非 human_verified 的当前状态
+    """
+    if current_status == STATUS_HUMAN_VERIFIED and respect_human:
+        if eval_status == STATUS_DISPUTED:
+            return current_status, "人工已确认但数据复核矛盾，建议重新核对"
+        return current_status, None
+    if eval_status == STATUS_PENDING:
+        return current_status, None
+    return eval_status, None
+
+
+def summarize(results: List[dict]) -> dict:
+    """验证结果汇总统计。"""
+    counter = {s: 0 for s in ALL_STATUSES}
+    for r in results:
+        counter[r.get("status", STATUS_PENDING)] = counter.get(r.get("status", STATUS_PENDING), 0) + 1
+    return {
+        "total": len(results),
+        "by_status": counter,
+        "data_verified_ratio": round(counter[STATUS_DATA_VERIFIED] / len(results), 3) if results else 0,
+    }