cma-management/backend/app/services/causality_verification.py

"""因果链验证核心逻辑 — 数据验证（Pearson相关性）+ 状态机 (2026-08-27 P2)

三层验证：
1. 数据验证（自动化）：kpi_values 历史值 → Pearson 相关系数 + 方向一致性 + 滞后对齐
2. AI/人工验证：战略回顾会人工打标 human_verified（API PUT /verify）
3. 状态机流转：
   pending（初始）→ data_verified / disputed（数据验证 cron）
   pending/data_verified/disputed → human_verified（人工确认，最终）
   任何矛盾 → disputed（待检）

判定规则（可解释、可测试）：
- 对齐后数据点 < MIN_POINTS(4) → pending（数据不足，无法统计验证）
- |r| >= CORR_THRESHOLD(0.5) 且方向与 direction 一致 → data_verified
- 否则（点数足够但弱相关/方向矛盾）→ disputed
- human_verified 为人工最终确认，脚本默认不覆盖（respect_human=True），
  但若数据矛盾会在报告中给出警示（disputed_note）

脚本入口: scripts/correlation-check.py
测试入口: tests/test_causality_verification.py
"""
from __future__ import annotations

import logging
import math
import re
from typing import Dict, List, Optional, Tuple

logger = logging.getLogger("causality-verification")

# 验证状态
STATUS_PENDING = "pending"
STATUS_DATA_VERIFIED = "data_verified"
STATUS_HUMAN_VERIFIED = "human_verified"
STATUS_DISPUTED = "disputed"
ALL_STATUSES = (STATUS_PENDING, STATUS_DATA_VERIFIED, STATUS_HUMAN_VERIFIED, STATUS_DISPUTED)

# 建链来源
SOURCE_MANUAL = "manual"
SOURCE_AI = "AI_suggested"
SOURCE_IMPORTED = "imported"
ALL_SOURCE_TYPES = (SOURCE_MANUAL, SOURCE_AI, SOURCE_IMPORTED)

# 判定参数
MIN_POINTS = 4          # 最少对齐数据点（少于则无法统计验证）
CORR_THRESHOLD = 0.5    # |r| 阈值：达到且方向一致 → 数据证实
VERIFIER_SCRIPT = "correlation-check"

# period 粒度（对齐时只允许同粒度配对，避免月度/年度量纲混用）
GRANULARITY_ORDER = ("month", "half", "year")

_PERIOD_RE = {
    "month": re.compile(r"^(\d{4})-(\d{2})$"),
    "half": re.compile(r"^(\d{4})-H([12])$"),
    "year": re.compile(r"^(\d{4})$"),
}


def parse_period(period: str) -> Optional[Tuple[str, int]]:
    """解析 period 为 (granularity, seq)。

    seq = year*12 + 月序号（0-indexed），可比较/做滞后偏移。
    - '2026-07' → ('month', 2026*12+6)
    - '2026-H1' → ('half', 2026*12+5)   (H1≈6月)
    - '2026-H2' → ('half', 2026*12+11)  (H2≈12月)
    - '2026'    → ('year', 2026*12+5)   (年中)
    无法解析 → None
    """
    if not period:
        return None
    s = str(period).strip()
    m = _PERIOD_RE["month"].match(s)
    if m:
        year, mon = int(m.group(1)), int(m.group(2))
        if 1 <= mon <= 12:
            return ("month", year * 12 + (mon - 1))
    m = _PERIOD_RE["half"].match(s)
    if m:
        year, half = int(m.group(1)), int(m.group(2))
        return ("half", year * 12 + (5 if half == 1 else 11))
    m = _PERIOD_RE["year"].match(s)
    if m:
        return ("year", int(m.group(1)) * 12 + 5)
    return None


def pearson(xs: List[float], ys: List[float]) -> Tuple[Optional[float], int]:
    """Pearson 相关系数。点数 < 2 返回 (None, n)。"""
    n = len(xs)
    if n < 2:
        return None, n
    mx = sum(xs) / n
    my = sum(ys) / n
    sxy = sum((x - mx) * (y - my) for x, y in zip(xs, ys))
    sxx = sum((x - mx) ** 2 for x in xs)
    syy = sum((y - my) ** 2 for y in ys)
    if sxx <= 0 or syy <= 0:  # 常数列 → 无相关
        return None, n
    r = sxy / math.sqrt(sxx * syy)
    # 数值保护：浮点误差可能略超 [-1,1]
    return max(-1.0, min(1.0, r)), n


def align_series(
    source_values: List[Tuple[str, float]],
    target_values: List[Tuple[str, float]],
    lag_months: int = 0,
) -> Tuple[Optional[str], List[Tuple[float, float]]]:
    """按滞后期对齐 source/target 时间序列，返回 (granularity, pairs)。

    语义：source 是因（先发生），target 是果（滞后 lag 月出现）。
    pair = (source[t], target[t + lag])。
    只使用同粒度（month/half/year）数据配对，避免量纲混用。
    按粒度优先级 month > half > year 选取数据点最多的粒度。
    """
    parsed_src: Dict[str, Dict[int, float]] = {g: {} for g in GRANULARITY_ORDER}
    parsed_tgt: Dict[str, Dict[int, float]] = {g: {} for g in GRANULARITY_ORDER}
    for period, val in source_values:
        if val is None:
            continue
        r = parse_period(period)
        if r:
            g, seq = r
            parsed_src[g][seq] = float(val)
    for period, val in target_values:
        if val is None:
            continue
        r = parse_period(period)
        if r:
            g, seq = r
            parsed_tgt[g][seq] = float(val)

    best_g, best_pairs = None, []
    for g in GRANULARITY_ORDER:
        src_map, tgt_map = parsed_src[g], parsed_tgt[g]
        pairs = []
        for seq, sv in sorted(src_map.items()):
            tv = tgt_map.get(seq + lag_months)
            if tv is not None:
                pairs.append((sv, tv))
        if len(pairs) > len(best_pairs):
            best_g, best_pairs = g, pairs
    return best_g, best_pairs


def evaluate_chain(
    source_values: List[Tuple[str, float]],
    target_values: List[Tuple[str, float]],
    lag_months: int = 0,
    direction: str = "positive",
    min_points: int = MIN_POINTS,
    corr_threshold: float = CORR_THRESHOLD,
) -> dict:
    """对单条因果链做数据验证。

    返回:
    {
      granularity, n, r, expected_sign, actual_sign, direction_consistent,
      status (pending/data_verified/disputed), reason
    }
    """
    granularity, pairs = align_series(source_values, target_values, lag_months)
    n = len(pairs)
    r = None
    if n >= 2:
        r, _ = pearson([p[0] for p in pairs], [p[1] for p in pairs])

    expected_sign = 1 if direction == "positive" else -1
    actual_sign = 1 if r is not None and r > 0 else (-1 if r is not None and r < 0 else 0)
    direction_consistent = r is not None and actual_sign == expected_sign

    if n < min_points or r is None:
        return {
            "granularity": granularity, "n": n, "r": r,
            "expected_sign": expected_sign, "actual_sign": actual_sign,
            "direction_consistent": direction_consistent,
            "status": STATUS_PENDING,
            "reason": f"数据不足（对齐后{n}点，需≥{min_points}点）" if n < min_points else "序列无方差，无法计算相关性",
        }

    abs_r = abs(r)
    if abs_r >= corr_threshold and direction_consistent:
        return {
            "granularity": granularity, "n": n, "r": r,
            "expected_sign": expected_sign, "actual_sign": actual_sign,
            "direction_consistent": True,
            "status": STATUS_DATA_VERIFIED,
            "reason": f"|r|={abs_r:.3f}≥{corr_threshold} 且方向一致({direction}) → 数据证实",
        }
    if not direction_consistent:
        return {
            "granularity": granularity, "n": n, "r": r,
            "expected_sign": expected_sign, "actual_sign": actual_sign,
            "direction_consistent": False,
            "status": STATUS_DISPUTED,
            "reason": f"方向矛盾: 声明{direction}但实际相关方向{'正' if r > 0 else '负'} (r={r:.3f})",
        }
    return {
        "granularity": granularity, "n": n, "r": r,
        "expected_sign": expected_sign, "actual_sign": actual_sign,
        "direction_consistent": True,
        "status": STATUS_DISPUTED,
        "reason": f"弱相关: |r|={abs_r:.3f}<{corr_threshold}，数据暂不能证实该因果强度",
    }


def apply_state_machine(
    current_status: str,
    eval_status: str,
    respect_human: bool = True,
) -> Tuple[str, Optional[str]]:
    """状态机：根据数据验证结果流转状态。

    规则:
    - human_verified 是人工最终确认: respect_human=True 时不被脚本覆盖
      （返回原状态 + disputed_note 警示）
    - 数据不足(pending) → 保持当前状态（不降级已有结论）
    - data_verified → 覆盖为非 human_verified 的当前状态
    - disputed → 覆盖为非 human_verified 的当前状态
    """
    if current_status == STATUS_HUMAN_VERIFIED and respect_human:
        if eval_status == STATUS_DISPUTED:
            return current_status, "人工已确认但数据复核矛盾，建议重新核对"
        return current_status, None
    if eval_status == STATUS_PENDING:
        return current_status, None
    return eval_status, None


def summarize(results: List[dict]) -> dict:
    """验证结果汇总统计。"""
    counter = {s: 0 for s in ALL_STATUSES}
    for r in results:
        counter[r.get("status", STATUS_PENDING)] = counter.get(r.get("status", STATUS_PENDING), 0) + 1
    return {
        "total": len(results),
        "by_status": counter,
        "data_verified_ratio": round(counter[STATUS_DATA_VERIFIED] / len(results), 3) if results else 0,
    }