feat: 因果链三层验证机制(P2) — 数据验证+人工确认+状态机
- kpi_causality 加列: source_type/verify_status/verified_at/verified_by/entity_id(回填)
- 核心服务: app/services/causality_verification.py (Pearson+滞后对齐+状态机)
- 数据验证脚本: scripts/correlation-check.py (月度cron, 输出JSON报告)
- API: create/update支持source_type, GET /verify-status, PUT /{id}/verify(人工确认)
- 全部端点按entity_id账套隔离, kpi/{id}/network/simulate补跨企业校验
- 前端: KPIDetail因果链页显示验证状态徽标(数据证实/存疑/待检)
- 测试: test_causality_verification.py 37用例 + 原因果链测试全过(59个)
- 50条因果链首轮验证: 1数据证实(#37渠补率到净利润lag1 r=-0.89), 4存疑, 45待检(数据不足)
This commit is contained in:
@@ -1,6 +1,7 @@
|
||||
"""KPI因果链建模 — 任务2
|
||||
KPI间因果关系网络 + 模拟推演
|
||||
KPI间因果关系网络 + 模拟推演 + 三层验证机制(数据/AI/人工) (2026-08-27 P2)
|
||||
"""
|
||||
from datetime import datetime
|
||||
from fastapi import APIRouter, Depends, HTTPException, Query
|
||||
from sqlalchemy.orm import Session
|
||||
from sqlalchemy import text
|
||||
@@ -11,6 +12,11 @@ from app.database import get_db
|
||||
from app.deps import get_entity_id
|
||||
from app.auth_middleware import require_auth, require_role
|
||||
from app.models import KPIDefinition, KPICausality, KPIValue, OperationLog
|
||||
from app.services.causality_verification import (
|
||||
ALL_STATUSES,
|
||||
ALL_SOURCE_TYPES,
|
||||
STATUS_PENDING,
|
||||
)
|
||||
|
||||
logger = logging.getLogger("kpi-causality")
|
||||
|
||||
@@ -30,8 +36,8 @@ def _to_dict(obj):
|
||||
|
||||
@router.get("/full-network")
|
||||
def get_full_network(db: Session = Depends(get_db), entity_id: int = Depends(get_entity_id)):
|
||||
"""获取全局因果网络数据(用于力导向图)— 账套隔离: 仅当前企业KPI的因果链 (2026-08-23 P1b)"""
|
||||
edges = db.query(KPICausality).join(KPIDefinition, KPIDefinition.id == KPICausality.source_kpi_id).filter(KPIDefinition.entity_id == entity_id).all()
|
||||
"""获取全局因果网络数据(用于力导向图)— 账套隔离: 仅当前企业KPI的因果链 (2026-08-23 P1b, 2026-08-27 用entity_id列)"""
|
||||
edges = db.query(KPICausality).filter(KPICausality.entity_id == entity_id).all()
|
||||
node_ids = set()
|
||||
edge_list = []
|
||||
for e in edges:
|
||||
@@ -61,14 +67,19 @@ def get_full_network(db: Session = Depends(get_db), entity_id: int = Depends(get
|
||||
|
||||
|
||||
@router.get("/kpi/{kpi_id}/network")
|
||||
def get_kpi_network(kpi_id: int, db: Session = Depends(get_db)):
|
||||
"""获取KPI的因果网络(上游驱动 + 下游影响)"""
|
||||
def get_kpi_network(kpi_id: int, db: Session = Depends(get_db), entity_id: int = Depends(get_entity_id)):
|
||||
"""获取KPI的因果网络(上游驱动 + 下游影响)— 账套隔离: 校验KPI属于当前企业"""
|
||||
kpi = db.query(KPIDefinition).filter(KPIDefinition.id == kpi_id).first()
|
||||
if not kpi:
|
||||
raise HTTPException(404, "KPI不存在")
|
||||
if kpi.entity_id != entity_id:
|
||||
raise HTTPException(404, "KPI不存在") # 跨企业不暴露存在性
|
||||
|
||||
# 上游(指向当前KPI的因果)
|
||||
upstream = db.query(KPICausality).filter(KPICausality.target_kpi_id == kpi_id).all()
|
||||
upstream = db.query(KPICausality).filter(
|
||||
KPICausality.target_kpi_id == kpi_id,
|
||||
KPICausality.entity_id == entity_id,
|
||||
).all()
|
||||
upstream_list = []
|
||||
for c in upstream:
|
||||
src = db.query(KPIDefinition).filter(KPIDefinition.id == c.source_kpi_id).first()
|
||||
@@ -78,10 +89,16 @@ def get_kpi_network(kpi_id: int, db: Session = Depends(get_db)):
|
||||
"kpi_id": src.id, "kpi_code": src.kpi_code, "kpi_name": src.kpi_name,
|
||||
"strength": c.strength, "lag_months": c.lag_months,
|
||||
"direction": c.direction, "formula": c.formula,
|
||||
"source_type": c.source_type, "verify_status": c.verify_status,
|
||||
"verified_at": c.verified_at.isoformat() if c.verified_at else None,
|
||||
"verified_by": c.verified_by,
|
||||
})
|
||||
|
||||
# 下游(当前KPI指向的因果)
|
||||
downstream = db.query(KPICausality).filter(KPICausality.source_kpi_id == kpi_id).all()
|
||||
downstream = db.query(KPICausality).filter(
|
||||
KPICausality.source_kpi_id == kpi_id,
|
||||
KPICausality.entity_id == entity_id,
|
||||
).all()
|
||||
downstream_list = []
|
||||
for c in downstream:
|
||||
tgt = db.query(KPIDefinition).filter(KPIDefinition.id == c.target_kpi_id).first()
|
||||
@@ -91,6 +108,9 @@ def get_kpi_network(kpi_id: int, db: Session = Depends(get_db)):
|
||||
"kpi_id": tgt.id, "kpi_code": tgt.kpi_code, "kpi_name": tgt.kpi_name,
|
||||
"strength": c.strength, "lag_months": c.lag_months,
|
||||
"direction": c.direction, "formula": c.formula,
|
||||
"source_type": c.source_type, "verify_status": c.verify_status,
|
||||
"verified_at": c.verified_at.isoformat() if c.verified_at else None,
|
||||
"verified_by": c.verified_by,
|
||||
})
|
||||
|
||||
return {
|
||||
@@ -101,7 +121,7 @@ def get_kpi_network(kpi_id: int, db: Session = Depends(get_db)):
|
||||
|
||||
|
||||
@router.post("/simulate")
|
||||
def simulate_causality(data: dict, db: Session = Depends(get_db)):
|
||||
def simulate_causality(data: dict, db: Session = Depends(get_db), entity_id: int = Depends(get_entity_id)):
|
||||
"""模拟推演: 修改一个KPI的值,预测对其他KPI的影响
|
||||
Body: { kpi_id: int, new_value: float, period: str }
|
||||
"""
|
||||
@@ -115,6 +135,8 @@ def simulate_causality(data: dict, db: Session = Depends(get_db)):
|
||||
source_kpi = db.query(KPIDefinition).filter(KPIDefinition.id == kpi_id).first()
|
||||
if not source_kpi:
|
||||
raise HTTPException(404, "KPI不存在")
|
||||
if source_kpi.entity_id != entity_id:
|
||||
raise HTTPException(404, "KPI不存在")
|
||||
|
||||
# 获取当前值
|
||||
current_value = None
|
||||
@@ -142,9 +164,10 @@ def simulate_causality(data: dict, db: Session = Depends(get_db)):
|
||||
continue
|
||||
visited.add(current_kpi_id)
|
||||
|
||||
# 查找从current_kpi_id出发的下游因果链
|
||||
# 查找从current_kpi_id出发的下游因果链(账套隔离: 仅本企业链)
|
||||
downstream = db.query(KPICausality).filter(
|
||||
KPICausality.source_kpi_id == current_kpi_id
|
||||
KPICausality.source_kpi_id == current_kpi_id,
|
||||
KPICausality.entity_id == entity_id,
|
||||
).all()
|
||||
|
||||
for edge in downstream:
|
||||
@@ -212,15 +235,18 @@ def simulate_causality(data: dict, db: Session = Depends(get_db)):
|
||||
def list_causalities(
|
||||
source_kpi_id: Optional[int] = None,
|
||||
target_kpi_id: Optional[int] = None,
|
||||
verify_status: Optional[str] = None,
|
||||
db: Session = Depends(get_db),
|
||||
entity_id: int = Depends(get_entity_id),
|
||||
):
|
||||
"""获取因果链列表(账套隔离: 仅当前企业KPI, 2026-08-23 P1b)"""
|
||||
query = db.query(KPICausality).join(KPIDefinition, KPIDefinition.id == KPICausality.source_kpi_id).filter(KPIDefinition.entity_id == entity_id)
|
||||
"""获取因果链列表(账套隔离: 仅当前企业KPI, 2026-08-23 P1b, 2026-08-27 支持verify_status筛选)"""
|
||||
query = db.query(KPICausality).filter(KPICausality.entity_id == entity_id)
|
||||
if source_kpi_id:
|
||||
query = query.filter(KPICausality.source_kpi_id == source_kpi_id)
|
||||
if target_kpi_id:
|
||||
query = query.filter(KPICausality.target_kpi_id == target_kpi_id)
|
||||
if verify_status:
|
||||
query = query.filter(KPICausality.verify_status == verify_status)
|
||||
items = query.order_by(KPICausality.id).all()
|
||||
|
||||
result = []
|
||||
@@ -236,11 +262,51 @@ def list_causalities(
|
||||
return {"data": result, "total": len(result)}
|
||||
|
||||
|
||||
@router.get("/verify-status")
|
||||
def get_verify_status(
|
||||
verify_status: Optional[str] = None,
|
||||
db: Session = Depends(get_db),
|
||||
entity_id: int = Depends(get_entity_id),
|
||||
):
|
||||
"""验证状态总览 — 按状态统计 + 链列表(2026-08-27 三层验证机制)
|
||||
|
||||
可选 ?verify_status=pending/data_verified/human_verified/disputed 筛选
|
||||
"""
|
||||
if verify_status and verify_status not in ALL_STATUSES:
|
||||
raise HTTPException(400, f"verify_status 必须为 {'/'.join(ALL_STATUSES)}")
|
||||
|
||||
query = db.query(KPICausality).filter(KPICausality.entity_id == entity_id)
|
||||
if verify_status:
|
||||
query = query.filter(KPICausality.verify_status == verify_status)
|
||||
items = query.order_by(KPICausality.id).all()
|
||||
|
||||
by_status = {s: 0 for s in ALL_STATUSES}
|
||||
data = []
|
||||
for c in items:
|
||||
by_status[c.verify_status] = by_status.get(c.verify_status, 0) + 1
|
||||
d = _to_dict(c)
|
||||
src = db.query(KPIDefinition).filter(KPIDefinition.id == c.source_kpi_id).first()
|
||||
tgt = db.query(KPIDefinition).filter(KPIDefinition.id == c.target_kpi_id).first()
|
||||
d["source_kpi_code"] = src.kpi_code if src else None
|
||||
d["source_kpi_name"] = src.kpi_name if src else None
|
||||
d["target_kpi_code"] = tgt.kpi_code if tgt else None
|
||||
d["target_kpi_name"] = tgt.kpi_name if tgt else None
|
||||
d["verified_at"] = c.verified_at.isoformat() if c.verified_at else None
|
||||
data.append(d)
|
||||
|
||||
return {
|
||||
"summary": {"total": len(items), "by_status": by_status},
|
||||
"data": data,
|
||||
}
|
||||
|
||||
|
||||
@router.get("/{causality_id}")
|
||||
def get_causality(causality_id: int, db: Session = Depends(get_db)):
|
||||
def get_causality(causality_id: int, db: Session = Depends(get_db), entity_id: int = Depends(get_entity_id)):
|
||||
c = db.query(KPICausality).filter(KPICausality.id == causality_id).first()
|
||||
if not c:
|
||||
raise HTTPException(404, "因果链不存在")
|
||||
if c.entity_id != entity_id:
|
||||
raise HTTPException(404, "因果链不存在") # 账套隔离
|
||||
d = _to_dict(c)
|
||||
src = db.query(KPIDefinition).filter(KPIDefinition.id == c.source_kpi_id).first()
|
||||
tgt = db.query(KPIDefinition).filter(KPIDefinition.id == c.target_kpi_id).first()
|
||||
@@ -251,7 +317,7 @@ def get_causality(causality_id: int, db: Session = Depends(get_db)):
|
||||
|
||||
@router.post("")
|
||||
def create_causality(data: dict, db: Session = Depends(get_db), user=WRITE_ROLES):
|
||||
"""创建因果链"""
|
||||
"""创建因果链(source_type标记来源, 2026-08-27)"""
|
||||
source_id = data.get("source_kpi_id")
|
||||
target_id = data.get("target_kpi_id")
|
||||
if not source_id or not target_id:
|
||||
@@ -262,6 +328,8 @@ def create_causality(data: dict, db: Session = Depends(get_db), user=WRITE_ROLES
|
||||
tgt = db.query(KPIDefinition).filter(KPIDefinition.id == target_id).first()
|
||||
if not src or not tgt:
|
||||
raise HTTPException(404, "KPI不存在")
|
||||
if src.entity_id != tgt.entity_id:
|
||||
raise HTTPException(400, "源KPI和目标KPI必须属于同一企业")
|
||||
|
||||
existing = db.query(KPICausality).filter(
|
||||
KPICausality.source_kpi_id == source_id,
|
||||
@@ -270,40 +338,91 @@ def create_causality(data: dict, db: Session = Depends(get_db), user=WRITE_ROLES
|
||||
if existing:
|
||||
raise HTTPException(400, f"因果链已存在: {src.kpi_code}→{tgt.kpi_code}")
|
||||
|
||||
source_type = data.get("source_type", "manual")
|
||||
if source_type not in ALL_SOURCE_TYPES:
|
||||
raise HTTPException(400, f"source_type 必须为 {'/'.join(ALL_SOURCE_TYPES)}")
|
||||
|
||||
c = KPICausality(
|
||||
entity_id=src.entity_id,
|
||||
source_kpi_id=source_id,
|
||||
target_kpi_id=target_id,
|
||||
strength=data.get("strength", 0.5),
|
||||
lag_months=data.get("lag_months", 1),
|
||||
formula=data.get("formula"),
|
||||
direction=data.get("direction", "positive"),
|
||||
source_type=source_type,
|
||||
verify_status=STATUS_PENDING,
|
||||
)
|
||||
db.add(c)
|
||||
db.commit()
|
||||
db.refresh(c)
|
||||
db.add(OperationLog(action="create", target_type="kpi_causality",
|
||||
detail=f"创建因果链: {src.kpi_code}→{tgt.kpi_code}"))
|
||||
detail=f"创建因果链: {src.kpi_code}→{tgt.kpi_code} (source={source_type})"))
|
||||
db.commit()
|
||||
return _to_dict(c)
|
||||
|
||||
|
||||
@router.put("/{causality_id}")
|
||||
def update_causality(causality_id: int, data: dict, db: Session = Depends(get_db), user=WRITE_ROLES):
|
||||
def update_causality(causality_id: int, data: dict, db: Session = Depends(get_db), user=WRITE_ROLES,
|
||||
entity_id: int = Depends(get_entity_id)):
|
||||
c = db.query(KPICausality).filter(KPICausality.id == causality_id).first()
|
||||
if not c:
|
||||
raise HTTPException(404, "因果链不存在")
|
||||
for field in ("strength", "lag_months", "formula", "direction"):
|
||||
if c.entity_id != entity_id:
|
||||
raise HTTPException(404, "因果链不存在") # 账套隔离
|
||||
for field in ("strength", "lag_months", "formula", "direction", "source_type"):
|
||||
if field in data:
|
||||
if field == "source_type" and data[field] not in ALL_SOURCE_TYPES:
|
||||
raise HTTPException(400, f"source_type 必须为 {'/'.join(ALL_SOURCE_TYPES)}")
|
||||
setattr(c, field, data[field])
|
||||
# 修改链定义后,验证状态回到待检(定义变了旧结论失效)
|
||||
if any(f in data for f in ("strength", "lag_months", "formula", "direction")):
|
||||
c.verify_status = STATUS_PENDING
|
||||
c.verified_at = None
|
||||
c.verified_by = None
|
||||
db.commit()
|
||||
db.refresh(c)
|
||||
return _to_dict(c)
|
||||
|
||||
|
||||
@router.put("/{causality_id}/verify")
|
||||
def verify_causality(causality_id: int, data: dict, db: Session = Depends(get_db), user=WRITE_ROLES,
|
||||
entity_id: int = Depends(get_entity_id)):
|
||||
"""人工确认(战略回顾会核对打标)— 2026-08-27 三层验证机制
|
||||
|
||||
Body: { verify_status: "human_verified"|"disputed", verified_by?: "任富海" }
|
||||
默认打标 human_verified(人工最终确认)。
|
||||
"""
|
||||
c = db.query(KPICausality).filter(KPICausality.id == causality_id).first()
|
||||
if not c:
|
||||
raise HTTPException(404, "因果链不存在")
|
||||
if c.entity_id != entity_id:
|
||||
raise HTTPException(404, "因果链不存在") # 账套隔离
|
||||
|
||||
target_status = data.get("verify_status", "human_verified")
|
||||
if target_status not in ("human_verified", "disputed"):
|
||||
raise HTTPException(400, "verify_status 必须为 human_verified 或 disputed")
|
||||
verified_by = data.get("verified_by") or user.name or user.username
|
||||
|
||||
c.verify_status = target_status
|
||||
c.verified_at = datetime.now()
|
||||
c.verified_by = str(verified_by)[:50]
|
||||
db.add(OperationLog(action="verify", target_type="kpi_causality",
|
||||
detail=f"因果链 #{causality_id} 人工确认: {target_status} (by {verified_by})"))
|
||||
db.commit()
|
||||
db.refresh(c)
|
||||
d = _to_dict(c)
|
||||
d["verified_at"] = c.verified_at.isoformat() if c.verified_at else None
|
||||
return d
|
||||
|
||||
|
||||
@router.delete("/{causality_id}")
|
||||
def delete_causality(causality_id: int, db: Session = Depends(get_db), user=WRITE_ROLES):
|
||||
def delete_causality(causality_id: int, db: Session = Depends(get_db), user=WRITE_ROLES,
|
||||
entity_id: int = Depends(get_entity_id)):
|
||||
c = db.query(KPICausality).filter(KPICausality.id == causality_id).first()
|
||||
if c:
|
||||
if c.entity_id != entity_id:
|
||||
raise HTTPException(404, "因果链不存在") # 账套隔离
|
||||
db.delete(c)
|
||||
db.commit()
|
||||
return {"message": "已删除"}
|
||||
|
||||
@@ -292,12 +292,17 @@ class KPICausality(Base):
|
||||
"""KPI因果链 — 记录KPI间的因果关系"""
|
||||
__tablename__ = "kpi_causality"
|
||||
id = Column(Integer, primary_key=True, index=True)
|
||||
entity_id = Column(Integer, default=1, comment="企业ID (多租户隔离 2026-08-27)")
|
||||
source_kpi_id = Column(Integer, ForeignKey("kpi_definitions.id"), nullable=False, comment="源KPI(因)")
|
||||
target_kpi_id = Column(Integer, ForeignKey("kpi_definitions.id"), nullable=False, comment="目标KPI(果)")
|
||||
strength = Column(Float, default=0.5, comment="影响强度 0~1")
|
||||
lag_months = Column(Integer, default=1, comment="滞后期(月)")
|
||||
formula = Column(String(500), nullable=True, comment="影响公式描述")
|
||||
direction = Column(String(10), default="positive", comment="positive/negative 正向/负向影响")
|
||||
source_type = Column(String(20), default="manual", comment="建链来源 AI_suggested/manual/imported (2026-08-27 验证机制)")
|
||||
verify_status = Column(String(20), default="pending", comment="验证状态 pending/data_verified/human_verified/disputed (2026-08-27)")
|
||||
verified_at = Column(DateTime, nullable=True, comment="验证时间")
|
||||
verified_by = Column(String(50), nullable=True, comment="验证人/AI/脚本")
|
||||
created_at = Column(DateTime, server_default=func.now())
|
||||
updated_at = Column(DateTime, server_default=func.now(), onupdate=func.now())
|
||||
|
||||
|
||||
@@ -0,0 +1,238 @@
|
||||
"""因果链验证核心逻辑 — 数据验证(Pearson相关性)+ 状态机 (2026-08-27 P2)
|
||||
|
||||
三层验证:
|
||||
1. 数据验证(自动化):kpi_values 历史值 → Pearson 相关系数 + 方向一致性 + 滞后对齐
|
||||
2. AI/人工验证:战略回顾会人工打标 human_verified(API PUT /verify)
|
||||
3. 状态机流转:
|
||||
pending(初始)→ data_verified / disputed(数据验证 cron)
|
||||
pending/data_verified/disputed → human_verified(人工确认,最终)
|
||||
任何矛盾 → disputed(待检)
|
||||
|
||||
判定规则(可解释、可测试):
|
||||
- 对齐后数据点 < MIN_POINTS(4) → pending(数据不足,无法统计验证)
|
||||
- |r| >= CORR_THRESHOLD(0.5) 且方向与 direction 一致 → data_verified
|
||||
- 否则(点数足够但弱相关/方向矛盾)→ disputed
|
||||
- human_verified 为人工最终确认,脚本默认不覆盖(respect_human=True),
|
||||
但若数据矛盾会在报告中给出警示(disputed_note)
|
||||
|
||||
脚本入口: scripts/correlation-check.py
|
||||
测试入口: tests/test_causality_verification.py
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import math
|
||||
import re
|
||||
from typing import Dict, List, Optional, Tuple
|
||||
|
||||
logger = logging.getLogger("causality-verification")
|
||||
|
||||
# 验证状态
|
||||
STATUS_PENDING = "pending"
|
||||
STATUS_DATA_VERIFIED = "data_verified"
|
||||
STATUS_HUMAN_VERIFIED = "human_verified"
|
||||
STATUS_DISPUTED = "disputed"
|
||||
ALL_STATUSES = (STATUS_PENDING, STATUS_DATA_VERIFIED, STATUS_HUMAN_VERIFIED, STATUS_DISPUTED)
|
||||
|
||||
# 建链来源
|
||||
SOURCE_MANUAL = "manual"
|
||||
SOURCE_AI = "AI_suggested"
|
||||
SOURCE_IMPORTED = "imported"
|
||||
ALL_SOURCE_TYPES = (SOURCE_MANUAL, SOURCE_AI, SOURCE_IMPORTED)
|
||||
|
||||
# 判定参数
|
||||
MIN_POINTS = 4 # 最少对齐数据点(少于则无法统计验证)
|
||||
CORR_THRESHOLD = 0.5 # |r| 阈值:达到且方向一致 → 数据证实
|
||||
VERIFIER_SCRIPT = "correlation-check"
|
||||
|
||||
# period 粒度(对齐时只允许同粒度配对,避免月度/年度量纲混用)
|
||||
GRANULARITY_ORDER = ("month", "half", "year")
|
||||
|
||||
_PERIOD_RE = {
|
||||
"month": re.compile(r"^(\d{4})-(\d{2})$"),
|
||||
"half": re.compile(r"^(\d{4})-H([12])$"),
|
||||
"year": re.compile(r"^(\d{4})$"),
|
||||
}
|
||||
|
||||
|
||||
def parse_period(period: str) -> Optional[Tuple[str, int]]:
|
||||
"""解析 period 为 (granularity, seq)。
|
||||
|
||||
seq = year*12 + 月序号(0-indexed),可比较/做滞后偏移。
|
||||
- '2026-07' → ('month', 2026*12+6)
|
||||
- '2026-H1' → ('half', 2026*12+5) (H1≈6月)
|
||||
- '2026-H2' → ('half', 2026*12+11) (H2≈12月)
|
||||
- '2026' → ('year', 2026*12+5) (年中)
|
||||
无法解析 → None
|
||||
"""
|
||||
if not period:
|
||||
return None
|
||||
s = str(period).strip()
|
||||
m = _PERIOD_RE["month"].match(s)
|
||||
if m:
|
||||
year, mon = int(m.group(1)), int(m.group(2))
|
||||
if 1 <= mon <= 12:
|
||||
return ("month", year * 12 + (mon - 1))
|
||||
m = _PERIOD_RE["half"].match(s)
|
||||
if m:
|
||||
year, half = int(m.group(1)), int(m.group(2))
|
||||
return ("half", year * 12 + (5 if half == 1 else 11))
|
||||
m = _PERIOD_RE["year"].match(s)
|
||||
if m:
|
||||
return ("year", int(m.group(1)) * 12 + 5)
|
||||
return None
|
||||
|
||||
|
||||
def pearson(xs: List[float], ys: List[float]) -> Tuple[Optional[float], int]:
|
||||
"""Pearson 相关系数。点数 < 2 返回 (None, n)。"""
|
||||
n = len(xs)
|
||||
if n < 2:
|
||||
return None, n
|
||||
mx = sum(xs) / n
|
||||
my = sum(ys) / n
|
||||
sxy = sum((x - mx) * (y - my) for x, y in zip(xs, ys))
|
||||
sxx = sum((x - mx) ** 2 for x in xs)
|
||||
syy = sum((y - my) ** 2 for y in ys)
|
||||
if sxx <= 0 or syy <= 0: # 常数列 → 无相关
|
||||
return None, n
|
||||
r = sxy / math.sqrt(sxx * syy)
|
||||
# 数值保护:浮点误差可能略超 [-1,1]
|
||||
return max(-1.0, min(1.0, r)), n
|
||||
|
||||
|
||||
def align_series(
|
||||
source_values: List[Tuple[str, float]],
|
||||
target_values: List[Tuple[str, float]],
|
||||
lag_months: int = 0,
|
||||
) -> Tuple[Optional[str], List[Tuple[float, float]]]:
|
||||
"""按滞后期对齐 source/target 时间序列,返回 (granularity, pairs)。
|
||||
|
||||
语义:source 是因(先发生),target 是果(滞后 lag 月出现)。
|
||||
pair = (source[t], target[t + lag])。
|
||||
只使用同粒度(month/half/year)数据配对,避免量纲混用。
|
||||
按粒度优先级 month > half > year 选取数据点最多的粒度。
|
||||
"""
|
||||
parsed_src: Dict[str, Dict[int, float]] = {g: {} for g in GRANULARITY_ORDER}
|
||||
parsed_tgt: Dict[str, Dict[int, float]] = {g: {} for g in GRANULARITY_ORDER}
|
||||
for period, val in source_values:
|
||||
if val is None:
|
||||
continue
|
||||
r = parse_period(period)
|
||||
if r:
|
||||
g, seq = r
|
||||
parsed_src[g][seq] = float(val)
|
||||
for period, val in target_values:
|
||||
if val is None:
|
||||
continue
|
||||
r = parse_period(period)
|
||||
if r:
|
||||
g, seq = r
|
||||
parsed_tgt[g][seq] = float(val)
|
||||
|
||||
best_g, best_pairs = None, []
|
||||
for g in GRANULARITY_ORDER:
|
||||
src_map, tgt_map = parsed_src[g], parsed_tgt[g]
|
||||
pairs = []
|
||||
for seq, sv in sorted(src_map.items()):
|
||||
tv = tgt_map.get(seq + lag_months)
|
||||
if tv is not None:
|
||||
pairs.append((sv, tv))
|
||||
if len(pairs) > len(best_pairs):
|
||||
best_g, best_pairs = g, pairs
|
||||
return best_g, best_pairs
|
||||
|
||||
|
||||
def evaluate_chain(
|
||||
source_values: List[Tuple[str, float]],
|
||||
target_values: List[Tuple[str, float]],
|
||||
lag_months: int = 0,
|
||||
direction: str = "positive",
|
||||
min_points: int = MIN_POINTS,
|
||||
corr_threshold: float = CORR_THRESHOLD,
|
||||
) -> dict:
|
||||
"""对单条因果链做数据验证。
|
||||
|
||||
返回:
|
||||
{
|
||||
granularity, n, r, expected_sign, actual_sign, direction_consistent,
|
||||
status (pending/data_verified/disputed), reason
|
||||
}
|
||||
"""
|
||||
granularity, pairs = align_series(source_values, target_values, lag_months)
|
||||
n = len(pairs)
|
||||
r = None
|
||||
if n >= 2:
|
||||
r, _ = pearson([p[0] for p in pairs], [p[1] for p in pairs])
|
||||
|
||||
expected_sign = 1 if direction == "positive" else -1
|
||||
actual_sign = 1 if r is not None and r > 0 else (-1 if r is not None and r < 0 else 0)
|
||||
direction_consistent = r is not None and actual_sign == expected_sign
|
||||
|
||||
if n < min_points or r is None:
|
||||
return {
|
||||
"granularity": granularity, "n": n, "r": r,
|
||||
"expected_sign": expected_sign, "actual_sign": actual_sign,
|
||||
"direction_consistent": direction_consistent,
|
||||
"status": STATUS_PENDING,
|
||||
"reason": f"数据不足(对齐后{n}点,需≥{min_points}点)" if n < min_points else "序列无方差,无法计算相关性",
|
||||
}
|
||||
|
||||
abs_r = abs(r)
|
||||
if abs_r >= corr_threshold and direction_consistent:
|
||||
return {
|
||||
"granularity": granularity, "n": n, "r": r,
|
||||
"expected_sign": expected_sign, "actual_sign": actual_sign,
|
||||
"direction_consistent": True,
|
||||
"status": STATUS_DATA_VERIFIED,
|
||||
"reason": f"|r|={abs_r:.3f}≥{corr_threshold} 且方向一致({direction}) → 数据证实",
|
||||
}
|
||||
if not direction_consistent:
|
||||
return {
|
||||
"granularity": granularity, "n": n, "r": r,
|
||||
"expected_sign": expected_sign, "actual_sign": actual_sign,
|
||||
"direction_consistent": False,
|
||||
"status": STATUS_DISPUTED,
|
||||
"reason": f"方向矛盾: 声明{direction}但实际相关方向{'正' if r > 0 else '负'} (r={r:.3f})",
|
||||
}
|
||||
return {
|
||||
"granularity": granularity, "n": n, "r": r,
|
||||
"expected_sign": expected_sign, "actual_sign": actual_sign,
|
||||
"direction_consistent": True,
|
||||
"status": STATUS_DISPUTED,
|
||||
"reason": f"弱相关: |r|={abs_r:.3f}<{corr_threshold},数据暂不能证实该因果强度",
|
||||
}
|
||||
|
||||
|
||||
def apply_state_machine(
|
||||
current_status: str,
|
||||
eval_status: str,
|
||||
respect_human: bool = True,
|
||||
) -> Tuple[str, Optional[str]]:
|
||||
"""状态机:根据数据验证结果流转状态。
|
||||
|
||||
规则:
|
||||
- human_verified 是人工最终确认: respect_human=True 时不被脚本覆盖
|
||||
(返回原状态 + disputed_note 警示)
|
||||
- 数据不足(pending) → 保持当前状态(不降级已有结论)
|
||||
- data_verified → 覆盖为非 human_verified 的当前状态
|
||||
- disputed → 覆盖为非 human_verified 的当前状态
|
||||
"""
|
||||
if current_status == STATUS_HUMAN_VERIFIED and respect_human:
|
||||
if eval_status == STATUS_DISPUTED:
|
||||
return current_status, "人工已确认但数据复核矛盾,建议重新核对"
|
||||
return current_status, None
|
||||
if eval_status == STATUS_PENDING:
|
||||
return current_status, None
|
||||
return eval_status, None
|
||||
|
||||
|
||||
def summarize(results: List[dict]) -> dict:
|
||||
"""验证结果汇总统计。"""
|
||||
counter = {s: 0 for s in ALL_STATUSES}
|
||||
for r in results:
|
||||
counter[r.get("status", STATUS_PENDING)] = counter.get(r.get("status", STATUS_PENDING), 0) + 1
|
||||
return {
|
||||
"total": len(results),
|
||||
"by_status": counter,
|
||||
"data_verified_ratio": round(counter[STATUS_DATA_VERIFIED] / len(results), 3) if results else 0,
|
||||
}
|
||||
Reference in New Issue
Block a user