feat: 因果链三层验证机制(P2) — 数据验证+人工确认+状态机

- kpi_causality 加列: source_type/verify_status/verified_at/verified_by/entity_id(回填) - 核心服务: app/services/causality_verification.py (Pearson+滞后对齐+状态机) - 数据验证脚本: scripts/correlation-check.py (月度cron, 输出JSON报告) - API: create/update支持source_type, GET /verify-status, PUT /{id}/verify(人工确认) - 全部端点按entity_id账套隔离, kpi/{id}/network/simulate补跨企业校验 - 前端: KPIDetail因果链页显示验证状态徽标(数据证实/存疑/待检) - 测试: test_causality_verification.py 37用例 + 原因果链测试全过(59个) - 50条因果链首轮验证: 1数据证实(#37渠补率到净利润lag1 r=-0.89), 4存疑, 45待检(数据不足)
2026-08-27 15:51:19 +08:00
parent 6b479bfe7d
commit fdc42d443d
10 changed files with 1063 additions and 22 deletions
@@ -0,0 +1,363 @@
+"""因果链验证机制测试 — 数据验证核心 + 状态机 + API (2026-08-27 P2)
+
+覆盖:
+1. 服务层: parse_period / pearson / align_series(滞后) / evaluate_chain / apply_state_machine
+2. API: create(source_type) / verify-status / verify(人工确认) / entity隔离 / 权限
+"""
+import hashlib
+
+import pytest
+from fastapi.testclient import TestClient
+from sqlalchemy.orm import Session
+
+from app.models import KPIDefinition, KPICausality, KPIValue, Entity, User
+from app.services.causality_verification import (
+    STATUS_DATA_VERIFIED,
+    STATUS_DISPUTED,
+    STATUS_HUMAN_VERIFIED,
+    STATUS_PENDING,
+    align_series,
+    apply_state_machine,
+    evaluate_chain,
+    parse_period,
+    pearson,
+    summarize,
+)
+from tests.conftest import create_test_user, get_token_for_user, auth_header
+
+BASE = "/api/cma/kpi-causality"
+
+
+def _seed_kpi(db: Session, code: str, name: str = None, dimension: str = "finance",
+              entity_id: int = 1) -> KPIDefinition:
+    kpi = KPIDefinition(
+        kpi_code=code, kpi_name=name or code, dimension=dimension,
+        entity_id=entity_id, status="active", target_value=100.0,
+    )
+    db.add(kpi)
+    db.commit()
+    db.refresh(kpi)
+    return kpi
+
+
+def _seed_chain(db: Session, source_type: str = "AI_suggested"):
+    src = _seed_kpi(db, "BH_REVENUE", "营业收入")
+    tgt = _seed_kpi(db, "BH_NET_PROFIT", "净利润")
+    c = KPICausality(entity_id=src.entity_id, source_kpi_id=src.id, target_kpi_id=tgt.id,
+                     strength=0.5, lag_months=0, direction="positive",
+                     source_type=source_type, verify_status=STATUS_PENDING)
+    db.add(c)
+    db.commit()
+    db.refresh(c)
+    return src, tgt, c
+
+
+class TestParsePeriod:
+    def test_month(self):
+        assert parse_period("2026-07") == ("month", 2026 * 12 + 6)
+
+    def test_half(self):
+        assert parse_period("2026-H1") == ("half", 2026 * 12 + 5)
+        assert parse_period("2026-H2") == ("half", 2026 * 12 + 11)
+
+    def test_year(self):
+        assert parse_period("2026") == ("year", 2026 * 12 + 5)
+
+    def test_invalid(self):
+        assert parse_period("abc") is None
+        assert parse_period("") is None
+        assert parse_period(None) is None
+
+
+class TestPearson:
+    def test_perfect_positive(self):
+        r, n = pearson([1, 2, 3, 4], [2, 4, 6, 8])
+        assert n == 4
+        assert abs(r - 1.0) < 1e-9
+
+    def test_perfect_negative(self):
+        r, n = pearson([1, 2, 3, 4], [8, 6, 4, 2])
+        assert abs(r + 1.0) < 1e-9
+
+    def test_known_value(self):
+        # 与 numpy 核对过的样例 (F_REVENUE / F_NET_PROFIT 7点)
+        xs = [180.87, 132.33, 120.15, 60.5, 90.09, 129.32, 81.08]
+        ys = [94.31, -85.06, -21.66, -3.74, -21.91, -45.04, -17.63]
+        r, n = pearson(xs, ys)
+        assert n == 7
+        assert abs(r - 0.394012) < 1e-4
+
+    def test_insufficient(self):
+        r, n = pearson([1], [2])
+        assert r is None and n == 1
+
+    def test_constant_series(self):
+        r, n = pearson([3, 3, 3], [1, 2, 3])
+        assert r is None and n == 3
+
+
+class TestAlignSeries:
+    def test_no_lag(self):
+        src = [("2026-01", 1), ("2026-02", 2), ("2026-03", 3)]
+        tgt = [("2026-01", 10), ("2026-02", 20), ("2026-03", 30)]
+        g, pairs = align_series(src, tgt, lag_months=0)
+        assert g == "month"
+        assert pairs == [(1, 10), (2, 20), (3, 30)]
+
+    def test_lag_alignment(self):
+        """source t 与 target t+lag 配对"""
+        src = [("2026-01", 1), ("2026-02", 2), ("2026-03", 3)]
+        tgt = [("2026-02", 10), ("2026-03", 20), ("2026-04", 30)]
+        g, pairs = align_series(src, tgt, lag_months=1)
+        assert pairs == [(1, 10), (2, 20), (3, 30)]
+
+    def test_granularity_filter(self):
+        """月度/半年度混用时只取同粒度（优先月）"""
+        src = [("2026-01", 1), ("2026-02", 2), ("2026-H1", 3)]
+        tgt = [("2026-01", 10), ("2026-02", 20), ("2026-H1", 30)]
+        g, pairs = align_series(src, tgt, lag_months=0)
+        assert g == "month"
+        assert pairs == [(1, 10), (2, 20)]
+
+
+class TestEvaluateChain:
+    def test_data_verified_positive(self):
+        src = [(f"2026-{m:02d}", m) for m in range(1, 9)]
+        tgt = [(f"2026-{m:02d}", m * 2) for m in range(1, 9)]
+        ev = evaluate_chain(src, tgt, lag_months=0, direction="positive")
+        assert ev["status"] == STATUS_DATA_VERIFIED
+        assert ev["direction_consistent"] is True
+        assert ev["n"] == 8
+
+    def test_data_verified_negative(self):
+        src = [(f"2026-{m:02d}", m) for m in range(1, 9)]
+        tgt = [(f"2026-{m:02d}", -m * 2) for m in range(1, 9)]
+        ev = evaluate_chain(src, tgt, lag_months=0, direction="negative")
+        assert ev["status"] == STATUS_DATA_VERIFIED
+
+    def test_direction_conflict(self):
+        """声明 positive 但实际负相关 → disputed"""
+        src = [(f"2026-{m:02d}", m) for m in range(1, 9)]
+        tgt = [(f"2026-{m:02d}", -m) for m in range(1, 9)]
+        ev = evaluate_chain(src, tgt, lag_months=0, direction="positive")
+        assert ev["status"] == STATUS_DISPUTED
+        assert "方向矛盾" in ev["reason"]
+
+    def test_weak_correlation(self):
+        """弱相关（方向一致但|r|<阈值）→ disputed"""
+        # numpy seed=1: x=[1..8], y=x+N(0,6) → r≈0.119 (弱正相关)
+        src = [(f"2026-{m:02d}", m) for m in range(1, 9)]
+        tgt = [(f"2026-{m:02d}", y) for m, y in enumerate(
+            [10.75, -1.67, -0.17, -2.44, 10.19, -7.81, 17.47, 3.43], start=1)]
+        ev = evaluate_chain(src, tgt, lag_months=0, direction="positive")
+        assert ev["status"] == STATUS_DISPUTED
+        assert "弱相关" in ev["reason"]
+
+    def test_insufficient_points(self):
+        """数据点不足 → pending"""
+        src = [("2026-01", 1), ("2026-02", 2)]
+        tgt = [("2026-01", 10), ("2026-02", 20)]
+        ev = evaluate_chain(src, tgt, lag_months=0, direction="positive")
+        assert ev["status"] == STATUS_PENDING
+
+    def test_no_shared_periods(self):
+        src = [("2026-01", 1)]
+        tgt = [("2026-02", 10)]
+        ev = evaluate_chain(src, tgt, lag_months=0, direction="positive")
+        assert ev["status"] == STATUS_PENDING
+
+
+class TestStateMachine:
+    def test_pending_to_verified(self):
+        st, note = apply_state_machine(STATUS_PENDING, STATUS_DATA_VERIFIED)
+        assert st == STATUS_DATA_VERIFIED and note is None
+
+    def test_pending_to_disputed(self):
+        st, _ = apply_state_machine(STATUS_PENDING, STATUS_DISPUTED)
+        assert st == STATUS_DISPUTED
+
+    def test_human_verified_not_overridden(self):
+        st, note = apply_state_machine(STATUS_HUMAN_VERIFIED, STATUS_DISPUTED)
+        assert st == STATUS_HUMAN_VERIFIED
+        assert note is not None  # 数据矛盾警示
+
+    def test_human_verified_positive_note_none(self):
+        st, note = apply_state_machine(STATUS_HUMAN_VERIFIED, STATUS_DATA_VERIFIED)
+        assert st == STATUS_HUMAN_VERIFIED and note is None
+
+    def test_insufficient_keeps_status(self):
+        st, _ = apply_state_machine(STATUS_PENDING, STATUS_PENDING)
+        assert st == STATUS_PENDING
+
+
+class TestSummarize:
+    def test_counts(self):
+        s = summarize([{"status": STATUS_DATA_VERIFIED}, {"status": STATUS_DISPUTED},
+                       {"status": STATUS_PENDING}, {"status": STATUS_HUMAN_VERIFIED}])
+        assert s["total"] == 4
+        assert s["by_status"][STATUS_DATA_VERIFIED] == 1
+        assert s["by_status"][STATUS_DISPUTED] == 1
+        assert s["by_status"][STATUS_PENDING] == 1
+        assert s["by_status"][STATUS_HUMAN_VERIFIED] == 1
+
+
+class TestCausalityVerificationAPI:
+    def test_create_with_source_type(self, client: TestClient, db: Session):
+        create_test_user(db)
+        token = get_token_for_user(client)
+        src = _seed_kpi(db, "BH_REVENUE", "营业收入")
+        tgt = _seed_kpi(db, "BH_NET_PROFIT", "净利润")
+
+        resp = client.post(BASE, headers=auth_header(token), json={
+            "source_kpi_id": src.id, "target_kpi_id": tgt.id,
+            "source_type": "AI_suggested",
+        })
+        assert resp.status_code == 200
+        body = resp.json()
+        assert body["source_type"] == "AI_suggested"
+        assert body["verify_status"] == STATUS_PENDING
+
+    def test_create_invalid_source_type(self, client: TestClient, db: Session):
+        create_test_user(db)
+        token = get_token_for_user(client)
+        src = _seed_kpi(db, "BH_REVENUE", "营业收入")
+        tgt = _seed_kpi(db, "BH_NET_PROFIT", "净利润")
+        resp = client.post(BASE, headers=auth_header(token), json={
+            "source_kpi_id": src.id, "target_kpi_id": tgt.id, "source_type": "unknown",
+        })
+        assert resp.status_code == 400
+
+    def test_update_resets_verification(self, client: TestClient, db: Session):
+        create_test_user(db)
+        token = get_token_for_user(client)
+        src, tgt, c = _seed_chain(db)
+        # 先人工确认
+        resp = client.put(f"{BASE}/{c.id}/verify", headers=auth_header(token),
+                          json={"verify_status": "human_verified", "verified_by": "任富海"})
+        assert resp.json()["verify_status"] == STATUS_HUMAN_VERIFIED
+        # 修改链定义 → 状态回到 pending
+        resp2 = client.put(f"{BASE}/{c.id}", headers=auth_header(token), json={"strength": 0.9})
+        assert resp2.json()["verify_status"] == STATUS_PENDING
+        assert resp2.json()["verified_by"] is None
+
+    def test_verify_status_endpoint(self, client: TestClient, db: Session):
+        create_test_user(db)
+        token = get_token_for_user(client)
+        src, tgt, c = _seed_chain(db)
+
+        resp = client.get(f"{BASE}/verify-status", headers=auth_header(token))
+        assert resp.status_code == 200
+        body = resp.json()
+        assert body["summary"]["total"] == 1
+        assert body["summary"]["by_status"][STATUS_PENDING] == 1
+        assert body["data"][0]["id"] == c.id
+        assert body["data"][0]["verify_status"] == STATUS_PENDING
+
+    def test_verify_status_filter(self, client: TestClient, db: Session):
+        create_test_user(db)
+        token = get_token_for_user(client)
+        src, tgt, c = _seed_chain(db)
+        client.put(f"{BASE}/{c.id}/verify", headers=auth_header(token),
+                   json={"verify_status": "human_verified"})
+
+        resp = client.get(f"{BASE}/verify-status?verify_status=human_verified",
+                          headers=auth_header(token))
+        assert resp.json()["summary"]["total"] == 1
+        resp2 = client.get(f"{BASE}/verify-status?verify_status=pending", headers=auth_header(token))
+        assert resp2.json()["summary"]["total"] == 0
+
+    def test_human_verify(self, client: TestClient, db: Session):
+        create_test_user(db)
+        token = get_token_for_user(client)
+        src, tgt, c = _seed_chain(db)
+
+        resp = client.put(f"{BASE}/{c.id}/verify", headers=auth_header(token),
+                          json={"verify_status": "human_verified", "verified_by": "任富海"})
+        assert resp.status_code == 200
+        body = resp.json()
+        assert body["verify_status"] == STATUS_HUMAN_VERIFIED
+        assert body["verified_by"] == "任富海"
+        assert body["verified_at"] is not None
+
+    def test_human_verify_default(self, client: TestClient, db: Session):
+        create_test_user(db)
+        token = get_token_for_user(client)
+        src, tgt, c = _seed_chain(db)
+        resp = client.put(f"{BASE}/{c.id}/verify", headers=auth_header(token), json={})
+        assert resp.json()["verify_status"] == STATUS_HUMAN_VERIFIED
+        assert resp.json()["verified_by"] is not None  # 默认取用户名
+
+    def test_verify_disputed(self, client: TestClient, db: Session):
+        create_test_user(db)
+        token = get_token_for_user(client)
+        src, tgt, c = _seed_chain(db)
+        resp = client.put(f"{BASE}/{c.id}/verify", headers=auth_header(token),
+                          json={"verify_status": "disputed"})
+        assert resp.json()["verify_status"] == STATUS_DISPUTED
+
+    def test_verify_invalid_status(self, client: TestClient, db: Session):
+        create_test_user(db)
+        token = get_token_for_user(client)
+        src, tgt, c = _seed_chain(db)
+        resp = client.put(f"{BASE}/{c.id}/verify", headers=auth_header(token),
+                          json={"verify_status": "bogus"})
+        assert resp.status_code == 400
+
+    def test_verify_not_found(self, client: TestClient, db: Session):
+        create_test_user(db)
+        token = get_token_for_user(client)
+        resp = client.put(f"{BASE}/99999/verify", headers=auth_header(token),
+                          json={"verify_status": "human_verified"})
+        assert resp.status_code == 404
+
+    def test_business_cannot_verify(self, client: TestClient, db: Session):
+        """business 角色无写权限 → 403"""
+        business = User(
+            username="business_verify", password_hash=hashlib.sha256("pass123".encode()).hexdigest(),
+            name="业务员", role="business",
+        )
+        db.add(business)
+        db.commit()
+        token = get_token_for_user(client, username="business_verify", password="pass123")
+        src, tgt, c = _seed_chain(db)
+        resp = client.put(f"{BASE}/{c.id}/verify", headers=auth_header(token),
+                          json={"verify_status": "human_verified"})
+        assert resp.status_code == 403
+
+    def test_entity_isolation(self, client: TestClient, db: Session):
+        """企业B看不到企业A的链，也不能verify企业A的链"""
+        create_test_user(db)
+        token = get_token_for_user(client)  # entity_id=1
+        _seed_entity2(db)
+        src2 = _seed_kpi(db, "BH2_REVENUE", "博海收入", entity_id=2)
+        tgt2 = _seed_kpi(db, "BH2_PROFIT", "博海利润", entity_id=2)
+        c2 = KPICausality(entity_id=2, source_kpi_id=src2.id, target_kpi_id=tgt2.id,
+                          strength=0.5, lag_months=0, direction="positive",
+                          source_type="manual", verify_status=STATUS_PENDING)
+        db.add(c2)
+        db.commit()
+
+        # entity1 的 verify-status 看不到 entity2 的链
+        resp = client.get(f"{BASE}/verify-status", headers=auth_header(token))
+        assert resp.json()["summary"]["total"] == 0
+        # entity1 的 token verify entity2 的链 → 404
+        resp2 = client.put(f"{BASE}/{c2.id}/verify", headers=auth_header(token),
+                           json={"verify_status": "human_verified"})
+        assert resp2.status_code == 404
+
+    def test_network_includes_verify_status(self, client: TestClient, db: Session):
+        create_test_user(db)
+        token = get_token_for_user(client)
+        src, tgt, c = _seed_chain(db)
+        resp = client.get(f"{BASE}/kpi/{src.id}/network", headers=auth_header(token))
+        assert resp.status_code == 200
+        downstream = resp.json()["downstream"]
+        assert downstream[0]["verify_status"] == STATUS_PENDING
+        assert downstream[0]["source_type"] == "AI_suggested"
+
+
+def _seed_entity2(db: Session) -> None:
+    ent = db.query(Entity).filter(Entity.id == 2).first()
+    if not ent:
+        db.add(Entity(id=2, name="博海网络科技", short_name="博海", status="active"))
+        db.commit()