cma-management/backend/scripts/verify_ext_input_ledgers_e2e.py

#!/usr/bin/env python3
"""外部输入台账 端到端验收电池（**试验田 = scratch 库**，零生产写入）
任务: 0-P0-project-20260915-cma-eval-m2-ext-input-tables（2026-09-15）

做什么：
  1. root@unix_socket 建库 cma_scratch_ext_<ts>（先 DROP IF EXISTS，保证可复跑）
  2. 跑真实迁移脚本（--check / --apply / 二次 --apply 幂等 / --show-create 证据），目标 = scratch
  3. scratch 上建其余表 + 最小夹具（entities / users / kpi_definitions），不复制生产数据
  4. 真实 HTTP（TestClient）跑：新增 / CSV 批量导入 ×2 / 按 (entity_id, period) 查询 /
     派生 ×2（幂等）/ 分母 0 不写值（税务·人事·证照三口径）/ 定义缺失跳过 / 跨账套 403 / 未认证 401
     —— **原始请求与响应 JSON 全文打印**
  5. 收尾 DROP scratch

边界自证（本单「生产库未写入」）：
  * 全程只创建 1 个 engine，URL 指向 scratch 库；脚本不读也不写 cma 库
  * 迁移脚本对 DB_NAME=='cma' 的 --apply/--down 默认拒绝（下方单独打印该守卫的实测输出）
  * 不复制生产数据（未使用 mysqldump）：夹具全部脚本内合成；token 走进程内存储（不写生产 Redis）

用法: cd /root/cma-management/backend && ./venv/bin/python3 scripts/verify_ext_input_ledgers_e2e.py
"""
from __future__ import annotations

import json
import os
import subprocess
import sys
import time

HERE = os.path.dirname(os.path.abspath(__file__))
BACKEND = os.path.dirname(HERE)
sys.path.insert(0, BACKEND)

import pymysql  # noqa: E402
from sqlalchemy import create_engine  # noqa: E402

SOCKET = "/var/run/mysqld/mysqld.sock"
SCRATCH = f"cma_scratch_ext_{time.strftime('%Y%m%d%H%M%S')}"
SCRATCH_URL = f"mysql+pymysql://root@localhost/{SCRATCH}?unix_socket={SOCKET}&charset=utf8mb4"
MIGRATE = os.path.join(HERE, "migrate_ext_input_ledgers_20260915.py")
VENV_PY = os.path.join(BACKEND, "venv", "bin", "python3")
FAILURES = []


def say(title: str, body: str = "") -> None:
    print(f"\n===== {title} =====")
    if body:
        print(body)


def check(name: str, cond: bool, detail: str = "") -> None:
    print(f"  [{'PASS' if cond else 'FAIL'}] {name}{(' | ' + detail) if detail else ''}")
    if not cond:
        FAILURES.append(f"{name} | {detail}")


def raw_req(label: str, resp) -> None:
    """原始请求/响应全文（验收要求「真实请求与响应原文贴出」）"""
    body = resp.text
    if len(body) > 3000:
        body = body[:3000] + "...(truncated)"
    print(f"  -> {label}: HTTP {resp.status_code}\n     {body}")


def root_conn(db=None):
    return pymysql.connect(unix_socket=SOCKET, user="root", database=db, charset="utf8mb4")


def count(db, model) -> int:
    """回查行数：先 rollback 结束当前事务 —— MySQL REPEATABLE READ 下 session 的长事务
    会固定快照，看不到 API 请求（另一连接）刚提交的行，会误判「写入 0 行」。"""
    db.rollback()
    return db.query(model).count()


def main() -> int:
    con = root_conn()
    with con.cursor() as cur:
        cur.execute(f"DROP DATABASE IF EXISTS {SCRATCH}")
        cur.execute(f"CREATE DATABASE {SCRATCH} DEFAULT CHARSET utf8mb4")
    con.close()
    print(f"scratch 库已建: {SCRATCH}")

    # ── 1. 迁移脚本：check → apply → 二次 apply（幂等）→ show-create ──────────
    def run_migrate(*args):
        p = subprocess.run([VENV_PY, MIGRATE, "--url", SCRATCH_URL] + list(args),
                           capture_output=True, text=True, cwd=BACKEND)
        return p.returncode, (p.stdout or "") + (p.stderr or "")

    rc, out = run_migrate("--check")
    say("迁移 --check（建表前，期望三表 exists=False）", out)
    check("apply 前 --check rc=0 且三表不存在", rc == 0 and out.count("exists=False") == 3)

    rc, out = run_migrate("--apply")
    say("迁移 --apply（建 3 张外部输入台账）", out)
    check("apply rc=0 且仍缺表为 0", rc == 0 and "仍缺表: []" in out)

    rc, out2 = run_migrate("--apply")
    say("迁移 --apply 二次（幂等复核）", out2)
    check("二次 apply 幂等（仍缺表: []，新增 0）", "仍缺表: []" in out2)

    rc, out = run_migrate("--check")
    table_lines = [ln for ln in out.splitlines()
                   if ln.strip().startswith(("tax_filing_log", "hr_monthly_ledger", "licence_ledger"))]
    check("--check 三表存在且缺列=[]",
          len(table_lines) == 3 and all("exists=True" in ln and "missing_cols=[]" in ln for ln in table_lines))

    rc, out = run_migrate("--show-create")
    say("SHOW CREATE TABLE 原文（验收证据）", out)

    # ── 1b. 生产守卫实测（目标库名=cma 时 --apply 必须被拒）──────────────────
    guard_env = dict(os.environ)
    guard_env.pop("CMA_DDL_APPROVED", None)
    p = subprocess.run([VENV_PY, MIGRATE, "--apply", "--url",
                        "mysql+pymysql://root@localhost/cma?charset=utf8mb4"],
                       capture_output=True, text=True, cwd=BACKEND, env=guard_env)
    say("生产守卫实测（--apply 目标=cma，期望拒绝且零执行）", (p.stdout or "") + (p.stderr or ""))
    check("生产库 --apply 被守卫拒绝（exit=3）",
          p.returncode == 3 and "拒绝执行" in (p.stdout or ""), f"exit={p.returncode}")

    # ── 2. 指向 scratch 的 engine + 建其余表 + 夹具 ─────────────────────────
    import app.database as dbmod
    dbmod.DATABASE_URL = SCRATCH_URL
    dbmod._engine = None
    dbmod._SessionLocal = None
    engine = dbmod.get_engine()
    check("app.database 指向 scratch（非生产）", engine.url.database == SCRATCH, str(engine.url.database))

    import app.auth_middleware as am
    am._redis_available = False        # token 走进程内内存，不碰生产 Redis
    am._token_store.clear()

    from app.database import Base
    import app.models  # noqa: F401  注册全部模型
    from app.models import (Entity, KPIDefinition, KPIValue, LicenceLedger, TaxFilingLog,
                            HrMonthlyLedger, User)

    Base.metadata.create_all(bind=engine)   # 三张新表已由迁移脚本建好 → checkfirst 跳过

    Session = dbmod.get_session_local()
    db = Session()
    db.add_all([Entity(id=1, name="测试酣客", short_name="酣客", status="active"),
                Entity(id=2, name="测试博海", short_name="博海", status="active")])
    db.add(User(id=1, username="vfy_admin", password_hash="x", name="验收管理员", role="ceo"))
    for code, name in (("P_TAX_TIMELY", "税务申报及时率"), ("L_EMPLOYEE_TURNOVER", "员工流失率"),
                       ("P_LICENCE_OK", "经营许可合规率")):
        db.add(KPIDefinition(entity_id=1, kpi_code=code, kpi_name=name, unit="%",
                             dimension="process", status="active"))
    db.commit()
    H1 = {"Authorization": f"Bearer {am.create_token(1, 1)}"}   # user 1 / entity 1
    H2 = {"Authorization": f"Bearer {am.create_token(1, 2)}"}   # user 1 / entity 2
    say("夹具", "entity 1/2；user id=1(ceo)；entity 1 有 3 个 KPI 定义、entity 2 无定义；"
                "token 走进程内存储（未写生产 Redis）")

    from fastapi.testclient import TestClient
    from app.main import app
    client = TestClient(app)

    # ── 3. 真实 HTTP：新增 / on_time 服务端派生 ───────────────────────────
    r = client.post("/api/cma/ext-ledger/tax-filings", headers=H1, json={
        "entity_id": 1, "period": "2026-08", "tax_type": "个人所得税",
        "legal_deadline": "2026-08-15", "filed_date": "2026-08-20",
        "on_time": 1, "evidence_no": "HZ-000", "registered_by": "财务部"})
    raw_req("POST /tax-filings（请求带 on_time=1，实际逾期）", r)
    b = r.json()
    check("on_time 服务端派生 = 0（手填值被忽略并回报）",
          r.status_code == 201 and b["data"]["on_time"] == 0 and b["on_time_ignored"] is True,
          json.dumps(b, ensure_ascii=False))

    r = client.post("/api/cma/ext-ledger/tax-filings", headers=H1, json={
        "period": "2026-08", "tax_type": "个人所得税", "legal_deadline": "2026-08-15",
        "filed_date": "2026-08-12"})
    raw_req("POST /tax-filings 重复（唯一键冲突）", r)
    check("唯一键冲突 → 409 且不新增", r.status_code == 409)

    # ── 4. CSV 批量导入 ×2（幂等）────────────────────────────────────────
    tax_csv = ("period,tax_type,legal_deadline,filed_date,evidence_no\n"
               "2026-08,增值税,2026-08-15,2026-08-12,HZ-001\n"
               "2026-08,附加税,2026-08-15,2026-08-12,HZ-002\n"
               "2026-08,印花税,2026-08-15,2026-08-20,HZ-003\n"
               "2026-07,增值税,2026-07-15,2026-07-10,HZ-004\n"
               "2026-07,附加税,2026-07-15,2026-07-10,HZ-005\n")
    r1 = client.post("/api/cma/ext-ledger/tax-filings/import", headers=H1,
                     json={"csv": tax_csv, "on_conflict": "skip"})
    raw_req("POST /tax-filings/import #1（CSV 5 行）", r1)
    r2 = client.post("/api/cma/ext-ledger/tax-filings/import", headers=H1,
                     json={"csv": tax_csv, "on_conflict": "skip"})
    raw_req("POST /tax-filings/import #2（同 CSV 复跑）", r2)
    check("CSV 导入幂等（#1 inserted=5；#2 inserted=0 skipped=5）",
          r1.json()["inserted"] == 5 and r2.json()["inserted"] == 0 and r2.json()["skipped"] == 5)
    n_tax = count(db, TaxFilingLog)
    check("台账行数 = 6（1 新增 + 5 导入，无重复行）", n_tax == 6, f"rows={n_tax}")

    r = client.post("/api/cma/ext-ledger/hr-monthly", headers=H1, json={
        "period": "2026-08", "headcount_begin": 100, "hires": 2, "separations": 1,
        "headcount_end": 99})
    raw_req("POST /hr-monthly（等式不成立）", r)
    check("人事等式不成立 → 400 拒绝写入", r.status_code == 400 and "等式不成立" in r.json()["detail"])
    check("被拒行未落库", count(db, HrMonthlyLedger) == 0)
    r = client.post("/api/cma/ext-ledger/hr-monthly/import", headers=H1, json={
        "csv": ("period,headcount_begin,hires,separations,headcount_end\n"
                "2026-08,120,2,4,118\n2026-07,118,3,1,120\n")})
    raw_req("POST /hr-monthly/import（2 行合法）", r)
    check("人事台账导入 2 行（2026-08: 120-4+2=118）", r.json()["inserted"] == 2)

    r = client.post("/api/cma/ext-ledger/licences/import", headers=H1, json={
        "csv": ("licence_name,licence_no,valid_from,valid_to,annual_check_status,issuer\n"
                "营业执照,91610100MA0001,2020-01-01,2030-01-01,已年检,市场监管局\n"
                "食品经营许可证,JY1610100001,2024-01-01,2026-03-31,过期,市场监管局\n"
                "酒类流通备案,HZ-2024-01,,2030-12-31,未到期,商务局\n")})
    raw_req("POST /licences/import（3 行，其中 1 张已过期）", r)
    check("证照台账导入 3 行", r.json()["inserted"] == 3)

    # ── 5. 按 (entity_id, period) 查询 ────────────────────────────────────
    r = client.get("/api/cma/ext-ledger/tax-filings?period=2026-08", headers=H1)
    raw_req("GET /tax-filings?period=2026-08", r)
    check("查询按期间过滤；及时率 2/4 = 50.0",
          r.json()["total"] == 4 and r.json()["summary"]["tax_timely_rate"] == 50.0,
          json.dumps(r.json()["summary"], ensure_ascii=False))

    r = client.get("/api/cma/ext-ledger/hr-monthly?period=2026-08", headers=H1)
    raw_req("GET /hr-monthly?period=2026-08", r)
    check("人事查询含流失率派生（4/120 = 3.33%）", r.json()["data"][0]["turnover_rate"] == 3.33)

    r = client.get("/api/cma/ext-ledger/licences?as_of=2026-06-30", headers=H1)
    raw_req("GET /licences?as_of=2026-06-30", r)
    check("证照合规率 2/3 = 66.7（过期证照不计入有效数）",
          r.json()["summary"]["licence_ok"] == 66.7 and r.json()["summary"]["valid"] == 2,
          json.dumps(r.json()["summary"], ensure_ascii=False))

    # ── 6. 派生 ×2（幂等）────────────────────────────────────────────────
    d1 = client.post("/api/cma/ext-ledger/derive", headers=H1, json={"period": "2026-08"})
    raw_req("POST /derive #1（2026-08）", d1)
    v1 = {k: v["actual_value"] for k, v in d1.json()["items"].items()}
    a1 = {k: v["action"] for k, v in d1.json()["items"].items()}
    check("派生值 = 税务 50.0 / 流失率 3.33 / 合规率 66.7(2026-Q3)",
          v1 == {"P_TAX_TIMELY": 50.0, "L_EMPLOYEE_TURNOVER": 3.33, "P_LICENCE_OK": 66.7},
          json.dumps(v1, ensure_ascii=False))
    check("首次派生全 insert（含季频期间归集 2026-08 → 2026-Q3）",
          set(a1.values()) == {"insert"} and d1.json()["items"]["P_LICENCE_OK"]["period"] == "2026-Q3",
          json.dumps(a1, ensure_ascii=False))
    kv1 = count(db, KPIValue)
    check("kpi_values 写入 3 行", kv1 == 3, f"rows={kv1}")

    d2 = client.post("/api/cma/ext-ledger/derive", headers=H1, json={"period": "2026-08"})
    raw_req("POST /derive #2（同期间复跑 → 幂等）", d2)
    a2 = {k: v["action"] for k, v in d2.json()["items"].items()}
    kv2 = count(db, KPIValue)
    check("派生幂等（第二次全 upsert；kpi_values 行数不变）",
          set(a2.values()) == {"upsert"} and kv1 == kv2, f"kv#1={kv1} kv#2={kv2} actions={a2}")

    # 分母 0（口径一/二）：2026-09 无任何税务/人事台账行 → None 且不写值
    d3 = client.post("/api/cma/ext-ledger/derive", headers=H1, json={
        "period": "2026-09", "kpi_codes": ["P_TAX_TIMELY", "L_EMPLOYEE_TURNOVER"]})
    raw_req("POST /derive（2026-09 台账空 → 分母 0）", d3)
    check("分母为 0 → actual_value=None 且 action=skipped_denominator_zero",
          all(v["actual_value"] is None and v["action"] == "skipped_denominator_zero"
              for v in d3.json()["items"].values()),
          json.dumps({k: v["action"] for k, v in d3.json()["items"].items()}, ensure_ascii=False))
    check("分母 0 未新增 kpi_values 行（无 0、无大数）", count(db, KPIValue) == kv2)

    # 分母 0（口径三）：证照台账清空后 P_LICENCE_OK 亦为 None（且回报旧行 stale）
    db.query(LicenceLedger).delete()
    db.commit()
    d4 = client.post("/api/cma/ext-ledger/derive", headers=H1, json={
        "period": "2026-08", "kpi_codes": ["P_LICENCE_OK"]})
    raw_req("POST /derive（证照台账已清空 → 应持证照数 0）", d4)
    it4 = d4.json()["items"]["P_LICENCE_OK"]
    check("证照分母 0 → None 且不覆盖/不新增值",
          it4["actual_value"] is None and it4["action"] == "skipped_denominator_zero"
          and it4["stale_row_present"] is True,
          json.dumps(it4, ensure_ascii=False)[:300])

    # 定义缺失：不新建定义
    d5 = client.post("/api/cma/ext-ledger/derive", headers=H2, json={"period": "2026-08"})
    raw_req("POST /derive（entity=2 无 KPI 定义）", d5)
    check("定义缺失 → skipped_no_definition（本模块不新建/不改动 KPI 定义）",
          all(v["action"] == "skipped_no_definition" for v in d5.json()["items"].values()))
    check("KPI 定义总数未变", count(db, KPIDefinition) == 3,
          f"defs={count(db, KPIDefinition)}")

    # ── 7. 边界：跨账套 403 / 未认证 401 ─────────────────────────────────
    r = client.post("/api/cma/ext-ledger/hr-monthly?entity_id=2", headers=H1, json={
        "period": "2026-08", "headcount_begin": 10, "hires": 0, "separations": 0, "headcount_end": 10})
    raw_req("POST /hr-monthly?entity_id=2（会话账套=1）", r)
    check("跨账套 → 403", r.status_code == 403 and "无权访问企业 entity_id=2" in r.json()["detail"])
    r = client.get("/api/cma/ext-ledger/options")
    raw_req("GET /options（无 token）", r)
    check("未认证 → 401", r.status_code == 401)

    # ── 8. 收尾 ─────────────────────────────────────────────────────────
    say("汇总断言", f"FAIL 数 = {len(FAILURES)}" + ("" if not FAILURES else "\n  " + "\n  ".join(FAILURES)))
    db.close()
    engine.dispose()
    con = root_conn()
    with con.cursor() as cur:
        cur.execute("SET SESSION lock_wait_timeout=8")
        for _ in range(3):
            try:
                cur.execute(f"DROP DATABASE IF EXISTS {SCRATCH}")
                break
            except Exception:
                time.sleep(1)
    con.close()
    print(f"\nscratch 库已删除: {SCRATCH}")
    print("边界自证: 全程 engine 数=1（URL 指向 scratch）；未连接/未读写生产库 cma；"
          "未使用 mysqldump 复制生产数据；token 未写生产 Redis")
    return 1 if FAILURES else 0


if __name__ == "__main__":
    sys.exit(main())