wecom-fullstack
885cfddbff
ci/woodpecker/push/main Pipeline failed
P0-A（任务 fullstack-20261017-warning-capability-build-p0）： - cash_thresholds.py：阈值迁 per-entity（复用 alert_rules.rule_type=cash_gap_threshold， 默认 20/10 万不动；无配置/非法/异常一律回落默认，fail-safe 不改预警行为） - cash_forecast_engine.py：阈值改从配置读取，响应新增 threshold_source 明写来源 - verify_cash_gap_replay.py：同输入双跑逐字段 diff=0 + 手算 3/3 + 端点逐字段对拍(--live) - cash_gap_narrative.py：复用 ai_brief.call_deepseek，只写原因/建议； 断言生成文本数字集合 ⊆ 规则层数字集合，违例即不带文案上线 - audit_warning_data_readiness.py：G0 只读核验（cma_reader SELECT only）， 行数/期间/枚举 + 数据质量缺口（期初全 0、负期末、单批次、无员工维度表） 未 push（push 即部署，须项目侧书面放行）；未改生产库/未动 cron。
410 lines
19 KiB
Python
410 lines
19 KiB
Python
#!/usr/bin/env python3
|
||
# -*- coding: utf-8 -*-
|
||
"""资金缺口复算对拍 — G1 验收件（P0-A）
|
||
|
||
任务：fullstack-20261017-warning-capability-build-p0（上游 yanxue-20261017-warning-capability-build-p0）
|
||
方案：`_reports/yanxue-20261017-warning-capability-build-p0.md` §G1 + §四「数字零错误（可复算）」
|
||
|
||
本脚本做三件事，全部机器可判：
|
||
1. **同输入双跑逐字段 diff=0**：固定输入（内存 SQLite 夹具 + 固定锚点日期）跑两遍
|
||
`forecast_cash_flow_with_plans`，把两次结果整体 JSON 归一后逐字段比对；
|
||
再用「独立重建的同一夹具」跑第三遍，证明无隐藏状态（跨会话也一致）。
|
||
2. **手算 3/3 案例对拍**：3 个案例的每日余额由人工算式预先算出并**写死为常量**
|
||
（不调用引擎反推期望值），逐字段断言：每日余额、缺口日、缺口金额、红黄绿、最小余额、
|
||
预警点、阈值来源。
|
||
3. **端点逐字段一致**（`--live`）：脚本内直接调引擎的结果 vs 运行中服务的
|
||
`GET /api/cma/cash/gap-forecast` 响应，逐字段比对（含 threshold_source）。
|
||
|
||
手算口径（与引擎实现逐条对应，锚点固定 2026-10-05 周一 09:00）：
|
||
锚点 today = 2026-10-05；offset 1..N 依次是其后 N 天。
|
||
无 KPI 历史 → ocf_avg 取引擎兜底 0.5（万元/日），趋势 0；无预算 → budget_ocf=None。
|
||
每日基础增量 = 0.5；`day>=25` 且无月底窗口（本例日期均在 10-06..10-15）；周末 ×0.5。
|
||
叠加收付款计划后：cash_i = cash_{i-1} + base_delta_i + planned_in_i - planned_out_i（逐步 round 2）。
|
||
|
||
用法：
|
||
夹具复算（默认，不连生产）:
|
||
python3 backend/scripts/verify_cash_gap_replay.py
|
||
端点对拍（需服务在线，默认 127.0.0.1:8010，读 .secrets 登录）:
|
||
python3 backend/scripts/verify_cash_gap_replay.py --live --base-url http://127.0.0.1:8010 --entity 1
|
||
退出码：0 = 全部 PASS（diff=0 且 3/3 案例 PASS，live 模式下端点逐字段一致）；1 = 有 FAIL
|
||
"""
|
||
from __future__ import annotations
|
||
|
||
import argparse
|
||
import json
|
||
import os
|
||
import sys
|
||
import urllib.request
|
||
from datetime import datetime as _RealDatetime
|
||
|
||
BACKEND_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||
if BACKEND_DIR not in sys.path:
|
||
sys.path.insert(0, BACKEND_DIR)
|
||
|
||
from sqlalchemy import create_engine # noqa: E402
|
||
from sqlalchemy.orm import sessionmaker # noqa: E402
|
||
|
||
import app.models as models # noqa: E402
|
||
from app.api.alert_rules import AlertRule # noqa: E402
|
||
from app.utils import cash_forecast_engine as engine # noqa: E402
|
||
from app.utils.cash_thresholds import ( # noqa: E402
|
||
CASH_GAP_RULE_TYPE,
|
||
DEFAULT_CRITICAL,
|
||
DEFAULT_WARNING,
|
||
)
|
||
|
||
# ── 固定锚点（2026-10-05 周一，保证手算窗口内无周末/无月底）────────────────────
|
||
FIXED_NOW = _RealDatetime(2026, 10, 5, 9, 0, 0)
|
||
ENTITY = 1
|
||
|
||
FIXTURE_TABLES = None
|
||
|
||
|
||
class FixedDatetime(_RealDatetime):
|
||
"""把引擎里的 datetime.now() 钉死在 FIXED_NOW，保证复算可重现。"""
|
||
|
||
@classmethod
|
||
def now(cls, tz=None): # noqa: D401
|
||
return FIXED_NOW
|
||
|
||
|
||
def build_fixture_session(with_config=None, cash_plans=None, budget=None):
|
||
"""内存 SQLite 夹具：建最小表集，按需插入收付款计划 / 阈值配置 / 预算。"""
|
||
from sqlalchemy.pool import StaticPool
|
||
|
||
global FIXTURE_TABLES
|
||
if FIXTURE_TABLES is None:
|
||
FIXTURE_TABLES = [
|
||
models.KPIDefinition.__table__,
|
||
models.KPIValue.__table__,
|
||
models.SystemConfig.__table__,
|
||
models.CashPlan.__table__,
|
||
models.BudgetPlan.__table__,
|
||
AlertRule.__table__,
|
||
]
|
||
eng = create_engine("sqlite://", connect_args={"check_same_thread": False},
|
||
poolclass=StaticPool)
|
||
models.Base.metadata.create_all(eng, tables=FIXTURE_TABLES)
|
||
db = sessionmaker(bind=eng)()
|
||
for plan in (cash_plans or []):
|
||
db.add(models.CashPlan(**plan))
|
||
if with_config:
|
||
db.add(AlertRule(entity_id=ENTITY, kpi_id=0, rule_type=CASH_GAP_RULE_TYPE,
|
||
trigger_on="actual", enabled=1, params=with_config))
|
||
if budget:
|
||
# 预算校准会走 KPI 查找；夹具无 KPI → _budget_monthly_ocf 恒 None（保持手算口径）
|
||
pass
|
||
db.commit()
|
||
return db
|
||
|
||
|
||
def normalise(obj):
|
||
"""JSON 归一（Decimal/日期 → 字符串，浮点保持原值），用于逐字段严格比对。"""
|
||
return json.loads(json.dumps(obj, ensure_ascii=False, default=str))
|
||
|
||
|
||
def deep_diff(left, right, path="root"):
|
||
"""返回逐字段差异列表；空列表 = diff=0。"""
|
||
diffs = []
|
||
if type(left) is not type(right):
|
||
return [{"path": path, "left": left, "right": right, "kind": "type"}]
|
||
if isinstance(left, dict):
|
||
for key in sorted(set(left) | set(right)):
|
||
if key not in left:
|
||
diffs.append({"path": f"{path}.{key}", "left": "<absent>", "right": right[key], "kind": "missing_left"})
|
||
elif key not in right:
|
||
diffs.append({"path": f"{path}.{key}", "left": left[key], "right": "<absent>", "kind": "missing_right"})
|
||
else:
|
||
diffs.extend(deep_diff(left[key], right[key], f"{path}.{key}"))
|
||
return diffs
|
||
if isinstance(left, list):
|
||
if len(left) != len(right):
|
||
diffs.append({"path": path + ".len", "left": len(left), "right": len(right), "kind": "length"})
|
||
for idx, (a, b) in enumerate(zip(left, right)):
|
||
diffs.extend(deep_diff(a, b, f"{path}[{idx}]"))
|
||
return diffs
|
||
if left != right:
|
||
diffs.append({"path": path, "left": left, "right": right, "kind": "value"})
|
||
return diffs
|
||
|
||
|
||
# ══════════════════════════════════════════════════════════════════════
|
||
# 手算案例（期望值写死；算式见各案例注释，禁止用引擎反推）
|
||
# ══════════════════════════════════════════════════════════════════════
|
||
HAND_CASES = [
|
||
{
|
||
"name": "H1 无计划·全绿",
|
||
"input": {"current_cash": 30.0, "days": 10},
|
||
"plans": [],
|
||
"config": None,
|
||
# 手算：cash_i = 30 + 0.5*i（工作日）；10-10 周六 / 10-11 周日 增量 ×0.5 = 0.25
|
||
# 30 → 30.5 → 31.0 → 31.5 → 32.0 →(六) 32.25 →(日) 32.5 → 33.0 → 33.5 → 34.0 → 34.5
|
||
"expect": {
|
||
"warning_line": 20.0,
|
||
"critical_line": 10.0,
|
||
"threshold_source": "default",
|
||
"predicted_cash": [30.5, 31.0, 31.5, 32.0, 32.25, 32.5, 33.0, 33.5, 34.0, 34.5],
|
||
"alert_status": ["green"] * 10,
|
||
"gap_days": 0,
|
||
"gap_dates": [],
|
||
"min_cash": 30.5,
|
||
"min_cash_date": "2026-10-06",
|
||
"pre_alerts": 0,
|
||
"planned_out_total": 0.0,
|
||
},
|
||
},
|
||
{
|
||
"name": "H2 单笔付款·缺口前1天预警",
|
||
"input": {"current_cash": 21.0, "days": 4},
|
||
"plans": [
|
||
# 10-08（Thu）对外付款 4.0 万
|
||
{"entity_id": ENTITY, "plan_type": "pay", "amount": 4.0, "status": "pending",
|
||
"plan_date": _RealDatetime(2026, 10, 8, 0, 0, 0), "counterparty": "手算案例供应商",
|
||
"description": "FIXTURE-H2", "source": "fixture"},
|
||
],
|
||
"config": None,
|
||
# 手算：21 → +0.5 = 21.5 → +0.5 = 22.0 → +0.5 -4.0 = 18.5 → +0.5 = 19.0
|
||
# 缺口日 = 余额 < 20 → 10-08(18.5)、10-09(19.0)，缺额 1.5 / 1.0
|
||
# 缺口区间首日 idx=2 → 前1天预警落在 idx=1（10-07，余额 22.0 为 green）
|
||
"expect": {
|
||
"warning_line": 20.0,
|
||
"critical_line": 10.0,
|
||
"threshold_source": "default",
|
||
"predicted_cash": [21.5, 22.0, 18.5, 19.0],
|
||
"alert_status": ["green", "green", "yellow", "yellow"],
|
||
"gap_days": 2,
|
||
"gap_dates": [
|
||
{"date": "2026-10-08", "predicted_cash": 18.5, "gap_amount": 1.5, "level": "yellow"},
|
||
{"date": "2026-10-09", "predicted_cash": 19.0, "gap_amount": 1.0, "level": "yellow"},
|
||
],
|
||
"min_cash": 18.5,
|
||
"min_cash_date": "2026-10-08",
|
||
"pre_alerts": 1,
|
||
"pre_alert_0": {"alert_date": "2026-10-07", "alert_offset": 2, "gap_date": "2026-10-08",
|
||
"gap_cash": 18.5, "gap_amount": 1.5, "worst_cash": 18.5,
|
||
"worst_date": "2026-10-08", "lead_days": 1, "level": "yellow"},
|
||
"planned_out_total": 4.0,
|
||
},
|
||
},
|
||
{
|
||
"name": "H3 配置阈值 25/15·全红",
|
||
"input": {"current_cash": 8.0, "days": 5},
|
||
"plans": [],
|
||
# alert_rules: cash_gap_threshold {"warning":25.0,"critical":15.0}
|
||
"config": {"warning": 25.0, "critical": 15.0},
|
||
# 手算：8 → 8.5 → 9.0 → 9.5 → 10.0 →(六) 10.25，全部 < 15（红灯）
|
||
# 缺额（相对黄灯线 25）：16.5 / 16.0 / 15.5 / 15.0 / 14.75
|
||
"expect": {
|
||
"warning_line": 25.0,
|
||
"critical_line": 15.0,
|
||
"threshold_source": "alert_rules",
|
||
"predicted_cash": [8.5, 9.0, 9.5, 10.0, 10.25],
|
||
"alert_status": ["red"] * 5,
|
||
"gap_days": 5,
|
||
"gap_dates_first": {"date": "2026-10-06", "predicted_cash": 8.5,
|
||
"gap_amount": 16.5, "level": "red"},
|
||
"min_cash": 8.5,
|
||
"min_cash_date": "2026-10-06",
|
||
"pre_alerts": 0, # 缺口从第一天开始 → 无「缺口前1天」可落
|
||
"planned_out_total": 0.0,
|
||
},
|
||
},
|
||
]
|
||
|
||
|
||
def run_case(case):
|
||
db = build_fixture_session(with_config=case.get("config"), cash_plans=case.get("plans"))
|
||
try:
|
||
result = engine.forecast_cash_flow_with_plans(
|
||
ENTITY, db, days=case["input"]["days"], current_cash=case["input"]["current_cash"])
|
||
finally:
|
||
db.close()
|
||
return normalise(result)
|
||
|
||
|
||
def assert_case(case, result, failures):
|
||
exp = case["expect"]
|
||
name = case["name"]
|
||
checks = [
|
||
("warning_line", result.get("warning_line"), exp["warning_line"]),
|
||
("critical_line", result.get("critical_line"), exp["critical_line"]),
|
||
("threshold_source.source", (result.get("threshold_source") or {}).get("source"), exp["threshold_source"]),
|
||
("min_cash", result.get("min_cash"), exp["min_cash"]),
|
||
("min_cash_date", result.get("min_cash_date"), exp["min_cash_date"]),
|
||
("gap_days", len(result.get("gap_dates") or []), exp["gap_days"]),
|
||
("pre_alerts", len(result.get("pre_alerts") or []), exp["pre_alerts"]),
|
||
("predicted_cash",
|
||
[row["predicted_cash"] for row in result.get("forecast", [])], exp["predicted_cash"]),
|
||
("alert_status",
|
||
[row["alert_status"] for row in result.get("forecast", [])], exp["alert_status"]),
|
||
("summary.total_planned_out",
|
||
round((result.get("summary") or {}).get("total_planned_out") or 0.0, 2), exp["planned_out_total"]),
|
||
]
|
||
if "gap_dates" in exp:
|
||
checks.append(("gap_dates", [
|
||
{k: g[k] for k in ("date", "predicted_cash", "gap_amount", "level")}
|
||
for g in (result.get("gap_dates") or [])
|
||
], exp["gap_dates"]))
|
||
if "gap_dates_first" in exp:
|
||
first = (result.get("gap_dates") or [{}])[0]
|
||
checks.append(("gap_dates[0]",
|
||
{k: first.get(k) for k in ("date", "predicted_cash", "gap_amount", "level")},
|
||
exp["gap_dates_first"]))
|
||
if "pre_alert_0" in exp:
|
||
first = (result.get("pre_alerts") or [{}])[0]
|
||
checks.append(("pre_alerts[0]",
|
||
{k: first.get(k) for k in exp["pre_alert_0"].keys()}, exp["pre_alert_0"]))
|
||
|
||
for label, got, want in checks:
|
||
if got != want:
|
||
failures.append({"case": name, "check": label, "expected": want, "got": got})
|
||
print(" FAIL %-8s %-28s expected=%r got=%r" % (name.split()[0], label, want, got))
|
||
else:
|
||
print(" PASS %-8s %-28s = %r" % (name.split()[0], label, got))
|
||
|
||
|
||
def assert_invalid_config_falls_back(failures):
|
||
"""非法配置（critical > warning）→ 必须回落默认 20/10（配置写坏不能静默生效）。"""
|
||
db = build_fixture_session(with_config={"warning": 10.0, "critical": 20.0})
|
||
try:
|
||
got = engine.get_cash_gap_thresholds(db, ENTITY)
|
||
finally:
|
||
db.close()
|
||
if got != (DEFAULT_WARNING, DEFAULT_CRITICAL):
|
||
failures.append({"case": "GUARD", "check": "invalid_config_fallback",
|
||
"expected": [DEFAULT_WARNING, DEFAULT_CRITICAL], "got": list(got)})
|
||
print(" FAIL GUARD invalid config must fall back to defaults, got=%r" % (got,))
|
||
else:
|
||
print(" PASS GUARD invalid config → default (%.1f, %.1f)" % got)
|
||
|
||
|
||
def http_json(base_url, path, method="GET", payload=None, token=None, timeout=30):
|
||
headers = {}
|
||
if token:
|
||
headers["Authorization"] = "Bearer " + token
|
||
data = json.dumps(payload).encode() if payload is not None else None
|
||
if data:
|
||
headers["Content-Type"] = "application/json"
|
||
req = urllib.request.Request(base_url.rstrip("/") + path, data=data, headers=headers, method=method)
|
||
with urllib.request.urlopen(req, timeout=timeout) as resp:
|
||
return json.loads(resp.read().decode())
|
||
|
||
|
||
def _dotenv(path):
|
||
env = {}
|
||
with open(path, encoding="utf-8") as fh:
|
||
for line in fh:
|
||
line = line.strip()
|
||
if line and "=" in line and not line.startswith("#"):
|
||
key, val = line.split("=", 1)
|
||
env[key.strip()] = val.strip().strip('"').strip("'")
|
||
return env
|
||
|
||
|
||
def live_compare(base_url, entity_id, days, failures):
|
||
"""脚本内引擎结果 vs 在线端点响应（逐字段，含 threshold_source）。"""
|
||
# 夹具阶段把 now() 钉死在 FIXED_NOW；对拍真实端点必须还原真实时钟，
|
||
# 否则日期序列整体错位（脚本 2026-10-06 vs 服务 2026-10-18）。
|
||
engine.datetime = _RealDatetime
|
||
reader_env = os.environ.get("CMA_READER_ENV",
|
||
"/root/.hermes/profiles/wecom-fullstack/.secrets/cma-reader.env")
|
||
admin_env = os.environ.get("CMA_ADMIN_ENV",
|
||
"/root/.hermes/profiles/wecom-fullstack/.secrets/cma-admin.env")
|
||
rd = _dotenv(reader_env)
|
||
url = ("mysql+pymysql://%s:%s@%s:%s/%s?charset=utf8mb4"
|
||
% (rd["CMA_READER_DB_USER"], rd["CMA_READER_DB_PASS"], rd["CMA_READER_DB_HOST"],
|
||
rd["CMA_READER_DB_PORT"], rd["CMA_READER_DB_NAME"]))
|
||
eng = create_engine(url, pool_pre_ping=True)
|
||
db = sessionmaker(bind=eng)()
|
||
try:
|
||
script_side = normalise(engine.forecast_cash_flow_with_plans(entity_id, db, days=days))
|
||
finally:
|
||
db.close()
|
||
|
||
adm = _dotenv(admin_env)
|
||
login = http_json(base_url, "/api/cma/auth/login", "POST",
|
||
{"username": adm.get("CMA_ADMIN_USER"), "password": adm.get("CMA_ADMIN_PASS"),
|
||
"entity_id": entity_id})
|
||
token = login.get("token")
|
||
if not token:
|
||
failures.append({"case": "LIVE", "check": "login", "got": login})
|
||
print(" FAIL LIVE login 未拿到 token")
|
||
return
|
||
api_side = normalise(http_json(base_url, "/api/cma/cash/gap-forecast?days=%d" % days, token=token))
|
||
|
||
diffs = deep_diff(script_side, api_side, path="gap-forecast")
|
||
print(" %s LIVE 引擎 vs 端点 /api/cma/cash/gap-forecast diff=%d" %
|
||
("PASS" if not diffs else "FAIL", len(diffs)))
|
||
for d in diffs[:20]:
|
||
print(" %(path)s script=%(left)r api=%(right)r" % d)
|
||
if diffs:
|
||
failures.append({"case": "LIVE", "check": "endpoint_field_parity", "diffs": diffs[:20],
|
||
"diff_count": len(diffs)})
|
||
|
||
|
||
def main():
|
||
ap = argparse.ArgumentParser()
|
||
ap.add_argument("--live", action="store_true", help="对拍运行中服务端点")
|
||
ap.add_argument("--base-url", default="http://127.0.0.1:8010")
|
||
ap.add_argument("--entity", type=int, default=ENTITY)
|
||
ap.add_argument("--days", type=int, default=30)
|
||
ap.add_argument("--json-out", default="")
|
||
args = ap.parse_args()
|
||
|
||
failures = []
|
||
summary = {"fixed_now": FIXED_NOW.isoformat(), "entity_id": args.entity, "cases": []}
|
||
|
||
# 固定锚点
|
||
engine.datetime = FixedDatetime
|
||
|
||
print("== 1. 同输入双跑逐字段 diff=0 ==")
|
||
for case in HAND_CASES:
|
||
first = run_case(case)
|
||
second = run_case(case)
|
||
third = run_case(case) # 独立重建夹具，验证无隐藏状态
|
||
diffs_12 = deep_diff(first, second, path="run1_vs_run2")
|
||
diffs_13 = deep_diff(first, third, path="run1_vs_run3")
|
||
print(" %s %-24s diff(run1,run2)=%d diff(run1,run3)=%d" %
|
||
("PASS" if not (diffs_12 or diffs_13) else "FAIL",
|
||
case["name"], len(diffs_12), len(diffs_13)))
|
||
for d in (diffs_12 + diffs_13)[:10]:
|
||
print(" %(path)s left=%(left)r right=%(right)r" % d)
|
||
if diffs_12 or diffs_13:
|
||
failures.append({"case": case["name"], "check": "double_run_diff",
|
||
"diffs": (diffs_12 + diffs_13)[:10]})
|
||
summary["cases"].append({"case": case["name"], "diff_run2": len(diffs_12),
|
||
"diff_run3": len(diffs_13)})
|
||
|
||
print("== 2. 手算案例对拍 ==")
|
||
for case in HAND_CASES:
|
||
assert_case(case, run_case(case), failures)
|
||
|
||
print("== 3. 配置健壮性 ==")
|
||
assert_invalid_config_falls_back(failures)
|
||
|
||
if args.live:
|
||
print("== 4. 端点逐字段一致（--live）==")
|
||
live_compare(args.base_url, args.entity, args.days, failures)
|
||
|
||
passed = 3 if not failures else 0
|
||
print("=" * 60)
|
||
print("RESULT: %s hand_calc=%d/%d failures=%d" %
|
||
("PASS" if not failures else "FAIL", passed, len(HAND_CASES), len(failures)))
|
||
if failures:
|
||
for f in failures[:5]:
|
||
print(" ", json.dumps(f, ensure_ascii=False)[:400])
|
||
if args.json_out:
|
||
payload = {"task": "fullstack-20261017-warning-capability-build-p0",
|
||
"stage": "G1 cash gap replay", "result": "PASS" if not failures else "FAIL",
|
||
"fixed_now": FIXED_NOW.isoformat(), "entity_id": args.entity,
|
||
"hand_cases_passed": passed, "hand_cases_total": len(HAND_CASES),
|
||
"failures": failures, "summary": summary}
|
||
with open(args.json_out, "w", encoding="utf-8") as fh:
|
||
fh.write(json.dumps(payload, ensure_ascii=False, indent=2, default=str) + "\n")
|
||
print("json evidence: %s" % args.json_out)
|
||
return 0 if not failures else 1
|
||
|
||
|
||
if __name__ == "__main__":
|
||
sys.exit(main())
|