fix(cpa): sales bill-type whitelist for customer revenue caliber (P0)
ci/woodpecker/push/main Pipeline failed
ci/woodpecker/push/main Pipeline failed
Customer revenue/concentration now filters bill_type IN (1,4,14) (XSD/XSTHD/XSHHD). Non-sales docs (JHD purchase, YFKD prepay, YSKD accept, QTRK, SKD, YBFYD) carry SUPPLIER unit_id and were polluting 'customer concentration' (old Top1/Top2 were our biggest suppliers; root cause of 6.04x base gap). Unify sign filter to (sum_money+sum_tax_money)>0 (equivalent to money_only on current data, diff 0 rows) and correct the withdrawn 407-row/17.2% wording in cpa_caliber docstring. Deriver: anchors from cpa_caliber single source; new --concentration overwrite and --only concentration (minimal 9-row write face) with pre-overwrite backup. Tests: fixture gains bill_type/sum_tax_money; +5 whitelist cases (44 passed targeted).
This commit is contained in:
@@ -8,8 +8,19 @@
|
||||
2) `kpi_values` 汇总口径(**只存汇总,避免 284 客户 × 期数污染 KPI 宽表**,任务 §2.1):
|
||||
* `C_CUST_REVENUE` 期内客户收入合计(万元)
|
||||
* `C_CUST_COUNT` 期内活跃客户数(个)
|
||||
* `C_CUST_CONCENTRATION` **补值**:仅对**库内尚无该期行**的期次写入(现网定义 = 前5大客户
|
||||
收入 ÷ 期内合计 ×100;已有 2026-01~09 九期**原样保留、不覆盖**)
|
||||
* `C_CUST_CONCENTRATION` 现网定义 = 前5大客户收入 ÷ 期内合计 ×100。
|
||||
`--concentration`(默认 `fill-gaps`)控制**已有期次**的处置:
|
||||
* `fill-gaps` = 仅补库内尚无该期行的期次(2026-01~09 九期原样保留)
|
||||
* `overwrite` = **重算并覆盖**指定期次(P0 口径纠错后重算九期用;
|
||||
属生产写,须走放行)
|
||||
|
||||
口径(2026-10-04 P0 纠错后)
|
||||
单据类型白名单 = **销售类 `bill_type IN (1,4,14)`**(XSD 销售单 / XSTHD 销售退货 /
|
||||
XSHHD 销售换货);非销售单据(进货/验收/入库/预付款/收款/费用)的 unit_id 是**供应商**,
|
||||
不得计入客户收入。取数口径单一事实源 = `app/utils/cpa_caliber.py`(与
|
||||
`alert_rules._customer_revenue_shares` 同一实现)。
|
||||
2026-01~09 对拍锚点:65 户 / 1,207.74 万 / Top1 22.5150%(u152) / Top2 13.9734%(u155) /
|
||||
Top4 46.9407% / Top10 69.7511%(`cpa_caliber.ANCHORS_2026_01_09`)。
|
||||
|
||||
fail-closed(本器**不写**以下列/值;理由见 `app/utils/cpa_caliber.py::FAIL_CLOSED_REASONS`)
|
||||
`cogs_alloc` / `chan_sub` / `contrib` / `contrib_rate` / `C_CUST_COGS_ALLOC` /
|
||||
@@ -53,9 +64,10 @@ FAIL_CLOSED_NOTE = (
|
||||
"cogs_alloc/chan_sub/contrib/contrib_rate = P1 fail-closed(未裁定口径/未打标),禁静默 0 或分摊值"
|
||||
)
|
||||
|
||||
ANCHOR_2026_01_09 = {"units": 73, "total_wan": 4883.52, "top1": 25.3835, "top2": 23.4905,
|
||||
"top4_cum": 63.3249, "top10_cum": 81.7675}
|
||||
ANCHOR_TOL = 0.1
|
||||
# 对拍锚点**单一事实源** = `cpa_caliber.ANCHORS_2026_01_09`(销售类白名单口径,2026-10-04 P0 纠错后)
|
||||
def _anchors():
|
||||
cal = _load_caliber()
|
||||
return cal.ANCHORS_2026_01_09, cal.ANCHOR_TOL
|
||||
|
||||
|
||||
def _engine(args):
|
||||
@@ -109,6 +121,30 @@ def _load_caliber():
|
||||
return cpa_caliber
|
||||
|
||||
|
||||
def _dump_prior_concentration(engine, entity_id, kpi_id, ts):
|
||||
"""**覆盖前备份**现网 `C_CUST_CONCENTRATION` 全部行(回退点),落 `scripts/backups/`。
|
||||
|
||||
返回备份文件绝对路径(写进结果 JSON,供审批串/回退引用)。
|
||||
"""
|
||||
import json as _json
|
||||
from sqlalchemy import text
|
||||
cols = ("id", "period", "actual_value", "data_status", "source_type", "source_batch",
|
||||
"remark", "entity_id")
|
||||
with engine.connect() as conn:
|
||||
rows = conn.execute(text(
|
||||
"SELECT id, period, actual_value, data_status, source_type, source_batch, remark, entity_id "
|
||||
"FROM kpi_values WHERE entity_id=:e AND kpi_id=:k ORDER BY period, id"),
|
||||
{"e": entity_id, "k": kpi_id}).fetchall()
|
||||
d = os.path.join(HERE, "backups")
|
||||
os.makedirs(d, exist_ok=True)
|
||||
path = os.path.join(d, "cpa-concentration-prior-%s.json" % ts.replace(":", "").replace(" ", "-"))
|
||||
with open(path, "w", encoding="utf-8") as f:
|
||||
_json.dump({"ts": ts, "entity_id": entity_id, "kpi_id": kpi_id,
|
||||
"rows": [dict(zip(cols, r)) for r in rows]}, f, ensure_ascii=False,
|
||||
indent=2, default=str)
|
||||
return path
|
||||
|
||||
|
||||
def compute(engine, dialect, entity_id):
|
||||
"""只读计算:{period: {unit_id: amount}} + 汇总。"""
|
||||
cal = _load_caliber()
|
||||
@@ -138,15 +174,24 @@ def anchor_check(summary):
|
||||
return pairs
|
||||
|
||||
|
||||
def run(engine, dialect, entity_id, batch, apply=False, verify=False, since=None, until=None):
|
||||
def run(engine, dialect, entity_id, batch, apply=False, verify=False, since=None, until=None,
|
||||
concentration="fill-gaps", only="all"):
|
||||
cal = _load_caliber()
|
||||
from sqlalchemy import text
|
||||
result = {"entity_id": entity_id, "batch": batch, "apply": apply, "verify": verify}
|
||||
result = {"entity_id": entity_id, "batch": batch, "apply": apply, "verify": verify,
|
||||
"concentration_mode": concentration, "only": only}
|
||||
if concentration not in ("fill-gaps", "overwrite"):
|
||||
raise SystemExit("FAIL-CLOSED: --concentration 仅支持 fill-gaps / overwrite")
|
||||
if only not in ("all", "concentration"):
|
||||
raise SystemExit("FAIL-CLOSED: --only 仅支持 all / concentration")
|
||||
if apply and only == "concentration" and concentration != "overwrite":
|
||||
raise SystemExit("FAIL-CLOSED: --only concentration 必须配 --concentration overwrite(否则九期被跳过=空写)")
|
||||
|
||||
with engine.connect() as conn:
|
||||
exists = _table_exists(conn, dialect)
|
||||
if apply and not exists:
|
||||
if apply and only != "concentration" and not exists:
|
||||
raise SystemExit("FAIL-CLOSED: 目标表不存在,请先跑 migrate_cpa_snapshot_20261004.py --apply")
|
||||
|
||||
rows_before = None
|
||||
if exists:
|
||||
rows_before = conn.execute(text(f"SELECT COUNT(*) FROM {SNAPSHOT_TABLE} WHERE entity_id=:e"),
|
||||
@@ -174,21 +219,27 @@ def run(engine, dialect, entity_id, batch, apply=False, verify=False, since=None
|
||||
|
||||
snap_rows = []
|
||||
kpi_rows = []
|
||||
conc_written_periods = []
|
||||
for p in periods:
|
||||
pairs = monthly[p]
|
||||
total_yuan = sum(a for _, a in pairs)
|
||||
for idx, (uid, amount) in enumerate(pairs, start=1):
|
||||
snap_rows.append({
|
||||
"entity_id": entity_id, "period": p, "unit_id": uid,
|
||||
"revenue": cal.to_wan(amount), "rank_no": idx,
|
||||
"calib_status": "revenue_only", "fail_closed_reason": FAIL_CLOSED_NOTE,
|
||||
})
|
||||
kpi_rows.append(("C_CUST_REVENUE", p, cal.to_wan(total_yuan)))
|
||||
kpi_rows.append(("C_CUST_COUNT", p, float(cal.active_customer_count(pairs))))
|
||||
if p not in existing_conc_periods:
|
||||
if only != "concentration":
|
||||
for idx, (uid, amount) in enumerate(pairs, start=1):
|
||||
snap_rows.append({
|
||||
"entity_id": entity_id, "period": p, "unit_id": uid,
|
||||
"revenue": cal.to_wan(amount), "rank_no": idx,
|
||||
"calib_status": "revenue_only", "fail_closed_reason": FAIL_CLOSED_NOTE,
|
||||
})
|
||||
kpi_rows.append(("C_CUST_REVENUE", p, cal.to_wan(total_yuan)))
|
||||
kpi_rows.append(("C_CUST_COUNT", p, float(cal.active_customer_count(pairs))))
|
||||
if concentration == "overwrite" or p not in existing_conc_periods:
|
||||
kpi_rows.append(("C_CUST_CONCENTRATION", p, cal.concentration_pct(pairs, 5)))
|
||||
if p in existing_conc_periods:
|
||||
conc_written_periods.append(p)
|
||||
result["only"] = only
|
||||
result["snapshot_rows"] = len(snap_rows)
|
||||
result["kpi_rows"] = len(kpi_rows)
|
||||
result["concentration_overwritten_periods"] = sorted(conc_written_periods)
|
||||
result["kpi_rows_skipped_no_definition"] = [c for c, i in kpi_ids.items() if i is None]
|
||||
|
||||
if not apply:
|
||||
@@ -204,11 +255,17 @@ def run(engine, dialect, entity_id, batch, apply=False, verify=False, since=None
|
||||
result["grants_ok"] = ok
|
||||
if not ok:
|
||||
raise SystemExit("FAIL-CLOSED: 应用账号缺 DML 授权(kpi_values / 目标表)。原始 SHOW GRANTS:\n" + grants)
|
||||
missing = [c for c, i in kpi_ids.items() if i is None]
|
||||
need = (["C_CUST_CONCENTRATION"] if only == "concentration"
|
||||
else ["C_CUST_REVENUE", "C_CUST_COUNT", "C_CUST_CONCENTRATION"])
|
||||
missing = [c for c in need if kpi_ids.get(c) is None]
|
||||
if missing:
|
||||
raise SystemExit("FAIL-CLOSED: KPI 定义缺失 %s —— 请先执行 scripts/sql/cpa_kpi_defs_20261004.sql" % missing)
|
||||
|
||||
now_ts = _dt.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
|
||||
if concentration == "overwrite" and kpi_ids.get("C_CUST_CONCENTRATION"):
|
||||
result["prior_backup"] = _dump_prior_concentration(engine, entity_id,
|
||||
kpi_ids["C_CUST_CONCENTRATION"], now_ts)
|
||||
|
||||
inserted_snap = updated_snap = 0
|
||||
kpi_inserted = kpi_updated = 0
|
||||
with engine.begin() as conn:
|
||||
@@ -236,10 +293,14 @@ def run(engine, dialect, entity_id, batch, apply=False, verify=False, since=None
|
||||
"SELECT id, actual_value, data_status FROM kpi_values "
|
||||
"WHERE entity_id=:e AND kpi_id=:k AND period=:p ORDER BY id DESC LIMIT 1"),
|
||||
{"e": entity_id, "k": kid, "p": period}).fetchone()
|
||||
remark = {"C_CUST_REVENUE": "口径=Σzxt_masterbill.sum_money(期内、unit_id>0、含税净额>0 取舍)",
|
||||
"C_CUST_COUNT": "口径=期内 zxt_masterbill 活跃往来单位数(unit_id 去重)",
|
||||
"C_CUST_CONCENTRATION": "口径=前5大客户收入/期内客户收入合计×100%(补值期次;"
|
||||
"2026-01~09 现网值原样保留)"}[code]
|
||||
conc_note = ("重算覆盖期次(P0 销售类白名单口径)"
|
||||
if concentration == "overwrite" and period in existing_conc_periods
|
||||
else "补值期次(库内原无该期行)")
|
||||
remark = {"C_CUST_REVENUE": "口径=Σzxt_masterbill.sum_money(销售类 bill_type IN (1,4,14)、"
|
||||
"期内、unit_id>0、符号过滤(sum_money+sum_tax_money)>0)",
|
||||
"C_CUST_COUNT": "口径=期内 zxt_masterbill 销售类单据活跃客户数(unit_id 去重)",
|
||||
"C_CUST_CONCENTRATION": f"口径=前5大客户收入/期内客户收入合计×100%({conc_note})"}[code]
|
||||
|
||||
if row:
|
||||
# 幂等:值不变则零写入;不降级 data_status
|
||||
if abs(float(row[1] or 0) - float(value)) < 1e-9 and row[2]:
|
||||
@@ -300,26 +361,36 @@ def _verify(engine, dialect, entity_id, monthly, periods, kpi_ids, existing_conc
|
||||
|
||||
|
||||
def anchor_assert(engine, entity_id):
|
||||
"""2026-01~09 窗口对拍锚点(任务验收标准逐项)。"""
|
||||
from sqlalchemy import text
|
||||
"""2026-01~09 窗口对拍锚点(任务验收标准逐项)。
|
||||
|
||||
取数**只走** `cpa_caliber` 单一事实源(不另写第二份聚合 SQL):逐月客户对按月求和
|
||||
= 窗口客户合计(每行只属一个 unit,故月度求和与窗口 GROUP BY 等价)。
|
||||
"""
|
||||
cal = _load_caliber()
|
||||
with engine.connect() as conn:
|
||||
rows = conn.execute(text(
|
||||
"SELECT unit_id, SUM(sum_money) AS amount FROM zxt_masterbill WHERE entity_id=:e "
|
||||
"AND unit_id IS NOT NULL AND unit_id>0 AND (sum_money + sum_tax_money) > 0 "
|
||||
"AND bill_date>='2026-01-01' AND bill_date<='2026-09-30' GROUP BY unit_id ORDER BY amount DESC"),
|
||||
{"e": entity_id}).fetchall()
|
||||
pairs = [(int(r[0]), float(r[1])) for r in rows]
|
||||
anchors, tol = _anchors()
|
||||
monthly = cal.fetch_monthly_revenue(engine, entity_id)
|
||||
by_unit = {}
|
||||
for p, pairs in monthly.items():
|
||||
if "2026-01" <= p <= "2026-09":
|
||||
for u, a in pairs:
|
||||
by_unit[u] = by_unit.get(u, 0.0) + a
|
||||
pairs = sorted(by_unit.items(), key=lambda x: x[1], reverse=True)
|
||||
total = sum(a for _, a in pairs)
|
||||
cum = lambda n: round(sum(a for _, a in pairs[:n]) / total * 100, 4) if total else None
|
||||
got = {"units": len(pairs), "total_wan": round(total / 10000, 2),
|
||||
"top1": round(pairs[0][1] / total * 100, 4) if pairs else None,
|
||||
"top1_unit": pairs[0][0] if pairs else None,
|
||||
"top2": round(pairs[1][1] / total * 100, 4) if len(pairs) > 1 else None,
|
||||
"top2_unit": pairs[1][0] if len(pairs) > 1 else None,
|
||||
"top4_cum": cum(4), "top10_cum": cum(10)}
|
||||
checks = {}
|
||||
for k, exp in ANCHOR_2026_01_09.items():
|
||||
for k, exp in anchors.items():
|
||||
v = got.get(k)
|
||||
checks[k] = {"expected": exp, "got": v, "pass": (v is not None and abs(float(v) - exp) <= ANCHOR_TOL)}
|
||||
if isinstance(exp, int) and not isinstance(exp, bool) and k.endswith("_unit"):
|
||||
checks[k] = {"expected": exp, "got": v, "pass": (v == exp)}
|
||||
continue
|
||||
checks[k] = {"expected": exp, "got": v,
|
||||
"pass": (v is not None and abs(float(v) - float(exp)) <= tol)}
|
||||
return {"anchor": got, "checks": checks, "ALL_PASS": all(c["pass"] for c in checks.values())}
|
||||
|
||||
|
||||
@@ -333,6 +404,11 @@ def main():
|
||||
ap.add_argument("--entity", type=int, default=1)
|
||||
ap.add_argument("--db-url")
|
||||
ap.add_argument("--batch", default=DEFAULT_BATCH)
|
||||
ap.add_argument("--concentration", choices=("fill-gaps", "overwrite"), default="fill-gaps",
|
||||
help="已有期次处置:fill-gaps=只补缺期(默认)/ overwrite=重算并覆盖(P0 纠错后重算九期用)")
|
||||
ap.add_argument("--only", choices=("all", "concentration"), default="all",
|
||||
help="写面:all=快照+三族汇总(默认)/ concentration=**仅**写 C_CUST_CONCENTRATION 期次"
|
||||
"(P0 九期重算最小写面;需配 --concentration overwrite)")
|
||||
ap.add_argument("--since")
|
||||
ap.add_argument("--until")
|
||||
ap.add_argument("--json", action="store_true")
|
||||
@@ -347,7 +423,8 @@ def main():
|
||||
return 0 if res["ALL_PASS"] else 1
|
||||
|
||||
res = run(engine, dialect, args.entity, args.batch, apply=args.apply,
|
||||
verify=args.verify, since=args.since, until=args.until)
|
||||
verify=args.verify, since=args.since, until=args.until,
|
||||
concentration=args.concentration, only=args.only)
|
||||
print(json.dumps(res, ensure_ascii=False, indent=2))
|
||||
return 0
|
||||
|
||||
|
||||
Reference in New Issue
Block a user