Hermes CI Fix
9d9a79545a
3 new tables + idempotent DDL migration (prod guard: refuses DB name cma without approval): tax_filing_log (P_TAX_TIMELY), hr_monthly_ledger (L_EMPLOYEE_TURNOVER), licence_ledger (P_LICENCE_OK). Entry API /api/cma/ext-ledger: create / CSV batch import / query by entity+period / derive trigger. on_time is server-derived only; HR rows rejected when end != begin + hires - separations. Deriver writes kpi_values only (source_type=derived, idempotent upsert); denominator 0 returns None and writes nothing; never creates or changes kpi_definitions. Tests: 22 new pytest cases (22 passed); scratch-DB e2e battery 28 assertions PASS with zero production write.
350 lines
15 KiB
Python
350 lines
15 KiB
Python
"""外部输入台账 → kpi_values 派生模块（能力评价 M2 准入3 派生，2026-09-15）
|
||
|
||
三张外部输入台账（app/models/ext_input_ledger.py）→ 三个能力评价指标值：
|
||
|
||
P_TAX_TIMELY = 当期 on_time=1 的税种数 ÷ 当期应申报税种数 × 100 （月频）
|
||
L_EMPLOYEE_TURNOVER = separations ÷ headcount_begin × 100 （月频）
|
||
P_LICENCE_OK = 有效证照数 ÷ 应持证照数 × 100 （季频，季末为基准日）
|
||
|
||
硬规则（任务 0-P0-project-20260915-cma-eval-m2-ext-input-tables，直接进验收）：
|
||
1. **分母为 0 必须返回 None（无定义 / 不参评），不得写 0、不得填大数。**
|
||
依据：F_INTEREST_COVER 已因「利息费用=0 → 分母 0」失去定义；同族错误不得在 3 张新台账上重演。
|
||
→ 值为 None 时**不写 kpi_values**；若该期已存在旧的 derived 行则只回报 stale_row_present，不删不改。
|
||
2. **只写 kpi_values**（entity_id / kpi_id / period / actual_value + 来源标识 source_type='derived'），
|
||
**不新建、不改动任何 KPI 定义**：定义缺失 → 跳过并回报 skipped_no_definition。
|
||
3. **幂等**：同一 (entity_id, kpi_id, period, source_type='derived') 重复运行只更新不新增
|
||
（与 scripts/kpi_bep_deriver.py 同范式）。
|
||
|
||
本模块只依赖「台账行对象」的属性（ORM 实例 / 简易对象皆可），纯计算函数可脱离数据库单测。
|
||
"""
|
||
from __future__ import annotations
|
||
|
||
import calendar
|
||
import logging
|
||
import re
|
||
from datetime import date, datetime
|
||
from typing import Iterable, Optional, Sequence
|
||
|
||
from sqlalchemy.orm import Session
|
||
|
||
from app.models import KPIDefinition, KPIValue, TaxFilingLog, HrMonthlyLedger, LicenceLedger
|
||
|
||
logger = logging.getLogger("cma.ext_input_derive")
|
||
|
||
TAX_TIMELY_CODE = "P_TAX_TIMELY"
|
||
EMPLOYEE_TURNOVER_CODE = "L_EMPLOYEE_TURNOVER"
|
||
LICENCE_OK_CODE = "P_LICENCE_OK"
|
||
|
||
ALL_KPI_CODES = (TAX_TIMELY_CODE, EMPLOYEE_TURNOVER_CODE, LICENCE_OK_CODE)
|
||
|
||
# 数据来源标识（写入 kpi_values.source_type；与 dashboard「最新派生期间」取数口径一致）
|
||
SOURCE_TYPE = "derived"
|
||
|
||
_MONTH_RE = re.compile(r"^(\d{4})-(0[1-9]|1[0-2])$")
|
||
_QUARTER_RE = re.compile(r"^(\d{4})-Q([1-4])$")
|
||
|
||
|
||
# ============================================================
|
||
# 期间工具
|
||
# ============================================================
|
||
|
||
def is_month_period(period: str) -> bool:
|
||
return bool(_MONTH_RE.match(str(period or "")))
|
||
|
||
|
||
def is_quarter_period(period: str) -> bool:
|
||
return bool(_QUARTER_RE.match(str(period or "")))
|
||
|
||
|
||
def month_to_quarter(period: str) -> str:
|
||
"""2026-06 → 2026-Q2（季频指标按「所在季度」归集）"""
|
||
m = _MONTH_RE.match(str(period or ""))
|
||
if not m:
|
||
raise ValueError(f"非法月期间: {period}（应为 YYYY-MM）")
|
||
quarter = (int(m.group(2)) - 1) // 3 + 1
|
||
return f"{m.group(1)}-Q{quarter}"
|
||
|
||
|
||
def quarter_end_date(period: str) -> date:
|
||
"""季末基准日。接受 2026-Q2（季频）或 2026-06（月频，自动归到所在季度）"""
|
||
q = _QUARTER_RE.match(str(period or ""))
|
||
if q:
|
||
year, quarter = int(q.group(1)), int(q.group(2))
|
||
else:
|
||
q2 = re.match(r"^(\d{4})-Q([1-4])$", month_to_quarter(period))
|
||
if not q2:
|
||
raise ValueError(f"非法季期间: {period}（应为 YYYY-Qn 或 YYYY-MM）")
|
||
year, quarter = int(q2.group(1)), int(q2.group(2))
|
||
last_month = quarter * 3
|
||
return date(year, last_month, calendar.monthrange(year, last_month)[1])
|
||
|
||
|
||
# ============================================================
|
||
# 1. 申报行为台账 → P_TAX_TIMELY
|
||
# ============================================================
|
||
|
||
def derive_on_time(filed_date: Optional[date], legal_deadline: Optional[date]) -> Optional[int]:
|
||
"""`on_time` 的**唯一**服务端派生规则（录入方不得手填）：
|
||
|
||
- 已申报且不晚于法定期限 → 1
|
||
- 已申报但晚于法定期限 → 0
|
||
- 未申报（filed_date 为空） → 0（逾期未申报，计入分母）
|
||
- 已申报但缺法定期限（无法判定）→ None（该行不计入分子；分母仍计）
|
||
"""
|
||
if filed_date is None:
|
||
return 0
|
||
if legal_deadline is None:
|
||
return None
|
||
return 1 if filed_date <= legal_deadline else 0
|
||
|
||
|
||
def compute_tax_timely_rate(rows: Iterable) -> Optional[float]:
|
||
"""P_TAX_TIMELY = on_time=1 的税种数 ÷ 应申报税种数 × 100；分母为 0 → None"""
|
||
rows = list(rows)
|
||
denominator = len(rows)
|
||
if denominator == 0:
|
||
return None
|
||
timely = sum(1 for r in rows if getattr(r, "on_time", None) == 1)
|
||
return round(timely / denominator * 100.0, 1)
|
||
|
||
|
||
def explain_tax_timely_rate(rows: Iterable) -> dict:
|
||
rows = list(rows)
|
||
value = compute_tax_timely_rate(rows)
|
||
return {
|
||
"denominator": len(rows),
|
||
"timely": sum(1 for r in rows if getattr(r, "on_time", None) == 1),
|
||
"unknown": sum(1 for r in rows if getattr(r, "on_time", None) is None),
|
||
"value": value,
|
||
"no_definition": value is None,
|
||
}
|
||
|
||
|
||
# ============================================================
|
||
# 2. 人事月度台账 → L_EMPLOYEE_TURNOVER
|
||
# ============================================================
|
||
|
||
def check_headcount_equation(headcount_begin, hires, separations, headcount_end) -> Optional[str]:
|
||
"""期末 = 期初 + 入职 − 离职。通过返回 None，不通过返回错误说明（调用方拒绝写入并回报）"""
|
||
try:
|
||
lhs = int(headcount_end)
|
||
rhs = int(headcount_begin) + int(hires) - int(separations)
|
||
except (TypeError, ValueError):
|
||
return "期初/入职/离职/期末必须为整数"
|
||
if lhs != rhs:
|
||
return (f"人事台账等式不成立: 期末({lhs}) ≠ 期初({headcount_begin}) + 入职({hires}) "
|
||
f"− 离职({separations}) = {rhs}（拒绝写入）")
|
||
return None
|
||
|
||
|
||
def compute_employee_turnover(row) -> Optional[float]:
|
||
"""L_EMPLOYEE_TURNOVER = separations ÷ headcount_begin × 100；期初为 0（或缺失）→ None"""
|
||
if row is None:
|
||
return None
|
||
try:
|
||
begin = int(getattr(row, "headcount_begin"))
|
||
separations = int(getattr(row, "separations"))
|
||
except (TypeError, ValueError):
|
||
return None
|
||
if begin == 0:
|
||
return None
|
||
return round(separations / begin * 100.0, 2)
|
||
|
||
|
||
def explain_employee_turnover(row) -> dict:
|
||
"""并列返回「期初期末平均在职」口径值，便于复核（平均在职为 0 → 该口径同样 None）"""
|
||
value = compute_employee_turnover(row)
|
||
detail = {
|
||
"headcount_begin": getattr(row, "headcount_begin", None),
|
||
"headcount_end": getattr(row, "headcount_end", None),
|
||
"hires": getattr(row, "hires", None),
|
||
"separations": getattr(row, "separations", None),
|
||
"value": value,
|
||
"no_definition": value is None,
|
||
"avg_headcount": None,
|
||
"value_on_avg": None,
|
||
}
|
||
if row is None:
|
||
return detail
|
||
try:
|
||
avg = (int(row.headcount_begin) + int(row.headcount_end)) / 2.0
|
||
except (TypeError, ValueError):
|
||
avg = None
|
||
detail["avg_headcount"] = avg
|
||
if avg:
|
||
detail["value_on_avg"] = round(int(row.separations) / avg * 100.0, 2)
|
||
return detail
|
||
|
||
|
||
# ============================================================
|
||
# 3. 证照台账 → P_LICENCE_OK
|
||
# ============================================================
|
||
|
||
def is_licence_valid(licence, as_of: date) -> bool:
|
||
"""有效证照判定：状态非「过期」且（valid_to 为空=长期有效 或 valid_to >= 基准日）"""
|
||
if str(getattr(licence, "annual_check_status", "") or "").strip() == "过期":
|
||
return False
|
||
valid_to = getattr(licence, "valid_to", None)
|
||
if valid_to is None:
|
||
return True
|
||
if isinstance(valid_to, datetime):
|
||
valid_to = valid_to.date()
|
||
return valid_to >= as_of
|
||
|
||
|
||
def compute_licence_ok(rows: Iterable, as_of: date) -> Optional[float]:
|
||
"""P_LICENCE_OK = 有效证照数 ÷ 应持证照数 × 100（基准日=季末）；应持证照数为 0 → None"""
|
||
rows = list(rows)
|
||
denominator = len(rows)
|
||
if denominator == 0:
|
||
return None
|
||
valid = sum(1 for r in rows if is_licence_valid(r, as_of))
|
||
return round(valid / denominator * 100.0, 1)
|
||
|
||
|
||
def explain_licence_ok(rows: Iterable, as_of: date) -> dict:
|
||
rows = list(rows)
|
||
value = compute_licence_ok(rows, as_of)
|
||
expired = [getattr(r, "licence_no", "") for r in rows if str(getattr(r, "annual_check_status", "")) == "过期"]
|
||
return {
|
||
"as_of": as_of.isoformat(),
|
||
"denominator": len(rows),
|
||
"valid": sum(1 for r in rows if is_licence_valid(r, as_of)),
|
||
"expired_or_outdated": expired,
|
||
"value": value,
|
||
"no_definition": value is None,
|
||
}
|
||
|
||
|
||
# ============================================================
|
||
# 落库（幂等 upsert；只写 kpi_values）
|
||
# ============================================================
|
||
|
||
def _resolve_kpi(db: Session, entity_id: int, kpi_code: str) -> Optional[KPIDefinition]:
|
||
"""按 (entity_id, kpi_code) 取定义（仅 active）。定义唯一性在库内非全局：
|
||
e1/e2 可各有一条同名 kpi_code，故必须带 entity_id 过滤 + order_by(id) 取稳定行。"""
|
||
return (db.query(KPIDefinition)
|
||
.filter(KPIDefinition.entity_id == entity_id,
|
||
KPIDefinition.kpi_code == kpi_code,
|
||
KPIDefinition.status == "active")
|
||
.order_by(KPIDefinition.id.asc())
|
||
.first())
|
||
|
||
|
||
def upsert_kpi_value(db: Session, entity_id: int, kpi: KPIDefinition, period: str,
|
||
value: float, remark: str, batch: str) -> str:
|
||
"""幂等写 kpi_values：命中 (kpi_id, period, source_type='derived') 则更新，返回 'insert'/'upsert'"""
|
||
existing = (db.query(KPIValue)
|
||
.filter(KPIValue.kpi_id == kpi.id,
|
||
KPIValue.period == period,
|
||
KPIValue.source_type == SOURCE_TYPE)
|
||
.order_by(KPIValue.id.asc())
|
||
.first())
|
||
if existing is not None:
|
||
action = "upsert"
|
||
existing.actual_value = value
|
||
existing.source_batch = batch
|
||
existing.data_status = "pending"
|
||
existing.remark = remark
|
||
else:
|
||
db.add(KPIValue(
|
||
entity_id=entity_id,
|
||
kpi_id=kpi.id,
|
||
period=period,
|
||
actual_value=value,
|
||
source_type=SOURCE_TYPE,
|
||
source_batch=batch,
|
||
data_status="pending",
|
||
remark=remark,
|
||
))
|
||
action = "insert"
|
||
db.flush()
|
||
return action
|
||
|
||
|
||
def run_derive(db: Session, entity_id: int, period: str,
|
||
kpis: Optional[Sequence[str]] = None, write: bool = True) -> dict:
|
||
"""按实体+期间派生 3 个指标并写 kpi_values（幂等）。
|
||
|
||
period：P_TAX_TIMELY / L_EMPLOYEE_TURNOVER 用 YYYY-MM；
|
||
P_LICENCE_OK 用 YYYY-Qn（传 YYYY-MM 时自动归到所在季度，落库 period 为季度）。
|
||
write=False → 只算不写（dry-run）。
|
||
返回 {'entity_id', 'period', 'batch', 'items': {kpi_code: {...}}}
|
||
"""
|
||
codes = list(kpis or ALL_KPI_CODES)
|
||
unknown = [c for c in codes if c not in ALL_KPI_CODES]
|
||
if unknown:
|
||
raise ValueError(f"不支持的指标编码: {unknown}（可用: {list(ALL_KPI_CODES)}）")
|
||
batch = f"ext-ledger-{datetime.now().strftime('%Y%m%d%H%M%S')}"
|
||
items: dict = {}
|
||
|
||
for code in codes:
|
||
kpi = _resolve_kpi(db, entity_id, code)
|
||
if kpi is None:
|
||
items[code] = {
|
||
"action": "skipped_no_definition",
|
||
"reason": f"entity={entity_id} 无 active 定义 {code}（定义变更属另一派单，本模块不新建）",
|
||
"actual_value": None,
|
||
}
|
||
continue
|
||
|
||
if code == TAX_TIMELY_CODE:
|
||
rows = (db.query(TaxFilingLog)
|
||
.filter(TaxFilingLog.entity_id == entity_id, TaxFilingLog.period == period)
|
||
.order_by(TaxFilingLog.id.asc()).all())
|
||
detail = explain_tax_timely_rate(rows)
|
||
target_period = period
|
||
remark = (f"台账派生: 应申报{detail['denominator']}个税种，按时{detail['timely']}个，"
|
||
f"无法判定{detail['unknown']}个（tax_filing_log {period}）")
|
||
elif code == EMPLOYEE_TURNOVER_CODE:
|
||
row = (db.query(HrMonthlyLedger)
|
||
.filter(HrMonthlyLedger.entity_id == entity_id, HrMonthlyLedger.period == period)
|
||
.order_by(HrMonthlyLedger.id.asc()).first())
|
||
detail = explain_employee_turnover(row)
|
||
target_period = period
|
||
remark = (f"台账派生: 离职{detail['separations']}/期初在职{detail['headcount_begin']}×100 "
|
||
f"(期初{detail['headcount_begin']}+入职{detail['hires']}-离职{detail['separations']}"
|
||
f"=期末{detail['headcount_end']})；并列口径 期初期末平均在职{detail['avg_headcount']}人 "
|
||
f"→ {detail['value_on_avg']}%")
|
||
else: # LICENCE_OK —— 季频
|
||
target_period = month_to_quarter(period) if is_month_period(period) else period
|
||
as_of = quarter_end_date(target_period)
|
||
rows = (db.query(LicenceLedger)
|
||
.filter(LicenceLedger.entity_id == entity_id)
|
||
.order_by(LicenceLedger.id.asc()).all())
|
||
detail = explain_licence_ok(rows, as_of)
|
||
remark = (f"台账派生: 季末基准日{detail['as_of']}，有效{detail['valid']}/应持证{detail['denominator']}"
|
||
f"（过期或失效: {detail['expired_or_outdated'] or '无'}）")
|
||
|
||
value = detail.get("value")
|
||
item = {
|
||
"kpi_id": kpi.id,
|
||
"kpi_code": code,
|
||
"period": target_period,
|
||
"actual_value": value,
|
||
"detail": detail,
|
||
}
|
||
if value is None:
|
||
# 硬规则：分母 0 → 无定义，不写 kpi_values（不写 0、不填大数）
|
||
stale = (db.query(KPIValue)
|
||
.filter(KPIValue.kpi_id == kpi.id, KPIValue.period == target_period,
|
||
KPIValue.source_type == SOURCE_TYPE)
|
||
.order_by(KPIValue.id.asc()).first())
|
||
item["action"] = "skipped_denominator_zero"
|
||
item["reason"] = "分母为 0 → 无定义（不参评），不写 kpi_values"
|
||
item["stale_row_present"] = stale is not None
|
||
items[code] = item
|
||
continue
|
||
|
||
if write:
|
||
item["action"] = upsert_kpi_value(db, entity_id, kpi, target_period, value, remark, batch)
|
||
else:
|
||
item["action"] = "dry-run"
|
||
item["remark"] = remark
|
||
items[code] = item
|
||
|
||
if write:
|
||
db.commit()
|
||
return {"entity_id": entity_id, "period": period, "batch": batch,
|
||
"write": write, "items": items}
|