feat(cma): external input ledgers for M2 capability eval (tax/hr/licence)

3 new tables + idempotent DDL migration (prod guard: refuses DB name cma without approval): tax_filing_log (P_TAX_TIMELY), hr_monthly_ledger (L_EMPLOYEE_TURNOVER), licence_ledger (P_LICENCE_OK).

Entry API /api/cma/ext-ledger: create / CSV batch import / query by entity+period / derive trigger. on_time is server-derived only; HR rows rejected when end != begin + hires - separations.

Deriver writes kpi_values only (source_type=derived, idempotent upsert); denominator 0 returns None and writes nothing; never creates or changes kpi_definitions.

Tests: 22 new pytest cases (22 passed); scratch-DB e2e battery 28 assertions PASS with zero production write.
This commit is contained in:
Hermes CI Fix
2026-09-15 18:12:05 +08:00
parent 5d0884cc3b
commit 9d9a79545a
9 changed files with 1956 additions and 1 deletions
+579
View File
@@ -0,0 +1,579 @@
"""外部输入台账录入/查询 API(能力评价 M2 准入3 派生,2026-09-15)
三张台账的单向录入接口(新增 / CSV 批量导入 / 按实体与期次查询)+ 台账→kpi_values 派生触发。
设计口径:
- **单向**:只提供「新增 + 批量导入 + 查询 + 派生」,不提供修改/删除接口
(历史登记行不可改;错登由人工在业务侧作废后重新登记一行,保持审计链)
- **on_time 由服务端派生**:请求里带 on_time 一律忽略并在响应回报 on_time_ignored
- **人事台账等式校验**:期末 ≠ 期初 + 入职 − 离职 → 拒绝写入并回报(HTTP 400 / 批量逐行 errors)
- **多租户**:所有端点经 app.deps.assert_entity_scope 收敛账套(显式 entity_id 与会话账套不一致 → 403)
- **鉴权**:router 级 require_auth(未认证 401);写端点额外 require_role(ceo/finance/it)(角色不足 403)
- 派生必须**幂等**(同 (entity_id, kpi_id, period) 重复运行只更新不新增),分母为 0 → 不写值
CSV 列头(UTF-8,可带 BOM;列序无关,多余列忽略):
申报台账 period,tax_type,legal_deadline,filed_date,evidence_no,registered_by,remark
人事台账 period,headcount_begin,hires,separations,headcount_end,registered_by,remark
证照台账 licence_name,licence_no,valid_from,valid_to,annual_check_status,issuer,owner_role,attachment_no,registered_by,remark
"""
from __future__ import annotations
import csv
import io
import logging
import re
from datetime import date, datetime
from typing import Optional
from fastapi import APIRouter, Depends, HTTPException, Query, Request
from sqlalchemy.orm import Session
from app.auth_middleware import require_auth, require_role
from app.database import get_db
from app.deps import assert_entity_scope
from app.models import HrMonthlyLedger, KPIDefinition, LicenceLedger, TaxFilingLog
from app.models.ext_input_ledger import ANNUAL_CHECK_STATUSES, MONTH_PERIOD_RE, QUARTER_PERIOD_RE
from app.services.ext_input_derive import (
ALL_KPI_CODES,
LICENCE_OK_CODE,
check_headcount_equation,
compute_employee_turnover,
derive_on_time,
explain_licence_ok,
explain_tax_timely_rate,
is_licence_valid,
month_to_quarter,
quarter_end_date,
run_derive,
)
logger = logging.getLogger("cma.ext_input_api")
router = APIRouter(
prefix="/api/cma/ext-ledger",
tags=["外部输入台账"],
dependencies=[Depends(require_auth)],
)
# 写权限(录入/导入/派生):ceo / finance / it(与 value_sources 等既有写端点同口径)
write_guard = Depends(require_role("ceo", "finance", "it"))
SUGGESTED_TAX_TYPES = ("增值税", "附加税", "企业所得税", "印花税", "个人所得税", "房产税", "土地使用税")
# CSV 全量列(文档口径)+ 必需列(列缺失才拒绝;可选列缺省即留空)
TAX_HEADERS = ("period", "tax_type", "legal_deadline", "filed_date", "evidence_no", "registered_by", "remark")
HR_HEADERS = ("period", "headcount_begin", "hires", "separations", "headcount_end", "registered_by", "remark")
LICENCE_HEADERS = ("licence_name", "licence_no", "valid_from", "valid_to", "annual_check_status",
"issuer", "owner_role", "attachment_no", "registered_by", "remark")
TAX_REQUIRED = ("period", "tax_type")
HR_REQUIRED = ("period", "headcount_begin", "hires", "separations", "headcount_end")
LICENCE_REQUIRED = ("licence_name", "licence_no", "annual_check_status")
# ============================================================
# 公共工具:账套收敛 / 字段校验
# ============================================================
def _resolve_entity(request: Request, db: Session, explicit: Optional[int]) -> int:
"""账套收敛:会话 token 绑定实体优先;显式 entity_id 与会话不一致 → 403(文案同 deps.get_entity_id)"""
session_entity = assert_entity_scope(request, explicit, db)
if explicit is not None:
return explicit
if session_entity is not None:
return session_entity
return 1
def _body_entity(payload: dict, query_entity: Optional[int]) -> Optional[int]:
raw = payload.get("entity_id") if isinstance(payload, dict) else None
if raw in (None, ""):
return query_entity
try:
return int(raw)
except (TypeError, ValueError):
raise HTTPException(400, f"entity_id 非法: {raw}")
def _req_str(value, name: str, max_len: int) -> str:
text = "" if value is None else str(value).strip()
if not text:
raise HTTPException(400, f"缺少必填字段: {name}")
if len(text) > max_len:
raise HTTPException(400, f"字段 {name} 超长({len(text)} > {max_len})")
return text
def _opt_str(value, max_len: int) -> Optional[str]:
if value is None:
return None
text = str(value).strip()
if not text:
return None
return text[:max_len]
def _parse_date(value, name: str) -> Optional[date]:
if value in (None, ""):
return None
if isinstance(value, datetime):
return value.date()
if isinstance(value, date):
return value
text = str(value).strip().replace("/", "-")
for fmt in ("%Y-%m-%d", "%Y-%m-%d %H:%M:%S", "%Y%m%d"):
try:
return datetime.strptime(text, fmt).date()
except ValueError:
continue
raise HTTPException(400, f"字段 {name} 日期格式非法: {value}(应为 YYYY-MM-DD)")
def _parse_int(value, name: str, required: bool = True) -> Optional[int]:
if value in (None, ""):
if required:
raise HTTPException(400, f"缺少必填字段: {name}")
return None
try:
return int(str(value).strip())
except (TypeError, ValueError):
raise HTTPException(400, f"字段 {name} 必须为整数: {value}")
def _month_period(value, name: str = "period") -> str:
text = _req_str(value, name, 20)
if not re.match(MONTH_PERIOD_RE, text):
raise HTTPException(400, f"字段 {name} 期次格式非法: {text}(应为 YYYY-MM)")
return text
def _target_period(kpi_code: str, period: str) -> str:
"""指标实际落库期间:P_LICENCE_OK 为季频(传月期间时归到所在季度)"""
if kpi_code == LICENCE_OK_CODE and re.match(MONTH_PERIOD_RE, period):
return month_to_quarter(period)
return period
# ============================================================
# 字段选项(前端下拉动态加载用;禁前端硬编码)
# ============================================================
@router.get("/options")
def get_options():
"""录入表单选项:年检状态枚举 + 建议税种 + 期次格式"""
return {
"annual_check_status": list(ANNUAL_CHECK_STATUSES),
"tax_type_suggested": list(SUGGESTED_TAX_TYPES),
"period_format": {"monthly": "YYYY-MM 例:2026-08", "quarterly": "YYYY-Qn 例:2026-Q2"},
"derivable_kpi_codes": list(ALL_KPI_CODES),
}
# ============================================================
# 1. 申报行为台账 tax_filing_log
# ============================================================
def _tax_values(payload: dict) -> dict:
period = _month_period(payload.get("period"))
tax_type = _req_str(payload.get("tax_type"), "tax_type", 50)
legal_deadline = _parse_date(payload.get("legal_deadline"), "legal_deadline")
filed_date = _parse_date(payload.get("filed_date"), "filed_date")
return {
"period": period,
"tax_type": tax_type,
"legal_deadline": legal_deadline,
"filed_date": filed_date,
# on_time 唯一来源 = 服务端派生(请求里的同名字段一律忽略)
"on_time": derive_on_time(filed_date, legal_deadline),
"evidence_no": _opt_str(payload.get("evidence_no"), 100),
"registered_by": _opt_str(payload.get("registered_by"), 100),
"remark": _opt_str(payload.get("remark"), 500),
}
def _tax_json(row: TaxFilingLog) -> dict:
return {
"id": row.id,
"entity_id": row.entity_id,
"period": row.period,
"tax_type": row.tax_type,
"legal_deadline": row.legal_deadline.isoformat() if row.legal_deadline else None,
"filed_date": row.filed_date.isoformat() if row.filed_date else None,
"on_time": row.on_time,
"evidence_no": row.evidence_no,
"registered_by": row.registered_by,
"remark": row.remark,
}
@router.get("/tax-filings")
def list_tax_filings(
request: Request,
period: Optional[str] = Query(None, description="属期 YYYY-MM"),
tax_type: Optional[str] = Query(None),
entity_id: Optional[int] = Query(None, ge=1),
db: Session = Depends(get_db),
):
"""按 (entity_id, period) 查询申报台账 + 及时率口径摘要"""
eid = _resolve_entity(request, db, entity_id)
q = db.query(TaxFilingLog).filter(TaxFilingLog.entity_id == eid)
if period:
q = q.filter(TaxFilingLog.period == period)
if tax_type:
q = q.filter(TaxFilingLog.tax_type == tax_type)
rows = q.order_by(TaxFilingLog.period.desc(), TaxFilingLog.id.asc()).all()
summary = explain_tax_timely_rate([r for r in rows if not period or r.period == period])
return {"entity_id": eid, "data": [_tax_json(r) for r in rows], "total": len(rows),
"summary": {"period": period, "tax_timely_rate": summary["value"],
"denominator": summary["denominator"], "timely": summary["timely"],
"no_definition": summary["no_definition"]}}
@router.post("/tax-filings", status_code=201, dependencies=[write_guard])
def create_tax_filing(
payload: dict,
request: Request,
entity_id: Optional[int] = Query(None, ge=1),
db: Session = Depends(get_db),
):
"""新增一条申报行为台账(on_time 服务端派生;唯一键冲突 → 409)"""
eid = _resolve_entity(request, db, _body_entity(payload, entity_id))
vals = _tax_values(payload)
exists = db.query(TaxFilingLog).filter(
TaxFilingLog.entity_id == eid, TaxFilingLog.period == vals["period"],
TaxFilingLog.tax_type == vals["tax_type"]).first()
if exists:
raise HTTPException(409, f"已存在同 (entity_id={eid}, period={vals['period']}, "
f"tax_type={vals['tax_type']}) 的登记行 id={exists.id}")
row = TaxFilingLog(entity_id=eid, **vals)
db.add(row)
db.commit()
db.refresh(row)
return {"data": _tax_json(row),
"on_time_derived": row.on_time,
"on_time_ignored": payload.get("on_time") not in (None, "")}
@router.post("/tax-filings/import", dependencies=[write_guard])
def import_tax_filings(
payload: dict,
request: Request,
entity_id: Optional[int] = Query(None, ge=1),
db: Session = Depends(get_db),
):
"""CSV 批量导入申报台账。on_conflict: skip(默认)/update;dry_run=true 只校验不落库"""
eid = _resolve_entity(request, db, _body_entity(payload, entity_id))
on_conflict = str(payload.get("on_conflict") or "skip").lower()
if on_conflict not in ("skip", "update"):
raise HTTPException(400, f"on_conflict 非法: {on_conflict}(skip/update)")
dry_run = bool(payload.get("dry_run"))
records, errors = _read_csv(payload.get("csv"), TAX_REQUIRED, _tax_values)
result = _apply_import(db, TaxFilingLog, eid, records, errors, on_conflict, dry_run,
key_fields=("period", "tax_type"))
return result
# ============================================================
# 2. 人事月度台账 hr_monthly_ledger
# ============================================================
def _hr_values(payload: dict) -> dict:
period = _month_period(payload.get("period"))
begin = _parse_int(payload.get("headcount_begin"), "headcount_begin")
hires = _parse_int(payload.get("hires"), "hires")
separations = _parse_int(payload.get("separations"), "separations")
end = _parse_int(payload.get("headcount_end"), "headcount_end")
err = check_headcount_equation(begin, hires, separations, end)
if err:
raise HTTPException(400, err)
return {
"period": period,
"headcount_begin": begin,
"hires": hires,
"separations": separations,
"headcount_end": end,
"registered_by": _opt_str(payload.get("registered_by"), 100),
"remark": _opt_str(payload.get("remark"), 500),
}
def _hr_json(row: HrMonthlyLedger) -> dict:
return {
"id": row.id, "entity_id": row.entity_id, "period": row.period,
"headcount_begin": row.headcount_begin, "hires": row.hires,
"separations": row.separations, "headcount_end": row.headcount_end,
"turnover_rate": compute_employee_turnover(row),
"registered_by": row.registered_by, "remark": row.remark,
}
@router.get("/hr-monthly")
def list_hr_monthly(
request: Request,
period: Optional[str] = Query(None, description="期间 YYYY-MM"),
entity_id: Optional[int] = Query(None, ge=1),
db: Session = Depends(get_db),
):
"""按 (entity_id, period) 查询人事月度台账(含流失率派生值)"""
eid = _resolve_entity(request, db, entity_id)
q = db.query(HrMonthlyLedger).filter(HrMonthlyLedger.entity_id == eid)
if period:
q = q.filter(HrMonthlyLedger.period == period)
rows = q.order_by(HrMonthlyLedger.period.desc(), HrMonthlyLedger.id.asc()).all()
return {"entity_id": eid, "data": [_hr_json(r) for r in rows], "total": len(rows)}
@router.post("/hr-monthly", status_code=201, dependencies=[write_guard])
def create_hr_monthly(
payload: dict,
request: Request,
entity_id: Optional[int] = Query(None, ge=1),
db: Session = Depends(get_db),
):
"""新增人事月度台账(等式校验不通过 → 400 拒绝写入)"""
eid = _resolve_entity(request, db, _body_entity(payload, entity_id))
vals = _hr_values(payload)
exists = db.query(HrMonthlyLedger).filter(
HrMonthlyLedger.entity_id == eid, HrMonthlyLedger.period == vals["period"]).first()
if exists:
raise HTTPException(409, f"已存在同 (entity_id={eid}, period={vals['period']}) 的登记行 id={exists.id}")
row = HrMonthlyLedger(entity_id=eid, **vals)
db.add(row)
db.commit()
db.refresh(row)
return {"data": _hr_json(row)}
@router.post("/hr-monthly/import", dependencies=[write_guard])
def import_hr_monthly(
payload: dict,
request: Request,
entity_id: Optional[int] = Query(None, ge=1),
db: Session = Depends(get_db),
):
"""CSV 批量导入人事月度台账(等式不成立的行被拒绝并在 errors 回报)"""
eid = _resolve_entity(request, db, _body_entity(payload, entity_id))
on_conflict = str(payload.get("on_conflict") or "skip").lower()
if on_conflict not in ("skip", "update"):
raise HTTPException(400, f"on_conflict 非法: {on_conflict}(skip/update)")
dry_run = bool(payload.get("dry_run"))
records, errors = _read_csv(payload.get("csv"), HR_REQUIRED, _hr_values)
return _apply_import(db, HrMonthlyLedger, eid, records, errors, on_conflict, dry_run,
key_fields=("period",))
# ============================================================
# 3. 证照台账 licence_ledger
# ============================================================
def _licence_values(payload: dict) -> dict:
name = _req_str(payload.get("licence_name"), "licence_name", 200)
no = _req_str(payload.get("licence_no"), "licence_no", 100)
status = _req_str(payload.get("annual_check_status"), "annual_check_status", 20)
if status not in ANNUAL_CHECK_STATUSES:
raise HTTPException(400, f"字段 annual_check_status 非法: {status}(合法值: {'/'.join(ANNUAL_CHECK_STATUSES)})")
return {
"licence_name": name,
"licence_no": no,
"valid_from": _parse_date(payload.get("valid_from"), "valid_from"),
"valid_to": _parse_date(payload.get("valid_to"), "valid_to"),
"annual_check_status": status,
"issuer": _opt_str(payload.get("issuer"), 200),
"owner_role": _opt_str(payload.get("owner_role"), 100),
"attachment_no": _opt_str(payload.get("attachment_no"), 100),
"registered_by": _opt_str(payload.get("registered_by"), 100),
"remark": _opt_str(payload.get("remark"), 500),
}
def _licence_json(row: LicenceLedger, as_of: Optional[date] = None) -> dict:
return {
"id": row.id, "entity_id": row.entity_id, "licence_name": row.licence_name,
"licence_no": row.licence_no,
"valid_from": row.valid_from.isoformat() if row.valid_from else None,
"valid_to": row.valid_to.isoformat() if row.valid_to else None,
"annual_check_status": row.annual_check_status, "issuer": row.issuer,
"owner_role": row.owner_role, "attachment_no": row.attachment_no,
"valid_as_of": is_licence_valid(row, as_of) if as_of else None,
"registered_by": row.registered_by, "remark": row.remark,
}
@router.get("/licences")
def list_licences(
request: Request,
as_of: Optional[str] = Query(None, description="有效性基准日 YYYY-MM-DD(默认今天)"),
licence_name: Optional[str] = Query(None),
valid_only: bool = Query(False, description="仅返回基准日有效的证照"),
entity_id: Optional[int] = Query(None, ge=1),
db: Session = Depends(get_db),
):
"""证照台账查询(有效/过期按基准日判定)"""
eid = _resolve_entity(request, db, entity_id)
base = _parse_date(as_of, "as_of") or date.today()
q = db.query(LicenceLedger).filter(LicenceLedger.entity_id == eid)
if licence_name:
q = q.filter(LicenceLedger.licence_name == licence_name)
rows = q.order_by(LicenceLedger.id.asc()).all()
# 合规率分母 = 应持证照数(不受 valid_only 过滤影响)
summary = explain_licence_ok(rows, base)
shown = [r for r in rows if is_licence_valid(r, base)] if valid_only else rows
return {"entity_id": eid, "as_of": base.isoformat(),
"data": [_licence_json(r, base) for r in shown], "total": len(shown),
"summary": {"licence_ok": summary["value"], "denominator": summary["denominator"],
"valid": summary["valid"], "no_definition": summary["no_definition"]}}
@router.post("/licences", status_code=201, dependencies=[write_guard])
def create_licence(
payload: dict,
request: Request,
entity_id: Optional[int] = Query(None, ge=1),
db: Session = Depends(get_db),
):
"""新增证照台账(状态白名单校验;唯一键冲突 → 409)"""
eid = _resolve_entity(request, db, _body_entity(payload, entity_id))
vals = _licence_values(payload)
exists = db.query(LicenceLedger).filter(
LicenceLedger.entity_id == eid, LicenceLedger.licence_name == vals["licence_name"],
LicenceLedger.licence_no == vals["licence_no"]).first()
if exists:
raise HTTPException(409, f"已存在同 (entity_id={eid}, licence_name={vals['licence_name']}, "
f"licence_no={vals['licence_no']}) 的登记行 id={exists.id}")
row = LicenceLedger(entity_id=eid, **vals)
db.add(row)
db.commit()
db.refresh(row)
return {"data": _licence_json(row, date.today())}
@router.post("/licences/import", dependencies=[write_guard])
def import_licences(
payload: dict,
request: Request,
entity_id: Optional[int] = Query(None, ge=1),
db: Session = Depends(get_db),
):
"""CSV 批量导入证照台账"""
eid = _resolve_entity(request, db, _body_entity(payload, entity_id))
on_conflict = str(payload.get("on_conflict") or "skip").lower()
if on_conflict not in ("skip", "update"):
raise HTTPException(400, f"on_conflict 非法: {on_conflict}(skip/update)")
dry_run = bool(payload.get("dry_run"))
records, errors = _read_csv(payload.get("csv"), LICENCE_REQUIRED, _licence_values)
return _apply_import(db, LicenceLedger, eid, records, errors, on_conflict, dry_run,
key_fields=("licence_name", "licence_no"))
# ============================================================
# 4. 台账 → kpi_values 派生
# ============================================================
@router.post("/derive", dependencies=[write_guard])
def derive_kpis(
request: Request,
payload: dict,
entity_id: Optional[int] = Query(None, ge=1),
db: Session = Depends(get_db),
):
"""触发台账→kpi_values 派生(幂等)。period: 月频 YYYY-MM;P_LICENCE_OK 接受 YYYY-Qn。
dry_run=true 只算不写。分母为 0 的指标 → 不写值,回报 skipped_denominator_zero。"""
eid = _resolve_entity(request, db, _body_entity(payload, entity_id))
period = str(payload.get("period") or "").strip()
if not period:
raise HTTPException(400, "缺少必填字段: period")
if not (re.match(MONTH_PERIOD_RE, period) or re.match(QUARTER_PERIOD_RE, period)):
raise HTTPException(400, f"period 非法: {period}(应为 YYYY-MM 或 YYYY-Qn)")
kpi_codes = payload.get("kpi_codes") or payload.get("kpis") or None
if kpi_codes is not None and not isinstance(kpi_codes, list):
raise HTTPException(400, "kpi_codes 必须为数组")
try:
result = run_derive(db, eid, period, kpis=kpi_codes, write=not bool(payload.get("dry_run")))
except ValueError as e:
raise HTTPException(400, str(e))
return result
@router.get("/derive/status")
def derive_status(
request: Request,
period: str = Query(..., description="月期间 YYYY-MM 或季期间 YYYY-Qn"),
entity_id: Optional[int] = Query(None, ge=1),
db: Session = Depends(get_db),
):
"""只读:查看该期派生指标的定义/现有 kpi_values 行(不写入)"""
eid = _resolve_entity(request, db, entity_id)
if not (re.match(MONTH_PERIOD_RE, period) or re.match(QUARTER_PERIOD_RE, period)):
raise HTTPException(400, f"period 非法: {period}(应为 YYYY-MM 或 YYYY-Qn)")
out = {}
from app.models import KPIValue
for code in ALL_KPI_CODES:
kpi = (db.query(KPIDefinition)
.filter(KPIDefinition.entity_id == eid, KPIDefinition.kpi_code == code,
KPIDefinition.status == "active")
.order_by(KPIDefinition.id.asc()).first())
item = {"kpi_defined": kpi is not None, "kpi_id": kpi.id if kpi else None,
"target_period": _target_period(code, period)}
if kpi is not None:
row = (db.query(KPIValue)
.filter(KPIValue.kpi_id == kpi.id, KPIValue.source_type == "derived",
KPIValue.period == item["target_period"])
.order_by(KPIValue.id.desc()).first())
item["latest_derived"] = ({"period": row.period, "actual_value": row.actual_value,
"data_status": row.data_status} if row else None)
out[code] = item
return {"entity_id": eid, "period": period, "items": out}
# ============================================================
# CSV 解析 / 批量落库(三张表共用;校验规则与单条新增完全同源)
# ============================================================
def _read_csv(csv_text, required_headers, builder):
"""解析 CSV:返回 (records, errors)。required_headers 为必需列(缺列即 400);其余列可选。"""
if not csv_text or not str(csv_text).strip():
raise HTTPException(400, "缺少必填字段: csv")
text = str(csv_text).lstrip("\ufeff")
reader = csv.DictReader(io.StringIO(text))
fieldnames = [(f or "").strip().lower() for f in (reader.fieldnames or [])]
if not fieldnames:
raise HTTPException(400, "CSV 缺少表头行")
missing = [h for h in required_headers if h not in fieldnames]
if missing:
raise HTTPException(400, f"CSV 缺少必需列: {missing}(必需列: {list(required_headers)})")
records, errors = [], []
for offset, raw in enumerate(reader, start=2): # 表头占第 1 行
row = {(k or "").strip().lower(): v for k, v in raw.items() if k}
if not any(str(v or "").strip() for v in row.values()):
continue # 跳过空行
try:
records.append({"line": offset, "values": builder(row)})
except HTTPException as e:
errors.append({"line": offset, "reason": e.detail})
return records, errors
def _apply_import(db: Session, model, entity_id: int, records, errors, on_conflict: str,
dry_run: bool, key_fields) -> dict:
"""逐行 upsert;返回 {inserted, updated, skipped, errors, dry_run}"""
inserted = updated = skipped = 0
for rec in records:
vals = rec["values"]
filters = [getattr(model, f) == vals[f] for f in key_fields] + [model.entity_id == entity_id]
existing = db.query(model).filter(*filters).first()
if existing is not None:
if on_conflict == "update":
for k, v in vals.items():
setattr(existing, k, v)
updated += 1
else:
skipped += 1
else:
db.add(model(entity_id=entity_id, **vals))
inserted += 1
if dry_run:
db.rollback()
else:
db.commit()
return {"entity_id": entity_id, "dry_run": dry_run, "on_conflict": on_conflict,
"inserted": inserted, "updated": updated, "skipped": skipped,
"errors": errors, "error_count": len(errors)}
+2 -1
View File
@@ -7,7 +7,7 @@ from fastapi.responses import JSONResponse
from fastapi.security import HTTPBearer, HTTPAuthorizationCredentials
from dotenv import load_dotenv
from app.database import init_db, get_db
from app.api import auth, kpis, kpi_governance, templates, maps, dashboard, data, alerts, ai_analysis, ai_suggestions, alert_rules, users, thresholds, notifications, permissions, action_plans, alignment, org, objectives, versions, budget, cost, predict, reports, security, knowledge, bot_bridge, bot_bridge_v2, bot_suggestions, lead, tenant, customer_dashboard, deviation_push, budget_generate, knowledge_articles, kpi_causality, data_quality, bi_reports, entities, bsc_layers, okr, okr_templates, subjects, driver_budget, bot_kpis, ontology, bot_iron_law, analysis_results, expenses, cash, tax_compliance, verify, growth_quality, products, data_classification, value_sources, zero_based, derivation_rules, cash_classify, cashflow_structure, cash_vouchers_json
from app.api import auth, kpis, kpi_governance, templates, maps, dashboard, data, alerts, ai_analysis, ai_suggestions, alert_rules, users, thresholds, notifications, permissions, action_plans, alignment, org, objectives, versions, budget, cost, predict, reports, security, knowledge, bot_bridge, bot_bridge_v2, bot_suggestions, lead, tenant, customer_dashboard, deviation_push, budget_generate, knowledge_articles, kpi_causality, data_quality, bi_reports, entities, bsc_layers, okr, okr_templates, subjects, driver_budget, bot_kpis, ontology, bot_iron_law, analysis_results, expenses, cash, tax_compliance, verify, growth_quality, products, data_classification, value_sources, zero_based, derivation_rules, cash_classify, cashflow_structure, cash_vouchers_json, ext_input_ledgers
from app.utils.cache import clear_all as clear_cache, delete as delete_cache
from scripts.erp_sync import run_sync as run_erp_sync
from app.auth_middleware import require_auth, require_auth_401
@@ -98,6 +98,7 @@ app.include_router(cash_vouchers_json.router)
app.include_router(data_classification.router)
app.include_router(tax_compliance.router)
app.include_router(verify.router)
app.include_router(ext_input_ledgers.router) # 外部输入台账(M2 准入3 派生,2026-09-15)
@app.exception_handler(Exception)
async def global_exception_handler(request: Request, exc: Exception):
+1
View File
@@ -7,6 +7,7 @@ from app.models.cost_model import StandardCost, ActualCost, AbcActivity, AbcAllo
from app.models.knowledge import KnowledgeEvent, KnowledgeSummary
from app.models.driver_budget import DriverFactorTemplate, DriverFactorBudget
from app.models.product_sales import ProductSales
from app.models.ext_input_ledger import TaxFilingLog, HrMonthlyLedger, LicenceLedger
class Entity(Base):
+101
View File
@@ -0,0 +1,101 @@
"""外部输入台账 — 三张**外部输入**业务表(能力评价 M2 准入3 派生,2026-09-15)
背景:能力评价 6 项缺口值经只读实测确认**无一项**能从既有系统合法取证;其中 3 项
只能来自新的人工登记台账(PRD v1.3 §4.0)。本模块只做「建表 + 结构化字段」,
录入/派生分别见 app/api/ext_input_ledgers.py 与 app/services/ext_input_derive.py。
tax_filing_log 申报行为台账 → P_TAX_TIMELY (A4 税务申报及时率, 月频)
hr_monthly_ledger 人事月度台账 → L_EMPLOYEE_TURNOVER (B5 员工流失率, 月频)
licence_ledger 证照台账 → P_LICENCE_OK (B2 经营许可合规率, 季频)
纪律(任务 0-P0-project-20260915-cma-eval-m2-ext-input-tables):
- 本模块**只新增表**,不改既有表、不改任何 KPI 定义(kpi_definitions 属准入2/4 同表串行)
- `on_time` 是**服务端派生**字段(唯一实现 app/services/ext_input_derive.derive_on_time),
录入方手填值一律被忽略并回报 on_time_ignored
- 每表 entity_id NOT NULL + 唯一键以 entity_id 打头(多租户隔离 + 索引可用性)
"""
from sqlalchemy import Column, Integer, String, Date, DateTime, UniqueConstraint, func
from app.database import Base
# 证照年检状态枚举(PRD v1.3 §4.0 原文口径,录入时白名单校验)
ANNUAL_CHECK_STATUSES = ("未到期", "已年检", "待年检", "过期")
# 期次格式(period)
MONTH_PERIOD_RE = r"^\d{4}-(0[1-9]|1[0-2])$" # 2026-08
QUARTER_PERIOD_RE = r"^\d{4}-Q[1-4]$" # 2026-Q2
class TaxFilingLog(Base):
"""申报行为台账 — P_TAX_TIMELY 的唯一取证来源
唯一键 (entity_id, period, tax_type):同一账套同一属期同一税种只允许一条登记行。
"""
__tablename__ = "tax_filing_log"
id = Column(Integer, primary_key=True, index=True)
entity_id = Column(Integer, nullable=False, index=True, comment="企业ID(账套)")
period = Column(String(20), nullable=False, comment="属期 YYYY-MM(申报所属月份)")
tax_type = Column(String(50), nullable=False, comment="税种: 增值税/附加税/企业所得税/印花税…")
legal_deadline = Column(Date, nullable=True, comment="法定申报期限")
filed_date = Column(Date, nullable=True, comment="实际申报日(未申报为 NULL)")
on_time = Column(Integer, nullable=True, comment="是否按时申报 1/0(**服务端派生**,录入方不得手填)")
evidence_no = Column(String(100), nullable=True, comment="申报回执/截图编号")
registered_by = Column(String(100), nullable=True, comment="登记人(职能岗位)")
remark = Column(String(500), nullable=True, comment="备注")
created_at = Column(DateTime, server_default=func.now())
updated_at = Column(DateTime, server_default=func.now(), onupdate=func.now())
__table_args__ = (
UniqueConstraint("entity_id", "period", "tax_type", name="uq_tax_filing_log_entity_period_type"),
)
class HrMonthlyLedger(Base):
"""人事月度台账 — L_EMPLOYEE_TURNOVER 的唯一取证来源
唯一键 (entity_id, period)。
硬校验:headcount_end == headcount_begin + hires - separations,
不通过时**拒绝写入并回报**(见 app/services/ext_input_derive.check_headcount_equation)。
"""
__tablename__ = "hr_monthly_ledger"
id = Column(Integer, primary_key=True, index=True)
entity_id = Column(Integer, nullable=False, index=True, comment="企业ID(账套)")
period = Column(String(20), nullable=False, comment="期间 YYYY-MM")
headcount_begin = Column(Integer, nullable=False, comment="期初在职人数")
hires = Column(Integer, nullable=False, default=0, comment="入职人数")
separations = Column(Integer, nullable=False, default=0, comment="离职人数")
headcount_end = Column(Integer, nullable=False, comment="期末在职人数(须= 期初+入职-离职)")
registered_by = Column(String(100), nullable=True, comment="登记人(职能岗位)")
remark = Column(String(500), nullable=True, comment="备注")
created_at = Column(DateTime, server_default=func.now())
updated_at = Column(DateTime, server_default=func.now(), onupdate=func.now())
__table_args__ = (
UniqueConstraint("entity_id", "period", name="uq_hr_monthly_ledger_entity_period"),
)
class LicenceLedger(Base):
"""证照台账 — P_LICENCE_OK 的唯一取证来源
唯一键 (entity_id, licence_name, licence_no)。
有效性口径(派生侧):valid_to >= 季末 且 年检状态 != '过期';
valid_to 为空 = 长期有效(仍受状态约束)。
"""
__tablename__ = "licence_ledger"
id = Column(Integer, primary_key=True, index=True)
entity_id = Column(Integer, nullable=False, index=True, comment="企业ID(账套)")
licence_name = Column(String(200), nullable=False, comment="证照名称")
licence_no = Column(String(100), nullable=False, comment="证照编号")
valid_from = Column(Date, nullable=True, comment="有效期起")
valid_to = Column(Date, nullable=True, comment="有效期止(空=长期有效)")
annual_check_status = Column(String(20), nullable=False, default="未到期",
comment="年检状态: 未到期/已年检/待年检/过期")
issuer = Column(String(200), nullable=True, comment="发证/主管机关")
owner_role = Column(String(100), nullable=True, comment="责任岗位(职能,不指定自然人)")
attachment_no = Column(String(100), nullable=True, comment="附件/扫描件编号")
registered_by = Column(String(100), nullable=True, comment="登记人(职能岗位)")
remark = Column(String(500), nullable=True, comment="备注")
created_at = Column(DateTime, server_default=func.now())
updated_at = Column(DateTime, server_default=func.now(), onupdate=func.now())
__table_args__ = (
UniqueConstraint("entity_id", "licence_name", "licence_no", name="uq_licence_ledger_entity_name_no"),
)
+349
View File
@@ -0,0 +1,349 @@
"""外部输入台账 → kpi_values 派生模块(能力评价 M2 准入3 派生,2026-09-15)
三张外部输入台账(app/models/ext_input_ledger.py)→ 三个能力评价指标值:
P_TAX_TIMELY = 当期 on_time=1 的税种数 ÷ 当期应申报税种数 × 100 (月频)
L_EMPLOYEE_TURNOVER = separations ÷ headcount_begin × 100 (月频)
P_LICENCE_OK = 有效证照数 ÷ 应持证照数 × 100 (季频,季末为基准日)
硬规则(任务 0-P0-project-20260915-cma-eval-m2-ext-input-tables,直接进验收):
1. **分母为 0 必须返回 None(无定义 / 不参评),不得写 0、不得填大数。**
依据:F_INTEREST_COVER 已因「利息费用=0 → 分母 0」失去定义;同族错误不得在 3 张新台账上重演。
→ 值为 None 时**不写 kpi_values**;若该期已存在旧的 derived 行则只回报 stale_row_present,不删不改。
2. **只写 kpi_values**(entity_id / kpi_id / period / actual_value + 来源标识 source_type='derived'),
**不新建、不改动任何 KPI 定义**:定义缺失 → 跳过并回报 skipped_no_definition。
3. **幂等**:同一 (entity_id, kpi_id, period, source_type='derived') 重复运行只更新不新增
(与 scripts/kpi_bep_deriver.py 同范式)。
本模块只依赖「台账行对象」的属性(ORM 实例 / 简易对象皆可),纯计算函数可脱离数据库单测。
"""
from __future__ import annotations
import calendar
import logging
import re
from datetime import date, datetime
from typing import Iterable, Optional, Sequence
from sqlalchemy.orm import Session
from app.models import KPIDefinition, KPIValue, TaxFilingLog, HrMonthlyLedger, LicenceLedger
logger = logging.getLogger("cma.ext_input_derive")
TAX_TIMELY_CODE = "P_TAX_TIMELY"
EMPLOYEE_TURNOVER_CODE = "L_EMPLOYEE_TURNOVER"
LICENCE_OK_CODE = "P_LICENCE_OK"
ALL_KPI_CODES = (TAX_TIMELY_CODE, EMPLOYEE_TURNOVER_CODE, LICENCE_OK_CODE)
# 数据来源标识(写入 kpi_values.source_type;与 dashboard「最新派生期间」取数口径一致)
SOURCE_TYPE = "derived"
_MONTH_RE = re.compile(r"^(\d{4})-(0[1-9]|1[0-2])$")
_QUARTER_RE = re.compile(r"^(\d{4})-Q([1-4])$")
# ============================================================
# 期间工具
# ============================================================
def is_month_period(period: str) -> bool:
return bool(_MONTH_RE.match(str(period or "")))
def is_quarter_period(period: str) -> bool:
return bool(_QUARTER_RE.match(str(period or "")))
def month_to_quarter(period: str) -> str:
"""2026-06 → 2026-Q2(季频指标按「所在季度」归集)"""
m = _MONTH_RE.match(str(period or ""))
if not m:
raise ValueError(f"非法月期间: {period}(应为 YYYY-MM)")
quarter = (int(m.group(2)) - 1) // 3 + 1
return f"{m.group(1)}-Q{quarter}"
def quarter_end_date(period: str) -> date:
"""季末基准日。接受 2026-Q2(季频)或 2026-06(月频,自动归到所在季度)"""
q = _QUARTER_RE.match(str(period or ""))
if q:
year, quarter = int(q.group(1)), int(q.group(2))
else:
q2 = re.match(r"^(\d{4})-Q([1-4])$", month_to_quarter(period))
if not q2:
raise ValueError(f"非法季期间: {period}(应为 YYYY-Qn 或 YYYY-MM)")
year, quarter = int(q2.group(1)), int(q2.group(2))
last_month = quarter * 3
return date(year, last_month, calendar.monthrange(year, last_month)[1])
# ============================================================
# 1. 申报行为台账 → P_TAX_TIMELY
# ============================================================
def derive_on_time(filed_date: Optional[date], legal_deadline: Optional[date]) -> Optional[int]:
"""`on_time` 的**唯一**服务端派生规则(录入方不得手填):
- 已申报且不晚于法定期限 → 1
- 已申报但晚于法定期限 → 0
- 未申报(filed_date 为空) → 0(逾期未申报,计入分母)
- 已申报但缺法定期限(无法判定)→ None(该行不计入分子;分母仍计)
"""
if filed_date is None:
return 0
if legal_deadline is None:
return None
return 1 if filed_date <= legal_deadline else 0
def compute_tax_timely_rate(rows: Iterable) -> Optional[float]:
"""P_TAX_TIMELY = on_time=1 的税种数 ÷ 应申报税种数 × 100;分母为 0 → None"""
rows = list(rows)
denominator = len(rows)
if denominator == 0:
return None
timely = sum(1 for r in rows if getattr(r, "on_time", None) == 1)
return round(timely / denominator * 100.0, 1)
def explain_tax_timely_rate(rows: Iterable) -> dict:
rows = list(rows)
value = compute_tax_timely_rate(rows)
return {
"denominator": len(rows),
"timely": sum(1 for r in rows if getattr(r, "on_time", None) == 1),
"unknown": sum(1 for r in rows if getattr(r, "on_time", None) is None),
"value": value,
"no_definition": value is None,
}
# ============================================================
# 2. 人事月度台账 → L_EMPLOYEE_TURNOVER
# ============================================================
def check_headcount_equation(headcount_begin, hires, separations, headcount_end) -> Optional[str]:
"""期末 = 期初 + 入职 − 离职。通过返回 None,不通过返回错误说明(调用方拒绝写入并回报)"""
try:
lhs = int(headcount_end)
rhs = int(headcount_begin) + int(hires) - int(separations)
except (TypeError, ValueError):
return "期初/入职/离职/期末必须为整数"
if lhs != rhs:
return (f"人事台账等式不成立: 期末({lhs}) ≠ 期初({headcount_begin}) + 入职({hires}) "
f"− 离职({separations}) = {rhs}(拒绝写入)")
return None
def compute_employee_turnover(row) -> Optional[float]:
"""L_EMPLOYEE_TURNOVER = separations ÷ headcount_begin × 100;期初为 0(或缺失)→ None"""
if row is None:
return None
try:
begin = int(getattr(row, "headcount_begin"))
separations = int(getattr(row, "separations"))
except (TypeError, ValueError):
return None
if begin == 0:
return None
return round(separations / begin * 100.0, 2)
def explain_employee_turnover(row) -> dict:
"""并列返回「期初期末平均在职」口径值,便于复核(平均在职为 0 → 该口径同样 None)"""
value = compute_employee_turnover(row)
detail = {
"headcount_begin": getattr(row, "headcount_begin", None),
"headcount_end": getattr(row, "headcount_end", None),
"hires": getattr(row, "hires", None),
"separations": getattr(row, "separations", None),
"value": value,
"no_definition": value is None,
"avg_headcount": None,
"value_on_avg": None,
}
if row is None:
return detail
try:
avg = (int(row.headcount_begin) + int(row.headcount_end)) / 2.0
except (TypeError, ValueError):
avg = None
detail["avg_headcount"] = avg
if avg:
detail["value_on_avg"] = round(int(row.separations) / avg * 100.0, 2)
return detail
# ============================================================
# 3. 证照台账 → P_LICENCE_OK
# ============================================================
def is_licence_valid(licence, as_of: date) -> bool:
"""有效证照判定:状态非「过期」且(valid_to 为空=长期有效 或 valid_to >= 基准日)"""
if str(getattr(licence, "annual_check_status", "") or "").strip() == "过期":
return False
valid_to = getattr(licence, "valid_to", None)
if valid_to is None:
return True
if isinstance(valid_to, datetime):
valid_to = valid_to.date()
return valid_to >= as_of
def compute_licence_ok(rows: Iterable, as_of: date) -> Optional[float]:
"""P_LICENCE_OK = 有效证照数 ÷ 应持证照数 × 100(基准日=季末);应持证照数为 0 → None"""
rows = list(rows)
denominator = len(rows)
if denominator == 0:
return None
valid = sum(1 for r in rows if is_licence_valid(r, as_of))
return round(valid / denominator * 100.0, 1)
def explain_licence_ok(rows: Iterable, as_of: date) -> dict:
rows = list(rows)
value = compute_licence_ok(rows, as_of)
expired = [getattr(r, "licence_no", "") for r in rows if str(getattr(r, "annual_check_status", "")) == "过期"]
return {
"as_of": as_of.isoformat(),
"denominator": len(rows),
"valid": sum(1 for r in rows if is_licence_valid(r, as_of)),
"expired_or_outdated": expired,
"value": value,
"no_definition": value is None,
}
# ============================================================
# 落库(幂等 upsert;只写 kpi_values)
# ============================================================
def _resolve_kpi(db: Session, entity_id: int, kpi_code: str) -> Optional[KPIDefinition]:
"""按 (entity_id, kpi_code) 取定义(仅 active)。定义唯一性在库内非全局:
e1/e2 可各有一条同名 kpi_code,故必须带 entity_id 过滤 + order_by(id) 取稳定行。"""
return (db.query(KPIDefinition)
.filter(KPIDefinition.entity_id == entity_id,
KPIDefinition.kpi_code == kpi_code,
KPIDefinition.status == "active")
.order_by(KPIDefinition.id.asc())
.first())
def upsert_kpi_value(db: Session, entity_id: int, kpi: KPIDefinition, period: str,
value: float, remark: str, batch: str) -> str:
"""幂等写 kpi_values:命中 (kpi_id, period, source_type='derived') 则更新,返回 'insert'/'upsert'"""
existing = (db.query(KPIValue)
.filter(KPIValue.kpi_id == kpi.id,
KPIValue.period == period,
KPIValue.source_type == SOURCE_TYPE)
.order_by(KPIValue.id.asc())
.first())
if existing is not None:
action = "upsert"
existing.actual_value = value
existing.source_batch = batch
existing.data_status = "pending"
existing.remark = remark
else:
db.add(KPIValue(
entity_id=entity_id,
kpi_id=kpi.id,
period=period,
actual_value=value,
source_type=SOURCE_TYPE,
source_batch=batch,
data_status="pending",
remark=remark,
))
action = "insert"
db.flush()
return action
def run_derive(db: Session, entity_id: int, period: str,
kpis: Optional[Sequence[str]] = None, write: bool = True) -> dict:
"""按实体+期间派生 3 个指标并写 kpi_values(幂等)。
period:P_TAX_TIMELY / L_EMPLOYEE_TURNOVER 用 YYYY-MM;
P_LICENCE_OK 用 YYYY-Qn(传 YYYY-MM 时自动归到所在季度,落库 period 为季度)。
write=False → 只算不写(dry-run)。
返回 {'entity_id', 'period', 'batch', 'items': {kpi_code: {...}}}
"""
codes = list(kpis or ALL_KPI_CODES)
unknown = [c for c in codes if c not in ALL_KPI_CODES]
if unknown:
raise ValueError(f"不支持的指标编码: {unknown}(可用: {list(ALL_KPI_CODES)})")
batch = f"ext-ledger-{datetime.now().strftime('%Y%m%d%H%M%S')}"
items: dict = {}
for code in codes:
kpi = _resolve_kpi(db, entity_id, code)
if kpi is None:
items[code] = {
"action": "skipped_no_definition",
"reason": f"entity={entity_id} 无 active 定义 {code}(定义变更属另一派单,本模块不新建)",
"actual_value": None,
}
continue
if code == TAX_TIMELY_CODE:
rows = (db.query(TaxFilingLog)
.filter(TaxFilingLog.entity_id == entity_id, TaxFilingLog.period == period)
.order_by(TaxFilingLog.id.asc()).all())
detail = explain_tax_timely_rate(rows)
target_period = period
remark = (f"台账派生: 应申报{detail['denominator']}个税种,按时{detail['timely']}个,"
f"无法判定{detail['unknown']}个(tax_filing_log {period})")
elif code == EMPLOYEE_TURNOVER_CODE:
row = (db.query(HrMonthlyLedger)
.filter(HrMonthlyLedger.entity_id == entity_id, HrMonthlyLedger.period == period)
.order_by(HrMonthlyLedger.id.asc()).first())
detail = explain_employee_turnover(row)
target_period = period
remark = (f"台账派生: 离职{detail['separations']}/期初在职{detail['headcount_begin']}×100 "
f"(期初{detail['headcount_begin']}+入职{detail['hires']}-离职{detail['separations']}"
f"=期末{detail['headcount_end']});并列口径 期初期末平均在职{detail['avg_headcount']}人 "
f"→ {detail['value_on_avg']}%")
else: # LICENCE_OK —— 季频
target_period = month_to_quarter(period) if is_month_period(period) else period
as_of = quarter_end_date(target_period)
rows = (db.query(LicenceLedger)
.filter(LicenceLedger.entity_id == entity_id)
.order_by(LicenceLedger.id.asc()).all())
detail = explain_licence_ok(rows, as_of)
remark = (f"台账派生: 季末基准日{detail['as_of']},有效{detail['valid']}/应持证{detail['denominator']}"
f"(过期或失效: {detail['expired_or_outdated'] or '无'})")
value = detail.get("value")
item = {
"kpi_id": kpi.id,
"kpi_code": code,
"period": target_period,
"actual_value": value,
"detail": detail,
}
if value is None:
# 硬规则:分母 0 → 无定义,不写 kpi_values(不写 0、不填大数)
stale = (db.query(KPIValue)
.filter(KPIValue.kpi_id == kpi.id, KPIValue.period == target_period,
KPIValue.source_type == SOURCE_TYPE)
.order_by(KPIValue.id.asc()).first())
item["action"] = "skipped_denominator_zero"
item["reason"] = "分母为 0 → 无定义(不参评),不写 kpi_values"
item["stale_row_present"] = stale is not None
items[code] = item
continue
if write:
item["action"] = upsert_kpi_value(db, entity_id, kpi, target_period, value, remark, batch)
else:
item["action"] = "dry-run"
item["remark"] = remark
items[code] = item
if write:
db.commit()
return {"entity_id": entity_id, "period": period, "batch": batch,
"write": write, "items": items}