343 lines
14 KiB
Python
343 lines
14 KiB
Python
"""KPI实际值自动归集采集器 — 管理会计OS (P1-④ 2026-08-28)
|
||
|
||
按 kpi_value_sources 取数映射，从源头表(科目余额/进销存/现金流水)自动汇总写入 kpi_values。
|
||
- 源头: voucher_details(网银凭证明细) / product_inventory(库存汇总) / product_inventory_detail(库存明细) / cash_plans(收付款计划)
|
||
- 严格按 entity_id + period 过滤，避免跨账套/跨期串数
|
||
- 幂等: 同 kpi_id+period 已有 auto_collect 记录则更新；人工 excel/manual 写入不覆盖
|
||
- 调度: 系统 crontab 每日 06:30 (参考 auto_verify_cron.py 模式)
|
||
|
||
用法:
|
||
/usr/bin/python3 scripts/kpi_value_collector.py # 全量采集当月
|
||
/usr/bin/python3 scripts/kpi_value_collector.py 2026-08 # 指定期间
|
||
/usr/bin/python3 scripts/kpi_value_collector.py 2026-08 5 # 指定期间+KPI
|
||
"""
|
||
import sys
|
||
import logging
|
||
from datetime import datetime
|
||
from typing import Optional, Tuple
|
||
|
||
logging.basicConfig(
|
||
level=logging.INFO,
|
||
format="%(asctime)s [%(levelname)s] %(name)s: %(message)s",
|
||
)
|
||
logger = logging.getLogger("cma.kpi_collector")
|
||
|
||
# 保证从 backend 目录直接运行时能 import app
|
||
if __name__ == "__main__":
|
||
import os
|
||
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||
|
||
from sqlalchemy import func
|
||
from app.database import get_session_local
|
||
from app.models import (
|
||
KPIValueSource, KPIValueCollectLog, KPIValue, KPIDefinition,
|
||
VoucherDetail, CashPlan,
|
||
)
|
||
|
||
|
||
# 源头表 → ORM模型映射（动态 import 避免循环依赖）
|
||
def _source_model(table: str):
|
||
if table == "voucher_details":
|
||
return VoucherDetail
|
||
if table == "cash_plans":
|
||
return CashPlan
|
||
# product_inventory / product_inventory_detail 无ORM模型 → SQLAlchemy Table 反射
|
||
from sqlalchemy import Table, MetaData
|
||
from app.database import get_engine
|
||
md = MetaData()
|
||
return Table(table, md, autoload_with=get_engine())
|
||
|
||
|
||
def _field_expression(model, field: str, aggregate: str = "sum"):
|
||
"""聚合表达式: sum/avg/count/max/min"""
|
||
col = getattr(model, field)
|
||
if aggregate == "count":
|
||
return func.count(col)
|
||
if aggregate == "avg":
|
||
return func.avg(col)
|
||
if aggregate == "max":
|
||
return func.max(col)
|
||
if aggregate == "min":
|
||
return func.min(col)
|
||
return func.sum(col)
|
||
|
||
|
||
def collect_for_mapping(db, mapping: KPIValueSource, period: str, write_kpi: bool = True) -> Tuple[Optional[float], str]:
|
||
"""执行单条取数映射，返回 (采集值, 说明)。write_kpi=False 时为试跑模式（不写库）。"""
|
||
table = mapping.source_table
|
||
field = mapping.source_field
|
||
aggregate = mapping.aggregate or "sum"
|
||
filter_rule = mapping.filter_rule or {}
|
||
period_field = mapping.period_field or "period"
|
||
unit = mapping.unit_conversion or 1
|
||
|
||
# P1③④ 激活(2026-09-14): 科目组合映射 —— 由 kpi_subject_map 推导的多科目加权归集
|
||
# filter_rule 形如 {"subject_codes":["5001%"],"measure":"credit","weights":{"5001%":1.0},
|
||
# "calc":"sum|ratio","denominator":"5001%","scale":100}
|
||
if isinstance(filter_rule, dict) and filter_rule.get("subject_codes"):
|
||
return collect_subject_group(db, mapping, period, write_kpi=write_kpi)
|
||
|
||
model = _source_model(table)
|
||
|
||
# 构建查询
|
||
col = getattr(model, field, None)
|
||
if col is None:
|
||
return None, f"字段 {field} 不存在于表 {table}"
|
||
|
||
q = db.query(_field_expression(model, field, aggregate))
|
||
|
||
# entity_id 过滤（所有源头表都有）
|
||
q = q.filter(model.entity_id == mapping.entity_id)
|
||
|
||
# 期间过滤
|
||
if period_field == "voucher_date":
|
||
# voucher_date 是 DATE 类型 → 按 %Y-%m 前缀匹配
|
||
q = q.filter(func.date_format(model.voucher_date, "%Y-%m") == period)
|
||
else:
|
||
pf = getattr(model, period_field, None)
|
||
if pf is None:
|
||
return None, f"期间字段 {period_field} 不存在于表 {table}"
|
||
q = q.filter(pf == period)
|
||
|
||
# 过滤规则: subject_code / direction / plan_type / carry_forward
|
||
subject_code = filter_rule.get("subject_code")
|
||
if subject_code and hasattr(model, "subject_code"):
|
||
q = q.filter(model.subject_code == subject_code)
|
||
direction = filter_rule.get("direction")
|
||
if direction:
|
||
# direction 覆盖: credit→只算贷方, debit→只算借方
|
||
if direction == "credit" and hasattr(model, "credit_amount"):
|
||
q = db.query(_field_expression(model, "credit_amount", aggregate))
|
||
q = q.filter(model.entity_id == mapping.entity_id)
|
||
if period_field == "voucher_date":
|
||
q = q.filter(func.date_format(model.voucher_date, "%Y-%m") == period)
|
||
else:
|
||
q = q.filter(getattr(model, period_field) == period)
|
||
field = "credit_amount"
|
||
elif direction == "debit" and hasattr(model, "debit_amount"):
|
||
q = db.query(_field_expression(model, "debit_amount", aggregate))
|
||
q = q.filter(model.entity_id == mapping.entity_id)
|
||
if period_field == "voucher_date":
|
||
q = q.filter(func.date_format(model.voucher_date, "%Y-%m") == period)
|
||
else:
|
||
q = q.filter(getattr(model, period_field) == period)
|
||
field = "debit_amount"
|
||
plan_type = filter_rule.get("plan_type")
|
||
if plan_type and hasattr(model, "plan_type"):
|
||
q = q.filter(model.plan_type == plan_type)
|
||
if filter_rule.get("exclude_carry_forward") and hasattr(model, "carry_forward"):
|
||
q = q.filter(model.carry_forward == 0)
|
||
|
||
value = q.scalar()
|
||
if value is None:
|
||
# 源表无记录 → 不写 0，交由 run_collector 记 skipped，避免污染 KPI 实际值
|
||
return None, "源表无记录"
|
||
value = float(value)
|
||
value = round(value * unit, 2)
|
||
|
||
message = f"表{table}.{field} {aggregate}(period={period}) × {unit}"
|
||
if subject_code:
|
||
message += f", 科目{subject_code}"
|
||
if direction:
|
||
message += f", 方向{direction}"
|
||
if plan_type:
|
||
message += f", 类型{plan_type}"
|
||
|
||
if write_kpi:
|
||
_upsert_auto_value(db, mapping, period, value, f"自动归集: {table}")
|
||
|
||
return value, message
|
||
|
||
|
||
def _upsert_auto_value(db, mapping, period: str, value: float, remark: str) -> None:
|
||
"""upsert kpi_values: 同 kpi_id+period 已有 auto_collect 记录则更新。
|
||
|
||
人工来源(excel/manual)行不匹配本查询 → 自动归集永不覆盖人工录入值。
|
||
"""
|
||
existing = db.query(KPIValue).filter(
|
||
KPIValue.kpi_id == mapping.kpi_id,
|
||
KPIValue.period == period,
|
||
KPIValue.source_type == "auto_collect",
|
||
).first()
|
||
if existing:
|
||
existing.actual_value = value
|
||
existing.source_batch = _batch_no()
|
||
existing.remark = remark
|
||
existing.data_status = "pending"
|
||
else:
|
||
db.add(KPIValue(
|
||
entity_id=mapping.entity_id,
|
||
kpi_id=mapping.kpi_id,
|
||
period=period,
|
||
actual_value=value,
|
||
source_type="auto_collect",
|
||
source_batch=_batch_no(),
|
||
data_status="pending",
|
||
remark=remark,
|
||
))
|
||
|
||
|
||
# ── 科目组合归集（P1③④ 激活 · 2026-09-14）─────────────────────────
|
||
# 背景: kpi_subject_map 用新准则科目(6xxx)，voucher_details 是智享通凭证明细(5xxx/1xxx/2xxx)，
|
||
# 编码体系不同 → 由 scripts/kpi_value_source_derive.py 先做「同代码/同名/子目前缀/结转科目」
|
||
# 机械桥接，把**实际存在的科目编码或前缀**写进 filter_rule.subject_codes，再由本函数归集。
|
||
_CREDIT_NORMAL_PREFIXES = ("2", "4", "60", "61", "63")
|
||
|
||
|
||
def _is_credit_normal(subject_code: str) -> bool:
|
||
return str(subject_code).startswith(_CREDIT_NORMAL_PREFIXES)
|
||
|
||
|
||
def _subject_amount(db, entity_id: int, pattern: str, period: str, measure: str):
|
||
"""单科目(支持 LIKE 前缀)期间金额 → (金额, 行数)。measure: credit|debit|net|debit_net"""
|
||
q = db.query(
|
||
func.coalesce(func.sum(VoucherDetail.credit_amount), 0),
|
||
func.coalesce(func.sum(VoucherDetail.debit_amount), 0),
|
||
func.count(VoucherDetail.id),
|
||
).filter(VoucherDetail.entity_id == entity_id)
|
||
if "%" in pattern:
|
||
q = q.filter(VoucherDetail.subject_code.like(pattern))
|
||
else:
|
||
q = q.filter(VoucherDetail.subject_code == pattern)
|
||
if period:
|
||
q = q.filter(VoucherDetail.period == period)
|
||
credit, debit, rows = q.first()
|
||
credit, debit = float(credit or 0), float(debit or 0)
|
||
if measure == "debit":
|
||
amount = debit
|
||
elif measure == "net":
|
||
amount = credit - debit
|
||
elif measure == "debit_net":
|
||
amount = debit - credit
|
||
else: # credit（默认）: 贷方发生额——损益类科目月结"结转"腿与本期发生额同额，取贷方即本期发生额
|
||
amount = credit
|
||
return amount, int(rows or 0)
|
||
|
||
|
||
def collect_subject_group(db, mapping, period: str, write_kpi: bool = True):
|
||
"""多科目加权归集（filter_rule.subject_codes 路径），返回 (采集值, 说明)。"""
|
||
rule = mapping.filter_rule or {}
|
||
codes = list(rule.get("subject_codes") or [])
|
||
weights = rule.get("weights") or {}
|
||
measure = rule.get("measure", "credit")
|
||
calc = rule.get("calc", "sum")
|
||
scale = float(rule.get("scale", 1) or 1)
|
||
unit = mapping.unit_conversion or 1
|
||
|
||
total, rows, parts, matched = 0.0, 0, [], 0
|
||
for code in codes:
|
||
amount, n = _subject_amount(db, mapping.entity_id, code, period, measure)
|
||
if n:
|
||
matched += 1
|
||
rows += n
|
||
weight = float(weights.get(code, 1) or 0)
|
||
total += weight * amount
|
||
parts.append(f"{code}×{weight:g}")
|
||
if rows == 0:
|
||
return None, f"源表无记录(period={period}, {len(codes)}个科目均无发生额)"
|
||
|
||
if calc == "ratio":
|
||
denom_code = rule.get("denominator") or (codes[0] if codes else None)
|
||
if not denom_code:
|
||
return None, "比率映射缺少 denominator"
|
||
denom, _ = _subject_amount(db, mapping.entity_id, str(denom_code), period, measure)
|
||
if not denom:
|
||
return None, f"比率分母 {denom_code} 期间无发生额"
|
||
value = round(float(total) / float(denom) * scale * unit, 4)
|
||
else:
|
||
value = round(float(total) * unit, 2)
|
||
|
||
message = (f"科目组合(period={period}, {measure}, 命中{matched}/{len(codes)}): "
|
||
+ " + ".join(parts))
|
||
if calc == "ratio":
|
||
message += f" ÷ {denom_code} × {scale:g}"
|
||
|
||
if write_kpi:
|
||
_upsert_auto_value(db, mapping, period, value,
|
||
f"自动归集: {mapping.source_table} 科目组合")
|
||
return value, message
|
||
|
||
|
||
def _batch_no() -> str:
|
||
return f"auto-{datetime.now().strftime('%Y%m%d%H%M%S')}"
|
||
|
||
|
||
def run_collector(db, entity_id: Optional[int] = None, period: Optional[str] = None, kpi_id: Optional[int] = None) -> dict:
|
||
"""运行采集器：遍历 active 映射 → 汇总 → upsert kpi_values → 写采集日志"""
|
||
if period is None:
|
||
period = datetime.now().strftime("%Y-%m")
|
||
|
||
query = db.query(KPIValueSource).filter(KPIValueSource.status == "active")
|
||
if entity_id is not None:
|
||
query = query.filter(KPIValueSource.entity_id == entity_id)
|
||
if kpi_id is not None:
|
||
query = query.filter(KPIValueSource.kpi_id == kpi_id)
|
||
mappings = query.all()
|
||
|
||
if not mappings:
|
||
return {"success": True, "collected": 0, "failed": 0, "message": "无激活取数映射"}
|
||
|
||
collected, failed = 0, 0
|
||
errors = []
|
||
for m in mappings:
|
||
try:
|
||
value, message = collect_for_mapping(db, m, period, write_kpi=True)
|
||
if value is None:
|
||
# 源表无记录 → 跳过，不落 pending=0 的 KPIValue
|
||
db.add(KPIValueCollectLog(
|
||
entity_id=m.entity_id,
|
||
kpi_id=m.kpi_id,
|
||
period=period,
|
||
source_table=m.source_table,
|
||
collected_value=None,
|
||
status="skipped",
|
||
message=message,
|
||
))
|
||
continue
|
||
db.add(KPIValueCollectLog(
|
||
entity_id=m.entity_id,
|
||
kpi_id=m.kpi_id,
|
||
period=period,
|
||
source_table=m.source_table,
|
||
collected_value=value,
|
||
status="success",
|
||
message=message,
|
||
))
|
||
collected += 1
|
||
except Exception as e:
|
||
failed += 1
|
||
errors.append({"kpi_id": m.kpi_id, "source_table": m.source_table, "error": str(e)})
|
||
db.add(KPIValueCollectLog(
|
||
entity_id=m.entity_id,
|
||
kpi_id=m.kpi_id,
|
||
period=period,
|
||
source_table=m.source_table,
|
||
collected_value=None,
|
||
status="failed",
|
||
message=str(e)[:500],
|
||
))
|
||
logger.error("采集失败 kpi=%s table=%s: %s", m.kpi_id, m.source_table, e)
|
||
|
||
db.commit()
|
||
logger.info("采集完成: 成功%s 失败%s (period=%s)", collected, failed, period)
|
||
return {
|
||
"success": failed == 0,
|
||
"collected": collected,
|
||
"failed": failed,
|
||
"period": period,
|
||
"errors": errors[:20],
|
||
}
|
||
|
||
|
||
if __name__ == "__main__":
|
||
period_arg = sys.argv[1] if len(sys.argv) > 1 else None
|
||
kpi_arg = int(sys.argv[2]) if len(sys.argv) > 2 and sys.argv[2].isdigit() else None
|
||
db = get_session_local()()
|
||
try:
|
||
r = run_collector(db, period=period_arg, kpi_id=kpi_arg)
|
||
print(f"实际值自动归集完成: 成功{r['collected']} 失败{r['failed']} (period={r.get('period')})")
|
||
for e in r.get("errors", []):
|
||
print(f" 失败: kpi={e['kpi_id']} table={e['source_table']} -> {e['error']}")
|
||
finally:
|
||
db.close()
|