cma-management/backend/scripts/kpi_value_collector.py

"""KPI实际值自动归集采集器 — 管理会计OS (P1-④ 2026-08-28)

按 kpi_value_sources 取数映射，从源头表(科目余额/进销存/现金流水)自动汇总写入 kpi_values。
- 源头: voucher_details(网银凭证明细) / product_inventory(库存汇总) / product_inventory_detail(库存明细) / cash_plans(收付款计划)
- 严格按 entity_id + period 过滤，避免跨账套/跨期串数
- 幂等: 同 kpi_id+period 已有 auto_collect 记录则更新；人工 excel/manual 写入不覆盖
- 调度: 系统 crontab 每日 06:30 (参考 auto_verify_cron.py 模式)

用法:
  /usr/bin/python3 scripts/kpi_value_collector.py            # 全量采集当月
  /usr/bin/python3 scripts/kpi_value_collector.py 2026-08    # 指定期间
  /usr/bin/python3 scripts/kpi_value_collector.py 2026-08 5  # 指定期间+KPI
"""
import sys
import logging
from datetime import datetime
from typing import Optional, Tuple

logging.basicConfig(
    level=logging.INFO,
    format="%(asctime)s [%(levelname)s] %(name)s: %(message)s",
)
logger = logging.getLogger("cma.kpi_collector")

# 保证从 backend 目录直接运行时能 import app
if __name__ == "__main__":
    import os
    sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))

from sqlalchemy import func
from app.database import get_session_local
from app.models import (
    KPIValueSource, KPIValueCollectLog, KPIValue, KPIDefinition,
    VoucherDetail, CashPlan,
)


# 源头表 → ORM模型映射（动态 import 避免循环依赖）
def _source_model(table: str):
    if table == "voucher_details":
        return VoucherDetail
    if table == "cash_plans":
        return CashPlan
    # product_inventory / product_inventory_detail 无ORM模型 → SQLAlchemy Table 反射
    from sqlalchemy import Table, MetaData
    from app.database import get_engine
    md = MetaData()
    return Table(table, md, autoload_with=get_engine())


def _field_expression(model, field: str, aggregate: str = "sum"):
    """聚合表达式: sum/avg/count/max/min"""
    col = getattr(model, field)
    if aggregate == "count":
        return func.count(col)
    if aggregate == "avg":
        return func.avg(col)
    if aggregate == "max":
        return func.max(col)
    if aggregate == "min":
        return func.min(col)
    return func.sum(col)


def collect_for_mapping(db, mapping: KPIValueSource, period: str, write_kpi: bool = True) -> Tuple[Optional[float], str]:
    """执行单条取数映射，返回 (采集值, 说明)。write_kpi=False 时为试跑模式（不写库）。"""
    table = mapping.source_table
    field = mapping.source_field
    aggregate = mapping.aggregate or "sum"
    filter_rule = mapping.filter_rule or {}
    period_field = mapping.period_field or "period"
    unit = mapping.unit_conversion or 1

    model = _source_model(table)

    # 构建查询
    col = getattr(model, field, None)
    if col is None:
        return None, f"字段 {field} 不存在于表 {table}"

    q = db.query(_field_expression(model, field, aggregate))

    # entity_id 过滤（所有源头表都有）
    q = q.filter(model.entity_id == mapping.entity_id)

    # 期间过滤
    if period_field == "voucher_date":
        # voucher_date 是 DATE 类型 → 按 %Y-%m 前缀匹配
        q = q.filter(func.date_format(model.voucher_date, "%Y-%m") == period)
    else:
        pf = getattr(model, period_field, None)
        if pf is None:
            return None, f"期间字段 {period_field} 不存在于表 {table}"
        q = q.filter(pf == period)

    # 过滤规则: subject_code / direction / plan_type / carry_forward
    subject_code = filter_rule.get("subject_code")
    if subject_code and hasattr(model, "subject_code"):
        q = q.filter(model.subject_code == subject_code)
    direction = filter_rule.get("direction")
    if direction:
        # direction 覆盖: credit→只算贷方, debit→只算借方
        if direction == "credit" and hasattr(model, "credit_amount"):
            q = db.query(_field_expression(model, "credit_amount", aggregate))
            q = q.filter(model.entity_id == mapping.entity_id)
            if period_field == "voucher_date":
                q = q.filter(func.date_format(model.voucher_date, "%Y-%m") == period)
            else:
                q = q.filter(getattr(model, period_field) == period)
            field = "credit_amount"
        elif direction == "debit" and hasattr(model, "debit_amount"):
            q = db.query(_field_expression(model, "debit_amount", aggregate))
            q = q.filter(model.entity_id == mapping.entity_id)
            if period_field == "voucher_date":
                q = q.filter(func.date_format(model.voucher_date, "%Y-%m") == period)
            else:
                q = q.filter(getattr(model, period_field) == period)
            field = "debit_amount"
    plan_type = filter_rule.get("plan_type")
    if plan_type and hasattr(model, "plan_type"):
        q = q.filter(model.plan_type == plan_type)
    if filter_rule.get("exclude_carry_forward") and hasattr(model, "carry_forward"):
        q = q.filter(model.carry_forward == 0)

    value = q.scalar()
    value = float(value or 0)
    value = round(value * unit, 2)

    message = f"表{table}.{field} {aggregate}(period={period}) × {unit}"
    if subject_code:
        message += f", 科目{subject_code}"
    if direction:
        message += f", 方向{direction}"
    if plan_type:
        message += f", 类型{plan_type}"

    if write_kpi:
        # upsert kpi_values: 同 kpi_id+period 已有 auto_collect 记录则更新
        existing = db.query(KPIValue).filter(
            KPIValue.kpi_id == mapping.kpi_id,
            KPIValue.period == period,
            KPIValue.source_type == "auto_collect",
        ).first()
        if existing:
            existing.actual_value = value
            existing.source_batch = _batch_no()
            existing.remark = f"自动归集: {table}"
            existing.data_status = "pending"
        else:
            db.add(KPIValue(
                entity_id=mapping.entity_id,
                kpi_id=mapping.kpi_id,
                period=period,
                actual_value=value,
                source_type="auto_collect",
                source_batch=_batch_no(),
                data_status="pending",
                remark=f"自动归集: {table}",
            ))

    return value, message


def _batch_no() -> str:
    return f"auto-{datetime.now().strftime('%Y%m%d%H%M%S')}"


def run_collector(db, entity_id: Optional[int] = None, period: Optional[str] = None, kpi_id: Optional[int] = None) -> dict:
    """运行采集器：遍历 active 映射 → 汇总 → upsert kpi_values → 写采集日志"""
    if period is None:
        period = datetime.now().strftime("%Y-%m")

    query = db.query(KPIValueSource).filter(KPIValueSource.status == "active")
    if entity_id is not None:
        query = query.filter(KPIValueSource.entity_id == entity_id)
    if kpi_id is not None:
        query = query.filter(KPIValueSource.kpi_id == kpi_id)
    mappings = query.all()

    if not mappings:
        return {"success": True, "collected": 0, "failed": 0, "message": "无激活取数映射"}

    collected, failed = 0, 0
    errors = []
    for m in mappings:
        try:
            value, message = collect_for_mapping(db, m, period, write_kpi=True)
            db.add(KPIValueCollectLog(
                entity_id=m.entity_id,
                kpi_id=m.kpi_id,
                period=period,
                source_table=m.source_table,
                collected_value=value,
                status="success",
                message=message,
            ))
            collected += 1
        except Exception as e:
            failed += 1
            errors.append({"kpi_id": m.kpi_id, "source_table": m.source_table, "error": str(e)})
            db.add(KPIValueCollectLog(
                entity_id=m.entity_id,
                kpi_id=m.kpi_id,
                period=period,
                source_table=m.source_table,
                collected_value=None,
                status="failed",
                message=str(e)[:500],
            ))
            logger.error("采集失败 kpi=%s table=%s: %s", m.kpi_id, m.source_table, e)

    db.commit()
    logger.info("采集完成: 成功%s 失败%s (period=%s)", collected, failed, period)
    return {
        "success": failed == 0,
        "collected": collected,
        "failed": failed,
        "period": period,
        "errors": errors[:20],
    }


if __name__ == "__main__":
    period_arg = sys.argv[1] if len(sys.argv) > 1 else None
    kpi_arg = int(sys.argv[2]) if len(sys.argv) > 2 and sys.argv[2].isdigit() else None
    db = get_session_local()()
    try:
        r = run_collector(db, period=period_arg, kpi_id=kpi_arg)
        print(f"实际值自动归集完成: 成功{r['collected']} 失败{r['failed']} (period={r.get('period')})")
        for e in r.get("errors", []):
            print(f"  失败: kpi={e['kpi_id']} table={e['source_table']} -> {e['error']}")
    finally:
        db.close()