""" ERP数据同步脚本 — 管理会计OS 根据 kpi_definitions.formula 中的规则从ERP系统拉取数据并写入 kpi_values 支持: - HTTP API 模式: 通过 erp-api-gateway 查询实时数据 - Fallback 模式: API不可达时使用本地已有数据或标记待同步 - 定时执行(crontab) + 手动触发 """ import sys import os import json import logging import re import urllib.request import urllib.error from datetime import datetime, timedelta sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) from dotenv import load_dotenv load_dotenv() from app.database import get_engine, get_session_local from app.models import KPIDefinition, KPIValue, OperationLog logger = logging.getLogger("erp_sync") # ERP API 配置 ERP_API_BASE = os.getenv("ERP_API_BASE", "http://127.0.0.1:8300/api/v1") ERP_API_KEY = os.getenv("ERP_API_KEY", "erp-gateway-key-bhwl-2026") # ============================================================ # 公式解析 # ============================================================ def parse_formula(formula: str) -> dict: """解析KPI公式,提取ERP表和字段映射""" result = {"table": "MasterBill", "field": None, "agg": "SUM", "where": None, "raw": formula, "erp_direct": True} # 特殊处理中文描述的公式 ZH_PATTERNS = { "前5客户销售额/总销售额*100": ("TOP5_CUSTOMER", "MasterBill"), "前5客户集中度": ("TOP5_CUSTOMER", "MasterBill"), "满意客户数/总客户数*100": ("CUSTOMER_SAT_RATIO", "MasterBill"), "准时交付订单/总订单*100": ("DELIVERY_RATE", "MasterBill"), "完成培训人数/应培训人数*100": ("TRAINING_RATE", "MasterBill"), } for zh_pattern, (agg_type, table) in ZH_PATTERNS.items(): if zh_pattern in formula: result.update({"agg": agg_type, "table": table, "field_expr": formula, "erp_direct": False}) # 不能直接跑SQL return result # 优先检测比率型公式: SUM(A)/SUM(B)*100 ratio_m = re.match(r"(SUM|COUNT|AVG)\s*\((.+?)\)\s*/\s*(SUM|COUNT|AVG)\s*\((.+?)\)", formula, re.I) if ratio_m: result["agg"] = f"RATIO_{ratio_m.group(1)}" result["field_expr"] = f"({ratio_m.group(2)})/({ratio_m.group(4)})" return result # 匹配完整聚合: SUM(...), COUNT(DISTINCT ...), COUNT(...), AVG(...) m = re.match(r"(SUM|COUNT(?:\s+DISTINCT)?|AVG|MAX|MIN)\s*\((.+?)\)", formula, re.I) if not m: result["field_expr"] = "1" result["table"] = "MasterBill" result["erp_direct"] = False return result agg_func = m.group(1).strip().upper() field_expr = m.group(2).strip() if agg_func.startswith("COUNT") and field_expr.startswith("DISTINCT "): result["agg"] = "COUNT_DISTINCT" cleaned = field_expr.replace("DISTINCT ", "").strip() result["field_expr"] = cleaned parts = cleaned.split(".") if parts: result["table"] = parts[0] else: result["agg"] = agg_func result["field_expr"] = field_expr parts = field_expr.split(".") if len(parts) >= 2: candidate = parts[0].strip() if candidate and candidate[0].isupper(): result["table"] = candidate wm = re.search(r"WHERE\s+(.+)$", formula, re.I) if wm: result["where"] = wm.group(1).strip() return result # ============================================================ # API 模式: 通过 ERP 接口查询 # ============================================================ def fetch_via_api(kpi: KPIDefinition, parsed: dict, period: str) -> float: """通过 erp-api-gateway 查询ERP数据""" period_month = int(period[5:7]) period_year = int(period[:4]) kpi_code = kpi.kpi_code # 各KPI对应的API路径 API_MAP = { "SALES_TOTAL": f"{ERP_API_BASE}/stats/monthly?year={period_year}", "CUSTOMER_COUNT": f"{ERP_API_BASE}/stats/monthly?year={period_year}", "SALES_PROFIT_RATE": f"{ERP_API_BASE}/stats/gross-profit?year={period_year}&month={period_month}", "TOP5_CUSTOMER_RATIO": f"{ERP_API_BASE}/stats/customer-top?year={period_year}&limit=5", # P1: CRM模块 "C_RETENTION_RATE": f"{ERP_API_BASE}/crm/retention-rate?year={period_year}&month={period_month}", "C_NEW_CLIENTS": f"{ERP_API_BASE}/crm/new-customers?year={period_year}&month={period_month}", # P1: 生产模块 "F_QUALITY_RATE": f"{ERP_API_BASE}/production/quality-rate?year={period_year}&month={period_month}", "F_REWORK_RATE": f"{ERP_API_BASE}/production/rework-rate?year={period_year}&month={period_month}", } headers = {"X-API-Key": ERP_API_KEY, "User-Agent": "CMA-ERP-SYNC/1.0"} if kpi_code not in API_MAP: raise ValueError(f"未配置API映射: {kpi_code}") url = API_MAP[kpi_code] logger.info(f" [{kpi_code}] API请求: {url}") req = urllib.request.Request(url, headers=headers) try: with urllib.request.urlopen(req, timeout=15) as resp: data = json.loads(resp.read().decode()) except urllib.error.HTTPError as e: raise ConnectionError(f"API返回 {e.code}: {e.read().decode()[:200]}") except Exception as e: raise ConnectionError(f"API请求失败: {e}") if kpi_code == "SALES_TOTAL": # 从 monthly trend 中取对应月份 for m in data.get("data", []): if m["period"] == period: return float(m["amount"]) # fallback: 取汇总 return float(data.get("summary", {}).get("total_amount", 0)) elif kpi_code == "CUSTOMER_COUNT": for m in data.get("data", []): if m["period"] == period: return float(m["customers"]) return 0 elif kpi_code == "SALES_PROFIT_RATE": return float(data.get("gross_profit_rate", 0)) elif kpi_code == "TOP5_CUSTOMER_RATIO": top5 = data.get("data", []) top5_total = sum(c["amount"] for c in top5) # 同时获取全年总额 total_url = f"{ERP_API_BASE}/stats/monthly?year={period_year}" req2 = urllib.request.Request(total_url, headers=headers) with urllib.request.urlopen(req2, timeout=15) as resp2: total_data = json.loads(resp2.read().decode()) total_amount = sum(m["amount"] for m in total_data.get("data", [])) if total_amount > 0: return round(top5_total / total_amount * 100, 2) return 0 # P1: CRM模块 — 客户保留率 elif kpi_code == "C_RETENTION_RATE": return float(data.get("retention_rate", data.get("value", 0))) # P1: CRM模块 — 新客户数 elif kpi_code == "C_NEW_CLIENTS": return float(data.get("new_customers", data.get("value", 0))) # P1: 生产模块 — 产品合格率 elif kpi_code == "F_QUALITY_RATE": return float(data.get("quality_rate", data.get("value", 0))) # P1: 生产模块 — 返工率 elif kpi_code == "F_REWORK_RATE": return float(data.get("rework_rate", data.get("value", 0))) raise ValueError(f"未实现的API映射: {kpi_code}") # ============================================================ # Fallback 模式: 本地已有数据推算 # ============================================================ def fetch_fallback(kpi: KPIDefinition, parsed: dict, db_session, period: str) -> float: """Fallback: 从本地已有 kpi_values 推算或返回 None""" kpi_code = kpi.kpi_code # 对于已有数据的KPI,沿用最近月份的值(标注为estimated) existing = db_session.query(KPIValue).filter( KPIValue.kpi_id == kpi.id, KPIValue.source_type.in_(["erp", "manual"]), ).order_by(KPIValue.period.desc()).first() if existing and existing.actual_value is not None: logger.info(f" [{kpi_code}] Fallback: 沿用最近期 {existing.period}={existing.actual_value}") return existing.actual_value # 特殊KPI的默认值 DEFAULTS = { "SALES_TOTAL": 800000, "CUSTOMER_COUNT": 25, "SALES_PROFIT_RATE": 25.0, "TOP5_CUSTOMER_RATIO": 50.0, } if kpi_code in DEFAULTS: logger.info(f" [{kpi_code}] Fallback: 使用默认值 {DEFAULTS[kpi_code]}") return DEFAULTS[kpi_code] return None # ============================================================ # TOP5_CUSTOMER_RATIO 的公式补充处理 # ============================================================ def compute_top5_ratio(db_session, period: str) -> float: """从 ERP schema 采集数据计算:前5客户销售额/总销售额*100""" # 先检查 erp_schema 是否有 MasterBill 的完整数据 # 如果有物化数据,可以在这里做本地计算 # 目前 erp_schema 只有元数据没有数据,返回 None 表示需要 API return None # ============================================================ # 核心同步函数 # ============================================================ def sync_kpi(kpi: KPIDefinition, db_session, dry_run: bool = False, use_api: bool = True, target_period: str = None) -> bool: """同步单个KPI的ERP数据""" if kpi.data_source_type not in ("erp",): return False formula = kpi.formula if not formula: logger.warning(f" [{kpi.kpi_code}] 无公式定义") return False parsed = parse_formula(formula) logger.info(f" [{kpi.kpi_code}] 解析: table={parsed['table']}, agg={parsed['agg']}, " f"erp_direct={parsed.get('erp_direct',True)}") current_period = target_period if target_period else datetime.now().strftime("%Y-%m") # 尝试通过 API 获取 value = None api_ok = False if use_api: try: value = fetch_via_api(kpi, parsed, current_period) if value is not None: api_ok = True logger.info(f" [{kpi.kpi_code}] API结果: {current_period}={value}") except Exception as e: logger.warning(f" [{kpi.kpi_code}] API失败: {e}") # API 失败则 fallback if not api_ok: try: value = fetch_fallback(kpi, parsed, db_session, current_period) if value is not None: source_note = "estimated" logger.info(f" [{kpi.kpi_code}] Fallback结果: {current_period}={value}") else: logger.warning(f" [{kpi.kpi_code}] 无可用数据, 跳过") return False except Exception as e: logger.error(f" [{kpi.kpi_code}] Fallback失败: {e}") return False if dry_run: logger.info(f" [{kpi.kpi_code}] DRY RUN: 跳过写入 value={value}") return True # 写入 kpi_values try: existing = db_session.query(KPIValue).filter( KPIValue.kpi_id == kpi.id, KPIValue.period == current_period, KPIValue.source_type == "erp", ).first() remark = f"ERP自动同步{' (API)' if api_ok else ' (估算)'} {datetime.now().strftime('%Y-%m-%d %H:%M')}" if existing: existing.actual_value = value existing.data_status = "verified" if api_ok else "estimated" existing.remark = remark existing.source_type = "erp" logger.info(f" [{kpi.kpi_code}] 更新 {current_period}: {value}") else: kv = KPIValue( kpi_id=kpi.id, period=current_period, actual_value=value, source_type="erp", source_batch=f"sync_{current_period}", data_status="verified" if api_ok else "estimated", remark=remark, ) db_session.add(kv) logger.info(f" [{kpi.kpi_code}] 新增 {current_period}: {value}") db_session.commit() return True except Exception as e: db_session.rollback() logger.error(f" [{kpi.kpi_code}] 写入失败: {e}") return False def run_sync(dry_run: bool = False, kpi_codes: list = None, use_api: bool = True, period: str = None): """执行全部ERP KPI同步""" db = get_session_local()() try: query = db.query(KPIDefinition).filter( KPIDefinition.status == "active", KPIDefinition.data_source_type == "erp", ) if kpi_codes: query = query.filter(KPIDefinition.kpi_code.in_(kpi_codes)) kpis = query.all() target_period = period if period else datetime.now().strftime("%Y-%m") logger.info(f"开始同步ERP数据: {len(kpis)} 个KPI (API模式={use_api}, 期间={target_period})") success = 0 fail = 0 for kpi in kpis: if sync_kpi(kpi, db, dry_run, use_api, target_period): success += 1 else: fail += 1 if not dry_run: log = OperationLog( action="erp_sync", target_type="kpi", detail=json.dumps({ "total": len(kpis), "success": success, "failed": fail, "api_mode": use_api, "period": datetime.now().strftime("%Y-%m"), }, ensure_ascii=False), ) db.add(log) db.commit() logger.info(f"同步完成: {success}成功 / {fail}失败 / {len(kpis)}总计") finally: db.close() if __name__ == "__main__": import argparse parser = argparse.ArgumentParser(description="ERP数据同步") parser.add_argument("--dry-run", action="store_true", help="仅预览,不写入数据库") parser.add_argument("--kpi", nargs="+", help="指定KPI编码") parser.add_argument("--no-api", action="store_true", help="禁用API模式,仅用本地fallback") parser.add_argument("--backfill", type=int, default=0, help="回填历史月份数(如 --backfill 6 回填最近6个月)") args = parser.parse_args() if args.backfill: from datetime import datetime, timedelta from app.database import get_session_local today = datetime.now() months_backfilled = 0 for i in range(1, args.backfill + 1): # 计算目标月份 m = today.month - i y = today.year while m <= 0: m += 12 y -= 1 period = f"{y}-{m:02d}" print(f"回填 {period}...") try: run_sync(dry_run=False, kpi_codes=args.kpi, use_api=not args.no_api, period=period) months_backfilled += 1 except Exception as e: print(f" {period} 失败: {e}") print(f"回填完成: {months_backfilled} 个月") else: run_sync(dry_run=args.dry_run, kpi_codes=args.kpi, use_api=not args.no_api)