Hermes CI Fix
3dddd36866
包含前后端完整代码： - 前端：Vue3+Vite+ElementPlus - 后端：FastAPI+SQLAlchemy - 模块：驾驶舱/KPI/战略地图/预警/预算/成本/预测/改善行动 - 当前版本：v1.0.0
375 lines
14 KiB
Python
375 lines
14 KiB
Python
"""
|
|
ERP数据同步脚本 — 管理会计OS
|
|
根据 kpi_definitions.formula 中的规则从ERP系统拉取数据并写入 kpi_values
|
|
支持:
|
|
- HTTP API 模式: 通过 erp-api-gateway 查询实时数据
|
|
- Fallback 模式: API不可达时使用本地已有数据或标记待同步
|
|
- 定时执行(crontab) + 手动触发
|
|
"""
|
|
|
|
import sys
|
|
import os
|
|
import json
|
|
import logging
|
|
import re
|
|
import urllib.request
|
|
import urllib.error
|
|
from datetime import datetime, timedelta
|
|
|
|
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
|
|
|
from dotenv import load_dotenv
|
|
load_dotenv()
|
|
|
|
from app.database import get_engine, get_session_local
|
|
from app.models import KPIDefinition, KPIValue, OperationLog
|
|
|
|
logger = logging.getLogger("erp_sync")
|
|
|
|
# ERP API 配置
|
|
ERP_API_BASE = os.getenv("ERP_API_BASE", "http://127.0.0.1:8300/api/v1")
|
|
ERP_API_KEY = os.getenv("ERP_API_KEY", "erp-gateway-key-bhwl-2026")
|
|
|
|
# ============================================================
|
|
# 公式解析
|
|
# ============================================================
|
|
|
|
def parse_formula(formula: str) -> dict:
|
|
"""解析KPI公式，提取ERP表和字段映射"""
|
|
result = {"table": "MasterBill", "field": None, "agg": "SUM",
|
|
"where": None, "raw": formula, "erp_direct": True}
|
|
|
|
# 特殊处理中文描述的公式
|
|
ZH_PATTERNS = {
|
|
"前5客户销售额/总销售额*100": ("TOP5_CUSTOMER", "MasterBill"),
|
|
"前5客户集中度": ("TOP5_CUSTOMER", "MasterBill"),
|
|
"满意客户数/总客户数*100": ("CUSTOMER_SAT_RATIO", "MasterBill"),
|
|
"准时交付订单/总订单*100": ("DELIVERY_RATE", "MasterBill"),
|
|
"完成培训人数/应培训人数*100": ("TRAINING_RATE", "MasterBill"),
|
|
}
|
|
for zh_pattern, (agg_type, table) in ZH_PATTERNS.items():
|
|
if zh_pattern in formula:
|
|
result.update({"agg": agg_type, "table": table, "field_expr": formula,
|
|
"erp_direct": False}) # 不能直接跑SQL
|
|
return result
|
|
|
|
# 优先检测比率型公式: SUM(A)/SUM(B)*100
|
|
ratio_m = re.match(r"(SUM|COUNT|AVG)\s*\((.+?)\)\s*/\s*(SUM|COUNT|AVG)\s*\((.+?)\)", formula, re.I)
|
|
if ratio_m:
|
|
result["agg"] = f"RATIO_{ratio_m.group(1)}"
|
|
result["field_expr"] = f"({ratio_m.group(2)})/({ratio_m.group(4)})"
|
|
return result
|
|
|
|
# 匹配完整聚合: SUM(...), COUNT(DISTINCT ...), COUNT(...), AVG(...)
|
|
m = re.match(r"(SUM|COUNT(?:\s+DISTINCT)?|AVG|MAX|MIN)\s*\((.+?)\)", formula, re.I)
|
|
if not m:
|
|
result["field_expr"] = "1"
|
|
result["table"] = "MasterBill"
|
|
result["erp_direct"] = False
|
|
return result
|
|
|
|
agg_func = m.group(1).strip().upper()
|
|
field_expr = m.group(2).strip()
|
|
|
|
if agg_func.startswith("COUNT") and field_expr.startswith("DISTINCT "):
|
|
result["agg"] = "COUNT_DISTINCT"
|
|
cleaned = field_expr.replace("DISTINCT ", "").strip()
|
|
result["field_expr"] = cleaned
|
|
parts = cleaned.split(".")
|
|
if parts:
|
|
result["table"] = parts[0]
|
|
else:
|
|
result["agg"] = agg_func
|
|
result["field_expr"] = field_expr
|
|
parts = field_expr.split(".")
|
|
if len(parts) >= 2:
|
|
candidate = parts[0].strip()
|
|
if candidate and candidate[0].isupper():
|
|
result["table"] = candidate
|
|
|
|
wm = re.search(r"WHERE\s+(.+)$", formula, re.I)
|
|
if wm:
|
|
result["where"] = wm.group(1).strip()
|
|
|
|
return result
|
|
|
|
|
|
# ============================================================
|
|
# API 模式: 通过 ERP 接口查询
|
|
# ============================================================
|
|
|
|
def fetch_via_api(kpi: KPIDefinition, parsed: dict, period: str) -> float:
|
|
"""通过 erp-api-gateway 查询ERP数据"""
|
|
period_month = int(period[5:7])
|
|
period_year = int(period[:4])
|
|
kpi_code = kpi.kpi_code
|
|
|
|
# 各KPI对应的API路径
|
|
API_MAP = {
|
|
"SALES_TOTAL": f"{ERP_API_BASE}/stats/monthly?year={period_year}",
|
|
"CUSTOMER_COUNT": f"{ERP_API_BASE}/stats/monthly?year={period_year}",
|
|
"SALES_PROFIT_RATE": f"{ERP_API_BASE}/stats/gross-profit?year={period_year}&month={period_month}",
|
|
"TOP5_CUSTOMER_RATIO": f"{ERP_API_BASE}/stats/customer-top?year={period_year}&limit=5",
|
|
}
|
|
|
|
headers = {"X-API-Key": ERP_API_KEY, "User-Agent": "CMA-ERP-SYNC/1.0"}
|
|
|
|
if kpi_code not in API_MAP:
|
|
raise ValueError(f"未配置API映射: {kpi_code}")
|
|
|
|
url = API_MAP[kpi_code]
|
|
logger.info(f" [{kpi_code}] API请求: {url}")
|
|
|
|
req = urllib.request.Request(url, headers=headers)
|
|
try:
|
|
with urllib.request.urlopen(req, timeout=15) as resp:
|
|
data = json.loads(resp.read().decode())
|
|
except urllib.error.HTTPError as e:
|
|
raise ConnectionError(f"API返回 {e.code}: {e.read().decode()[:200]}")
|
|
except Exception as e:
|
|
raise ConnectionError(f"API请求失败: {e}")
|
|
|
|
if kpi_code == "SALES_TOTAL":
|
|
# 从 monthly trend 中取对应月份
|
|
for m in data.get("data", []):
|
|
if m["period"] == period:
|
|
return float(m["amount"])
|
|
# fallback: 取汇总
|
|
return float(data.get("summary", {}).get("total_amount", 0))
|
|
|
|
elif kpi_code == "CUSTOMER_COUNT":
|
|
for m in data.get("data", []):
|
|
if m["period"] == period:
|
|
return float(m["customers"])
|
|
return 0
|
|
|
|
elif kpi_code == "SALES_PROFIT_RATE":
|
|
return float(data.get("gross_profit_rate", 0))
|
|
|
|
elif kpi_code == "TOP5_CUSTOMER_RATIO":
|
|
top5 = data.get("data", [])
|
|
top5_total = sum(c["amount"] for c in top5)
|
|
# 同时获取全年总额
|
|
total_url = f"{ERP_API_BASE}/stats/monthly?year={period_year}"
|
|
req2 = urllib.request.Request(total_url, headers=headers)
|
|
with urllib.request.urlopen(req2, timeout=15) as resp2:
|
|
total_data = json.loads(resp2.read().decode())
|
|
total_amount = sum(m["amount"] for m in total_data.get("data", []))
|
|
if total_amount > 0:
|
|
return round(top5_total / total_amount * 100, 2)
|
|
return 0
|
|
|
|
raise ValueError(f"未实现的API映射: {kpi_code}")
|
|
|
|
|
|
# ============================================================
|
|
# Fallback 模式: 本地已有数据推算
|
|
# ============================================================
|
|
|
|
def fetch_fallback(kpi: KPIDefinition, parsed: dict, db_session, period: str) -> float:
|
|
"""Fallback: 从本地已有 kpi_values 推算或返回 None"""
|
|
kpi_code = kpi.kpi_code
|
|
|
|
# 对于已有数据的KPI，沿用最近月份的值（标注为estimated）
|
|
existing = db_session.query(KPIValue).filter(
|
|
KPIValue.kpi_id == kpi.id,
|
|
KPIValue.source_type.in_(["erp", "manual"]),
|
|
).order_by(KPIValue.period.desc()).first()
|
|
|
|
if existing and existing.actual_value is not None:
|
|
logger.info(f" [{kpi_code}] Fallback: 沿用最近期 {existing.period}={existing.actual_value}")
|
|
return existing.actual_value
|
|
|
|
# 特殊KPI的默认值
|
|
DEFAULTS = {
|
|
"SALES_TOTAL": 800000,
|
|
"CUSTOMER_COUNT": 25,
|
|
"SALES_PROFIT_RATE": 25.0,
|
|
"TOP5_CUSTOMER_RATIO": 50.0,
|
|
}
|
|
if kpi_code in DEFAULTS:
|
|
logger.info(f" [{kpi_code}] Fallback: 使用默认值 {DEFAULTS[kpi_code]}")
|
|
return DEFAULTS[kpi_code]
|
|
|
|
return None
|
|
|
|
|
|
# ============================================================
|
|
# TOP5_CUSTOMER_RATIO 的公式补充处理
|
|
# ============================================================
|
|
|
|
def compute_top5_ratio(db_session, period: str) -> float:
|
|
"""从 ERP schema 采集数据计算：前5客户销售额/总销售额*100"""
|
|
# 先检查 erp_schema 是否有 MasterBill 的完整数据
|
|
# 如果有物化数据，可以在这里做本地计算
|
|
# 目前 erp_schema 只有元数据没有数据，返回 None 表示需要 API
|
|
return None
|
|
|
|
|
|
# ============================================================
|
|
# 核心同步函数
|
|
# ============================================================
|
|
|
|
def sync_kpi(kpi: KPIDefinition, db_session, dry_run: bool = False,
|
|
use_api: bool = True, target_period: str = None) -> bool:
|
|
"""同步单个KPI的ERP数据"""
|
|
if kpi.data_source_type not in ("erp",):
|
|
return False
|
|
|
|
formula = kpi.formula
|
|
if not formula:
|
|
logger.warning(f" [{kpi.kpi_code}] 无公式定义")
|
|
return False
|
|
|
|
parsed = parse_formula(formula)
|
|
logger.info(f" [{kpi.kpi_code}] 解析: table={parsed['table']}, agg={parsed['agg']}, "
|
|
f"erp_direct={parsed.get('erp_direct',True)}")
|
|
|
|
current_period = target_period if target_period else datetime.now().strftime("%Y-%m")
|
|
|
|
# 尝试通过 API 获取
|
|
value = None
|
|
api_ok = False
|
|
if use_api:
|
|
try:
|
|
value = fetch_via_api(kpi, parsed, current_period)
|
|
if value is not None:
|
|
api_ok = True
|
|
logger.info(f" [{kpi.kpi_code}] API结果: {current_period}={value}")
|
|
except Exception as e:
|
|
logger.warning(f" [{kpi.kpi_code}] API失败: {e}")
|
|
|
|
# API 失败则 fallback
|
|
if not api_ok:
|
|
try:
|
|
value = fetch_fallback(kpi, parsed, db_session, current_period)
|
|
if value is not None:
|
|
source_note = "estimated"
|
|
logger.info(f" [{kpi.kpi_code}] Fallback结果: {current_period}={value}")
|
|
else:
|
|
logger.warning(f" [{kpi.kpi_code}] 无可用数据, 跳过")
|
|
return False
|
|
except Exception as e:
|
|
logger.error(f" [{kpi.kpi_code}] Fallback失败: {e}")
|
|
return False
|
|
|
|
if dry_run:
|
|
logger.info(f" [{kpi.kpi_code}] DRY RUN: 跳过写入 value={value}")
|
|
return True
|
|
|
|
# 写入 kpi_values
|
|
try:
|
|
existing = db_session.query(KPIValue).filter(
|
|
KPIValue.kpi_id == kpi.id,
|
|
KPIValue.period == current_period,
|
|
KPIValue.source_type == "erp",
|
|
).first()
|
|
|
|
remark = f"ERP自动同步{' (API)' if api_ok else ' (估算)'} {datetime.now().strftime('%Y-%m-%d %H:%M')}"
|
|
|
|
if existing:
|
|
existing.actual_value = value
|
|
existing.data_status = "verified" if api_ok else "estimated"
|
|
existing.remark = remark
|
|
existing.source_type = "erp"
|
|
logger.info(f" [{kpi.kpi_code}] 更新 {current_period}: {value}")
|
|
else:
|
|
kv = KPIValue(
|
|
kpi_id=kpi.id,
|
|
period=current_period,
|
|
actual_value=value,
|
|
source_type="erp",
|
|
source_batch=f"sync_{current_period}",
|
|
data_status="verified" if api_ok else "estimated",
|
|
remark=remark,
|
|
)
|
|
db_session.add(kv)
|
|
logger.info(f" [{kpi.kpi_code}] 新增 {current_period}: {value}")
|
|
|
|
db_session.commit()
|
|
return True
|
|
|
|
except Exception as e:
|
|
db_session.rollback()
|
|
logger.error(f" [{kpi.kpi_code}] 写入失败: {e}")
|
|
return False
|
|
|
|
|
|
def run_sync(dry_run: bool = False, kpi_codes: list = None, use_api: bool = True, period: str = None):
|
|
"""执行全部ERP KPI同步"""
|
|
db = get_session_local()()
|
|
try:
|
|
query = db.query(KPIDefinition).filter(
|
|
KPIDefinition.status == "active",
|
|
KPIDefinition.data_source_type == "erp",
|
|
)
|
|
if kpi_codes:
|
|
query = query.filter(KPIDefinition.kpi_code.in_(kpi_codes))
|
|
|
|
kpis = query.all()
|
|
target_period = period if period else datetime.now().strftime("%Y-%m")
|
|
logger.info(f"开始同步ERP数据: {len(kpis)} 个KPI (API模式={use_api}, 期间={target_period})")
|
|
|
|
success = 0
|
|
fail = 0
|
|
for kpi in kpis:
|
|
if sync_kpi(kpi, db, dry_run, use_api, target_period):
|
|
success += 1
|
|
else:
|
|
fail += 1
|
|
|
|
if not dry_run:
|
|
log = OperationLog(
|
|
action="erp_sync",
|
|
target_type="kpi",
|
|
detail=json.dumps({
|
|
"total": len(kpis), "success": success,
|
|
"failed": fail, "api_mode": use_api,
|
|
"period": datetime.now().strftime("%Y-%m"),
|
|
}, ensure_ascii=False),
|
|
)
|
|
db.add(log)
|
|
db.commit()
|
|
|
|
logger.info(f"同步完成: {success}成功 / {fail}失败 / {len(kpis)}总计")
|
|
|
|
finally:
|
|
db.close()
|
|
|
|
|
|
if __name__ == "__main__":
|
|
import argparse
|
|
parser = argparse.ArgumentParser(description="ERP数据同步")
|
|
parser.add_argument("--dry-run", action="store_true", help="仅预览，不写入数据库")
|
|
parser.add_argument("--kpi", nargs="+", help="指定KPI编码")
|
|
parser.add_argument("--no-api", action="store_true", help="禁用API模式，仅用本地fallback")
|
|
parser.add_argument("--backfill", type=int, default=0,
|
|
help="回填历史月份数（如 --backfill 6 回填最近6个月）")
|
|
args = parser.parse_args()
|
|
|
|
if args.backfill:
|
|
from datetime import datetime, timedelta
|
|
from app.database import get_session_local
|
|
|
|
today = datetime.now()
|
|
months_backfilled = 0
|
|
for i in range(1, args.backfill + 1):
|
|
# 计算目标月份
|
|
m = today.month - i
|
|
y = today.year
|
|
while m <= 0:
|
|
m += 12
|
|
y -= 1
|
|
period = f"{y}-{m:02d}"
|
|
|
|
print(f"回填 {period}...")
|
|
try:
|
|
run_sync(dry_run=False, kpi_codes=args.kpi, use_api=not args.no_api, period=period)
|
|
months_backfilled += 1
|
|
except Exception as e:
|
|
print(f" {period} 失败: {e}")
|
|
|
|
print(f"回填完成: {months_backfilled} 个月")
|
|
else:
|
|
run_sync(dry_run=args.dry_run, kpi_codes=args.kpi, use_api=not args.no_api)
|