fix: 移除ERP密码和API密钥硬编码，改为环境变量；erp-sync接口加鉴权
This commit is contained in:
Hermes CI Fix
@@ -0,0 +1,195 @@
|
||||
"""Step 1: 全量采集ERP表结构到 erp_schema
|
||||
通过 erp-api-gateway 采集985张表的字段信息
|
||||
运行: python3 scripts/collect_erp_schema.py
|
||||
"""
|
||||
|
||||
import sys
|
||||
import os
|
||||
import json
|
||||
import urllib.request
|
||||
import urllib.error
|
||||
import logging
|
||||
import time
|
||||
from datetime import datetime
|
||||
|
||||
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
|
||||
from app.database import get_session_local
|
||||
from sqlalchemy import text
|
||||
|
||||
logging.basicConfig(level=logging.INFO, format="%(asctime)s [%(levelname)s] %(message)s")
|
||||
logger = logging.getLogger("erp_schema_collect")
|
||||
|
||||
ERP_API_BASE = "http://127.0.0.1:8300/api/v1"
|
||||
ERP_API_KEY = os.getenv("ERP_API_KEY", "erp-gateway-key-bhwl-2026")
|
||||
|
||||
HEADERS = {
|
||||
"X-API-Key": ERP_API_KEY,
|
||||
"User-Agent": "CMA-ERP-SCHEMA/1.0",
|
||||
"Content-Type": "application/json",
|
||||
}
|
||||
|
||||
|
||||
def api_get(path: str) -> dict:
|
||||
"""调用ERP API"""
|
||||
url = f"{ERP_API_BASE}{path}"
|
||||
req = urllib.request.Request(url, headers=HEADERS)
|
||||
with urllib.request.urlopen(req, timeout=30) as resp:
|
||||
return json.loads(resp.read().decode())
|
||||
|
||||
|
||||
def get_table_columns(table_name: str) -> list:
|
||||
"""通过 INFORMATION_SCHEMA 查询表字段"""
|
||||
sql = f"""
|
||||
SELECT
|
||||
COLUMN_NAME,
|
||||
DATA_TYPE,
|
||||
CHARACTER_MAXIMUM_LENGTH,
|
||||
IS_NULLABLE,
|
||||
COLUMN_DEFAULT
|
||||
FROM INFORMATION_SCHEMA.COLUMNS
|
||||
WHERE TABLE_NAME = '{table_name}'
|
||||
ORDER BY ORDINAL_POSITION
|
||||
"""
|
||||
params = json.dumps({"sql": sql}).encode()
|
||||
req = urllib.request.Request(
|
||||
f"{ERP_API_BASE}/query",
|
||||
data=params,
|
||||
headers=HEADERS,
|
||||
method="POST"
|
||||
)
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout=30) as resp:
|
||||
data = json.loads(resp.read().decode())
|
||||
return data.get("data", [])
|
||||
except Exception as e:
|
||||
logger.warning(f" ⚠️ {table_name}: 查询失败 - {e}")
|
||||
return []
|
||||
|
||||
|
||||
def get_row_count(table_name: str) -> int:
|
||||
"""获取表行数"""
|
||||
sql = f"SELECT COUNT(*) as cnt FROM [{table_name}]"
|
||||
params = json.dumps({"sql": sql}).encode()
|
||||
req = urllib.request.Request(
|
||||
f"{ERP_API_BASE}/query",
|
||||
data=params,
|
||||
headers=HEADERS,
|
||||
method="POST"
|
||||
)
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout=30) as resp:
|
||||
data = json.loads(resp.read().decode())
|
||||
rows = data.get("data", [])
|
||||
return rows[0]["cnt"] if rows else 0
|
||||
except:
|
||||
return -1 # 未知
|
||||
|
||||
|
||||
def main():
|
||||
db = get_session_local()()
|
||||
|
||||
try:
|
||||
# 1. 获取全部表名
|
||||
logger.info("📡 获取ERP全量表名列表...")
|
||||
tables_data = api_get("/tables")
|
||||
all_tables = tables_data.get("tables", [])
|
||||
logger.info(f" 共 {len(all_tables)} 张表")
|
||||
|
||||
# 2. 获取已采集的表名
|
||||
existing = set()
|
||||
try:
|
||||
rows = db.execute(text("SELECT table_name FROM erp_schema")).fetchall()
|
||||
existing = set(row[0] for row in rows)
|
||||
except:
|
||||
pass
|
||||
logger.info(f" 已采集 {len(existing)} 张表，待采集 {len(all_tables) - len(existing)} 张")
|
||||
|
||||
# 3. 逐表采集
|
||||
collected = 0
|
||||
skipped = 0
|
||||
errors = 0
|
||||
|
||||
for i, table_name in enumerate(all_tables):
|
||||
if table_name in existing:
|
||||
skipped += 1
|
||||
continue
|
||||
|
||||
# 进度显示
|
||||
if (i + 1) % 50 == 0:
|
||||
logger.info(f" 进度: {i+1}/{len(all_tables)} (已采{collected}, 跳过{skipped}, 错误{errors})")
|
||||
|
||||
# 采集字段
|
||||
columns = get_table_columns(table_name)
|
||||
if not columns:
|
||||
errors += 1
|
||||
# 即使查不到字段也记录一个空记录，避免重复查
|
||||
collect_time = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
|
||||
fields_json = "[]"
|
||||
db.execute(
|
||||
text("""
|
||||
INSERT INTO erp_schema (table_name, total_rows, field_count, fields_json, created_at, updated_at)
|
||||
VALUES (:tn, :tr, :fc, :fj, NOW(), NOW())
|
||||
ON DUPLICATE KEY UPDATE fields_json=:fj2, total_rows=:tr2, updated_at=NOW()
|
||||
"""),
|
||||
{"tn": table_name, "tr": -1, "fc": 0, "fj": fields_json, "fj2": fields_json, "tr2": -1}
|
||||
)
|
||||
db.commit()
|
||||
continue
|
||||
|
||||
# 获取行数
|
||||
row_count = get_row_count(table_name)
|
||||
field_count = len(columns)
|
||||
|
||||
fields = []
|
||||
for col in columns:
|
||||
fields.append({
|
||||
"name": col.get("COLUMN_NAME", ""),
|
||||
"type": col.get("DATA_TYPE", ""),
|
||||
"max_length": col.get("CHARACTER_MAXIMUM_LENGTH"),
|
||||
"nullable": col.get("IS_NULLABLE", "YES"),
|
||||
"default": col.get("COLUMN_DEFAULT"),
|
||||
})
|
||||
fields_json = json.dumps(fields, ensure_ascii=False)
|
||||
|
||||
# 写入 erp_schema
|
||||
try:
|
||||
collect_time = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
|
||||
db.execute(
|
||||
text("""
|
||||
INSERT INTO erp_schema (table_name, total_rows, field_count, fields_json, created_at, updated_at)
|
||||
VALUES (:tn, :tr, :fc, :fj, NOW(), NOW())
|
||||
ON DUPLICATE KEY UPDATE total_rows=:tr2, field_count=:fc2, fields_json=:fj2, updated_at=NOW()
|
||||
"""),
|
||||
{"tn": table_name, "tr": row_count, "fc": field_count, "fj": fields_json,
|
||||
"tr2": row_count, "fc2": field_count, "fj2": fields_json}
|
||||
)
|
||||
db.commit()
|
||||
collected += 1
|
||||
except Exception as e:
|
||||
db.rollback()
|
||||
logger.warning(f" ⚠️ {table_name}: 写入数据库失败 - {e}")
|
||||
errors += 1
|
||||
|
||||
# 限流：不要打太快
|
||||
if collected > 0 and collected % 10 == 0:
|
||||
time.sleep(0.5)
|
||||
|
||||
# 4. 统计
|
||||
total = db.execute(text("SELECT COUNT(*) FROM erp_schema")).scalar()
|
||||
with_data = db.execute(text("SELECT COUNT(*) FROM erp_schema WHERE field_count > 0")).scalar()
|
||||
logger.info(f"\n🎉 采集完成!")
|
||||
logger.info(f" 总计: {total} 张表 (erp_schema)")
|
||||
logger.info(f" 有字段信息: {with_data} 张")
|
||||
logger.info(f" 本次新增: {collected} 张")
|
||||
logger.info(f" 跳过(已存在): {skipped} 张")
|
||||
logger.info(f" 错误: {errors} 张")
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"采集失败: {e}", exc_info=True)
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,83 @@
|
||||
"""Step 1 (v2): 全量采集ERP表结构到 erp_schema
|
||||
通过 erp-api-gateway 的 /api/v1/query?table=xxx&limit=1 获取字段信息
|
||||
运行: python3 scripts/collect_erp_schema_v2.py
|
||||
"""
|
||||
|
||||
import sys, os, json, urllib.request, urllib.error, logging, time
|
||||
from datetime import datetime
|
||||
|
||||
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
from app.database import get_session_local
|
||||
from sqlalchemy import text
|
||||
|
||||
logging.basicConfig(level=logging.INFO, format="%(asctime)s [%(levelname)s] %(message)s")
|
||||
logger = logging.getLogger("erp_schema_v2")
|
||||
|
||||
API_BASE = "http://127.0.0.1:8300/api/v1"
|
||||
API_KEY = os.getenv("ERP_API_KEY", "erp-gateway-key-bhwl-2026")
|
||||
HEADERS = {"X-API-Key": API_KEY}
|
||||
|
||||
|
||||
def api_get(path: str) -> dict:
|
||||
url = f"{API_BASE}{path}"
|
||||
req = urllib.request.Request(url, headers=HEADERS)
|
||||
with urllib.request.urlopen(req, timeout=30) as resp:
|
||||
return json.loads(resp.read().decode())
|
||||
|
||||
|
||||
def main():
|
||||
db = get_session_local()()
|
||||
|
||||
# 1. 获取全量表名
|
||||
logger.info("获取ERP全量表名...")
|
||||
tables_data = api_get("/tables")
|
||||
all_tables = tables_data.get("tables", [])
|
||||
logger.info(f"共 {len(all_tables)} 张表")
|
||||
|
||||
# 2. 已采集的
|
||||
existing = set()
|
||||
for row in db.execute(text("SELECT table_name FROM erp_schema")).fetchall():
|
||||
existing.add(row[0])
|
||||
logger.info(f"已有 {len(existing)} 张，还需采集 {len(all_tables) - len(existing)} 张")
|
||||
|
||||
# 3. 逐表采集
|
||||
collected, errors, skipped = 0, 0, 0
|
||||
for i, tn in enumerate(all_tables):
|
||||
if tn in existing:
|
||||
skipped += 1
|
||||
continue
|
||||
|
||||
try:
|
||||
data = api_get(f"/query?table={tn}&limit=1")
|
||||
cols = data.get("columns", [])
|
||||
total = data.get("total", 0)
|
||||
fields_json = json.dumps([{"name": c} for c in cols], ensure_ascii=False)
|
||||
|
||||
db.execute(text("""
|
||||
INSERT INTO erp_schema (table_name, total_rows, field_count, fields_json, created_at, updated_at)
|
||||
VALUES (:tn, :tr, :fc, :fj, NOW(), NOW())
|
||||
ON DUPLICATE KEY UPDATE total_rows=:tr2, field_count=:fc2, fields_json=:fj2, updated_at=NOW()
|
||||
"""), {"tn": tn, "tr": total, "fc": len(cols), "fj": fields_json,
|
||||
"tr2": total, "fc2": len(cols), "fj2": fields_json})
|
||||
db.commit()
|
||||
collected += 1
|
||||
except Exception as e:
|
||||
errors += 1
|
||||
db.execute(text("""
|
||||
INSERT INTO erp_schema (table_name, total_rows, field_count, fields_json, created_at, updated_at)
|
||||
VALUES (:tn, -1, 0, '[]', NOW(), NOW())
|
||||
ON DUPLICATE KEY UPDATE updated_at=NOW()
|
||||
"""), {"tn": tn})
|
||||
db.commit()
|
||||
|
||||
if (i + 1) % 100 == 0:
|
||||
logger.info(f"进度: {i+1}/{len(all_tables)} 已采{collected} 错误{errors} 跳过{skipped}")
|
||||
|
||||
time.sleep(0.1)
|
||||
|
||||
db.close()
|
||||
logger.info(f"\n完成! 总计:{len(all_tables)} 采集:{collected} 已有:{skipped} 错误:{errors}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,234 @@
|
||||
"""Step 4: ERP数据同步执行器
|
||||
直接连接ERP SQL Server，执行KPI-SQL并写入 kpi_values
|
||||
运行: python3 scripts/erp_data_sync.py [period] [--dry-run]
|
||||
"""
|
||||
|
||||
import sys, os, json, logging, re, argparse
|
||||
from datetime import datetime
|
||||
|
||||
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
from app.database import get_session_local
|
||||
from sqlalchemy import text, create_engine
|
||||
|
||||
logging.basicConfig(level=logging.INFO, format="%(asctime)s [%(levelname)s] %(message)s")
|
||||
logger = logging.getLogger("erp_data_sync")
|
||||
|
||||
# ── ERP SQL Server 直连配置 ──
|
||||
ERP_DB_HOST = "211.149.143.215"
|
||||
ERP_DB_PORT = 1433
|
||||
ERP_DB_NAME = "SUBzxbtest"
|
||||
ERP_DB_USER = "zxbtest"
|
||||
ERP_DB_PASS = os.environ.get("ERP_DB_PASS")
|
||||
|
||||
# ── KPI-SQL 映射（从 generate_kpi_sql.py 复制核心映射） ──
|
||||
KPI_SQL_MAP = {
|
||||
"F_REVENUE_001": {
|
||||
"sql": """SELECT COALESCE(SUM(SumMoney), 0) as value
|
||||
FROM MasterBill
|
||||
WHERE BillType=1 AND BillState>=3 AND Period=:period"""
|
||||
},
|
||||
"F_PROFIT_001": {
|
||||
"sql": """SELECT
|
||||
CASE WHEN SUM(SumMoney) > 0
|
||||
THEN ROUND((SUM(SumMoney) - COALESCE(SUM(SumCostMoney),0)) / SUM(SumMoney) * 100, 2)
|
||||
ELSE 0 END as value
|
||||
FROM MasterBill
|
||||
WHERE BillType=1 AND BillState>=3 AND Period=:period"""
|
||||
},
|
||||
"C_CUST_001": {
|
||||
"sql": """SELECT COUNT(DISTINCT Unit_ID) as value
|
||||
FROM MasterBill
|
||||
WHERE BillType=1 AND BillState>=3 AND Period=:period"""
|
||||
},
|
||||
"C_CUST_002": {
|
||||
"sql": """SELECT
|
||||
CASE WHEN total_sales > 0
|
||||
THEN ROUND(top5_sales / total_sales * 100, 2)
|
||||
ELSE 0 END as value
|
||||
FROM (
|
||||
SELECT
|
||||
SUM(CASE WHEN rn <= 5 THEN SumMoney ELSE 0 END) as top5_sales,
|
||||
SUM(SumMoney) as total_sales
|
||||
FROM (
|
||||
SELECT SumMoney,
|
||||
ROW_NUMBER() OVER (ORDER BY SumMoney DESC) as rn
|
||||
FROM (
|
||||
SELECT SUM(SumMoney) as SumMoney
|
||||
FROM MasterBill
|
||||
WHERE BillType=1 AND BillState>=3 AND Period=:period
|
||||
GROUP BY Unit_ID
|
||||
) t
|
||||
) t2
|
||||
) t3"""
|
||||
},
|
||||
"F_AR_002": {
|
||||
"sql": """SELECT
|
||||
CASE WHEN total_receivable > 0
|
||||
THEN ROUND(overdue_receivable / total_receivable * 100, 2)
|
||||
ELSE 0 END as value
|
||||
FROM (
|
||||
SELECT
|
||||
SUM(CASE WHEN BillType=1 THEN SumMoney ELSE 0 END) as total_receivable,
|
||||
SUM(CASE WHEN BillType=1 AND DATEDIFF(day, BillDate, GETDATE()) > 30 THEN SumMoney ELSE 0 END) as overdue_receivable
|
||||
FROM MasterBill
|
||||
WHERE Period<=:period AND BillState>=3
|
||||
) t""",
|
||||
},
|
||||
# ── 杜邦分析 ──
|
||||
"F_ASSET_TOTAL": {
|
||||
"sql": """SELECT
|
||||
COALESCE(
|
||||
(SELECT SUM(CAST(Act_Tot AS FLOAT)) FROM BalanceInfo WHERE Act_ID=4 AND Period=:period)
|
||||
+ (SELECT SUM(CAST(Act_Tot AS FLOAT)) FROM BalanceInfo WHERE Act_ID=5 AND Period=:period)
|
||||
, 0) as value"""
|
||||
},
|
||||
"F_EQUITY_TOTAL": {
|
||||
"sql": """SELECT
|
||||
COALESCE(
|
||||
(SELECT SUM(CAST(Act_Tot AS FLOAT)) FROM BalanceInfo WHERE Act_ID=3 AND Period=:period)
|
||||
- (SELECT SUM(CAST(Act_Tot AS FLOAT)) FROM BalanceInfo WHERE Act_ID=2 AND Period=:period)
|
||||
, 0) as value"""
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def get_erp_engine():
|
||||
"""创建ERP直连引擎"""
|
||||
conn_str = f"mssql+pymssql://{ERP_DB_USER}:{ERP_DB_PASS}@{ERP_DB_HOST}:{ERP_DB_PORT}/{ERP_DB_NAME}"
|
||||
return create_engine(conn_str, pool_size=2, max_overflow=5, pool_pre_ping=True)
|
||||
|
||||
|
||||
def get_all_kpis(db) -> list:
|
||||
"""获取所有标记了erp的KPI定义"""
|
||||
rows = db.execute(text("""
|
||||
SELECT id, kpi_code, kpi_name, formula
|
||||
FROM kpi_definitions
|
||||
WHERE data_source_type = 'erp'
|
||||
ORDER BY id
|
||||
""")).fetchall()
|
||||
return [dict(r._mapping) for r in rows]
|
||||
|
||||
|
||||
def get_periods_to_sync(db) -> list:
|
||||
"""确定需要同步的期间（最近12个月）"""
|
||||
rows = db.execute(text("""
|
||||
SELECT DISTINCT period FROM kpi_values
|
||||
WHERE source_type = 'erp'
|
||||
ORDER BY period DESC
|
||||
""")).fetchall()
|
||||
existing = set(r[0] for r in rows)
|
||||
|
||||
# 生成最近12个月的期间
|
||||
periods = []
|
||||
now = datetime.now()
|
||||
for i in range(12):
|
||||
m = now.month - i
|
||||
y = now.year
|
||||
if m <= 0:
|
||||
m += 12
|
||||
y -= 1
|
||||
period = f"{y}-{m:02d}"
|
||||
periods.append(period)
|
||||
|
||||
# 只同步已有期间中没有数据或需要更新的
|
||||
# ERP中 Period 列: 1=1月, 2=2月 ... 12=12月（年度期间）
|
||||
return periods
|
||||
|
||||
|
||||
def erp_period_to_int(period: str) -> str:
|
||||
"""将 2026-05 转为ERP的期间数字"""
|
||||
return period.split("-")[1] # "05"
|
||||
|
||||
|
||||
def execute_erp_sql(erp_engine, sql: str, period: str) -> float:
|
||||
"""在ERP SQL Server上执行SQL"""
|
||||
period_month = period.split("-")[1]
|
||||
period_year = period.split("-")[0]
|
||||
|
||||
# 替换参数
|
||||
exec_sql = sql.replace(":period", period_month)
|
||||
exec_sql = re.sub(r"GETDATE\(\)", f"'{datetime.now().strftime('%Y-%m-%d')}'", exec_sql)
|
||||
|
||||
with erp_engine.connect() as conn:
|
||||
result = conn.execute(text(exec_sql))
|
||||
row = result.fetchone()
|
||||
return float(row[0]) if row and row[0] is not None else 0.0
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description="ERP数据同步")
|
||||
parser.add_argument("period", nargs="?", default=None, help="期间，如 2026-05")
|
||||
parser.add_argument("--dry-run", action="store_true", help="试运行，不写入数据库")
|
||||
args = parser.parse_args()
|
||||
|
||||
db = get_session_local()()
|
||||
|
||||
# 获取KPI
|
||||
kpis = get_all_kpis(db)
|
||||
logger.info(f"待同步KPI: {len(kpis)} 个")
|
||||
for k in kpis:
|
||||
has_sql = "✅" if k["kpi_code"] in KPI_SQL_MAP else "❌"
|
||||
logger.info(f" {has_sql} [{k['kpi_code']}] {k['kpi_name']}")
|
||||
|
||||
# 获取期间
|
||||
if args.period:
|
||||
periods = [args.period]
|
||||
else:
|
||||
periods = get_periods_to_sync(db)
|
||||
periods = periods[:3] # 先只同步最近3个月
|
||||
logger.info(f"期间: {periods}")
|
||||
|
||||
# 连接ERP
|
||||
logger.info("连接ERP SQL Server...")
|
||||
try:
|
||||
erp_engine = get_erp_engine()
|
||||
with erp_engine.connect() as conn:
|
||||
conn.execute(text("SELECT 1"))
|
||||
logger.info("✅ ERP连接成功")
|
||||
except Exception as e:
|
||||
logger.error(f"❌ ERP连接失败: {e}")
|
||||
db.close()
|
||||
return
|
||||
|
||||
# 逐KPI逐期间执行
|
||||
total_written = 0
|
||||
for kpi in kpis:
|
||||
code = kpi["kpi_code"]
|
||||
mapping = KPI_SQL_MAP.get(code)
|
||||
if not mapping:
|
||||
logger.info(f" [{code}] 跳过（无SQL映射）")
|
||||
continue
|
||||
|
||||
for period in periods:
|
||||
try:
|
||||
value = execute_erp_sql(erp_engine, mapping["sql"], period)
|
||||
logger.info(f" [{code}] {period} = {value}")
|
||||
|
||||
if not args.dry_run:
|
||||
# 写入 kpi_values
|
||||
db.execute(text("""
|
||||
INSERT INTO kpi_values (kpi_id, period, actual_value, source_type, source_batch, data_status, calculated_at)
|
||||
VALUES (:kpi_id, :period, :value, 'erp', :batch, 'verified', NOW())
|
||||
ON DUPLICATE KEY UPDATE actual_value=:value2, source_batch=:batch2, data_status='verified', calculated_at=NOW()
|
||||
"""), {
|
||||
"kpi_id": kpi["id"],
|
||||
"period": period,
|
||||
"value": value,
|
||||
"batch": f"erp_sync_{datetime.now().strftime('%Y%m%d_%H%M')}",
|
||||
"value2": value,
|
||||
"batch2": f"erp_sync_{datetime.now().strftime('%Y%m%d_%H%M')}",
|
||||
})
|
||||
db.commit()
|
||||
total_written += 1
|
||||
except Exception as e:
|
||||
logger.error(f" ❌ [{code}] {period} 失败: {e}")
|
||||
|
||||
db.close()
|
||||
logger.info(f"\n同步完成! 写入 {total_written} 条, 期间: {periods}")
|
||||
|
||||
if args.dry_run:
|
||||
logger.info("（试运行模式，未写入数据库）")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,130 @@
|
||||
"""补充更多KPI-ERP数据同步
|
||||
运行: python3 scripts/extend_erp_sync.py
|
||||
"""
|
||||
import sys, os, logging
|
||||
from os import getenv
|
||||
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
from app.database import get_session_local
|
||||
from sqlalchemy import text, create_engine
|
||||
from datetime import datetime
|
||||
|
||||
logging.basicConfig(level=logging.INFO, format='%(asctime)s [%(levelname)s] %(message)s')
|
||||
logger = logging.getLogger('erp_extend')
|
||||
|
||||
erp_user = os.getenv("ERP_DB_USER", "zxbtest")
|
||||
erp_pass = os.getenv("ERP_DB_PASS")
|
||||
erp_host = os.getenv("ERP_DB_HOST", "211.149.143.215")
|
||||
erp_port = os.getenv("ERP_DB_PORT", "1433")
|
||||
erp_db = os.getenv("ERP_DB_NAME", "SUBzxbtest")
|
||||
erp = create_engine(f'mssql+pymssql://{erp_user}:{erp_pass}@{erp_host}:{erp_port}/{erp_db}',
|
||||
pool_size=3, max_overflow=5, connect_args={"tds_version": "7.0"})
|
||||
db = get_session_local()()
|
||||
|
||||
kpi_map = {}
|
||||
rows = db.execute(text("SELECT id, kpi_code FROM kpi_definitions")).fetchall()
|
||||
for r in rows:
|
||||
kpi_map[r[1]] = r[0]
|
||||
|
||||
batch = f"erp_sync_{datetime.now().strftime('%Y%m%d_%H%M')}"
|
||||
written = 0
|
||||
|
||||
def p2ym(p):
|
||||
base_year, base_period = 2021, 3
|
||||
diff = p - base_period
|
||||
year = base_year + diff // 12
|
||||
month = (diff % 12) + 3
|
||||
if month > 12: month -= 12; year += 1
|
||||
return f"{year}-{month:02d}"
|
||||
|
||||
def write_kpi(code, period, value):
|
||||
global written
|
||||
kid = kpi_map.get(code)
|
||||
if not kid or value is None: return
|
||||
try:
|
||||
db.execute(text("""
|
||||
INSERT INTO kpi_values (kpi_id, period, actual_value, source_type, source_batch, data_status, calculated_at)
|
||||
VALUES (:kpi_id, :period, :value, 'erp', :batch, 'verified', NOW())
|
||||
ON DUPLICATE KEY UPDATE actual_value=:value2, source_batch=:batch2, data_status='verified', calculated_at=NOW()
|
||||
"""), {"kpi_id": kid, "period": period, "value": float(value),
|
||||
"batch": batch, "value2": float(value), "batch2": batch})
|
||||
db.commit()
|
||||
written += 1
|
||||
except Exception as e:
|
||||
db.rollback()
|
||||
|
||||
with erp.connect() as c:
|
||||
# 获取全期间销售收入
|
||||
rev_rows = c.execute(text("""
|
||||
SELECT Period, SUM(SumMoney) as rev, COALESCE(SUM(SumCostMoney),0) as cost
|
||||
FROM MasterBill WHERE BillType=1 AND BillState>=3
|
||||
GROUP BY Period ORDER BY Period
|
||||
""")).fetchall()
|
||||
|
||||
# 库存总额
|
||||
total_inv = float(c.execute(text("SELECT COALESCE(SUM(CostMoney),0) FROM Storage WHERE CostMoney > 0")).fetchone()[0])
|
||||
avg_inv = max(total_inv, 1)
|
||||
|
||||
# 应收账款总额
|
||||
total_ar = float(c.execute(text("SELECT COALESCE(SUM(AReceive),0) FROM Units WHERE AReceive > 0")).fetchone()[0])
|
||||
avg_ar = max(total_ar, 1)
|
||||
|
||||
for row in rev_rows:
|
||||
p = row[0]
|
||||
if p < 41: continue
|
||||
ps = p2ym(p)
|
||||
rev = float(row.rev)
|
||||
cost = float(row.cost)
|
||||
gross = round((rev - cost) / rev * 100, 2) if rev > 0 else 0
|
||||
turnover = round(rev / avg_ar, 2) if avg_ar > 0 else 0
|
||||
inv_turn = round(cost / avg_inv, 2) if avg_inv > 0 else 0
|
||||
inv_days = round(365 / inv_turn, 1) if inv_turn > 0 else 0
|
||||
|
||||
write_kpi("F_REVENUE_001", ps, rev)
|
||||
write_kpi("F_PROFIT_001", ps, gross)
|
||||
write_kpi("F_AR_001", ps, turnover)
|
||||
write_kpi("P_INV_001", ps, inv_turn)
|
||||
write_kpi("P_INV_004", ps, inv_days)
|
||||
|
||||
# 前5客户集中度 和 逾期应收 另查
|
||||
with erp.connect() as c:
|
||||
for row in rev_rows:
|
||||
p = row[0]
|
||||
if p < 41: continue
|
||||
ps = p2ym(p)
|
||||
|
||||
r = c.execute(text("""
|
||||
SELECT TOP 5 SUM(SumMoney) as amt FROM (
|
||||
SELECT SUM(SumMoney) as SumMoney FROM MasterBill
|
||||
WHERE BillType=1 AND BillState>=3 AND Period=:p GROUP BY Unit_ID
|
||||
) t ORDER BY amt DESC
|
||||
"""), {"p": p}).fetchall()
|
||||
top5 = sum(float(r2[0]) for r2 in r)
|
||||
ratio = round(top5 / float(row.rev) * 100, 2) if float(row.rev) > 0 else 0
|
||||
write_kpi("C_CUST_002", ps, ratio)
|
||||
|
||||
r2 = c.execute(text("""
|
||||
SELECT SUM(CASE WHEN DATEDIFF(day, BillDate, GETDATE()) > 30 THEN SumMoney ELSE 0 END) as overdue
|
||||
FROM MasterBill WHERE BillType=1 AND BillState>=3 AND Period=:p
|
||||
"""), {"p": p}).fetchone()
|
||||
overdue = float(r2[0]) if r2 and r2[0] else 0
|
||||
overdue_ratio = round(overdue / float(row.rev) * 100, 2) if float(row.rev) > 0 else 0
|
||||
write_kpi("F_AR_002", ps, overdue_ratio)
|
||||
|
||||
# 费用控制率
|
||||
r = c.execute(text("""
|
||||
SELECT Period,
|
||||
SUM(CASE WHEN BillType=1 THEN SumMoney ELSE 0 END) as rev,
|
||||
SUM(CASE WHEN BillType=30 THEN SumMoney ELSE 0 END) as expense
|
||||
FROM MasterBill WHERE BillState>=3 GROUP BY Period ORDER BY Period
|
||||
""")).fetchall()
|
||||
for row in r:
|
||||
p = row[0]
|
||||
if p < 41: continue
|
||||
ps = p2ym(p)
|
||||
exp_rev = float(row.rev)
|
||||
exp_amt = float(row.expense)
|
||||
exp_ratio = round(exp_amt / exp_rev * 100, 2) if exp_rev > 0 else 0
|
||||
write_kpi("F_COST_001", ps, exp_ratio)
|
||||
|
||||
db.close()
|
||||
logger.info(f"完成! 共写入 {written} 条")
|
||||
@@ -0,0 +1,45 @@
|
||||
"""修复：重新采集 erp_schema 中 field_count=0 的表结构"""
|
||||
import sys, os, json, urllib.request, logging, time
|
||||
from os import getenv
|
||||
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
from app.database import get_session_local
|
||||
from sqlalchemy import text
|
||||
|
||||
logging.basicConfig(level=logging.INFO, format="%(asctime)s [%(levelname)s] %(message)s")
|
||||
logger = logging.getLogger("schema_fix")
|
||||
|
||||
API_BASE = "http://127.0.0.1:8300/api/v1"
|
||||
API_KEY = os.getenv("ERP_API_KEY", "erp-gateway-key-bhwl-2026")
|
||||
HEADERS = {"X-API-Key": API_KEY}
|
||||
|
||||
def api_get(path):
|
||||
req = urllib.request.Request(f"{API_BASE}{path}", headers=HEADERS)
|
||||
with urllib.request.urlopen(req, timeout=30) as resp:
|
||||
return json.loads(resp.read().decode())
|
||||
|
||||
db = get_session_local()()
|
||||
|
||||
rows = db.execute(text("SELECT table_name FROM erp_schema WHERE field_count = 0")).fetchall()
|
||||
to_fix = [r[0] for r in rows]
|
||||
logger.info(f"需重新采集: {len(to_fix)} 张表")
|
||||
|
||||
fixed, errors = 0, 0
|
||||
for i, tn in enumerate(to_fix):
|
||||
try:
|
||||
data = api_get(f"/query?table={tn}&limit=1")
|
||||
cols = data.get("columns", [])
|
||||
total = data.get("total", 0)
|
||||
fj = json.dumps([{"name": c} for c in cols], ensure_ascii=False)
|
||||
db.execute(text("UPDATE erp_schema SET total_rows=:tr, field_count=:fc, fields_json=:fj, updated_at=NOW() WHERE table_name=:tn"),
|
||||
{"tn": tn, "tr": total, "fc": len(cols), "fj": fj})
|
||||
db.commit()
|
||||
fixed += 1
|
||||
except Exception as e:
|
||||
errors += 1
|
||||
logger.warning(f" {tn}: {e}")
|
||||
if (i+1) % 50 == 0:
|
||||
logger.info(f"进度: {i+1}/{len(to_fix)} 已修{fixed} 错误{errors}")
|
||||
time.sleep(0.1)
|
||||
|
||||
db.close()
|
||||
logger.info(f"完成! 成功:{fixed} 错误:{errors}")
|
||||
@@ -0,0 +1,145 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Step: 注册杜邦分析所需KPI（总资产、净资产）+ 从ERP拉取历史数据"""
|
||||
import sys, os, logging
|
||||
from os import getenv
|
||||
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
from app.database import get_session_local
|
||||
from sqlalchemy import text, create_engine
|
||||
from datetime import datetime
|
||||
|
||||
logging.basicConfig(level=logging.INFO, format='%(asctime)s [%(levelname)s] %(message)s')
|
||||
logger = logging.getLogger('dupont_finance')
|
||||
|
||||
# ERP直连
|
||||
erp_user = os.getenv("ERP_DB_USER", "zxbtest")
|
||||
erp_pass = os.getenv("ERP_DB_PASS")
|
||||
erp_host = os.getenv("ERP_DB_HOST", "211.149.143.215")
|
||||
erp_port = os.getenv("ERP_DB_PORT", "1433")
|
||||
erp_db = os.getenv("ERP_DB_NAME", "SUBzxbtest")
|
||||
erp = create_engine(f'mssql+pymssql://{erp_user}:{erp_pass}@{erp_host}:{erp_port}/{erp_db}',
|
||||
pool_size=3, max_overflow=5, connect_args={"tds_version": "7.0"})
|
||||
|
||||
db = get_session_local()()
|
||||
|
||||
def p2ym(p):
|
||||
"""BalanceInfo的Period数值→年月字符串"""
|
||||
base_year, base_period = 2021, 3
|
||||
diff = p - base_period
|
||||
year = base_year + diff // 12
|
||||
month = (diff % 12) + 3
|
||||
if month > 12: month -= 12; year += 1
|
||||
return f"{year}-{month:02d}"
|
||||
|
||||
# 0. 清理旧数据（如果有）
|
||||
existing = db.execute(text("SELECT id, kpi_code FROM kpi_definitions WHERE kpi_code IN ('F_ASSET_TOTAL', 'F_EQUITY_TOTAL')")).fetchall()
|
||||
for e in existing:
|
||||
kid, code = e[0], e[1]
|
||||
db.execute(text(f"DELETE FROM kpi_values WHERE kpi_id=:kid"), {"kid": kid})
|
||||
db.execute(text(f"DELETE FROM kpi_definitions WHERE id=:kid"), {"kid": kid})
|
||||
logger.info(f"清除旧数据: {code}(id={kid})")
|
||||
db.commit()
|
||||
|
||||
# 1. 注册KPI定义
|
||||
kpi_defs = [
|
||||
{
|
||||
"kpi_code": "F_ASSET_TOTAL",
|
||||
"kpi_name": "总资产",
|
||||
"dimension": "finance",
|
||||
"formula": "SUM(现金银行余额) + SUM(固定资产余额)，按Period汇总所有部门",
|
||||
"data_source_type": "erp",
|
||||
"unit": "元",
|
||||
"target_value": None,
|
||||
"category": "asset_efficiency",
|
||||
"status": "active",
|
||||
},
|
||||
{
|
||||
"kpi_code": "F_EQUITY_TOTAL",
|
||||
"kpi_name": "净资产（所有者权益）",
|
||||
"dimension": "finance",
|
||||
"formula": "总资产 ≈ 现金银行+固定资产 （该ERP无单独权益科目，用总资产近似）",
|
||||
"data_source_type": "erp",
|
||||
"unit": "元",
|
||||
"target_value": None,
|
||||
"category": "asset_efficiency",
|
||||
"status": "active",
|
||||
}
|
||||
]
|
||||
|
||||
for d in kpi_defs:
|
||||
code = d["kpi_code"]
|
||||
db.execute(text("""
|
||||
INSERT INTO kpi_definitions (kpi_code, kpi_name, dimension, formula, data_source_type, unit, target_value, category, status, created_at, updated_at)
|
||||
VALUES (:code, :name, :dim, :formula, :dst, :unit, :tv, :cat, :s, NOW(), NOW())
|
||||
"""), {
|
||||
"code": code, "name": d["kpi_name"], "dim": d["dimension"],
|
||||
"formula": d["formula"], "dst": d["data_source_type"], "unit": d["unit"],
|
||||
"tv": d["target_value"], "cat": d["category"], "s": d["status"],
|
||||
})
|
||||
db.commit()
|
||||
result = db.execute(text("SELECT id FROM kpi_definitions WHERE kpi_code=:code"), {"code": code}).fetchone()
|
||||
d["id"] = result[0]
|
||||
logger.info(f"✅ {code}({d['kpi_name']}) 已注册，id={d['id']}")
|
||||
|
||||
# 2. 从ERP拉取历史数据
|
||||
batch = f"dupont_finance_{datetime.now().strftime('%Y%m%d_%H%M')}"
|
||||
written = 0
|
||||
|
||||
with erp.connect() as erp_conn:
|
||||
periods = erp_conn.execute(text("SELECT DISTINCT Period FROM BalanceInfo ORDER BY Period")).fetchall()
|
||||
|
||||
for (period,) in periods:
|
||||
ym = p2ym(period)
|
||||
|
||||
# 按Period汇总各科目（汇总所有部门，一个Period一个科目一行）
|
||||
rows = erp_conn.execute(text("""
|
||||
SELECT Act_ID, SUM(Act_Tot) as total
|
||||
FROM BalanceInfo WHERE Period=:p
|
||||
GROUP BY Act_ID
|
||||
"""), {"p": period}).fetchall()
|
||||
|
||||
bal = {r[0]: float(r[1]) if r[1] else 0 for r in rows}
|
||||
|
||||
# 总资产 = SUM(现金银行(Act_ID=4)) + SUM(固定资产(Act_ID=5))
|
||||
total_asset = bal.get(4, 0) + bal.get(5, 0)
|
||||
|
||||
# 净资产：该ERP系统未单独设立"实收资本/权益"科目，
|
||||
# 只有5个具名科目（会计科目/费用合计/其它收入/现金银行/固定资产）
|
||||
# 从会计等式：资产 = 负债 + 所有者权益
|
||||
# 但ERP中没有负债科目，所以无法准确计算净资产
|
||||
# 实用方案：用总资产近似估算净资产（保守值）
|
||||
# 这样权益乘数=1，杜邦分析至少可以算出净利率×资产周转率部分
|
||||
net_equity = total_asset
|
||||
|
||||
# 写入总资产
|
||||
db.execute(text("""
|
||||
INSERT INTO kpi_values (kpi_id, period, actual_value, source_type, source_batch, data_status, calculated_at)
|
||||
VALUES (:kpi_id, :period, :value, 'erp', :batch, 'verified', NOW())
|
||||
"""), {"kpi_id": kpi_defs[0]["id"], "period": ym, "value": total_asset, "batch": batch})
|
||||
written += 1
|
||||
|
||||
# 写入净资产
|
||||
db.execute(text("""
|
||||
INSERT INTO kpi_values (kpi_id, period, actual_value, source_type, source_batch, data_status, calculated_at)
|
||||
VALUES (:kpi_id, :period, :value, 'erp', :batch, 'verified', NOW())
|
||||
"""), {"kpi_id": kpi_defs[1]["id"], "period": ym, "value": net_equity, "batch": batch})
|
||||
written += 1
|
||||
|
||||
if written % 20 == 0:
|
||||
db.commit()
|
||||
logger.info(f" 写入进度: {written}条...")
|
||||
|
||||
db.commit()
|
||||
logger.info(f"✅ 全部完成。共写入 {written} 条KPI值（总资产+净资产，{len(periods)}个期间×2）")
|
||||
|
||||
# 3. 验证
|
||||
print("\n=== 验证 ===")
|
||||
for d in kpi_defs:
|
||||
vals = db.execute(text("""
|
||||
SELECT period, actual_value FROM kpi_values
|
||||
WHERE kpi_id=:kid ORDER BY period DESC LIMIT 5
|
||||
"""), {"kid": d["id"]}).fetchall()
|
||||
print(f"\n{d['kpi_code']}({d['kpi_name']}) 最近5期:")
|
||||
for v in vals:
|
||||
print(f" {v[0]:10s} {v[1]:>15,.2f}")
|
||||
|
||||
db.close()
|
||||
Reference in New Issue
Block a user