84 lines
3.0 KiB
Python
84 lines
3.0 KiB
Python
"""Step 1 (v2): 全量采集ERP表结构到 erp_schema
|
|
通过 erp-api-gateway 的 /api/v1/query?table=xxx&limit=1 获取字段信息
|
|
运行: python3 scripts/collect_erp_schema_v2.py
|
|
"""
|
|
|
|
import sys, os, json, urllib.request, urllib.error, logging, time
|
|
from datetime import datetime
|
|
|
|
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
|
from app.database import get_session_local
|
|
from sqlalchemy import text
|
|
|
|
logging.basicConfig(level=logging.INFO, format="%(asctime)s [%(levelname)s] %(message)s")
|
|
logger = logging.getLogger("erp_schema_v2")
|
|
|
|
API_BASE = "http://127.0.0.1:8300/api/v1"
|
|
API_KEY = os.getenv("ERP_API_KEY", "erp-gateway-key-bhwl-2026")
|
|
HEADERS = {"X-API-Key": API_KEY}
|
|
|
|
|
|
def api_get(path: str) -> dict:
|
|
url = f"{API_BASE}{path}"
|
|
req = urllib.request.Request(url, headers=HEADERS)
|
|
with urllib.request.urlopen(req, timeout=30) as resp:
|
|
return json.loads(resp.read().decode())
|
|
|
|
|
|
def main():
|
|
db = get_session_local()()
|
|
|
|
# 1. 获取全量表名
|
|
logger.info("获取ERP全量表名...")
|
|
tables_data = api_get("/tables")
|
|
all_tables = tables_data.get("tables", [])
|
|
logger.info(f"共 {len(all_tables)} 张表")
|
|
|
|
# 2. 已采集的
|
|
existing = set()
|
|
for row in db.execute(text("SELECT table_name FROM erp_schema")).fetchall():
|
|
existing.add(row[0])
|
|
logger.info(f"已有 {len(existing)} 张，还需采集 {len(all_tables) - len(existing)} 张")
|
|
|
|
# 3. 逐表采集
|
|
collected, errors, skipped = 0, 0, 0
|
|
for i, tn in enumerate(all_tables):
|
|
if tn in existing:
|
|
skipped += 1
|
|
continue
|
|
|
|
try:
|
|
data = api_get(f"/query?table={tn}&limit=1")
|
|
cols = data.get("columns", [])
|
|
total = data.get("total", 0)
|
|
fields_json = json.dumps([{"name": c} for c in cols], ensure_ascii=False)
|
|
|
|
db.execute(text("""
|
|
INSERT INTO erp_schema (table_name, total_rows, field_count, fields_json, created_at, updated_at)
|
|
VALUES (:tn, :tr, :fc, :fj, NOW(), NOW())
|
|
ON DUPLICATE KEY UPDATE total_rows=:tr2, field_count=:fc2, fields_json=:fj2, updated_at=NOW()
|
|
"""), {"tn": tn, "tr": total, "fc": len(cols), "fj": fields_json,
|
|
"tr2": total, "fc2": len(cols), "fj2": fields_json})
|
|
db.commit()
|
|
collected += 1
|
|
except Exception as e:
|
|
errors += 1
|
|
db.execute(text("""
|
|
INSERT INTO erp_schema (table_name, total_rows, field_count, fields_json, created_at, updated_at)
|
|
VALUES (:tn, -1, 0, '[]', NOW(), NOW())
|
|
ON DUPLICATE KEY UPDATE updated_at=NOW()
|
|
"""), {"tn": tn})
|
|
db.commit()
|
|
|
|
if (i + 1) % 100 == 0:
|
|
logger.info(f"进度: {i+1}/{len(all_tables)} 已采{collected} 错误{errors} 跳过{skipped}")
|
|
|
|
time.sleep(0.1)
|
|
|
|
db.close()
|
|
logger.info(f"\n完成! 总计:{len(all_tables)} 采集:{collected} 已有:{skipped} 错误:{errors}")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|