fix: 移除ERP密码和API密钥硬编码，改为环境变量；erp-sync接口加鉴权

2026-06-05 07:51:20 +08:00
parent 2c92168e0e
commit 7b9ef8ac27
9 changed files with 844 additions and 8 deletions
@@ -0,0 +1,83 @@
+"""Step 1 (v2): 全量采集ERP表结构到 erp_schema
+通过 erp-api-gateway 的 /api/v1/query?table=xxx&limit=1 获取字段信息
+运行: python3 scripts/collect_erp_schema_v2.py
+"""
+
+import sys, os, json, urllib.request, urllib.error, logging, time
+from datetime import datetime
+
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+from app.database import get_session_local
+from sqlalchemy import text
+
+logging.basicConfig(level=logging.INFO, format="%(asctime)s [%(levelname)s] %(message)s")
+logger = logging.getLogger("erp_schema_v2")
+
+API_BASE = "http://127.0.0.1:8300/api/v1"
+API_KEY = os.getenv("ERP_API_KEY", "erp-gateway-key-bhwl-2026")
+HEADERS = {"X-API-Key": API_KEY}
+
+
+def api_get(path: str) -> dict:
+    url = f"{API_BASE}{path}"
+    req = urllib.request.Request(url, headers=HEADERS)
+    with urllib.request.urlopen(req, timeout=30) as resp:
+        return json.loads(resp.read().decode())
+
+
+def main():
+    db = get_session_local()()
+
+    # 1. 获取全量表名
+    logger.info("获取ERP全量表名...")
+    tables_data = api_get("/tables")
+    all_tables = tables_data.get("tables", [])
+    logger.info(f"共 {len(all_tables)} 张表")
+
+    # 2. 已采集的
+    existing = set()
+    for row in db.execute(text("SELECT table_name FROM erp_schema")).fetchall():
+        existing.add(row[0])
+    logger.info(f"已有 {len(existing)} 张，还需采集 {len(all_tables) - len(existing)} 张")
+
+    # 3. 逐表采集
+    collected, errors, skipped = 0, 0, 0
+    for i, tn in enumerate(all_tables):
+        if tn in existing:
+            skipped += 1
+            continue
+
+        try:
+            data = api_get(f"/query?table={tn}&limit=1")
+            cols = data.get("columns", [])
+            total = data.get("total", 0)
+            fields_json = json.dumps([{"name": c} for c in cols], ensure_ascii=False)
+
+            db.execute(text("""
+                INSERT INTO erp_schema (table_name, total_rows, field_count, fields_json, created_at, updated_at)
+                VALUES (:tn, :tr, :fc, :fj, NOW(), NOW())
+                ON DUPLICATE KEY UPDATE total_rows=:tr2, field_count=:fc2, fields_json=:fj2, updated_at=NOW()
+            """), {"tn": tn, "tr": total, "fc": len(cols), "fj": fields_json,
+                   "tr2": total, "fc2": len(cols), "fj2": fields_json})
+            db.commit()
+            collected += 1
+        except Exception as e:
+            errors += 1
+            db.execute(text("""
+                INSERT INTO erp_schema (table_name, total_rows, field_count, fields_json, created_at, updated_at)
+                VALUES (:tn, -1, 0, '[]', NOW(), NOW())
+                ON DUPLICATE KEY UPDATE updated_at=NOW()
+            """), {"tn": tn})
+            db.commit()
+
+        if (i + 1) % 100 == 0:
+            logger.info(f"进度: {i+1}/{len(all_tables)} 已采{collected} 错误{errors} 跳过{skipped}")
+
+        time.sleep(0.1)
+
+    db.close()
+    logger.info(f"\n完成! 总计:{len(all_tables)} 采集:{collected} 已有:{skipped} 错误:{errors}")
+
+
+if __name__ == "__main__":
+    main()