feat: 智能导入——BOT自动识别报表类型/列名/期间/匹配KPI,无需手动映射
前端: 简化上传界面,自动模式(importExcelSmart) 后端: 新增/import-excel-smart端点,自动检测: - 列名: 科目/编码→kpi_code, 本期金额→value, 期间→period - 文件名: 提取期间(2026-06)和报表类型(利润表/现金流量表/资产负债表) - KPI匹配: 编码精确→别名→中文名模糊匹配
This commit is contained in:
Binary file not shown.
+180
-1
@@ -1,6 +1,6 @@
|
||||
"""数据对接 API"""
|
||||
import pandas as pd
|
||||
import io, json, hashlib
|
||||
import io, json, hashlib, re
|
||||
from datetime import datetime
|
||||
from fastapi import APIRouter, Depends, HTTPException, Query, UploadFile, File
|
||||
from sqlalchemy.orm import Session
|
||||
@@ -75,6 +75,185 @@ async def import_excel(file: UploadFile = File(...),
|
||||
msg += f"\n...还有{len(skipped)-10}条"
|
||||
return {"message": msg, "batch": batch, "total": count, "skipped": len(skipped)}
|
||||
|
||||
|
||||
# ── 智能导入(BOT自动识别,无需手动映射) ──
|
||||
|
||||
_SMART_MAP = {
|
||||
# KPI编码列匹配模式 → 标准kpi_code
|
||||
"kpi_code_patterns": [
|
||||
re.compile(r'^(kpi_?code|指标编码|编码)$', re.I),
|
||||
re.compile(r'^(科目|项目|账户|报表项目|项目名称)$'),
|
||||
re.compile(r'^(指标名称?|kpi名称?|name)$', re.I),
|
||||
],
|
||||
# 期间列匹配
|
||||
"period_patterns": [
|
||||
re.compile(r'^(period|期间|月份?|年月|日期|会计期间)$', re.I),
|
||||
re.compile(r'^(报表期[间]?|所属期)$'),
|
||||
],
|
||||
# 数值列匹配
|
||||
"value_patterns": [
|
||||
re.compile(r'^(actual_?value|数值|实际值|实际金额)$', re.I),
|
||||
re.compile(r'^(本期金额|本月数|本期|期末余额|期末数)$'),
|
||||
re.compile(r'^(金额|数据|value)$', re.I),
|
||||
],
|
||||
# 文件名→期间提取
|
||||
"period_in_filename": re.compile(r'[-_]?(\d{4})[-_]?(\d{1,2})'),
|
||||
# 文件名→报表类型
|
||||
"statement_types": {
|
||||
"利润表": "PL",
|
||||
"利润": "PL",
|
||||
"income": "PL",
|
||||
"现金流量表": "CF",
|
||||
"现金流": "CF",
|
||||
"cashflow": "CF",
|
||||
"cash_flow": "CF",
|
||||
"资产负债表": "BS",
|
||||
"资产负": "BS",
|
||||
"balance": "BS",
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def _smart_detect_kpi_col(cols: list[str]) -> str | None:
|
||||
for pat in _SMART_MAP["kpi_code_patterns"]:
|
||||
for c in cols:
|
||||
if pat.match(c.strip()):
|
||||
return c
|
||||
return None
|
||||
|
||||
|
||||
def _smart_detect_period_col(cols: list[str]) -> str | None:
|
||||
for pat in _SMART_MAP["period_patterns"]:
|
||||
for c in cols:
|
||||
if pat.match(c.strip()):
|
||||
return c
|
||||
return None
|
||||
|
||||
|
||||
def _smart_detect_value_col(cols: list[str]) -> str | None:
|
||||
for pat in _SMART_MAP["value_patterns"]:
|
||||
for c in cols:
|
||||
if pat.match(c.strip()):
|
||||
return c
|
||||
return None
|
||||
|
||||
|
||||
def _smart_extract_period_from_filename(filename: str) -> str | None:
|
||||
m = _SMART_MAP["period_in_filename"].search(filename)
|
||||
if m:
|
||||
return f"{m.group(1)}-{int(m.group(2)):02d}"
|
||||
return None
|
||||
|
||||
|
||||
def _smart_detect_statement_type(filename: str) -> str | None:
|
||||
for kw, tp in _SMART_MAP["statement_types"].items():
|
||||
if kw in filename:
|
||||
return tp
|
||||
return None
|
||||
|
||||
|
||||
@router.post("/import-excel-smart")
|
||||
async def import_excel_smart(file: UploadFile = File(...), db: Session = Depends(get_db)):
|
||||
"""智能导入 — BOT自动识别列名/期间/报表类型,无需手动映射"""
|
||||
content = await file.read()
|
||||
fname = file.filename or "未知文件"
|
||||
|
||||
try:
|
||||
df = pd.read_excel(io.BytesIO(content))
|
||||
except Exception as e:
|
||||
raise HTTPException(400, f"无法读取Excel文件: {e}")
|
||||
|
||||
if len(df) == 0:
|
||||
raise HTTPException(400, "Excel文件为空")
|
||||
|
||||
cols = list(df.columns)
|
||||
if len(cols) < 2:
|
||||
raise HTTPException(400, f"Excel列数过少: {cols}")
|
||||
|
||||
# 4. 智能检测列
|
||||
kpi_col = _smart_detect_kpi_col(cols) or cols[0]
|
||||
value_col = _smart_detect_value_col(cols) or cols[-1]
|
||||
period_col = _smart_detect_period_col(cols)
|
||||
|
||||
# 5. 从文件名提取期间
|
||||
period = _smart_extract_period_from_filename(fname) if not period_col else None
|
||||
|
||||
# 6. 检测报表类型(用于自动生成KPI编码前缀)
|
||||
stype = _smart_detect_statement_type(fname)
|
||||
|
||||
# 7. 预加载KPI字典
|
||||
from app.models import KPIDefinition
|
||||
kpis = {k.kpi_code: k for k in db.query(KPIDefinition).all()}
|
||||
known_codes = set(kpis.keys())
|
||||
# 构建别名映射(去掉空格/大小写/特殊字符)
|
||||
alias_map: dict[str, str] = {}
|
||||
for code in known_codes:
|
||||
clean = re.sub(r'[\s\-_()()]', '', code).lower()
|
||||
alias_map[clean] = code
|
||||
|
||||
# 8. 遍历导入
|
||||
batch = hashlib.md5(str(datetime.now().timestamp()).encode()).hexdigest()[:12]
|
||||
imported = 0
|
||||
skipped_rows = []
|
||||
|
||||
for idx, row in df.iterrows():
|
||||
raw_kpi = str(row.get(kpi_col, "")).strip()
|
||||
raw_val = row.get(value_col)
|
||||
raw_period = str(row.get(period_col, period or "")).strip() if period_col else (period or "")
|
||||
|
||||
if not raw_kpi or pd.isna(raw_val):
|
||||
skipped_rows.append(f"第{idx+2}行: 缺数据")
|
||||
continue
|
||||
if not raw_period:
|
||||
skipped_rows.append(f"第{idx+2}行: 无法确定期间")
|
||||
continue
|
||||
|
||||
# 智能匹配KPI编码
|
||||
kpi_code = None
|
||||
if raw_kpi in known_codes:
|
||||
kpi_code = raw_kpi
|
||||
else:
|
||||
# 别名匹配
|
||||
clean_key = re.sub(r'[\s\-_()()]', '', raw_kpi).lower()
|
||||
kpi_code = alias_map.get(clean_key)
|
||||
# 模糊匹配(中文科目名→KPI编码)
|
||||
if not kpi_code:
|
||||
for code, kpi_obj in kpis.items():
|
||||
if raw_kpi in kpi_obj.kpi_name or kpi_obj.kpi_name in raw_kpi:
|
||||
kpi_code = code
|
||||
break
|
||||
|
||||
if not kpi_code:
|
||||
skipped_rows.append(f"第{idx+2}行: 「{raw_kpi}」未匹配到KPI")
|
||||
continue
|
||||
|
||||
try:
|
||||
val = float(raw_val)
|
||||
except:
|
||||
skipped_rows.append(f"第{idx+2}行: 数值格式错误「{raw_val}」")
|
||||
continue
|
||||
|
||||
db.add(KPIValue(
|
||||
kpi_id=kpis[kpi_code].id,
|
||||
period=raw_period,
|
||||
actual_value=val,
|
||||
source_type="excel",
|
||||
source_batch=batch,
|
||||
data_status="verified",
|
||||
))
|
||||
imported += 1
|
||||
|
||||
db.commit()
|
||||
|
||||
# 9. 返回汇总
|
||||
stype_label = {"PL": "利润表", "CF": "现金流量表", "BS": "资产负债表"}.get(stype or "", "数据表")
|
||||
msg = f"✅ {stype_label}识别成功,导入{imported}条"
|
||||
if skipped_rows:
|
||||
msg += f",{len(skipped_rows)}条跳过:\n" + "\n".join(skipped_rows[:8])
|
||||
if len(skipped_rows) > 8:
|
||||
msg += f"\n...还有{len(skipped_rows) - 8}条"
|
||||
return {"message": msg, "batch": batch, "total": imported, "skipped": len(skipped_rows)}
|
||||
|
||||
@router.get("/sources")
|
||||
def list_sources(db: Session = Depends(get_db)):
|
||||
sources = db.query(DataSourceConfig).all()
|
||||
|
||||
@@ -67,6 +67,11 @@ export const dataApi = {
|
||||
form.append('file', file)
|
||||
return api.post(`/data/import-excel${qs ? '?' + qs : ''}`, form)
|
||||
},
|
||||
importExcelSmart: (file: File) => {
|
||||
const form = new FormData()
|
||||
form.append('file', file)
|
||||
return api.post('/data/import-excel-smart', form, { timeout: 60000 })
|
||||
},
|
||||
listSources: () => api.get('/data/sources'),
|
||||
createSource: (data: any) => api.post('/data/sources', data),
|
||||
updateSource: (id: number, data: any) => api.put(`/data/sources/${id}`, data),
|
||||
|
||||
@@ -5,52 +5,17 @@
|
||||
<el-tabs v-model="activeTab" style="margin-top:16px;">
|
||||
<el-tab-pane label="Excel导入" name="import">
|
||||
<el-card>
|
||||
<template #header>Excel导入</template>
|
||||
<template #header>Excel导入 <el-tag type="success" size="small">🤖 智能识别</el-tag></template>
|
||||
<el-upload drag :auto-upload="false" :on-change="handleFile" :on-remove="handleRemove" multiple accept=".xlsx,.xls">
|
||||
<el-icon :size="32"><UploadFilled /></el-icon>
|
||||
<div>拖拽或点击上传Excel文件</div>
|
||||
<div style="color:#999;font-size:12px;">支持 .xlsx .xls,可一次选择多个文件(按住Ctrl/Shift多选)</div>
|
||||
<div style="color:#999;font-size:12px;">支持 .xlsx .xls,自动识别利润表/现金流量表/资产负债表/KPI数据表</div>
|
||||
</el-upload>
|
||||
<div v-if="files.length > 0" style="margin-top:8px;">
|
||||
<el-tag v-for="(f, i) in files" :key="i" closable @close="removeFile(i)" style="margin:2px;">{{ f.name }}</el-tag>
|
||||
</div>
|
||||
|
||||
<!-- 列映射设置 -->
|
||||
<el-card v-if="files.length > 0" style="margin-top:12px;" size="small">
|
||||
<template #header>📐 列映射设置(如Excel列名与标准不同,请在此映射)</template>
|
||||
<el-form :model="colMap" label-width="120px" size="small">
|
||||
<el-row :gutter="16">
|
||||
<el-col :span="8">
|
||||
<el-form-item label="KPI编码列">
|
||||
<el-select v-model="colMap.kpi" allow-create filterable clearable placeholder="kpi_code">
|
||||
<el-option v-for="c in detectedCols" :key="c" :label="c" :value="c" />
|
||||
</el-select>
|
||||
</el-form-item>
|
||||
</el-col>
|
||||
<el-col :span="8">
|
||||
<el-form-item label="期间列(可选)">
|
||||
<el-select v-model="colMap.period" allow-create filterable clearable placeholder="period">
|
||||
<el-option v-for="c in detectedCols" :key="c" :label="c" :value="c" />
|
||||
<el-option label="(统一设置)" value="__fixed__" />
|
||||
</el-select>
|
||||
</el-form-item>
|
||||
</el-col>
|
||||
<el-col :span="8">
|
||||
<el-form-item label="数值列">
|
||||
<el-select v-model="colMap.value" allow-create filterable clearable placeholder="actual_value">
|
||||
<el-option v-for="c in detectedCols" :key="c" :label="c" :value="c" />
|
||||
</el-select>
|
||||
</el-form-item>
|
||||
</el-col>
|
||||
</el-row>
|
||||
<el-form-item v-if="colMap.period === '__fixed__'" label="统一期间">
|
||||
<el-input v-model="colMap.defaultPeriod" placeholder="如 2026-06" style="width:200px;" />
|
||||
</el-form-item>
|
||||
</el-form>
|
||||
</el-card>
|
||||
|
||||
<el-button v-if="files.length > 0" type="primary" style="margin-top:12px;" :loading="uploading" @click="doImport">导入 {{ files.length }} 个文件</el-button>
|
||||
<div v-if="result" style="margin-top:12px;"><el-alert :title="result" type="success" show-icon /></div>
|
||||
<el-button v-if="files.length > 0" type="primary" style="margin-top:12px;" :loading="uploading" @click="doImport">🤖 智能导入 {{ files.length }} 个文件</el-button>
|
||||
<div v-if="result" style="margin-top:12px;white-space:pre-wrap;"><el-alert :title="result" type="success" show-icon /></div>
|
||||
</el-card>
|
||||
</el-tab-pane>
|
||||
|
||||
@@ -77,7 +42,7 @@
|
||||
</el-table-column>
|
||||
<el-table-column prop="status" label="状态" width="70">
|
||||
<template #default="{ row }">
|
||||
<el-tag :type="row.status==='active'?'success':'danger'" size="small">{{ row.status }}</el-tag>
|
||||
<el-tag :type="row.status==='active'?'success':'danger'">{{ row.status }}</el-tag>
|
||||
</template>
|
||||
</el-table-column>
|
||||
<el-table-column prop="last_sync_at" label="上次同步" width="160" />
|
||||
@@ -135,14 +100,10 @@ const activeTab = ref('import')
|
||||
const files = ref<any[]>([])
|
||||
const uploading = ref(false)
|
||||
const result = ref('')
|
||||
const detectedCols = ref<string[]>([])
|
||||
const colMap = ref({ kpi: 'kpi_code', period: 'period', value: 'actual_value', defaultPeriod: '' })
|
||||
|
||||
function handleFile(f: any) {
|
||||
if (f.raw && !files.value.some(ex => ex.name === f.raw.name && ex.size === f.raw.size)) {
|
||||
files.value.push(f.raw)
|
||||
// 从第一个文件检测列名
|
||||
if (files.value.length === 1) detectColumns(f.raw)
|
||||
}
|
||||
}
|
||||
function handleRemove(f: any) {
|
||||
@@ -151,51 +112,21 @@ function handleRemove(f: any) {
|
||||
function removeFile(i: number) {
|
||||
files.value.splice(i, 1)
|
||||
}
|
||||
async function detectColumns(file: File) {
|
||||
// 用 Web 方式读 Excel 列名(仅第一行),供用户映射
|
||||
try {
|
||||
const buf = await file.arrayBuffer()
|
||||
const XLSX = (await import('xlsx')).default
|
||||
const wb = XLSX.read(buf, { type: 'array' })
|
||||
const ws = wb.Sheets[wb.SheetNames[0]]
|
||||
const rows: any[] = XLSX.utils.sheet_to_json(ws, { header: 1 })
|
||||
if (rows.length > 0) {
|
||||
detectedCols.value = (rows[0] as string[]).filter(Boolean)
|
||||
// 尝试智能匹配
|
||||
const all = detectedCols.value.map(c => c.toLowerCase().replace(/[\\s ]/g, ''))
|
||||
const kpiIdx = all.findIndex(c => /^(kpi_?code|编码|科目|项目|指标名称?)$/.test(c))
|
||||
const periodIdx = all.findIndex(c => /^(period|期间|月份?|日期|年月)$/.test(c))
|
||||
const valIdx = all.findIndex(c => /^(actual_?value|金额|数值|本月数|本期金额?|实际值)$/.test(c))
|
||||
if (kpiIdx >= 0) colMap.value.kpi = detectedCols.value[kpiIdx]
|
||||
if (periodIdx >= 0) colMap.value.period = detectedCols.value[periodIdx]
|
||||
if (valIdx >= 0) colMap.value.value = detectedCols.value[valIdx]
|
||||
}
|
||||
} catch (_) { /* 预览列名失败不影响导入 */ }
|
||||
}
|
||||
async function doImport() {
|
||||
if (files.value.length === 0) return
|
||||
uploading.value = true
|
||||
result.value = ''
|
||||
const msgs: string[] = []
|
||||
const params: Record<string, string> = {
|
||||
kpi_col: colMap.value.kpi || 'kpi_code',
|
||||
value_col: colMap.value.value || 'actual_value',
|
||||
}
|
||||
if (colMap.value.period && colMap.value.period !== '__fixed__') params['period_col'] = colMap.value.period
|
||||
if (colMap.value.period === '__fixed__' && colMap.value.defaultPeriod) params['default_period'] = colMap.value.defaultPeriod
|
||||
const qs = new URLSearchParams(params).toString()
|
||||
for (const f of files.value) {
|
||||
try {
|
||||
const r: any = await dataApi.importExcel(f, qs)
|
||||
const skipped = r.skipped || 0
|
||||
const total = r.total || 0
|
||||
msgs.push(`📄 ${f.name}: ${total}条导入${skipped > 0 ? `, ${skipped}条跳过` : ''}`)
|
||||
const r: any = await dataApi.importExcelSmart(f)
|
||||
const msg = r.message || ''
|
||||
msgs.push(`📄 ${f.name}: ${msg}`)
|
||||
} catch (e: any) {
|
||||
msgs.push(`❌ ${f.name}: ${e?.response?.data?.detail || '导入失败'}`)
|
||||
}
|
||||
}
|
||||
files.value = []
|
||||
detectedCols.value = []
|
||||
result.value = msgs.join('\n')
|
||||
uploading.value = false
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user