feat: 智能导入——BOT自动识别报表类型/列名/期间/匹配KPI,无需手动映射

前端: 简化上传界面,自动模式(importExcelSmart)
后端: 新增/import-excel-smart端点,自动检测:
  - 列名: 科目/编码→kpi_code, 本期金额→value, 期间→period
  - 文件名: 提取期间(2026-06)和报表类型(利润表/现金流量表/资产负债表)
  - KPI匹配: 编码精确→别名→中文名模糊匹配
This commit is contained in:
Hermes CI Fix
2026-07-13 15:36:22 +08:00
parent da9aceb567
commit d3edd71e60
4 changed files with 193 additions and 78 deletions
Binary file not shown.
+180 -1
View File
@@ -1,6 +1,6 @@
"""数据对接 API"""
import pandas as pd
import io, json, hashlib
import io, json, hashlib, re
from datetime import datetime
from fastapi import APIRouter, Depends, HTTPException, Query, UploadFile, File
from sqlalchemy.orm import Session
@@ -75,6 +75,185 @@ async def import_excel(file: UploadFile = File(...),
msg += f"\n...还有{len(skipped)-10}"
return {"message": msg, "batch": batch, "total": count, "skipped": len(skipped)}
# ── 智能导入(BOT自动识别,无需手动映射) ──
_SMART_MAP = {
# KPI编码列匹配模式 → 标准kpi_code
"kpi_code_patterns": [
re.compile(r'^(kpi_?code|指标编码|编码)$', re.I),
re.compile(r'^(科目|项目|账户|报表项目|项目名称)$'),
re.compile(r'^(指标名称?|kpi名称?|name)$', re.I),
],
# 期间列匹配
"period_patterns": [
re.compile(r'^(period|期间|月份?|年月|日期|会计期间)$', re.I),
re.compile(r'^(报表期[间]?|所属期)$'),
],
# 数值列匹配
"value_patterns": [
re.compile(r'^(actual_?value|数值|实际值|实际金额)$', re.I),
re.compile(r'^(本期金额|本月数|本期|期末余额|期末数)$'),
re.compile(r'^(金额|数据|value)$', re.I),
],
# 文件名→期间提取
"period_in_filename": re.compile(r'[-_]?(\d{4})[-_]?(\d{1,2})'),
# 文件名→报表类型
"statement_types": {
"利润表": "PL",
"利润": "PL",
"income": "PL",
"现金流量表": "CF",
"现金流": "CF",
"cashflow": "CF",
"cash_flow": "CF",
"资产负债表": "BS",
"资产负": "BS",
"balance": "BS",
},
}
def _smart_detect_kpi_col(cols: list[str]) -> str | None:
for pat in _SMART_MAP["kpi_code_patterns"]:
for c in cols:
if pat.match(c.strip()):
return c
return None
def _smart_detect_period_col(cols: list[str]) -> str | None:
for pat in _SMART_MAP["period_patterns"]:
for c in cols:
if pat.match(c.strip()):
return c
return None
def _smart_detect_value_col(cols: list[str]) -> str | None:
for pat in _SMART_MAP["value_patterns"]:
for c in cols:
if pat.match(c.strip()):
return c
return None
def _smart_extract_period_from_filename(filename: str) -> str | None:
m = _SMART_MAP["period_in_filename"].search(filename)
if m:
return f"{m.group(1)}-{int(m.group(2)):02d}"
return None
def _smart_detect_statement_type(filename: str) -> str | None:
for kw, tp in _SMART_MAP["statement_types"].items():
if kw in filename:
return tp
return None
@router.post("/import-excel-smart")
async def import_excel_smart(file: UploadFile = File(...), db: Session = Depends(get_db)):
"""智能导入 — BOT自动识别列名/期间/报表类型,无需手动映射"""
content = await file.read()
fname = file.filename or "未知文件"
try:
df = pd.read_excel(io.BytesIO(content))
except Exception as e:
raise HTTPException(400, f"无法读取Excel文件: {e}")
if len(df) == 0:
raise HTTPException(400, "Excel文件为空")
cols = list(df.columns)
if len(cols) < 2:
raise HTTPException(400, f"Excel列数过少: {cols}")
# 4. 智能检测列
kpi_col = _smart_detect_kpi_col(cols) or cols[0]
value_col = _smart_detect_value_col(cols) or cols[-1]
period_col = _smart_detect_period_col(cols)
# 5. 从文件名提取期间
period = _smart_extract_period_from_filename(fname) if not period_col else None
# 6. 检测报表类型(用于自动生成KPI编码前缀)
stype = _smart_detect_statement_type(fname)
# 7. 预加载KPI字典
from app.models import KPIDefinition
kpis = {k.kpi_code: k for k in db.query(KPIDefinition).all()}
known_codes = set(kpis.keys())
# 构建别名映射(去掉空格/大小写/特殊字符)
alias_map: dict[str, str] = {}
for code in known_codes:
clean = re.sub(r'[\s\-_()()]', '', code).lower()
alias_map[clean] = code
# 8. 遍历导入
batch = hashlib.md5(str(datetime.now().timestamp()).encode()).hexdigest()[:12]
imported = 0
skipped_rows = []
for idx, row in df.iterrows():
raw_kpi = str(row.get(kpi_col, "")).strip()
raw_val = row.get(value_col)
raw_period = str(row.get(period_col, period or "")).strip() if period_col else (period or "")
if not raw_kpi or pd.isna(raw_val):
skipped_rows.append(f"{idx+2}行: 缺数据")
continue
if not raw_period:
skipped_rows.append(f"{idx+2}行: 无法确定期间")
continue
# 智能匹配KPI编码
kpi_code = None
if raw_kpi in known_codes:
kpi_code = raw_kpi
else:
# 别名匹配
clean_key = re.sub(r'[\s\-_()()]', '', raw_kpi).lower()
kpi_code = alias_map.get(clean_key)
# 模糊匹配(中文科目名→KPI编码)
if not kpi_code:
for code, kpi_obj in kpis.items():
if raw_kpi in kpi_obj.kpi_name or kpi_obj.kpi_name in raw_kpi:
kpi_code = code
break
if not kpi_code:
skipped_rows.append(f"{idx+2}行: 「{raw_kpi}」未匹配到KPI")
continue
try:
val = float(raw_val)
except:
skipped_rows.append(f"{idx+2}行: 数值格式错误「{raw_val}")
continue
db.add(KPIValue(
kpi_id=kpis[kpi_code].id,
period=raw_period,
actual_value=val,
source_type="excel",
source_batch=batch,
data_status="verified",
))
imported += 1
db.commit()
# 9. 返回汇总
stype_label = {"PL": "利润表", "CF": "现金流量表", "BS": "资产负债表"}.get(stype or "", "数据表")
msg = f"{stype_label}识别成功,导入{imported}"
if skipped_rows:
msg += f"{len(skipped_rows)}条跳过:\n" + "\n".join(skipped_rows[:8])
if len(skipped_rows) > 8:
msg += f"\n...还有{len(skipped_rows) - 8}"
return {"message": msg, "batch": batch, "total": imported, "skipped": len(skipped_rows)}
@router.get("/sources")
def list_sources(db: Session = Depends(get_db)):
sources = db.query(DataSourceConfig).all()
+5
View File
@@ -67,6 +67,11 @@ export const dataApi = {
form.append('file', file)
return api.post(`/data/import-excel${qs ? '?' + qs : ''}`, form)
},
importExcelSmart: (file: File) => {
const form = new FormData()
form.append('file', file)
return api.post('/data/import-excel-smart', form, { timeout: 60000 })
},
listSources: () => api.get('/data/sources'),
createSource: (data: any) => api.post('/data/sources', data),
updateSource: (id: number, data: any) => api.put(`/data/sources/${id}`, data),
+8 -77
View File
@@ -5,52 +5,17 @@
<el-tabs v-model="activeTab" style="margin-top:16px;">
<el-tab-pane label="Excel导入" name="import">
<el-card>
<template #header>Excel导入</template>
<template #header>Excel导入 <el-tag type="success" size="small">🤖 智能识别</el-tag></template>
<el-upload drag :auto-upload="false" :on-change="handleFile" :on-remove="handleRemove" multiple accept=".xlsx,.xls">
<el-icon :size="32"><UploadFilled /></el-icon>
<div>拖拽或点击上传Excel文件</div>
<div style="color:#999;font-size:12px;">支持 .xlsx .xls可一次选择多个文件按住Ctrl/Shift多选</div>
<div style="color:#999;font-size:12px;">支持 .xlsx .xls自动识别利润表/现金流量表/资产负债表/KPI数据表</div>
</el-upload>
<div v-if="files.length > 0" style="margin-top:8px;">
<el-tag v-for="(f, i) in files" :key="i" closable @close="removeFile(i)" style="margin:2px;">{{ f.name }}</el-tag>
</div>
<!-- 列映射设置 -->
<el-card v-if="files.length > 0" style="margin-top:12px;" size="small">
<template #header>📐 列映射设置如Excel列名与标准不同请在此映射</template>
<el-form :model="colMap" label-width="120px" size="small">
<el-row :gutter="16">
<el-col :span="8">
<el-form-item label="KPI编码列">
<el-select v-model="colMap.kpi" allow-create filterable clearable placeholder="kpi_code">
<el-option v-for="c in detectedCols" :key="c" :label="c" :value="c" />
</el-select>
</el-form-item>
</el-col>
<el-col :span="8">
<el-form-item label="期间列(可选)">
<el-select v-model="colMap.period" allow-create filterable clearable placeholder="period">
<el-option v-for="c in detectedCols" :key="c" :label="c" :value="c" />
<el-option label="(统一设置)" value="__fixed__" />
</el-select>
</el-form-item>
</el-col>
<el-col :span="8">
<el-form-item label="数值列">
<el-select v-model="colMap.value" allow-create filterable clearable placeholder="actual_value">
<el-option v-for="c in detectedCols" :key="c" :label="c" :value="c" />
</el-select>
</el-form-item>
</el-col>
</el-row>
<el-form-item v-if="colMap.period === '__fixed__'" label="统一期间">
<el-input v-model="colMap.defaultPeriod" placeholder="如 2026-06" style="width:200px;" />
</el-form-item>
</el-form>
</el-card>
<el-button v-if="files.length > 0" type="primary" style="margin-top:12px;" :loading="uploading" @click="doImport">导入 {{ files.length }} 个文件</el-button>
<div v-if="result" style="margin-top:12px;"><el-alert :title="result" type="success" show-icon /></div>
<el-button v-if="files.length > 0" type="primary" style="margin-top:12px;" :loading="uploading" @click="doImport">🤖 智能导入 {{ files.length }} 个文件</el-button>
<div v-if="result" style="margin-top:12px;white-space:pre-wrap;"><el-alert :title="result" type="success" show-icon /></div>
</el-card>
</el-tab-pane>
@@ -77,7 +42,7 @@
</el-table-column>
<el-table-column prop="status" label="状态" width="70">
<template #default="{ row }">
<el-tag :type="row.status==='active'?'success':'danger'" size="small">{{ row.status }}</el-tag>
<el-tag :type="row.status==='active'?'success':'danger'">{{ row.status }}</el-tag>
</template>
</el-table-column>
<el-table-column prop="last_sync_at" label="上次同步" width="160" />
@@ -135,14 +100,10 @@ const activeTab = ref('import')
const files = ref<any[]>([])
const uploading = ref(false)
const result = ref('')
const detectedCols = ref<string[]>([])
const colMap = ref({ kpi: 'kpi_code', period: 'period', value: 'actual_value', defaultPeriod: '' })
function handleFile(f: any) {
if (f.raw && !files.value.some(ex => ex.name === f.raw.name && ex.size === f.raw.size)) {
files.value.push(f.raw)
// 从第一个文件检测列名
if (files.value.length === 1) detectColumns(f.raw)
}
}
function handleRemove(f: any) {
@@ -151,51 +112,21 @@ function handleRemove(f: any) {
function removeFile(i: number) {
files.value.splice(i, 1)
}
async function detectColumns(file: File) {
// 用 Web 方式读 Excel 列名(仅第一行),供用户映射
try {
const buf = await file.arrayBuffer()
const XLSX = (await import('xlsx')).default
const wb = XLSX.read(buf, { type: 'array' })
const ws = wb.Sheets[wb.SheetNames[0]]
const rows: any[] = XLSX.utils.sheet_to_json(ws, { header: 1 })
if (rows.length > 0) {
detectedCols.value = (rows[0] as string[]).filter(Boolean)
// 尝试智能匹配
const all = detectedCols.value.map(c => c.toLowerCase().replace(/[\\s ]/g, ''))
const kpiIdx = all.findIndex(c => /^(kpi_?code|编码|科目|项目|指标名称?)$/.test(c))
const periodIdx = all.findIndex(c => /^(period|期间|月份?|日期|年月)$/.test(c))
const valIdx = all.findIndex(c => /^(actual_?value|金额|数值|本月数|本期金额?|实际值)$/.test(c))
if (kpiIdx >= 0) colMap.value.kpi = detectedCols.value[kpiIdx]
if (periodIdx >= 0) colMap.value.period = detectedCols.value[periodIdx]
if (valIdx >= 0) colMap.value.value = detectedCols.value[valIdx]
}
} catch (_) { /* 预览列名失败不影响导入 */ }
}
async function doImport() {
if (files.value.length === 0) return
uploading.value = true
result.value = ''
const msgs: string[] = []
const params: Record<string, string> = {
kpi_col: colMap.value.kpi || 'kpi_code',
value_col: colMap.value.value || 'actual_value',
}
if (colMap.value.period && colMap.value.period !== '__fixed__') params['period_col'] = colMap.value.period
if (colMap.value.period === '__fixed__' && colMap.value.defaultPeriod) params['default_period'] = colMap.value.defaultPeriod
const qs = new URLSearchParams(params).toString()
for (const f of files.value) {
try {
const r: any = await dataApi.importExcel(f, qs)
const skipped = r.skipped || 0
const total = r.total || 0
msgs.push(`📄 ${f.name}: ${total}条导入${skipped > 0 ? `, ${skipped}条跳过` : ''}`)
const r: any = await dataApi.importExcelSmart(f)
const msg = r.message || ''
msgs.push(`📄 ${f.name}: ${msg}`)
} catch (e: any) {
msgs.push(`${f.name}: ${e?.response?.data?.detail || '导入失败'}`)
}
}
files.value = []
detectedCols.value = []
result.value = msgs.join('\n')
uploading.value = false
}