Files
Hermes CI Fix 2cff58ca6a feat(docs-center): report reading section (type=report) on /api/cma/docs
Increment #2 over PoC ede0524 (task yanxue-20260924-cma-report-source).

- backend derives reports from existing drop points (wisdom/_reports + every profile pending-tasks/_reports), read live per request, no cache, no rebuild

- gate = front-matter plus non-empty conclusion; missing either means silently out of the list (receipts/audits never enter); title/date/domain/sources missing are marked incomplete instead of rejected

- entity + role partitioning (ceo/it cross-entity; others own entity plus group/generic); kpi_refs resolve to a one-hop metric link

- new GET /api/cma/docs/report/{rid} detail endpoint; knowledge branch unchanged (type defaults to knowledge)

- frontend: CMAKnowledge adds a report reading pane (filters + detail dialog), reuses the existing markdown renderer, no new deps

(cherry picked from commit 7c6e0857e68133f5ba2a539fd3569aeb82547911)
2026-10-18 16:31:52 +08:00

556 lines
22 KiB
Python
Raw Permalink Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
"""CMA 文档中心 API —— md 单一来源(任务 yanxue-20260924-cma-docs-center-poc)
口径(任总 2026-09-24 00:54 拍板):**指标进 UI、结论出 md、机器走 API/MCP。**
设计要点
--------
1. 内容单一来源 = 运行期目录(默认 `/srv/hermes/shared/cma-content`,可用 `CMA_DOCS_DIR` 覆盖),
一篇一个 `docs/<slug>.md`,front-matter 带 title/category/category_name/icon/summary/order。
2. **故意不放在仓库内**:CMA 部署链是「部署树 = origin/main 已提交树」(deploy.sh `[0]/[1]`
`git checkout -f` + `git clean -fdx`):仓内内容目录会被每次部署重置,且内容改动会让
部署树出现「未提交修改」而被 `[1.1]` 判失败、**阻断整个部署**。放仓外 ⇒ 改内容不需要
git/部署/重启,dev 树与部署树天然共读同一份。
3. **本模块每次请求实时读盘,无缓存、不做 build-time 打包** ⇒ 新增/修改一个 md 后,
页面刷新即变,前端无需改码、无需 rebuild。
4. 只**新增**端点,不改任何既有接口语义;内容为系统级共享知识(与 `knowledge_articles`
`entity_id IS NULL` 的全局共享同族),不做账套分区,故不接 `entity_id`。
5. 写口 = 文件系统(Bot/Agent 直接写 md),**不提供写 API**,避免出现"第二来源"。
"""
from __future__ import annotations
import logging
import os
import re
from datetime import datetime, timezone
from pathlib import Path
from typing import Optional
from fastapi import APIRouter, Depends, HTTPException, Query, Request
from sqlalchemy.orm import Session
from urllib.parse import quote
from app.auth_middleware import require_auth, require_role
from app.database import get_db
from app.deps import get_session_entity
from app.models import Entity, User, UserEntity
logger = logging.getLogger("cma.doc_center_api")
DEFAULT_DOCS_DIR = "/srv/hermes/shared/cma-content"
SLUG_RE = re.compile(r"^[a-z0-9][a-z0-9-]{0,63}$")
router = APIRouter(
prefix="/api/cma/docs",
tags=["文档中心"],
dependencies=[Depends(require_role("ceo", "finance", "business", "it"))],
)
# ────────────────────────── 内容目录(运行期读盘)──────────────────────────
def content_root() -> Path:
"""内容根目录(每次调用实时解析,便于测试/迁移换目录)"""
return Path(os.environ.get("CMA_DOCS_DIR", DEFAULT_DOCS_DIR))
def docs_dir() -> Path:
return content_root() / "docs"
def _parse_scalar(raw: str):
raw = raw.strip()
if len(raw) >= 2 and raw[0] == raw[-1] and raw[0] in "\"'":
body = raw[1:-1]
return body.replace('\\"', '"').replace("\\\\", "\\")
if re.fullmatch(r"-?\d+", raw or ""):
return int(raw)
return raw
def parse_front_matter(text: str) -> Optional[dict]:
"""极简 YAML front-matter 解析(只支持 `key: value` 标量,避免引入新依赖)"""
if not text.startswith("---"):
return None
end = text.find("\n---", 3)
if end < 0:
return None
meta = {}
for line in text[3:end].splitlines():
if not line.strip() or line.lstrip().startswith("#"):
continue
key, sep, val = line.partition(":")
if not sep:
continue
meta[key.strip()] = _parse_scalar(val)
body = text[end + 4:]
if body.startswith("\n"):
body = body[1:]
return {"meta": meta, "body": body}
def _category_registry() -> list:
"""分类注册表(categories.json):决定分类顺序与展示名/图标"""
path = content_root() / "categories.json"
cats = []
if path.is_file():
import json
try:
data = json.loads(path.read_text(encoding="utf-8"))
cats = list(data.get("categories") or [])
except Exception as exc: # 注册表损坏不阻断读内容,只在响应里暴露
logger.warning("categories.json 解析失败: %s", exc)
return []
return cats
def _read_doc(path: Path):
try:
text = path.read_text(encoding="utf-8")
except Exception as exc:
return None, f"读取失败: {exc}"
parsed = parse_front_matter(text)
if parsed is None:
return None, "缺少 front-matter(--- 包裹的 title/category/... )"
meta = parsed["meta"]
for field in ("title", "category", "summary"):
if not str(meta.get(field, "")).strip():
return None, f"front-matter 缺必填字段: {field}"
stat = path.stat()
return {
"slug": path.stem,
"title": str(meta["title"]),
"category": str(meta["category"]),
"summary": str(meta["summary"]),
"order": int(meta.get("order") or 0),
"icon": str(meta.get("icon") or ""),
"category_name": str(meta.get("category_name") or ""),
"bytes": stat.st_size,
"updated_at": datetime.fromtimestamp(stat.st_mtime, tz=timezone.utc).isoformat(),
"content": parsed["body"],
}, None
def load_docs() -> tuple:
"""实时读盘:返回 (docs, skipped, available)。docs 已按 (category order, doc order, slug) 排序"""
d = docs_dir()
if not d.is_dir():
return [], [], False
cats = _category_registry()
cat_order = {c.get("key"): int(c.get("order") or 0) for c in cats}
docs, skipped = [], []
for path in sorted(d.glob("*.md")):
if not SLUG_RE.match(path.stem):
skipped.append({"file": path.name, "reason": "文件名不符合 slug 规则 ^[a-z0-9][a-z0-9-]{0,63}$"})
continue
doc, err = _read_doc(path)
if err:
skipped.append({"file": path.name, "reason": err})
continue
docs.append(doc)
docs.sort(key=lambda x: (cat_order.get(x["category"], 10_000), x["category"], x["order"], x["slug"]))
return docs, skipped, True
def _categories_view(docs: list) -> list:
"""分类视图:按注册表 `order` 升序(未登记分类补在末尾,不丢内容)"""
cats = _category_registry()
seen, out, seq = set(), [], 0
for c in cats:
key = c.get("key")
if not key or key in seen:
continue
seen.add(key)
seq += 1
out.append({
"key": key,
"name": c.get("name") or key,
"icon": c.get("icon") or "",
"order": int(c.get("order") or 0),
"_seq": seq,
"count": len([d for d in docs if d["category"] == key]),
})
for d in docs:
if d["category"] in seen:
continue
seen.add(d["category"])
seq += 1
out.append({
"key": d["category"],
"name": d["category_name"] or d["category"],
"icon": d["icon"] or "",
"order": 10_000,
"_seq": seq,
"count": len([x for x in docs if x["category"] == d["category"]]),
})
out.sort(key=lambda c: (c["order"], c["_seq"]))
for c in out:
c.pop("_seq")
return out
def _meta_only(doc: dict) -> dict:
return {k: v for k, v in doc.items() if k != "content"}
# ────────────────────────── 端点(只新增)──────────────────────────
@router.get("")
def list_docs(
request: Request,
with_content: bool = Query(False, description="是否带上 md 正文(默认只返回元数据)"),
category: Optional[str] = Query(None, description="按分类 key 过滤"),
type: str = Query("knowledge", description="内容类型:knowledge(缺省,知识库)/ report(报告解读)"),
domain: Optional[str] = Query(None, description="type=report:按域过滤"),
entity: Optional[str] = Query(None, description="type=report:按账套归属过滤"),
kind: Optional[str] = Query(None, description="type=report:按报告类型过滤"),
date_from: Optional[str] = Query(None, description="type=report:起始日(含)YYYY-MM-DD"),
date_to: Optional[str] = Query(None, description="type=report:结束日(含)YYYY-MM-DD"),
diagnose: bool = Query(False, description="type=report:附带被排除件明细(自诊断)"),
db: Session = Depends(get_db),
current_user: User = Depends(require_auth),
):
"""文档列表(实时读盘)。
- `data`:按「分类顺序 → 篇内 order → slug」排序的文档元数据(`with_content=true` 时含正文)
- `categories`:分类视图(name/icon/count),前端按它分组渲染
- `source`:内容目录绝对路径(机器侧取内容用同一来源,满足「机器走 API」)
- `skipped`:被跳过的 md 及原因(缺 front-matter/文件名非法),便于 Bot 自诊断
"""
if str(type or "").strip().lower() == "report":
return _list_reports(
request=request, db=db, user=current_user,
domain=domain, entity=entity, kind=kind,
date_from=date_from, date_to=date_to, diagnose=bool(diagnose),
)
docs, skipped, available = load_docs()
all_docs = docs
if category:
docs = [d for d in docs if d["category"] == category]
return {
"count": len(docs),
"available": available,
"source": str(docs_dir()),
"categories": _categories_view(all_docs), # 分类视图按全量内容统计(不受 category 过滤影响)
"data": docs if with_content else [_meta_only(d) for d in docs],
"skipped": skipped,
"format": "markdown",
}
@router.get("/{slug}")
def get_doc(slug: str):
"""单篇文档(含 md 正文,实时读盘)"""
if not SLUG_RE.match(slug or ""):
raise HTTPException(400, "无效的文档标识(只允许 [a-z0-9-],1-64 位,字母数字开头)")
path = docs_dir() / f"{slug}.md"
if not path.is_file():
raise HTTPException(404, "文档不存在")
doc, err = _read_doc(path)
if err:
raise HTTPException(422, f"文档格式不合法: {err}")
return doc
# ══════════════════════ 报告解读(type=report)══════════════════════════
# 口径(规范 /srv/hermes/wisdom/_decisions/report-frontmatter-spec-20260924.md,本单按其实现):
# ① 进门标记 = front-matter + `conclusion` 非空(§二「front-matter 本身就是进门标记」
# + §三「conclusion 是唯一硬字段」):缺任一 ⇒ **不进列表、不报错**。
# 回执/审计/施工/工作件天然不带 front-matter ⇒ 天然不进;**不搬家、不新建内容目录**。
# ② 规范 §四 另列 title/date/domain/sources 为必填:缺失**不拒收**,标 `incomplete` 并按
# 兜底值展示(若按字面拒收,页面只能是空壳,与 §一「无体之用」的立意相悖)。
# ③ 只**新增** type=report 分支与新端点 /report/{rid};type 缺省走知识库分支,语义逐字不变。
# ④ 权限:按 entity + CMA 既有 roles 分区 —— ceo/it 可跨账套读结论,其余角色只看本账套 + 集团/通用。
# ⑤ 读口 = 文件系统(实时读盘,改一篇 md 刷新即生效),无写 API(与知识库同口径,避免第二来源)。
WISDOM_REPORTS_DIR = "/srv/hermes/wisdom/_reports"
PROFILES_ROOT = "/root/.hermes/profiles"
REPORT_SPEC_PATH = "/srv/hermes/wisdom/_decisions/report-frontmatter-spec-20260924.md"
REPORT_HEAD_BYTES = 8192
REPORT_DIRS_ENV = "CMA_REPORTS_DIRS"
DOMAIN_LABELS = {
"finance": "财务", "customer": "客户", "process": "流程", "learning": "学习成长",
"tech": "技术", "ops": "运维", "market": "市场", "gov": "政务",
}
KIND_LABELS = {"report": "报告", "analysis": "分析", "brief": "摘要", "warning": "预警"}
GLOBAL_ENTITIES = ("集团", "通用")
CROSS_ENTITY_ROLES = ("ceo", "it")
ENTITY_NAME_FALLBACK = {1: "酣客", 2: "博海"}
_SAFE_STEM_RE = re.compile(r"^[^/\\\x00]{1,160}$")
def _as_list(val) -> list:
"""front-matter 值 → 列表(支持行内数组 [a, b] / 逗号分隔 / 单值)"""
if val is None:
return []
if isinstance(val, list):
return [str(v).strip() for v in val if str(v).strip()]
s = str(val).strip()
if not s:
return []
if s.startswith("[") and s.endswith("]"):
s = s[1:-1]
return [p.strip().strip("\"'") for p in s.split(",") if p.strip()]
def _kpi_link(raw: str) -> dict:
"""kpi_refs 项 → 一跳链(数字 / kpi:<id> → 指标详情页;其余 → 指标字典页关键词)"""
v = str(raw).strip()
m = re.fullmatch(r"(?:kpi[:/]?)?(\d{1,9})", v, flags=re.IGNORECASE)
if m:
return {"raw": v, "url": f"/kpis/{m.group(1)}", "resolved": "detail"}
return {"raw": v, "url": "/kpis?keyword=" + quote(v), "resolved": "list"}
def reports_dirs() -> list:
"""[(source_key, Path)] —— 既有报告落点的发现结果。
env `CMA_REPORTS_DIRS` 覆盖优先(`:`/`;` 分隔,条目可写 `key=path`,供测试与隔离实测);
缺省 = 自动发现:`/srv/hermes/wisdom/_reports` + 各 profile 的 `pending-tasks/_reports`
(按 resolve 后路径去重,避免 pending-tasks 为符号链接时重复收录)。
"""
raw = (os.environ.get(REPORT_DIRS_ENV) or "").strip()
out, seen = [], set()
if raw:
for i, item in enumerate(x for x in re.split(r"[:;]", raw) if x.strip()):
item = item.strip()
key, sep, path = item.partition("=")
if not sep:
key, path = f"src{i}", item
out.append(((key or f"src{i}").strip(), Path(path)))
return out
cands = []
wisdom = Path(WISDOM_REPORTS_DIR)
if wisdom.is_dir():
cands.append(("wisdom", wisdom))
for d in sorted(Path(PROFILES_ROOT).glob("*/pending-tasks/_reports")):
if not d.is_dir():
continue
prof = d.parent.parent.name
cands.append((prof[6:] if prof.startswith("wecom-") else prof, d))
for key, path in cands:
try:
rp = path.resolve()
except OSError:
rp = path
if rp in seen:
continue
seen.add(rp)
out.append((key, path))
return out
def _parse_report_text(text: str, path: Path, source_key: str):
"""报告件 → (record, None) 或 (None, 排除码)。
排除码:`no_frontmatter`(无 --- 进门标记)/ `missing_conclusion`(缺唯一硬字段)/
`unreadable:<err>`。**排除即静默**(不进列表、不报错),这正是负对照的判据。
"""
parsed = parse_front_matter(text) if text.startswith("---") else None
if parsed is None:
return None, "no_frontmatter"
meta = parsed["meta"]
if not str(meta.get("conclusion") or "").strip():
return None, "missing_conclusion"
try:
stat = path.stat()
except OSError as exc:
return None, f"unreadable:{exc}"
refs = _as_list(meta.get("kpi_refs"))
rec = {
"id": f"{source_key}:{path.stem}",
"title": str(meta.get("title") or "").strip() or path.stem,
"date": (str(meta.get("date") or "").strip()
or datetime.fromtimestamp(stat.st_mtime).date().isoformat()),
"domain": str(meta.get("domain") or "").strip() or "未分类",
"entity": str(meta.get("entity") or "").strip(),
"kind": str(meta.get("kind") or "").strip() or "report",
"conclusion": str(meta["conclusion"]).strip(),
"sources": _as_list(meta.get("sources")),
"impact": str(meta.get("impact") or "").strip(),
"decisions": _as_list(meta.get("decisions")),
"kpi_refs": refs,
"kpi_links": [_kpi_link(r) for r in refs],
"file": path.name,
"source": source_key,
"path": str(path),
"bytes": stat.st_size,
"updated_at": datetime.fromtimestamp(stat.st_mtime, tz=timezone.utc).isoformat(),
"incomplete": [f for f in ("title", "date", "domain", "sources")
if not str(meta.get(f) or "").strip()],
"content": parsed["body"],
}
return rec, None
def _read_report(path: Path, source_key: str, full: bool = False):
try:
if full:
text = path.read_text(encoding="utf-8", errors="replace")
else:
with open(path, "rb") as fh:
text = fh.read(REPORT_HEAD_BYTES).decode("utf-8", errors="replace")
if text.startswith("---") and text.find("\n---", 3) < 0:
text = path.read_text(encoding="utf-8", errors="replace")
except OSError as exc:
return None, f"unreadable:{exc}"
return _parse_report_text(text, path, source_key)
def _entity_scope(db: Session, user: User, request: Request) -> dict:
"""当前会话的可见范围:账套 id / 账套名称集合 / 是否跨账套角色"""
entity_id = get_session_entity(request)
if entity_id is None:
rel = db.query(UserEntity).filter(UserEntity.user_id == user.id).first()
entity_id = rel.entity_id if rel else None
names = set()
if entity_id is not None:
ent = db.query(Entity).filter(Entity.id == entity_id).first()
if ent:
names |= {str(ent.name or "").strip(), str(ent.short_name or "").strip()}
fb = ENTITY_NAME_FALLBACK.get(int(entity_id))
if fb:
names.add(fb)
names.discard("")
return {
"role": user.role,
"entity_id": entity_id,
"entity_names": sorted(names),
"cross_entity": user.role in CROSS_ENTITY_ROLES,
}
def _report_visible(rec: dict, scope: dict) -> bool:
"""按 entity 分区:跨账套角色(ceo/it)→ 全可见;其余 → 本账套名 + 集团/通用。
未标 entity 的件按「通用」处理(entity 属选填字段,不设门槛);
显式 entity 数字与会话账套一致也算命中(兼容 `entity: 2` 写法)。
"""
if scope["cross_entity"]:
return True
e = str(rec.get("entity") or "").strip()
if not e or e in GLOBAL_ENTITIES:
return True
if e in set(scope["entity_names"]):
return True
if e.isdigit() and scope["entity_id"] is not None and int(e) == int(scope["entity_id"]):
return True
return False
def _facet(items: list, field: str, labels: dict) -> list:
cnt = {}
for r in items:
k = str(r.get(field) or "").strip() or "(未标)"
cnt[k] = cnt.get(k, 0) + 1
return [{"key": k, "name": labels.get(k, k), "count": v}
for k, v in sorted(cnt.items(), key=lambda kv: (-kv[1], kv[0]))]
def _list_reports(request: Request, db: Session, user: User,
domain=None, entity=None, kind=None,
date_from=None, date_to=None, diagnose: bool = False) -> dict:
"""报告解读列表:扫描既有报告落点 → 只收带 front-matter + conclusion 的件 → 按 entity 分区 → 筛选"""
scope = _entity_scope(db, user, request)
included, sources = [], []
excluded = {"no_frontmatter": 0, "missing_conclusion": 0, "other": 0}
detail = []
for key, base in reports_dirs():
exists = base.is_dir()
found = 0
if exists:
for path in sorted(base.glob("*.md")):
found += 1
rec, reason = _read_report(path, key)
if reason:
if reason == "no_frontmatter":
excluded["no_frontmatter"] += 1
elif reason == "missing_conclusion":
excluded["missing_conclusion"] += 1
else:
excluded["other"] += 1
if diagnose and len(detail) < 200:
detail.append({"file": path.name, "source": key, "reason": reason})
continue
if not _report_visible(rec, scope):
continue
included.append(rec)
sources.append({"key": key, "path": str(base),
"exists": exists, "md_count": found})
visible_total = len(included)
def _keep(r: dict) -> bool:
if domain and r["domain"] != domain:
return False
if entity and r["entity"] != entity:
return False
if kind and r["kind"] != kind:
return False
if date_from and (r["date"] or "") < date_from:
return False
if date_to and (r["date"] or "") > date_to:
return False
return True
rows = [r for r in included if _keep(r)]
rows.sort(key=lambda r: (r["date"] or "", r["title"]), reverse=True)
dates = sorted({r["date"] for r in included if r["date"]})
data = []
for r in rows:
r = dict(r)
r.pop("content", None)
data.append(r)
payload = {
"type": "report",
"count": len(data),
"available": any(s["exists"] for s in sources),
"scope": scope,
"sources": sources,
"excluded": excluded,
"facets": {
"domain": _facet(included, "domain", DOMAIN_LABELS),
"entity": _facet(included, "entity", {}),
"kind": _facet(included, "kind", KIND_LABELS),
"date": {"min": dates[0] if dates else "", "max": dates[-1] if dates else ""},
},
"totals": {"visible": visible_total, "filtered_out": visible_total - len(data)},
"data": data,
"format": "markdown",
"spec": REPORT_SPEC_PATH,
}
if diagnose:
payload["excluded_detail"] = detail
return payload
@router.get("/report/{rid}")
def get_report(rid: str, request: Request,
db: Session = Depends(get_db),
current_user: User = Depends(require_auth)):
"""单篇报告(含 md 正文,实时读盘)。`rid` = `<source_key>:<文件名主干>`"""
key, sep, stem = (rid or "").partition(":")
if not sep or not key:
raise HTTPException(400, "无效的报告标识(应为 <source>:<文件名>)")
if not _SAFE_STEM_RE.match(stem) or stem.startswith(".") or ".." in stem:
raise HTTPException(400, "无效的报告标识(文件名非法)")
base = {k: p for k, p in reports_dirs()}.get(key)
if base is None:
raise HTTPException(404, "报告不存在")
path = base / f"{stem}.md"
if path.parent != base or not path.is_file():
raise HTTPException(404, "报告不存在")
rec, reason = _read_report(path, key, full=True)
if reason:
raise HTTPException(422, f"报告格式不合法: {reason}")
scope = _entity_scope(db, current_user, request)
if not _report_visible(rec, scope):
raise HTTPException(
403,
f"无权访问企业 {rec.get('entity') or '(未标注)'} 的报告(当前账套: {scope.get('entity_names') or '未绑定'})",
)
return rec