sxbh_admin/cma-management
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(okr): 季度复盘提醒审计口径与失败感知加固 — dry_run/sent/failed/skipped 状态语义 + 失败本地留痕+中继告警+退出码契约(0/1/2) (fullstack-okr-reminder-hardening-20260911)

Browse Source
This commit is contained in:
Hermes CI Fix
2026-09-12 01:03:31 +08:00
parent 38968750a3
commit 998da4ebd4
1 changed files with 147 additions and 27 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/scripts/okr_review_reminder.py
+142 -22
View File
@@ -11,11 +11,47 @@
cd /root/cma-management/backend && PYTHONPATH=. /usr/bin/python3 scripts/okr_review_reminder.py # dry-run
cd /root/cma-management/backend && PYTHONPATH=. /usr/bin/python3 scripts/okr_review_reminder.py --push # 推送
... --entity-id 1 # 仅某企业
────────────────────────────────────────────────────────────────────────
审计口径（2026-09-12 fullstack-okr-reminder-hardening）：dry-run 与真实推送可区分
- notification_logs.status 语义固定为四类：
'dry_run' 仅打印，未产生任何外部通知（不调用中继）
'skipped' --push 但当期无目标，未投递
'sent' 真实投递成功（唯一会在中继侧留下消息的状态）
'failed' 真实投递失败
- 历史遗留：2026-09-11 的两行 status='pending' 实为 dry-run 记录，
已迁移为 'dry_run'（'pending' 语义含混：既像"待推送"又像"已入队"，已废弃不再写入）。
- 退出码契约（供 cron 感知失败）：
0 = 正常（dry_run / sent / skipped）
1 = --push 投递失败（中继不可用或返回错误）
2 = 执行异常（DB 连接失败、脚本内部错误等）
────────────────────────────────────────────────────────────────────────
失败感知（2026-09-12 加固：首次触发失败不再无感知）
- main() 全量 try/except 兜底：任何异常 → 写本地失败日志 + 尽力经同一中继发一条运维告警 + 退出码 2
- --push 投递失败 → 同样写本地失败日志 + 尽力告警 + 退出码 1（原先静默退出 0）
- 退出码非 0 会被 Hermes cron（no_agent）判定为脚本失败，产生
"⚠ Cron watchdog 'OKR季度复盘提醒' script failed" 告警；该 job 的 failure_deliver
已指向运维通道，告警不再只落本地（cron job 60d4f6fc0ec5）。
- 本地失败日志（兜底留痕，中继也不可用时唯一可查痕迹）：
/root/cma-management/backend/logs/okr_review_reminder_failures.log
────────────────────────────────────────────────────────────────────────
口径说明（季度为空的历史目标：**有意为之**，非遗漏）
- 筛选条件显式包含 `quarter IS NULL OR quarter = ''`：截至 2026-09-12，44 个 active 目标中
only 5 个标了 2026Q3，其余 39 个是战略地图导入的历史遗留（无季度标记）。
- 一并纳入提醒的意图 = 推动这些历史目标补标季度或清理归档（每季度提醒一次直到被处理），
而不是"提醒漏了过滤条件"。
- 若日后不再需要，把 _build_query 里的 `OR o.quarter IS NULL OR o.quarter = ''` 去掉即可
（改动点单点集中，勿改其他口径）。
测试钩子（仅本机验证用；不设置时全部走生产默认值，cron 运行时无感）
- OKR_REVIEW_RELAY_URL 覆盖中继地址（默认 http://127.0.0.1:8800/send）
- OKR_REVIEW_FAILLOG 覆盖失败日志路径（默认见上）
"""
import argparse
import json
import os
import sys
import traceback
import urllib.parse
import urllib.request
from datetime import datetime
@@ -26,8 +62,15 @@ from sqlalchemy import text # noqa: E402
from app.database import get_session_local # noqa: E402
RELAY_URL = "http://127.0.0.1:8800/send" # 与 daily_push.py 同一中继通道（不新造通道）
RELAY_URL = os.environ.get("OKR_REVIEW_RELAY_URL", "http://127.0.0.1:8800/send")
SOURCE = "管理会计OS"
JOB_ID = "60d4f6fc0ec5" # Hermes cron job（OKR季度复盘提醒），失败告警里带上便于定位
FAIL_LOG = os.environ.get("OKR_REVIEW_FAILLOG",
"/root/cma-management/backend/logs/okr_review_reminder_failures.log")
EXIT_OK = 0
EXIT_PUSH_FAILED = 1
EXIT_ERROR = 2
def current_quarter(d=None):
@@ -68,52 +111,129 @@ def push_wecom(msg):
return {"ok": False, "error": "推送异常: %s" % e}
def main():
ap = argparse.ArgumentParser()
ap.add_argument("--push", action="store_true", help="真正推送到企微中继（默认 dry-run 只打印）")
ap.add_argument("--entity-id", type=int, default=None, help="仅指定企业")
args = ap.parse_args()
# ── 失败留痕 / 告警（2026-09-12 加固）────────────────────────────────────
quarter = current_quarter()
db = get_session_local()()
def append_fail_log(line):
"""本地兜底留痕：中继也不可用时，这里是唯一可查的失败痕迹。"""
try:
sql = ("SELECT o.id, o.title, o.owner, o.progress, o.confidence, o.quarter, o.entity_id, e.name AS entity_name "
d = os.path.dirname(FAIL_LOG)
if d:
os.makedirs(d, exist_ok=True)
with open(FAIL_LOG, "a", encoding="utf-8") as f:
f.write("[%s] %s\n" % (datetime.now().strftime("%Y-%m-%d %H:%M:%S"), line))
return True
except Exception as e: # 兜底不能再抛异常
print("== 本地失败日志写入失败: %s" % e)
return False
def alert_failure(reason):
"""失败告警：本地留痕（必做，不依赖网络）+ 尽力经同一中继推送运维告警。
约束：不新增通道 —— 告警复用与提醒完全相同的中继（RELAY_URL / SOURCE）。
返回 (logged, delivered)。
"""
append_fail_log("FAIL | %s" % reason)
msg = ("## ❗ 管理会计OS · 运维告警\n\n"
"**OKR 季度复盘提醒执行失败**（cron job `%s`）\n\n"
"- 时间：%s\n"
"- 原因：%s\n"
"- 中继：%s\n\n"
"排查：`scripts/okr_review_reminder.py`（本地失败日志见 backend/logs/"
"okr_review_reminder_failures.log）。" % (
JOB_ID, datetime.now().strftime("%Y-%m-%d %H:%M:%S"), reason, RELAY_URL))
try:
res = push_wecom(msg)
delivered = bool(res.get("ok", True)) and not res.get("error")
except Exception as e: # push_wecom 已自吞异常，这里再兜一层
res, delivered = {"error": str(e)}, False
append_fail_log("ALERT | delivered=%s | %s" % (
delivered, json.dumps(res, ensure_ascii=False)[:300]))
print("== 失败告警: delivered=%s | %s" % (delivered, json.dumps(res, ensure_ascii=False)[:200]))
return True, delivered
def _build_query(entity_id, quarter):
sql = ("SELECT o.id, o.title, o.owner, o.progress, o.confidence, o.quarter, o.entity_id, "
"e.name AS entity_name "
"FROM objectives o LEFT JOIN entities e ON e.id = o.entity_id "
"WHERE o.status = 'active'")
params = {}
if args.entity_id is not None:
if entity_id is not None:
sql += " AND o.entity_id = :eid"
params["eid"] = args.entity_id
# 本季度 + 未设季度（历史遗留）的目标都纳入提醒
params["eid"] = entity_id
# 本季度 + 未设季度（历史遗留）的目标都纳入提醒 —— 有意为之，见模块 docstring「口径说明」
sql += " AND (o.quarter = :q OR o.quarter IS NULL OR o.quarter = '')"
params["q"] = quarter
sql += " ORDER BY o.entity_id, o.id"
return sql, params
def _record(db, quarter, msg, status, err, sent_at):
"""复用既有 notifications 机制留存审计（notification_logs）"""
db.execute(text(
"INSERT INTO notification_logs (channel, recipient, title, content, status, error_msg, "
"sent_at, created_at) VALUES ('wecom', :r, :t, :c, :s, :e, :sa, NOW())"),
{"r": "企微群(中继)", "t": "OKR 季度复盘提醒（%s）" % quarter, "c": msg,
"s": status, "e": err, "sa": sent_at})
db.commit()
def _run(args):
"""返回退出码（0=正常 1=投递失败）；异常交给 main() 兜底。"""
quarter = current_quarter()
db = get_session_local()()
try:
sql, params = _build_query(args.entity_id, quarter)
rows = [dict(r._mapping) for r in db.execute(text(sql), params).fetchall()]
msg = build_message(rows, quarter)
print("== 目标数: %d（季度 %s；entity_id=%s）" % (len(rows), quarter, args.entity_id))
print(msg)
sent_at = None
if args.push and rows:
res = push_wecom(msg)
ok = bool(res.get("ok", True)) and not res.get("error")
print("== 推送结果:", json.dumps(res, ensure_ascii=False)[:300])
status = "sent" if ok else "failed"
err = None if ok else str(res.get("error"))[:400]
if ok:
status, err, sent_at, rc = "sent", None, datetime.now(), EXIT_OK
else:
status, err = "pending", None
status, err, rc = "failed", str(res.get("error"))[:400], EXIT_PUSH_FAILED
elif args.push:
status, err, rc = "skipped", None, EXIT_OK
print("== 无目标，跳过投递")
else:
status, err, rc = "dry_run", None, EXIT_OK
print("== dry-run：不调用中继、不产生外部通知")
# 复用既有 notifications 机制留存审计（notification_logs）
db.execute(text(
"INSERT INTO notification_logs (channel, recipient, title, content, status, error_msg, sent_at, created_at) "
"VALUES ('wecom', :r, :t, :c, :s, :e, :sa, NOW())"),
{"r": "企微群(中继)", "t": "OKR 季度复盘提醒（%s）" % quarter, "c": msg,
"s": status, "e": err, "sa": datetime.now() if status == "sent" else None})
db.commit()
_record(db, quarter, msg, status, err, sent_at)
print("== notification_logs 已记录: status=%s" % status)
if rc != EXIT_OK:
alert_failure("企微中继投递失败: %s" % err)
return rc
finally:
db.close()
def main():
ap = argparse.ArgumentParser()
ap.add_argument("--push", action="store_true", help="真正推送到企微中继（默认 dry-run 只打印）")
ap.add_argument("--entity-id", type=int, default=None, help="仅指定企业")
args = ap.parse_args()
try:
rc = _run(args)
except SystemExit:
raise
except BaseException as exc: # 含 DB 连接失败；保证任何异常都有留痕+告警+非零退出码
print("== 执行异常: %s" % exc)
print(traceback.format_exc())
alert_failure("执行异常: %s: %s" % (type(exc).__name__, exc))
sys.exit(EXIT_ERROR)
sys.exit(rc)
if __name__ == "__main__":
main()