fix(okr): 季度复盘提醒审计口径与失败感知加固 — dry_run/sent/failed/skipped 状态语义 + 失败本地留痕+中继告警+退出码契约(0/1/2) (fullstack-okr-reminder-hardening-20260911)

This commit is contained in:
Hermes CI Fix
2026-09-12 01:03:31 +08:00
parent 38968750a3
commit 998da4ebd4
+142 -22
View File
@@ -11,11 +11,47 @@
cd /root/cma-management/backend && PYTHONPATH=. /usr/bin/python3 scripts/okr_review_reminder.py # dry-run cd /root/cma-management/backend && PYTHONPATH=. /usr/bin/python3 scripts/okr_review_reminder.py # dry-run
cd /root/cma-management/backend && PYTHONPATH=. /usr/bin/python3 scripts/okr_review_reminder.py --push # 推送 cd /root/cma-management/backend && PYTHONPATH=. /usr/bin/python3 scripts/okr_review_reminder.py --push # 推送
... --entity-id 1 # 仅某企业 ... --entity-id 1 # 仅某企业
────────────────────────────────────────────────────────────────────────
审计口径(2026-09-12 fullstack-okr-reminder-hardening):dry-run 与真实推送可区分
- notification_logs.status 语义固定为四类:
'dry_run' 仅打印,未产生任何外部通知(不调用中继)
'skipped' --push 但当期无目标,未投递
'sent' 真实投递成功(唯一会在中继侧留下消息的状态)
'failed' 真实投递失败
- 历史遗留:2026-09-11 的两行 status='pending' 实为 dry-run 记录,
已迁移为 'dry_run'('pending' 语义含混:既像"待推送"又像"已入队",已废弃不再写入)。
- 退出码契约(供 cron 感知失败):
0 = 正常(dry_run / sent / skipped)
1 = --push 投递失败(中继不可用或返回错误)
2 = 执行异常(DB 连接失败、脚本内部错误等)
────────────────────────────────────────────────────────────────────────
失败感知(2026-09-12 加固:首次触发失败不再无感知)
- main() 全量 try/except 兜底:任何异常 → 写本地失败日志 + 尽力经同一中继发一条运维告警 + 退出码 2
- --push 投递失败 → 同样写本地失败日志 + 尽力告警 + 退出码 1(原先静默退出 0)
- 退出码非 0 会被 Hermes cron(no_agent)判定为脚本失败,产生
"⚠ Cron watchdog 'OKR季度复盘提醒' script failed" 告警;该 job 的 failure_deliver
已指向运维通道,告警不再只落本地(cron job 60d4f6fc0ec5)。
- 本地失败日志(兜底留痕,中继也不可用时唯一可查痕迹):
/root/cma-management/backend/logs/okr_review_reminder_failures.log
────────────────────────────────────────────────────────────────────────
口径说明(季度为空的历史目标:**有意为之**,非遗漏)
- 筛选条件显式包含 `quarter IS NULL OR quarter = ''`:截至 2026-09-12,44 个 active 目标中
only 5 个标了 2026Q3,其余 39 个是战略地图导入的历史遗留(无季度标记)。
- 一并纳入提醒的意图 = 推动这些历史目标补标季度或清理归档(每季度提醒一次直到被处理),
而不是"提醒漏了过滤条件"。
- 若日后不再需要,把 _build_query 里的 `OR o.quarter IS NULL OR o.quarter = ''` 去掉即可
(改动点单点集中,勿改其他口径)。
测试钩子(仅本机验证用;不设置时全部走生产默认值,cron 运行时无感)
- OKR_REVIEW_RELAY_URL 覆盖中继地址(默认 http://127.0.0.1:8800/send)
- OKR_REVIEW_FAILLOG 覆盖失败日志路径(默认见上)
""" """
import argparse import argparse
import json import json
import os import os
import sys import sys
import traceback
import urllib.parse import urllib.parse
import urllib.request import urllib.request
from datetime import datetime from datetime import datetime
@@ -26,8 +62,15 @@ from sqlalchemy import text # noqa: E402
from app.database import get_session_local # noqa: E402 from app.database import get_session_local # noqa: E402
RELAY_URL = "http://127.0.0.1:8800/send" # 与 daily_push.py 同一中继通道(不新造通道) RELAY_URL = os.environ.get("OKR_REVIEW_RELAY_URL", "http://127.0.0.1:8800/send")
SOURCE = "管理会计OS" SOURCE = "管理会计OS"
JOB_ID = "60d4f6fc0ec5" # Hermes cron job(OKR季度复盘提醒),失败告警里带上便于定位
FAIL_LOG = os.environ.get("OKR_REVIEW_FAILLOG",
"/root/cma-management/backend/logs/okr_review_reminder_failures.log")
EXIT_OK = 0
EXIT_PUSH_FAILED = 1
EXIT_ERROR = 2
def current_quarter(d=None): def current_quarter(d=None):
@@ -68,52 +111,129 @@ def push_wecom(msg):
return {"ok": False, "error": "推送异常: %s" % e} return {"ok": False, "error": "推送异常: %s" % e}
def main(): # ── 失败留痕 / 告警(2026-09-12 加固)────────────────────────────────────
ap = argparse.ArgumentParser()
ap.add_argument("--push", action="store_true", help="真正推送到企微中继(默认 dry-run 只打印)")
ap.add_argument("--entity-id", type=int, default=None, help="仅指定企业")
args = ap.parse_args()
quarter = current_quarter() def append_fail_log(line):
db = get_session_local()() """本地兜底留痕:中继也不可用时,这里是唯一可查的失败痕迹。"""
try: try:
sql = ("SELECT o.id, o.title, o.owner, o.progress, o.confidence, o.quarter, o.entity_id, e.name AS entity_name " d = os.path.dirname(FAIL_LOG)
if d:
os.makedirs(d, exist_ok=True)
with open(FAIL_LOG, "a", encoding="utf-8") as f:
f.write("[%s] %s\n" % (datetime.now().strftime("%Y-%m-%d %H:%M:%S"), line))
return True
except Exception as e: # 兜底不能再抛异常
print("== 本地失败日志写入失败: %s" % e)
return False
def alert_failure(reason):
"""失败告警:本地留痕(必做,不依赖网络)+ 尽力经同一中继推送运维告警。
约束:不新增通道 —— 告警复用与提醒完全相同的中继(RELAY_URL / SOURCE)。
返回 (logged, delivered)。
"""
append_fail_log("FAIL | %s" % reason)
msg = ("## ❗ 管理会计OS · 运维告警\n\n"
"**OKR 季度复盘提醒执行失败**(cron job `%s`)\n\n"
"- 时间:%s\n"
"- 原因:%s\n"
"- 中继:%s\n\n"
"排查:`scripts/okr_review_reminder.py`(本地失败日志见 backend/logs/"
"okr_review_reminder_failures.log)。" % (
JOB_ID, datetime.now().strftime("%Y-%m-%d %H:%M:%S"), reason, RELAY_URL))
try:
res = push_wecom(msg)
delivered = bool(res.get("ok", True)) and not res.get("error")
except Exception as e: # push_wecom 已自吞异常,这里再兜一层
res, delivered = {"error": str(e)}, False
append_fail_log("ALERT | delivered=%s | %s" % (
delivered, json.dumps(res, ensure_ascii=False)[:300]))
print("== 失败告警: delivered=%s | %s" % (delivered, json.dumps(res, ensure_ascii=False)[:200]))
return True, delivered
def _build_query(entity_id, quarter):
sql = ("SELECT o.id, o.title, o.owner, o.progress, o.confidence, o.quarter, o.entity_id, "
"e.name AS entity_name "
"FROM objectives o LEFT JOIN entities e ON e.id = o.entity_id " "FROM objectives o LEFT JOIN entities e ON e.id = o.entity_id "
"WHERE o.status = 'active'") "WHERE o.status = 'active'")
params = {} params = {}
if args.entity_id is not None: if entity_id is not None:
sql += " AND o.entity_id = :eid" sql += " AND o.entity_id = :eid"
params["eid"] = args.entity_id params["eid"] = entity_id
# 本季度 + 未设季度(历史遗留)的目标都纳入提醒 # 本季度 + 未设季度(历史遗留)的目标都纳入提醒 —— 有意为之,见模块 docstring「口径说明」
sql += " AND (o.quarter = :q OR o.quarter IS NULL OR o.quarter = '')" sql += " AND (o.quarter = :q OR o.quarter IS NULL OR o.quarter = '')"
params["q"] = quarter params["q"] = quarter
sql += " ORDER BY o.entity_id, o.id" sql += " ORDER BY o.entity_id, o.id"
return sql, params
def _record(db, quarter, msg, status, err, sent_at):
"""复用既有 notifications 机制留存审计(notification_logs)"""
db.execute(text(
"INSERT INTO notification_logs (channel, recipient, title, content, status, error_msg, "
"sent_at, created_at) VALUES ('wecom', :r, :t, :c, :s, :e, :sa, NOW())"),
{"r": "企微群(中继)", "t": "OKR 季度复盘提醒(%s)" % quarter, "c": msg,
"s": status, "e": err, "sa": sent_at})
db.commit()
def _run(args):
"""返回退出码(0=正常 1=投递失败);异常交给 main() 兜底。"""
quarter = current_quarter()
db = get_session_local()()
try:
sql, params = _build_query(args.entity_id, quarter)
rows = [dict(r._mapping) for r in db.execute(text(sql), params).fetchall()] rows = [dict(r._mapping) for r in db.execute(text(sql), params).fetchall()]
msg = build_message(rows, quarter) msg = build_message(rows, quarter)
print("== 目标数: %d(季度 %s;entity_id=%s)" % (len(rows), quarter, args.entity_id)) print("== 目标数: %d(季度 %s;entity_id=%s)" % (len(rows), quarter, args.entity_id))
print(msg) print(msg)
sent_at = None
if args.push and rows: if args.push and rows:
res = push_wecom(msg) res = push_wecom(msg)
ok = bool(res.get("ok", True)) and not res.get("error") ok = bool(res.get("ok", True)) and not res.get("error")
print("== 推送结果:", json.dumps(res, ensure_ascii=False)[:300]) print("== 推送结果:", json.dumps(res, ensure_ascii=False)[:300])
status = "sent" if ok else "failed" if ok:
err = None if ok else str(res.get("error"))[:400] status, err, sent_at, rc = "sent", None, datetime.now(), EXIT_OK
else: else:
status, err = "pending", None status, err, rc = "failed", str(res.get("error"))[:400], EXIT_PUSH_FAILED
elif args.push:
status, err, rc = "skipped", None, EXIT_OK
print("== 无目标,跳过投递")
else:
status, err, rc = "dry_run", None, EXIT_OK
print("== dry-run:不调用中继、不产生外部通知")
# 复用既有 notifications 机制留存审计(notification_logs) _record(db, quarter, msg, status, err, sent_at)
db.execute(text(
"INSERT INTO notification_logs (channel, recipient, title, content, status, error_msg, sent_at, created_at) "
"VALUES ('wecom', :r, :t, :c, :s, :e, :sa, NOW())"),
{"r": "企微群(中继)", "t": "OKR 季度复盘提醒(%s)" % quarter, "c": msg,
"s": status, "e": err, "sa": datetime.now() if status == "sent" else None})
db.commit()
print("== notification_logs 已记录: status=%s" % status) print("== notification_logs 已记录: status=%s" % status)
if rc != EXIT_OK:
alert_failure("企微中继投递失败: %s" % err)
return rc
finally: finally:
db.close() db.close()
def main():
ap = argparse.ArgumentParser()
ap.add_argument("--push", action="store_true", help="真正推送到企微中继(默认 dry-run 只打印)")
ap.add_argument("--entity-id", type=int, default=None, help="仅指定企业")
args = ap.parse_args()
try:
rc = _run(args)
except SystemExit:
raise
except BaseException as exc: # 含 DB 连接失败;保证任何异常都有留痕+告警+非零退出码
print("== 执行异常: %s" % exc)
print(traceback.format_exc())
alert_failure("执行异常: %s: %s" % (type(exc).__name__, exc))
sys.exit(EXIT_ERROR)
sys.exit(rc)
if __name__ == "__main__": if __name__ == "__main__":
main() main()