fix(okr): 季度复盘提醒审计口径与失败感知加固 — dry_run/sent/failed/skipped 状态语义 + 失败本地留痕+中继告警+退出码契约(0/1/2) (fullstack-okr-reminder-hardening-20260911)
This commit is contained in:
Hermes CI Fix
@@ -11,11 +11,47 @@
|
||||
cd /root/cma-management/backend && PYTHONPATH=. /usr/bin/python3 scripts/okr_review_reminder.py # dry-run
|
||||
cd /root/cma-management/backend && PYTHONPATH=. /usr/bin/python3 scripts/okr_review_reminder.py --push # 推送
|
||||
... --entity-id 1 # 仅某企业
|
||||
|
||||
────────────────────────────────────────────────────────────────────────
|
||||
审计口径（2026-09-12 fullstack-okr-reminder-hardening）：dry-run 与真实推送可区分
|
||||
- notification_logs.status 语义固定为四类：
|
||||
'dry_run' 仅打印，未产生任何外部通知（不调用中继）
|
||||
'skipped' --push 但当期无目标，未投递
|
||||
'sent' 真实投递成功（唯一会在中继侧留下消息的状态）
|
||||
'failed' 真实投递失败
|
||||
- 历史遗留：2026-09-11 的两行 status='pending' 实为 dry-run 记录，
|
||||
已迁移为 'dry_run'（'pending' 语义含混：既像"待推送"又像"已入队"，已废弃不再写入）。
|
||||
- 退出码契约（供 cron 感知失败）：
|
||||
0 = 正常（dry_run / sent / skipped）
|
||||
1 = --push 投递失败（中继不可用或返回错误）
|
||||
2 = 执行异常（DB 连接失败、脚本内部错误等）
|
||||
────────────────────────────────────────────────────────────────────────
|
||||
失败感知（2026-09-12 加固：首次触发失败不再无感知）
|
||||
- main() 全量 try/except 兜底：任何异常 → 写本地失败日志 + 尽力经同一中继发一条运维告警 + 退出码 2
|
||||
- --push 投递失败 → 同样写本地失败日志 + 尽力告警 + 退出码 1（原先静默退出 0）
|
||||
- 退出码非 0 会被 Hermes cron（no_agent）判定为脚本失败，产生
|
||||
"⚠ Cron watchdog 'OKR季度复盘提醒' script failed" 告警；该 job 的 failure_deliver
|
||||
已指向运维通道，告警不再只落本地（cron job 60d4f6fc0ec5）。
|
||||
- 本地失败日志（兜底留痕，中继也不可用时唯一可查痕迹）：
|
||||
/root/cma-management/backend/logs/okr_review_reminder_failures.log
|
||||
────────────────────────────────────────────────────────────────────────
|
||||
口径说明（季度为空的历史目标：**有意为之**，非遗漏）
|
||||
- 筛选条件显式包含 `quarter IS NULL OR quarter = ''`：截至 2026-09-12，44 个 active 目标中
|
||||
only 5 个标了 2026Q3，其余 39 个是战略地图导入的历史遗留（无季度标记）。
|
||||
- 一并纳入提醒的意图 = 推动这些历史目标补标季度或清理归档（每季度提醒一次直到被处理），
|
||||
而不是"提醒漏了过滤条件"。
|
||||
- 若日后不再需要，把 _build_query 里的 `OR o.quarter IS NULL OR o.quarter = ''` 去掉即可
|
||||
（改动点单点集中，勿改其他口径）。
|
||||
|
||||
测试钩子（仅本机验证用；不设置时全部走生产默认值，cron 运行时无感）
|
||||
- OKR_REVIEW_RELAY_URL 覆盖中继地址（默认 http://127.0.0.1:8800/send）
|
||||
- OKR_REVIEW_FAILLOG 覆盖失败日志路径（默认见上）
|
||||
"""
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import traceback
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
from datetime import datetime
|
||||
@@ -26,8 +62,15 @@ from sqlalchemy import text # noqa: E402
|
||||
|
||||
from app.database import get_session_local # noqa: E402
|
||||
|
||||
RELAY_URL = "http://127.0.0.1:8800/send" # 与 daily_push.py 同一中继通道（不新造通道）
|
||||
RELAY_URL = os.environ.get("OKR_REVIEW_RELAY_URL", "http://127.0.0.1:8800/send")
|
||||
SOURCE = "管理会计OS"
|
||||
JOB_ID = "60d4f6fc0ec5" # Hermes cron job（OKR季度复盘提醒），失败告警里带上便于定位
|
||||
FAIL_LOG = os.environ.get("OKR_REVIEW_FAILLOG",
|
||||
"/root/cma-management/backend/logs/okr_review_reminder_failures.log")
|
||||
|
||||
EXIT_OK = 0
|
||||
EXIT_PUSH_FAILED = 1
|
||||
EXIT_ERROR = 2
|
||||
|
||||
|
||||
def current_quarter(d=None):
|
||||
@@ -68,52 +111,129 @@ def push_wecom(msg):
|
||||
return {"ok": False, "error": "推送异常: %s" % e}
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--push", action="store_true", help="真正推送到企微中继（默认 dry-run 只打印）")
|
||||
ap.add_argument("--entity-id", type=int, default=None, help="仅指定企业")
|
||||
args = ap.parse_args()
|
||||
# ── 失败留痕 / 告警（2026-09-12 加固）────────────────────────────────────
|
||||
|
||||
def append_fail_log(line):
|
||||
"""本地兜底留痕：中继也不可用时，这里是唯一可查的失败痕迹。"""
|
||||
try:
|
||||
d = os.path.dirname(FAIL_LOG)
|
||||
if d:
|
||||
os.makedirs(d, exist_ok=True)
|
||||
with open(FAIL_LOG, "a", encoding="utf-8") as f:
|
||||
f.write("[%s] %s\n" % (datetime.now().strftime("%Y-%m-%d %H:%M:%S"), line))
|
||||
return True
|
||||
except Exception as e: # 兜底不能再抛异常
|
||||
print("== 本地失败日志写入失败: %s" % e)
|
||||
return False
|
||||
|
||||
|
||||
def alert_failure(reason):
|
||||
"""失败告警：本地留痕（必做，不依赖网络）+ 尽力经同一中继推送运维告警。
|
||||
|
||||
约束：不新增通道 —— 告警复用与提醒完全相同的中继（RELAY_URL / SOURCE）。
|
||||
返回 (logged, delivered)。
|
||||
"""
|
||||
append_fail_log("FAIL | %s" % reason)
|
||||
msg = ("## ❗ 管理会计OS · 运维告警\n\n"
|
||||
"**OKR 季度复盘提醒执行失败**（cron job `%s`）\n\n"
|
||||
"- 时间：%s\n"
|
||||
"- 原因：%s\n"
|
||||
"- 中继：%s\n\n"
|
||||
"排查：`scripts/okr_review_reminder.py`（本地失败日志见 backend/logs/"
|
||||
"okr_review_reminder_failures.log）。" % (
|
||||
JOB_ID, datetime.now().strftime("%Y-%m-%d %H:%M:%S"), reason, RELAY_URL))
|
||||
try:
|
||||
res = push_wecom(msg)
|
||||
delivered = bool(res.get("ok", True)) and not res.get("error")
|
||||
except Exception as e: # push_wecom 已自吞异常，这里再兜一层
|
||||
res, delivered = {"error": str(e)}, False
|
||||
append_fail_log("ALERT | delivered=%s | %s" % (
|
||||
delivered, json.dumps(res, ensure_ascii=False)[:300]))
|
||||
print("== 失败告警: delivered=%s | %s" % (delivered, json.dumps(res, ensure_ascii=False)[:200]))
|
||||
return True, delivered
|
||||
|
||||
|
||||
def _build_query(entity_id, quarter):
|
||||
sql = ("SELECT o.id, o.title, o.owner, o.progress, o.confidence, o.quarter, o.entity_id, "
|
||||
"e.name AS entity_name "
|
||||
"FROM objectives o LEFT JOIN entities e ON e.id = o.entity_id "
|
||||
"WHERE o.status = 'active'")
|
||||
params = {}
|
||||
if entity_id is not None:
|
||||
sql += " AND o.entity_id = :eid"
|
||||
params["eid"] = entity_id
|
||||
# 本季度 + 未设季度（历史遗留）的目标都纳入提醒 —— 有意为之，见模块 docstring「口径说明」
|
||||
sql += " AND (o.quarter = :q OR o.quarter IS NULL OR o.quarter = '')"
|
||||
params["q"] = quarter
|
||||
sql += " ORDER BY o.entity_id, o.id"
|
||||
return sql, params
|
||||
|
||||
|
||||
def _record(db, quarter, msg, status, err, sent_at):
|
||||
"""复用既有 notifications 机制留存审计（notification_logs）"""
|
||||
db.execute(text(
|
||||
"INSERT INTO notification_logs (channel, recipient, title, content, status, error_msg, "
|
||||
"sent_at, created_at) VALUES ('wecom', :r, :t, :c, :s, :e, :sa, NOW())"),
|
||||
{"r": "企微群(中继)", "t": "OKR 季度复盘提醒（%s）" % quarter, "c": msg,
|
||||
"s": status, "e": err, "sa": sent_at})
|
||||
db.commit()
|
||||
|
||||
|
||||
def _run(args):
|
||||
"""返回退出码（0=正常 1=投递失败）；异常交给 main() 兜底。"""
|
||||
quarter = current_quarter()
|
||||
db = get_session_local()()
|
||||
try:
|
||||
sql = ("SELECT o.id, o.title, o.owner, o.progress, o.confidence, o.quarter, o.entity_id, e.name AS entity_name "
|
||||
"FROM objectives o LEFT JOIN entities e ON e.id = o.entity_id "
|
||||
"WHERE o.status = 'active'")
|
||||
params = {}
|
||||
if args.entity_id is not None:
|
||||
sql += " AND o.entity_id = :eid"
|
||||
params["eid"] = args.entity_id
|
||||
# 本季度 + 未设季度（历史遗留）的目标都纳入提醒
|
||||
sql += " AND (o.quarter = :q OR o.quarter IS NULL OR o.quarter = '')"
|
||||
params["q"] = quarter
|
||||
sql += " ORDER BY o.entity_id, o.id"
|
||||
sql, params = _build_query(args.entity_id, quarter)
|
||||
rows = [dict(r._mapping) for r in db.execute(text(sql), params).fetchall()]
|
||||
|
||||
msg = build_message(rows, quarter)
|
||||
print("== 目标数: %d（季度 %s；entity_id=%s）" % (len(rows), quarter, args.entity_id))
|
||||
print(msg)
|
||||
|
||||
sent_at = None
|
||||
if args.push and rows:
|
||||
res = push_wecom(msg)
|
||||
ok = bool(res.get("ok", True)) and not res.get("error")
|
||||
print("== 推送结果:", json.dumps(res, ensure_ascii=False)[:300])
|
||||
status = "sent" if ok else "failed"
|
||||
err = None if ok else str(res.get("error"))[:400]
|
||||
if ok:
|
||||
status, err, sent_at, rc = "sent", None, datetime.now(), EXIT_OK
|
||||
else:
|
||||
status, err, rc = "failed", str(res.get("error"))[:400], EXIT_PUSH_FAILED
|
||||
elif args.push:
|
||||
status, err, rc = "skipped", None, EXIT_OK
|
||||
print("== 无目标，跳过投递")
|
||||
else:
|
||||
status, err = "pending", None
|
||||
status, err, rc = "dry_run", None, EXIT_OK
|
||||
print("== dry-run：不调用中继、不产生外部通知")
|
||||
|
||||
# 复用既有 notifications 机制留存审计（notification_logs）
|
||||
db.execute(text(
|
||||
"INSERT INTO notification_logs (channel, recipient, title, content, status, error_msg, sent_at, created_at) "
|
||||
"VALUES ('wecom', :r, :t, :c, :s, :e, :sa, NOW())"),
|
||||
{"r": "企微群(中继)", "t": "OKR 季度复盘提醒（%s）" % quarter, "c": msg,
|
||||
"s": status, "e": err, "sa": datetime.now() if status == "sent" else None})
|
||||
db.commit()
|
||||
_record(db, quarter, msg, status, err, sent_at)
|
||||
print("== notification_logs 已记录: status=%s" % status)
|
||||
|
||||
if rc != EXIT_OK:
|
||||
alert_failure("企微中继投递失败: %s" % err)
|
||||
return rc
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--push", action="store_true", help="真正推送到企微中继（默认 dry-run 只打印）")
|
||||
ap.add_argument("--entity-id", type=int, default=None, help="仅指定企业")
|
||||
args = ap.parse_args()
|
||||
|
||||
try:
|
||||
rc = _run(args)
|
||||
except SystemExit:
|
||||
raise
|
||||
except BaseException as exc: # 含 DB 连接失败；保证任何异常都有留痕+告警+非零退出码
|
||||
print("== 执行异常: %s" % exc)
|
||||
print(traceback.format_exc())
|
||||
alert_failure("执行异常: %s: %s" % (type(exc).__name__, exc))
|
||||
sys.exit(EXIT_ERROR)
|
||||
sys.exit(rc)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
|
||||
Reference in New Issue
Block a user