opsBot
6a48477d71
ci/woodpecker/push/main Pipeline was canceled
- 起因: 项目Bot 复验 t2 时 caseD(挂死桩/上限10s) exit=1 elapsed=50s - 根因: 目标接受TCP但不响应 -> 单次探测耗满 --max-time 2s, 次数上限(x2) 使 10s 上限实跑 50s, 默认 30s 最坏 150s - 修正: while 条件改墙钟 [ $(date +%s) - READY_T0 -lt SMOKE_READY_MAX ], deploy.sh 与 .woodpecker/main.yml 同步 - 复验: A(慢起6s,cap30)=exit0/6s; B(空端口,cap3)=exit1/3s; C(200+degraded,cap3)=exit1/3s; D(挂死,cap10)=exit1/10s(修正前50s) - task: t2-cma-deploy-readiness-poll-20260915 (follow-up)
167 lines
7.0 KiB
Bash
Executable File
167 lines
7.0 KiB
Bash
Executable File
#!/bin/bash
|
||
# 管理会计OS — 一键拉取、构建并部署到生产
|
||
set -e
|
||
|
||
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
|
||
|
||
echo "===== 管理会计OS 生产部署 ====="
|
||
echo ""
|
||
|
||
# 0. 提交纪律检查：本地有未提交修改则中止，防止 git pull 覆盖丢失
|
||
echo "[0/5] 提交纪律检查..."
|
||
cd "$SCRIPT_DIR"
|
||
DIRTY=$(git status --porcelain 2>/dev/null | grep -v "__pycache__" | head -20)
|
||
if [ -n "$DIRTY" ]; then
|
||
echo "❌ 检测到本地未提交的修改，中止部署（防止 git pull 覆盖丢失代码）:"
|
||
echo "$DIRTY"
|
||
echo ""
|
||
echo "请先提交代码: git add -A && git commit -m '...' && git push origin main"
|
||
exit 1
|
||
fi
|
||
echo " ✓ 工作区干净，无未提交修改"
|
||
|
||
# 1. 拉取最新代码
|
||
echo "[1/5] 拉取最新代码..."
|
||
git pull origin main
|
||
echo " ✓ 代码已更新"
|
||
|
||
# 1b. 鉴权面审计闸门（T1, 2026-09-15）：B 类缺口(用户端点无鉴权)存在即中止部署
|
||
echo "[1b/5] 鉴权面审计闸门..."
|
||
cd "$SCRIPT_DIR/backend"
|
||
if ! ./venv/bin/python3 scripts/audit_api_auth_surface.py > /tmp/cma-auth-audit.log 2>&1; then
|
||
echo "❌ 鉴权面审计发现 B 类缺口（用户端点缺鉴权），中止部署。详见 /tmp/cma-auth-audit.log"
|
||
tail -20 /tmp/cma-auth-audit.log
|
||
exit 1
|
||
fi
|
||
cd "$SCRIPT_DIR"
|
||
echo " ✓ 鉴权面审计通过（无 B 类缺口）"
|
||
|
||
# 2. 构建前端
|
||
echo "[2/5] 构建前端..."
|
||
cd "$SCRIPT_DIR/frontend"
|
||
pnpm build
|
||
echo " ✓ 构建完成"
|
||
|
||
# 3. 部署前端到 Nginx
|
||
echo "[3/5] 部署前端..."
|
||
rm -rf /var/www/cma/assets/
|
||
cp -r dist/* /var/www/cma/
|
||
echo " ✓ 部署完成"
|
||
|
||
# 4. 更新后端依赖（失败不阻断 — 依赖已在系统Python中，避免PEP668中断部署）
|
||
echo "[4/5] 更新后端依赖..."
|
||
cd "$SCRIPT_DIR/backend"
|
||
if [ -f venv/bin/activate ]; then
|
||
source venv/bin/activate
|
||
pip install -r requirements.txt --quiet --no-cache-dir || echo " ⚠️ 依赖更新失败（不阻断）"
|
||
deactivate
|
||
else
|
||
echo " ⚠️ venv为软链精简版，跳过依赖更新"
|
||
fi
|
||
echo " ✓ 依赖检查完成"
|
||
|
||
# 5. 重启后端服务
|
||
echo "[5/5] 重启后端服务..."
|
||
systemctl daemon-reload
|
||
systemctl restart cma-backend
|
||
systemctl reload nginx 2>/dev/null || nginx -s reload
|
||
echo " ✓ 服务重启完成"
|
||
|
||
# ── 部署后冒烟测试 ──
|
||
echo ""
|
||
echo "===== 冒烟测试 ====="
|
||
# 就绪轮询（2026-09-15 t2-cma-deploy-readiness-poll）：journalctl 实测冷启动就绪间隔 2/2/3/3/5s，
|
||
# 固定 sleep 3 会让贴边/超时的冷启动误判为冒烟 ❌（假告警，污染后续判断）。
|
||
# 判据 = HTTP 200 **且** 响应体含 {"status":"ok"}；每 0.5s 探一次，上限 SMOKE_READY_MAX 秒（默认 30）。
|
||
# 超时 → 明确报错 + exit 非 0（既不静默放行，也不让后续 4 项冒烟产出次生噪音掩盖根因）。
|
||
# SMOKE_HEALTH_URL / SMOKE_READY_MAX 可覆盖，供离线演练（桩服务 / 空端口 / 短上限）。
|
||
# >>> cma-readiness-poll（最小可抽取块：scripts/tests 按行首标记 awk 抽取后原样执行，改结构请沿用标记）
|
||
SMOKE_READY_MAX="${SMOKE_READY_MAX:-30}"
|
||
HEALTH_URL="${SMOKE_HEALTH_URL:-http://127.0.0.1:8010/health}"
|
||
READY_TRY=0
|
||
READY_T0=$(date +%s)
|
||
READY_OK=0
|
||
READY_BODY_FILE="$(mktemp -t cma-ready-body.XXXXXX)"
|
||
# 上限按**墙钟**判定（不是轮询次数）：目标"接受连接但不响应"时单次探测要耗满 --max-time 2s，
|
||
# 用次数当上限会让"上限 30s"实际跑到 150s。墙钟判定下最坏 = SMOKE_READY_MAX + 单次探测(≤2s) 的余量。
|
||
while [ $(( $(date +%s) - READY_T0 )) -lt "$SMOKE_READY_MAX" ]; do
|
||
READY_CODE=$(curl -s -o "$READY_BODY_FILE" -w '%{http_code}' --max-time 2 "$HEALTH_URL" 2>/dev/null || echo 000)
|
||
if [ "$READY_CODE" = "200" ] && grep -q '"status"[[:space:]]*:[[:space:]]*"ok"' "$READY_BODY_FILE" 2>/dev/null; then
|
||
READY_OK=1
|
||
break
|
||
fi
|
||
sleep 0.5
|
||
READY_TRY=$((READY_TRY + 1))
|
||
done
|
||
READY_ELAPSED=$(( $(date +%s) - READY_T0 ))
|
||
if [ "$READY_OK" -ne 1 ]; then
|
||
READY_LAST=$(head -c 200 "$READY_BODY_FILE" 2>/dev/null)
|
||
rm -f "$READY_BODY_FILE"
|
||
echo " ⏱ 就绪探测: 耗时 ${READY_ELAPSED}s / 轮询 ${READY_TRY} 次（上限 ${SMOKE_READY_MAX}s）→ ❌ 未就绪"
|
||
echo ""
|
||
echo "🚨 后端就绪超时（${SMOKE_READY_MAX}s）：$HEALTH_URL 未返回 HTTP 200 + {\"status\":\"ok\"}"
|
||
echo " 最近响应: ${READY_LAST:-<无响应>}"
|
||
echo " 请检查后端日志: journalctl -u cma-backend -n 50"
|
||
exit 1
|
||
fi
|
||
rm -f "$READY_BODY_FILE"
|
||
# <<< cma-readiness-poll
|
||
echo " ⏱ 就绪探测: 耗时 ${READY_ELAPSED}s / 轮询 ${READY_TRY} 次（上限 ${SMOKE_READY_MAX}s，命中就绪即停）"
|
||
SMOKE_FAIL=0
|
||
echo " ✅ 后端健康检查通过（${HEALTH_URL} → status=ok）"
|
||
if curl -sf http://127.0.0.1:8010/api/cma/bot/ping > /dev/null 2>&1; then
|
||
echo " ✅ BOT桥接API通过"
|
||
else
|
||
echo " ❌ BOT桥接API失败"; SMOKE_FAIL=1
|
||
fi
|
||
# Schema 一致性检查（数据库与ORM结构对齐）
|
||
if /root/cma-management/backend/venv/bin/python3 /root/cma-management/backend/scripts/schema_check.py > /dev/null 2>&1; then
|
||
echo " ✅ Schema一致性通过"
|
||
else
|
||
echo " ❌ Schema不一致，请检查: cd /root/cma-management/backend && venv/bin/python3 scripts/schema_check.py"
|
||
SMOKE_FAIL=1
|
||
fi
|
||
# 凭据同源（2026-09-14 cma-defaultcred-code-fix）：登录冒烟不再硬编码默认账号/口令 —— 轮换后必须自动跟随。
|
||
# 落点：$CMA_ADMIN_ENV 或 600 密钥文件（默认 wecom-ops 主副本）；缺失/缺键 → 显式报错并中止，禁止静默退回默认口令。
|
||
CMA_ADMIN_ENV_FILE="${CMA_ADMIN_ENV:-/root/.hermes/profiles/wecom-ops/.secrets/cma-admin.env}"
|
||
if [ ! -f "$CMA_ADMIN_ENV_FILE" ]; then
|
||
echo " ❌ 管理员凭据文件缺失: $CMA_ADMIN_ENV_FILE"
|
||
echo " 请准备 600 权限密钥文件（键名 CMA_ADMIN_USER / CMA_ADMIN_PASS），或用 CMA_ADMIN_ENV 指定路径"
|
||
exit 1
|
||
fi
|
||
set -a
|
||
# shellcheck disable=SC1090
|
||
. "$CMA_ADMIN_ENV_FILE"
|
||
set +a
|
||
if [ -z "${CMA_ADMIN_USER:-}" ] || [ -z "${CMA_ADMIN_PASS:-}" ]; then
|
||
echo " ❌ 凭据文件缺少 CMA_ADMIN_USER / CMA_ADMIN_PASS: $CMA_ADMIN_ENV_FILE"
|
||
exit 1
|
||
fi
|
||
echo " ℹ️ 登录冒烟凭据来源: $CMA_ADMIN_ENV_FILE（账号 ${CMA_ADMIN_USER}，口令不回显）"
|
||
TOKEN=$(curl -s -X POST http://127.0.0.1:8010/api/cma/auth/login \
|
||
-H "Content-Type: application/json" \
|
||
-d "{\"username\":\"${CMA_ADMIN_USER}\",\"password\":\"${CMA_ADMIN_PASS}\",\"entity_id\":1}" | python3 -c "import sys,json; print(json.load(sys.stdin).get('token',''))" 2>/dev/null)
|
||
if [ -n "$TOKEN" ]; then
|
||
echo " ✅ 登录API通过"
|
||
if curl -sf -H "Authorization: Bearer $TOKEN" http://127.0.0.1:8010/api/cma/kpis > /dev/null 2>&1; then
|
||
echo " ✅ KPI接口通过"
|
||
else
|
||
echo " ❌ KPI接口失败"; SMOKE_FAIL=1
|
||
fi
|
||
else
|
||
echo " ❌ 登录API失败"; SMOKE_FAIL=1
|
||
fi
|
||
|
||
if [ "$SMOKE_FAIL" -eq 1 ]; then
|
||
echo ""
|
||
echo "🚨 冒烟测试未全部通过！请检查后端日志: journalctl -u cma-backend -n 50"
|
||
exit 1
|
||
fi
|
||
echo " ✅ 全部冒烟测试通过"
|
||
|
||
echo ""
|
||
echo "===== 部署完成 ====="
|
||
echo " 前端: https://cma.sxbh.ltd"
|
||
echo " 后端: http://127.0.0.1:8010/docs"
|
||
echo " 后端状态: systemctl status cma-backend"
|