fix(deploy): 就绪轮询上限改墙钟判定(次数上限在挂死目标下会超时 5 倍)
ci/woodpecker/push/main Pipeline was canceled

- 起因: 项目Bot 复验 t2 时 caseD(挂死桩/上限10s) exit=1 elapsed=50s
- 根因: 目标接受TCP但不响应 -> 单次探测耗满 --max-time 2s, 次数上限(x2) 使 10s 上限实跑 50s, 默认 30s 最坏 150s
- 修正: while 条件改墙钟 [ $(date +%s) - READY_T0 -lt SMOKE_READY_MAX ], deploy.sh 与 .woodpecker/main.yml 同步
- 复验: A(慢起6s,cap30)=exit0/6s; B(空端口,cap3)=exit1/3s; C(200+degraded,cap3)=exit1/3s; D(挂死,cap10)=exit1/10s(修正前50s)
- task: t2-cma-deploy-readiness-poll-20260915 (follow-up)
This commit is contained in:
opsBot
2026-09-15 17:11:35 +08:00
parent b7950249a7
commit 6a48477d71
2 changed files with 4 additions and 3 deletions
+1 -1
View File
@@ -69,6 +69,6 @@ steps:
commands:
- *ssh_setup
- |
ssh root@git.sxbh.ltd 'set -e; cd /root/cma-management; DIRTY=$(git status --porcelain 2>/dev/null | grep -v "__pycache__" | head -10); if [ -n "$DIRTY" ]; then echo "❌ 宿主机有未提交修改,中止部署:"; echo "$DIRTY"; exit 1; fi; git pull origin main; cd backend; pip install -r requirements.txt --quiet --no-cache-dir; systemctl restart cma-backend; SMOKE_READY_MAX="${SMOKE_READY_MAX:-30}"; READY_TRY=0; READY_OK=0; READY_BODY=/tmp/cma-ci-ready-body.json; while [ "$READY_TRY" -lt $(( SMOKE_READY_MAX * 2 )) ]; do CODE=$(curl -s -o "$READY_BODY" -w "%{http_code}" --max-time 2 http://127.0.0.1:8010/health || echo 000); if [ "$CODE" = "200" ] && grep -q "\"status\":\"ok\"" "$READY_BODY" 2>/dev/null; then READY_OK=1; break; fi; sleep 0.5; READY_TRY=$(( READY_TRY + 1 )); done; if [ "$READY_OK" != "1" ]; then echo "🚨 后端就绪超时 ${SMOKE_READY_MAX}s(未达 HTTP200 + status=ok)"; cat "$READY_BODY" 2>/dev/null; exit 1; fi; echo "✅ 后端就绪(轮询 ${READY_TRY} 次,上限 ${SMOKE_READY_MAX}s)"; cat "$READY_BODY"'
ssh root@git.sxbh.ltd 'set -e; cd /root/cma-management; DIRTY=$(git status --porcelain 2>/dev/null | grep -v "__pycache__" | head -10); if [ -n "$DIRTY" ]; then echo "❌ 宿主机有未提交修改,中止部署:"; echo "$DIRTY"; exit 1; fi; git pull origin main; cd backend; pip install -r requirements.txt --quiet --no-cache-dir; systemctl restart cma-backend; SMOKE_READY_MAX="${SMOKE_READY_MAX:-30}"; READY_TRY=0; READY_OK=0; READY_T0=$(date +%s); READY_BODY=/tmp/cma-ci-ready-body.json; while [ $(( $(date +%s) - READY_T0 )) -lt "$SMOKE_READY_MAX" ]; do CODE=$(curl -s -o "$READY_BODY" -w "%{http_code}" --max-time 2 http://127.0.0.1:8010/health || echo 000); if [ "$CODE" = "200" ] && grep -q "\"status\":\"ok\"" "$READY_BODY" 2>/dev/null; then READY_OK=1; break; fi; sleep 0.5; READY_TRY=$(( READY_TRY + 1 )); done; if [ "$READY_OK" != "1" ]; then echo "🚨 后端就绪超时 ${SMOKE_READY_MAX}s(未达 HTTP200 + status=ok)"; cat "$READY_BODY" 2>/dev/null; exit 1; fi; echo "✅ 后端就绪(轮询 ${READY_TRY} 次,上限 ${SMOKE_READY_MAX}s)"; cat "$READY_BODY"'
when:
- path: backend/**
+3 -2
View File
@@ -78,12 +78,13 @@ echo "===== 冒烟测试 ====="
# >>> cma-readiness-poll(最小可抽取块:scripts/tests 按行首标记 awk 抽取后原样执行,改结构请沿用标记)
SMOKE_READY_MAX="${SMOKE_READY_MAX:-30}"
HEALTH_URL="${SMOKE_HEALTH_URL:-http://127.0.0.1:8010/health}"
READY_TRIES=$(( SMOKE_READY_MAX * 2 ))
READY_TRY=0
READY_T0=$(date +%s)
READY_OK=0
READY_BODY_FILE="$(mktemp -t cma-ready-body.XXXXXX)"
while [ "$READY_TRY" -lt "$READY_TRIES" ]; do
# 上限按**墙钟**判定(不是轮询次数):目标"接受连接但不响应"时单次探测要耗满 --max-time 2s,
# 用次数当上限会让"上限 30s"实际跑到 150s。墙钟判定下最坏 = SMOKE_READY_MAX + 单次探测(≤2s) 的余量。
while [ $(( $(date +%s) - READY_T0 )) -lt "$SMOKE_READY_MAX" ]; do
READY_CODE=$(curl -s -o "$READY_BODY_FILE" -w '%{http_code}' --max-time 2 "$HEALTH_URL" 2>/dev/null || echo 000)
if [ "$READY_CODE" = "200" ] && grep -q '"status"[[:space:]]*:[[:space:]]*"ok"' "$READY_BODY_FILE" 2>/dev/null; then
READY_OK=1