Hermes CI Fix
443a779ece
ci/woodpecker/push/main Pipeline failed
根治 2026-09-16 发布中断（新增模型触发启动 create_all，cma_api 无 CREATE → 崩溃循环）： - backend/app/database.py: 新增 CMA_AUTO_CREATE 开关（**默认关**），生产启动不再执行 DDL； 运行库账号因此无需 CREATE（最小权限可保持） - backend/scripts/db_migrate.py: 特权迁移步（--check/--apply/--rollback/--status）； 幂等（checkfirst）+ journal 留痕 + 可回滚（dump 失败/表有数据即拒绝，fail-closed）； 显式拒绝用应用运行账号 cma_api 执行 - deploy.sh: 新增 [4b] 结构迁移（**重启前**执行；失败即中止部署，后端旧码继续服务） - backend/scripts/tests/db-migrate-sim-e2e.sh: 17 项 scratch 复现/验收（负控制复现 CREATE command denied） - docs/db-migration-runbook.md: 运行手册（含 cma_api CREATE 回收步骤） 验收：scratch E2E 17/17；pytest 1113 passed/1 xfailed；生产 --check 缺失表 0 / 列级差异 0； --apply 幂等空跑 created_new=0（journal 留痕）
155 lines
9.2 KiB
Bash
155 lines
9.2 KiB
Bash
#!/usr/bin/env bash
|
||
# CMA「启动 create_all × 运行库最小权限」根治 —— 复现 + 验收 E2E
|
||
# 任务: cma-create-all-minpriv-rootfix-20260916
|
||
#
|
||
# 零生产触碰：全程只用 scratch 库 cma_migrate_sim + scratch 账号 cma_sim
|
||
# （权限画像与运行库 cma_api 一致：只有 SELECT/INSERT/UPDATE/DELETE，**无 CREATE/DROP**）。
|
||
#
|
||
# 用例：
|
||
# T1 负控制（复现事故）：旧行为（CMA_AUTO_CREATE=1）用无 DDL 权限账号启动 → CREATE command denied → 启动失败
|
||
# T2 迁移步幂等：--apply 首跑建全表，二跑 created_new=0
|
||
# T3 修复后启动：CMA_AUTO_CREATE 未设（默认关）→ 无 DDL、启动成功（/health=200）
|
||
# T4 验收关键路径：模拟"新增模型未迁移"（删一张表）→ 应用照常启动不中断；随后 --apply 补齐
|
||
# T5 回滚：journal --rollback 删除该次新建表；有数据的表拒绝回滚（exit 3），--force 才放行
|
||
# T6 清理：drop scratch 库 + 账号
|
||
#
|
||
# 用法: bash backend/scripts/tests/db-migrate-sim-e2e.sh （需 root，因走 mysql unix socket）
|
||
set -uo pipefail
|
||
|
||
SIM_DB="${SIM_DB:-cma_migrate_sim}"
|
||
SIM_USER="${SIM_USER:-cma_sim}"
|
||
SIM_PASS="$(openssl rand -hex 12)"
|
||
SIM_PORT="${SIM_PORT:-8013}"
|
||
SIM_SOCKET="${SIM_SOCKET:-/var/run/mysqld/mysqld.sock}"
|
||
BACKEND="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
|
||
WORK="$(mktemp -d -t cma-migrate-sim.XXXXXX)"
|
||
JOURNAL_DIR="$WORK/journal"
|
||
URL="http://127.0.0.1:${SIM_PORT}/health"
|
||
|
||
PASS=0; FAIL=0
|
||
ok() { echo " PASS $*"; PASS=$((PASS+1)); }
|
||
bad() { echo " FAIL $*"; FAIL=$((FAIL+1)); }
|
||
hdr() { echo; echo "── $* ──"; }
|
||
|
||
MYSQL_ROOT="mysql --socket=$SIM_SOCKET -u root"
|
||
SIM_ENV=(env "CMA_DB_HOST=127.0.0.1" "CMA_DB_PORT=3306" "CMA_DB_USER=$SIM_USER" "CMA_DB_PASS=$SIM_PASS" "CMA_DB_NAME=$SIM_DB")
|
||
MIG_ENV=(env "CMA_DB_NAME=$SIM_DB" "CMA_MIGRATE_JOURNAL_DIR=$JOURNAL_DIR" "CMA_MIGRATE_DB_SOCKET=$SIM_SOCKET")
|
||
|
||
wait_health() { # 轮询 /health，最多 $1 秒
|
||
local max="${1:-40}" t0=$SECONDS
|
||
while [ $((SECONDS-t0)) -lt "$max" ]; do
|
||
if curl -s -o "$WORK/health.json" -w '%{http_code}' --max-time 2 "$URL" 2>/dev/null | grep -q '^200$'; then
|
||
grep -q '"status"[[:space:]]*:[[:space:]]*"ok"' "$WORK/health.json" && return 0
|
||
fi
|
||
sleep 0.5
|
||
done
|
||
return 1
|
||
}
|
||
|
||
boot() { # boot <logfile> [extra env...] —— 后台起应用，返回 pid
|
||
local log="$1"; shift
|
||
( cd "$BACKEND" && "${SIM_ENV[@]}" "$@" ./venv/bin/python3 -m uvicorn app.main:app \
|
||
--host 127.0.0.1 --port "$SIM_PORT" --no-access-log ) >"$log" 2>&1 &
|
||
echo $!
|
||
}
|
||
foreground_boot() { # 前台起应用（预期失败用）
|
||
local log="$1"; shift
|
||
( cd "$BACKEND" && "${SIM_ENV[@]}" "$@" timeout 60 ./venv/bin/python3 -m uvicorn app.main:app \
|
||
--host 127.0.0.1 --port "$SIM_PORT" --no-access-log ) >"$log" 2>&1
|
||
}
|
||
|
||
cleanup() {
|
||
[ -n "${APP_PID:-}" ] && kill "$APP_PID" 2>/dev/null
|
||
pkill -f "uvicorn app.main:app --host 127.0.0.1 --port $SIM_PORT" 2>/dev/null
|
||
$MYSQL_ROOT -e "DROP DATABASE IF EXISTS \`$SIM_DB\`; DROP USER IF EXISTS '$SIM_USER'@'localhost', '$SIM_USER'@'127.0.0.1';" >/dev/null 2>&1
|
||
echo; echo "（scratch 清理完成：库 $SIM_DB / 账号 $SIM_USER 已删除；工作目录 $WORK 保留供证据）"
|
||
}
|
||
trap cleanup EXIT
|
||
|
||
echo "════ CMA 特权迁移步 E2E（scratch: $SIM_DB / $SIM_USER / port $SIM_PORT）════"
|
||
[ "$(id -u)" = "0" ] || { echo "需要 root（mysql unix socket）"; exit 2; }
|
||
|
||
# ── T0 setup ──
|
||
hdr "T0 setup：scratch 库 + 无 DDL 权限账号（画像同 cma_api）"
|
||
$MYSQL_ROOT -e "
|
||
DROP DATABASE IF EXISTS \`$SIM_DB\`;
|
||
CREATE DATABASE \`$SIM_DB\` DEFAULT CHARACTER SET utf8mb4;
|
||
DROP USER IF EXISTS '$SIM_USER'@'localhost', '$SIM_USER'@'127.0.0.1';
|
||
CREATE USER '$SIM_USER'@'localhost' IDENTIFIED BY '$SIM_PASS';
|
||
CREATE USER '$SIM_USER'@'127.0.0.1' IDENTIFIED BY '$SIM_PASS';
|
||
GRANT SELECT, INSERT, UPDATE, DELETE ON \`$SIM_DB\`.* TO '$SIM_USER'@'localhost';
|
||
GRANT SELECT, INSERT, UPDATE, DELETE ON \`$SIM_DB\`.* TO '$SIM_USER'@'127.0.0.1';
|
||
FLUSH PRIVILEGES;" 2>/dev/null
|
||
GRANTS=$($MYSQL_ROOT -N -e "SHOW GRANTS FOR '$SIM_USER'@'127.0.0.1';" 2>/dev/null)
|
||
echo "$GRANTS" | grep -q "CREATE" && bad "scratch 账号不应有 CREATE：$GRANTS" || ok "scratch 账号无 CREATE/DROP（权限画像同 cma_api）"
|
||
|
||
# ── T1 负控制 ──
|
||
hdr "T1 负控制（旧行为复现）：CMA_AUTO_CREATE=1 + 无 DDL 权限账号"
|
||
foreground_boot "$WORK/t1-old-behavior.log" "CMA_AUTO_CREATE=1"
|
||
if grep -q "CREATE command denied" "$WORK/t1-old-behavior.log"; then
|
||
ok "复现成功：$(grep -m1 -o "CREATE command denied to user '[^']*' for table '[^']*'" "$WORK/t1-old-behavior.log")"
|
||
else
|
||
bad "未复现 CREATE command denied（见 $WORK/t1-old-behavior.log）"
|
||
fi
|
||
grep -q "Application startup complete" "$WORK/t1-old-behavior.log" \
|
||
&& bad "旧行为下应用竟启动成功（与事故不符）" \
|
||
|| ok "旧行为下应用启动失败（= 2026-09-16 崩溃循环根因）"
|
||
|
||
# ── T2 迁移步幂等 ──
|
||
hdr "T2 特权迁移步：--check / --apply 首跑 / --apply 二跑"
|
||
CHECK1="$(cd "$BACKEND" && "${MIG_ENV[@]}" ./venv/bin/python3 scripts/db_migrate.py --check 2>&1)"
|
||
echo "$CHECK1" | grep -q "缺失表" && ok "T2a --check 列出缺失表：$(echo "$CHECK1" | grep -m1 '缺失表')" || bad "T2a --check 无输出"
|
||
APPLY1="$(cd "$BACKEND" && "${MIG_ENV[@]}" ./venv/bin/python3 scripts/db_migrate.py --apply 2>&1)"; RC1=$?
|
||
N1="$(echo "$APPLY1" | grep -m1 -o '新建 [0-9]* 张表' | grep -o '[0-9]*')"
|
||
echo "$APPLY1" | sed 's/^/ /'
|
||
[ "$RC1" = 0 ] && ok "T2b 首跑 exit=0，created_new=$N1" || bad "T2b 首跑 exit=$RC1"
|
||
APPLY2="$(cd "$BACKEND" && "${MIG_ENV[@]}" ./venv/bin/python3 scripts/db_migrate.py --apply 2>&1)"; RC2=$?
|
||
N2="$(echo "$APPLY2" | grep -m1 -o '新建 [0-9]* 张表' | grep -o '[0-9]*')"
|
||
echo "$APPLY2" | sed 's/^/ /'
|
||
[ "$RC2" = 0 ] && [ "$N2" = "0" ] && ok "T2c 二跑幂等：exit=0，created_new=0" || bad "T2c 二跑 exit=$RC2 created_new=$N2"
|
||
|
||
# ── T3 修复后启动（默认关）──
|
||
hdr "T3 修复后启动：CMA_AUTO_CREATE 未设（默认关）"
|
||
APP_PID=$(boot "$WORK/t3-fixed-boot.log")
|
||
if wait_health 40; then ok "启动成功：/health=200 status=ok（无 DDL）"; else bad "启动失败（见 $WORK/t3-fixed-boot.log）"; fi
|
||
grep -q "启动建表已关闭" "$WORK/t3-fixed-boot.log" && ok "日志确认：启动建表已关闭" || bad "未见'启动建表已关闭'日志"
|
||
kill "$APP_PID" 2>/dev/null; wait "$APP_PID" 2>/dev/null; APP_PID=""
|
||
|
||
# ── T4 验收关键路径：新增模型未迁移 ──
|
||
hdr "T4 新增模型未迁移：应用重启不中断 + 迁移步补齐"
|
||
$MYSQL_ROOT -e "DROP TABLE \`$SIM_DB\`.treasury_balances;" 2>/dev/null
|
||
ok "已模拟'新模型未迁移'（DROP TABLE treasury_balances）"
|
||
APP_PID=$(boot "$WORK/t4-boot-no-ddl.log")
|
||
if wait_health 40; then ok "T4a 表缺失下应用**照常启动**（无 CREATE command denied → 不再中断）"; else bad "T4a 启动失败（见 $WORK/t4-boot-no-ddl.log）"; fi
|
||
grep -q "CREATE command denied" "$WORK/t4-boot-no-ddl.log" && bad "T4a 启动期仍有 DDL 尝试" || ok "T4a 启动期零 DDL 尝试"
|
||
kill "$APP_PID" 2>/dev/null; wait "$APP_PID" 2>/dev/null; APP_PID=""
|
||
APPLY3="$(cd "$BACKEND" && "${MIG_ENV[@]}" ./venv/bin/python3 scripts/db_migrate.py --apply 2>&1)"; RC3=$?
|
||
N3="$(echo "$APPLY3" | grep -m1 -o '新建 [0-9]* 张表' | grep -o '[0-9]*')"
|
||
[ "$RC3" = 0 ] && [ "$N3" = "1" ] && ok "T4b 迁移步补齐缺失表：created_new=1（treasury_balances）" || { bad "T4b exit=$RC3 created_new=$N3"; echo "$APPLY3" | sed 's/^/ /'; }
|
||
APP_PID=$(boot "$WORK/t4-boot-migrated.log")
|
||
if wait_health 40; then ok "T4c 迁移后应用正常（表就位）"; else bad "T4c 迁移后启动失败"; fi
|
||
kill "$APP_PID" 2>/dev/null; wait "$APP_PID" 2>/dev/null; APP_PID=""
|
||
|
||
# ── T5 回滚 ──
|
||
hdr "T5 回滚（journal 驱动；有数据 fail-closed）"
|
||
LAST_J="$(ls -t "$JOURNAL_DIR"/*-applied.json | head -1)"
|
||
ROLL1="$(cd "$BACKEND" && "${MIG_ENV[@]}" ./venv/bin/python3 scripts/db_migrate.py --rollback "$LAST_J" 2>&1)"; RCR1=$?
|
||
echo "$ROLL1" | sed 's/^/ /'
|
||
[ "$RCR1" = 0 ] && ok "T5a 回滚成功（空表，含 dump）" || bad "T5a 回滚 exit=$RCR1"
|
||
$MYSQL_ROOT -N -e "SELECT COUNT(*) FROM information_schema.tables WHERE table_schema='$SIM_DB' AND table_name='treasury_balances';" | grep -q '^0$' \
|
||
&& ok "T5b 表确已删除（information_schema 回查=0）" || bad "T5b 表仍在"
|
||
# 有数据 → 拒绝
|
||
(cd "$BACKEND" && "${MIG_ENV[@]}" ./venv/bin/python3 scripts/db_migrate.py --apply >/dev/null 2>&1)
|
||
$MYSQL_ROOT -e "INSERT INTO \`$SIM_DB\`.treasury_balances (entity_id, account_id, period, balance, source, data_status) VALUES (1,1,'2026-09',1.0,'manual','estimated');" 2>/dev/null
|
||
ROWS=$($MYSQL_ROOT -N -e "SELECT COUNT(*) FROM \`$SIM_DB\`.treasury_balances;" 2>/dev/null)
|
||
[ "${ROWS:-0}" -gt 0 ] && ok "T5c 前置：已造 1 行数据（rows=$ROWS）" || bad "T5c 前置造数失败"
|
||
LAST_J="$(ls -t "$JOURNAL_DIR"/*-applied.json | head -1)"
|
||
ROLL2="$(cd "$BACKEND" && "${MIG_ENV[@]}" ./venv/bin/python3 scripts/db_migrate.py --rollback "$LAST_J" 2>&1)"; RCR2=$?
|
||
echo "$ROLL2" | sed 's/^/ /'
|
||
if [ "$RCR2" = 3 ]; then ok "T5c 有数据的表拒绝回滚（exit=3，fail-closed）"; else bad "T5c 期望 exit=3，实得 $RCR2"; fi
|
||
|
||
# ── 汇总 ──
|
||
hdr "汇总"
|
||
echo " PASS=$PASS FAIL=$FAIL （工作目录: $WORK）"
|
||
[ "$FAIL" = 0 ] && { echo " ✅ 全部通过"; exit 0; } || { echo " ❌ 有失败项"; exit 1; }
|