P2-ops: migration alignment, deploy test list, monitor alerts/single-host, backup/rollback closure, Caddy template
- migrations: MIGRATION_TABLES now equals POSTGRES_SCHEMA tables
(uploads, task_attachments, task_followups, environments added);
SEQUENCE_TABLES matches IDENTITY tables; task_followups
sender_task_id column ensured before insert; UNMIGRATED_TABLES
and migration_coverage() report tables with no PG schema instead
of silently skipping them; CLI prints the coverage report.
- deploy: DEFAULT_TEST_SUITES gains deploy/tests (discover -s only,
never package path); regression tests pin the default list.
- monitor: MONITOR_SINGLE_HOST=1 makes the server check single-host;
default stays dual to avoid silent downgrade; server_mode is in
the JSON. Optional fail-only alert outlet (webhook or mail,
default off, alert body is pass/fail JSON only, no secrets).
- backup/rollback: README and restore-drill document what is
recoverable (DB, code) vs not (worktrees, traces, uploads, host
configs); rollback.sh shares the server-unit list and dual-port
health success standard with host_deploy.sh (sync test guards it).
- public entry: Caddyfile uses native {$DOMAIN}/{$ACME_EMAIL}
variables plus a caddy.service template; README documents the
steps and the current BT-nginx production status.
This commit is contained in:
host committed
1 parent
ef532a38c2
commit
64995fafab
14 files changed
+953
-44
No files matched your search
+53
-14
@@ -10,9 +10,23 @@
|
||||
#
|
||||
# 回滚边界：只回代码，不回数据库（理由见 deploy.sh 头注释与
|
||||
# deploy/README.md：数据库迁移是只加列的幂等操作，老代码仍能读新列）。
|
||||
# 备份包来自 deploy.sh 每次部署前在主机上打的
|
||||
# 备份包来自每次部署前在主机上打的
|
||||
# /opt/codex-app-server-backups/codex-app-server-<时间戳>.tar.gz，
|
||||
# 本脚本恢复其中最新的一份；主机上没有备份包时直接报错退出，不编造。
|
||||
#
|
||||
# 与 host_deploy.sh 共享的约定（P2-ops，两脚本改动时必须同步，
|
||||
# deploy/tests/test_p2_ops.py 会双向核对，见文件尾注释）：
|
||||
# - server unit 清单：codex-cloud-server 必含，codex-cloud-server-2
|
||||
# 以 systemctl cat 探测为准（存在才纳入停/起与健康检查）；
|
||||
# - worker：autoscaler active 时停/起 autoscaler，否则 cloud-worker
|
||||
# （与 host_deploy.sh 的 worker_unit() 同款）；
|
||||
# - 成功标准：清单里每个 server 端口的 health 都通过（8765 必查，
|
||||
# server-2 存在时 8766 也必查），只通 8765 不算回滚成功——
|
||||
# 否则 LB 上游的 server-2 仍跑故障代码，回滚等于没回完。
|
||||
#
|
||||
# 仿真覆盖（仅测试用，生产用默认值）：CODEX_ROLLBACK_APP_DIR、
|
||||
# CODEX_ROLLBACK_BACKUP_DIR、CODEX_ROLLBACK_HEALTH_ATTEMPTS、
|
||||
# CODEX_ROLLBACK_HEALTH_INTERVAL。默认值与旧行为完全一致。
|
||||
set -eu
|
||||
|
||||
HOST=${1:-}
|
||||
@@ -33,12 +47,30 @@ ssh_cmd() {
|
||||
|
||||
ssh_cmd "$USER_ARG@$HOST" 'sh -s' <<'HOST_SCRIPT'
|
||||
set -eu
|
||||
APP_DIR=/opt/codex-app-server
|
||||
BACKUP_DIR=/opt/codex-app-server-backups
|
||||
APP_DIR=${CODEX_ROLLBACK_APP_DIR:-/opt/codex-app-server}
|
||||
BACKUP_DIR=${CODEX_ROLLBACK_BACKUP_DIR:-/opt/codex-app-server-backups}
|
||||
HEALTH_ATTEMPTS=${CODEX_ROLLBACK_HEALTH_ATTEMPTS:-12}
|
||||
HEALTH_INTERVAL=${CODEX_ROLLBACK_HEALTH_INTERVAL:-5}
|
||||
|
||||
# 与 host_deploy.sh 共享的 unit 清单约定（见本文件头注释）：
|
||||
# server-1 必含，server-2 以 unit 存在为准；改动清单时两脚本同步改。
|
||||
SERVER_UNITS="codex-cloud-server"
|
||||
HEALTH_PORTS="8765"
|
||||
if systemctl cat codex-cloud-server-2 >/dev/null 2>&1; then
|
||||
SERVER_UNITS="$SERVER_UNITS codex-cloud-server-2"
|
||||
HEALTH_PORTS="$HEALTH_PORTS 8766"
|
||||
fi
|
||||
if systemctl is-active -q codex-autoscaler 2>/dev/null; then
|
||||
WORKER_UNIT=codex-autoscaler
|
||||
else
|
||||
WORKER_UNIT=codex-cloud-worker
|
||||
fi
|
||||
|
||||
check_health() {
|
||||
OUT=$(curl -s --max-time 5 http://127.0.0.1:8765/health || true)
|
||||
[ -n "$OUT" ] && printf '%s' "$OUT" | python3 -c '
|
||||
for port in $HEALTH_PORTS; do
|
||||
OUT=$(curl -s --max-time 5 "http://127.0.0.1:$port/health" || true)
|
||||
[ -n "$OUT" ] || return 1
|
||||
if ! printf '%s' "$OUT" | python3 -c '
|
||||
import json, sys
|
||||
try:
|
||||
d = json.load(sys.stdin)
|
||||
@@ -46,14 +78,18 @@ except Exception:
|
||||
sys.exit(1)
|
||||
sys.exit(0 if (d.get("ok") is True and d.get("backend") == "postgres"
|
||||
and d.get("db_ok") is True) else 1)
|
||||
'
|
||||
'; then
|
||||
return 1
|
||||
fi
|
||||
done
|
||||
return 0
|
||||
}
|
||||
|
||||
wait_health() {
|
||||
i=0
|
||||
while [ "$i" -lt 12 ]; do
|
||||
while [ "$i" -lt "$HEALTH_ATTEMPTS" ]; do
|
||||
if check_health; then return 0; fi
|
||||
sleep 5
|
||||
sleep "$HEALTH_INTERVAL"
|
||||
i=$((i + 1))
|
||||
done
|
||||
return 1
|
||||
@@ -66,15 +102,18 @@ if [ -z "$BACKUP_FILE" ]; then
|
||||
exit 1
|
||||
fi
|
||||
echo "将恢复备份包：$BACKUP_FILE"
|
||||
systemctl stop codex-cloud-worker codex-cloud-server || true
|
||||
echo "停服务：worker_unit=$WORKER_UNIT server_units=$SERVER_UNITS"
|
||||
systemctl stop "$WORKER_UNIT" $SERVER_UNITS || true
|
||||
find "$APP_DIR" -mindepth 1 -maxdepth 1 ! -name work -exec rm -rf {} +
|
||||
tar -xzf "$BACKUP_FILE" -C /opt
|
||||
chown -R codexsvc:codexsvc "$APP_DIR"
|
||||
systemctl start codex-cloud-server codex-cloud-worker
|
||||
tar -xzf "$BACKUP_FILE" -C "$(dirname "$APP_DIR")"
|
||||
if id codexsvc >/dev/null 2>&1; then
|
||||
chown -R codexsvc:codexsvc "$APP_DIR"
|
||||
fi
|
||||
systemctl start $SERVER_UNITS "$WORKER_UNIT"
|
||||
if wait_health; then
|
||||
echo "回滚成功，health 通过，当前代码为备份包：$BACKUP_FILE"
|
||||
echo "回滚成功，全部端口 health 通过（$HEALTH_PORTS），当前代码为备份包：$BACKUP_FILE"
|
||||
exit 0
|
||||
fi
|
||||
echo "回滚后 health 仍失败，需人工介入" >&2
|
||||
echo "回滚后 health 仍失败（需全部端口通过），需人工介入" >&2
|
||||
exit 1
|
||||
HOST_SCRIPT
|
||||
Reference in new issue
Block a user