host
64995fafab
- migrations: MIGRATION_TABLES now equals POSTGRES_SCHEMA tables
(uploads, task_attachments, task_followups, environments added);
SEQUENCE_TABLES matches IDENTITY tables; task_followups
sender_task_id column ensured before insert; UNMIGRATED_TABLES
and migration_coverage() report tables with no PG schema instead
of silently skipping them; CLI prints the coverage report.
- deploy: DEFAULT_TEST_SUITES gains deploy/tests (discover -s only,
never package path); regression tests pin the default list.
- monitor: MONITOR_SINGLE_HOST=1 makes the server check single-host;
default stays dual to avoid silent downgrade; server_mode is in
the JSON. Optional fail-only alert outlet (webhook or mail,
default off, alert body is pass/fail JSON only, no secrets).
- backup/rollback: README and restore-drill document what is
recoverable (DB, code) vs not (worktrees, traces, uploads, host
configs); rollback.sh shares the server-unit list and dual-port
health success standard with host_deploy.sh (sync test guards it).
- public entry: Caddyfile uses native {$DOMAIN}/{$ACME_EMAIL}
variables plus a caddy.service template; README documents the
steps and the current BT-nginx production status.
120 lines
4.4 KiB
Bash
Executable File
120 lines
4.4 KiB
Bash
Executable File
#!/bin/sh
|
||
# 回滚：VM 侧运行，把主机恢复到最近一份代码备份包并重启验证。
|
||
#
|
||
# 用法：deploy/rollback.sh <host> [user]
|
||
# host：主机 IP 或域名（必填）；user：SSH 用户（默认 root）。
|
||
#
|
||
# 密码：与 deploy.sh 同款规则——绝不接受密码参数、不读 env 文件、
|
||
# 不打印任何密钥；SSH 密码由交互提示或调用方通道提供；需要代理时
|
||
# 用环境变量 SSH_PROXY_CMD 传入 ProxyCommand，未设则直连。
|
||
#
|
||
# 回滚边界：只回代码，不回数据库（理由见 deploy.sh 头注释与
|
||
# deploy/README.md：数据库迁移是只加列的幂等操作，老代码仍能读新列）。
|
||
# 备份包来自每次部署前在主机上打的
|
||
# /opt/codex-app-server-backups/codex-app-server-<时间戳>.tar.gz，
|
||
# 本脚本恢复其中最新的一份；主机上没有备份包时直接报错退出，不编造。
|
||
#
|
||
# 与 host_deploy.sh 共享的约定（P2-ops，两脚本改动时必须同步，
|
||
# deploy/tests/test_p2_ops.py 会双向核对，见文件尾注释）：
|
||
# - server unit 清单：codex-cloud-server 必含，codex-cloud-server-2
|
||
# 以 systemctl cat 探测为准（存在才纳入停/起与健康检查）；
|
||
# - worker：autoscaler active 时停/起 autoscaler，否则 cloud-worker
|
||
# （与 host_deploy.sh 的 worker_unit() 同款）；
|
||
# - 成功标准：清单里每个 server 端口的 health 都通过（8765 必查，
|
||
# server-2 存在时 8766 也必查），只通 8765 不算回滚成功——
|
||
# 否则 LB 上游的 server-2 仍跑故障代码，回滚等于没回完。
|
||
#
|
||
# 仿真覆盖（仅测试用，生产用默认值）：CODEX_ROLLBACK_APP_DIR、
|
||
# CODEX_ROLLBACK_BACKUP_DIR、CODEX_ROLLBACK_HEALTH_ATTEMPTS、
|
||
# CODEX_ROLLBACK_HEALTH_INTERVAL。默认值与旧行为完全一致。
|
||
set -eu
|
||
|
||
HOST=${1:-}
|
||
USER_ARG=${2:-root}
|
||
if [ -z "$HOST" ]; then
|
||
echo "用法：deploy/rollback.sh <host> [user]" >&2
|
||
exit 2
|
||
fi
|
||
|
||
ssh_cmd() {
|
||
if [ -n "${SSH_PROXY_CMD:-}" ]; then
|
||
ssh -o "ProxyCommand $SSH_PROXY_CMD" -o StrictHostKeyChecking=no \
|
||
-o UserKnownHostsFile=/dev/null "$@"
|
||
else
|
||
ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null "$@"
|
||
fi
|
||
}
|
||
|
||
ssh_cmd "$USER_ARG@$HOST" 'sh -s' <<'HOST_SCRIPT'
|
||
set -eu
|
||
APP_DIR=${CODEX_ROLLBACK_APP_DIR:-/opt/codex-app-server}
|
||
BACKUP_DIR=${CODEX_ROLLBACK_BACKUP_DIR:-/opt/codex-app-server-backups}
|
||
HEALTH_ATTEMPTS=${CODEX_ROLLBACK_HEALTH_ATTEMPTS:-12}
|
||
HEALTH_INTERVAL=${CODEX_ROLLBACK_HEALTH_INTERVAL:-5}
|
||
|
||
# 与 host_deploy.sh 共享的 unit 清单约定（见本文件头注释）：
|
||
# server-1 必含，server-2 以 unit 存在为准；改动清单时两脚本同步改。
|
||
SERVER_UNITS="codex-cloud-server"
|
||
HEALTH_PORTS="8765"
|
||
if systemctl cat codex-cloud-server-2 >/dev/null 2>&1; then
|
||
SERVER_UNITS="$SERVER_UNITS codex-cloud-server-2"
|
||
HEALTH_PORTS="$HEALTH_PORTS 8766"
|
||
fi
|
||
if systemctl is-active -q codex-autoscaler 2>/dev/null; then
|
||
WORKER_UNIT=codex-autoscaler
|
||
else
|
||
WORKER_UNIT=codex-cloud-worker
|
||
fi
|
||
|
||
check_health() {
|
||
for port in $HEALTH_PORTS; do
|
||
OUT=$(curl -s --max-time 5 "http://127.0.0.1:$port/health" || true)
|
||
[ -n "$OUT" ] || return 1
|
||
if ! printf '%s' "$OUT" | python3 -c '
|
||
import json, sys
|
||
try:
|
||
d = json.load(sys.stdin)
|
||
except Exception:
|
||
sys.exit(1)
|
||
sys.exit(0 if (d.get("ok") is True and d.get("backend") == "postgres"
|
||
and d.get("db_ok") is True) else 1)
|
||
'; then
|
||
return 1
|
||
fi
|
||
done
|
||
return 0
|
||
}
|
||
|
||
wait_health() {
|
||
i=0
|
||
while [ "$i" -lt "$HEALTH_ATTEMPTS" ]; do
|
||
if check_health; then return 0; fi
|
||
sleep "$HEALTH_INTERVAL"
|
||
i=$((i + 1))
|
||
done
|
||
return 1
|
||
}
|
||
|
||
BACKUP_FILE=$(ls -t "$BACKUP_DIR"/codex-app-server-*.tar.gz 2>/dev/null \
|
||
| head -n 1 || true)
|
||
if [ -z "$BACKUP_FILE" ]; then
|
||
echo "主机上没有代码备份包，无法回滚：$BACKUP_DIR" >&2
|
||
exit 1
|
||
fi
|
||
echo "将恢复备份包：$BACKUP_FILE"
|
||
echo "停服务：worker_unit=$WORKER_UNIT server_units=$SERVER_UNITS"
|
||
systemctl stop "$WORKER_UNIT" $SERVER_UNITS || true
|
||
find "$APP_DIR" -mindepth 1 -maxdepth 1 ! -name work -exec rm -rf {} +
|
||
tar -xzf "$BACKUP_FILE" -C "$(dirname "$APP_DIR")"
|
||
if id codexsvc >/dev/null 2>&1; then
|
||
chown -R codexsvc:codexsvc "$APP_DIR"
|
||
fi
|
||
systemctl start $SERVER_UNITS "$WORKER_UNIT"
|
||
if wait_health; then
|
||
echo "回滚成功，全部端口 health 通过（$HEALTH_PORTS），当前代码为备份包：$BACKUP_FILE"
|
||
exit 0
|
||
fi
|
||
echo "回滚后 health 仍失败（需全部端口通过），需人工介入" >&2
|
||
exit 1
|
||
HOST_SCRIPT
|