host
1f4463209b
The host backup tar runs with -C /opt codex-app-server, so its members are named codex-app-server/..., and the old patterns ./work and ./worker-heartbeat.json never matched anything. The live work/ dir (host-state, worktrees rewritten by running workers) was therefore included, tar hit 'file changed as we read it', exited 1, and set -e aborted the deploy. Patterns are now codex-app-server/work and codex-app-server/worker-heartbeat.json, which actually match.
171 lines
7.0 KiB
Bash
Executable File
171 lines
7.0 KiB
Bash
Executable File
#!/bin/sh
|
||
# 部署自动化：VM 侧运行，把本地代码同步到外部主机并重启验证，失败自动回滚。
|
||
#
|
||
# 用法：
|
||
# deploy/deploy.sh <host> [user] [--dry-run]
|
||
# host：主机 IP 或域名（必填）；user：SSH 用户（默认 root）。
|
||
# --dry-run：只打印将要做的步骤，不 scp、不 ssh、不改主机。
|
||
#
|
||
# 密码：本脚本绝不接受密码参数、不读任何 env 文件、不打印任何密钥。
|
||
# SSH 密码由 ssh/scp 交互提示输入，或由调用方自己的 SSH 通道提供
|
||
# （例如已配好的 key）。需要经代理时，用环境变量 SSH_PROXY_CMD
|
||
# 传入 ProxyCommand 内容（ssh 会自行展开 %h %p），未设则直连：
|
||
# SSH_PROXY_CMD="python3 /tmp/proxy_tunnel.py %h %p" deploy/deploy.sh <host>
|
||
#
|
||
# 步骤：
|
||
# 1. 本机打代码 tar 包（排除 bin/、work/、.git、__pycache__、state、
|
||
# host-config-real.json，与之前手工同步的排除规则一致）。
|
||
# 2. scp 到主机 /tmp/codex-app-server-code.tar.gz。
|
||
# 3. ssh 到主机：先把当前 /opt/codex-app-server 打备份包到
|
||
# /opt/codex-app-server-backups/（时间戳命名，只留最近 3 份，
|
||
# 备份排除 work/ 目录以免过大）；解包新代码并 chown codexsvc；
|
||
# 重启 server+worker；循环 curl localhost:8765/health 最多 60 秒，
|
||
# 要求 ok=true 且 backend=postgres 且 db_ok=true。
|
||
# 4. 健康检查失败时自动恢复刚打的备份包、重启、再验 health；回滚后
|
||
# health 恢复也算部署失败（退出非零），因为新代码没上去；回滚后
|
||
# health 仍失败则退出非零并明说“已回滚但 health 仍失败，需人工介入”。
|
||
# 5. 成功时只打印新代码的 git commit（从本机 git 取，不编造版本号）。
|
||
#
|
||
# 回滚边界：回滚只回代码，不回数据库。数据库迁移是只加列的幂等操作
|
||
# （projects.repo_url、tasks 的 6 列等），回滚代码后老代码仍能读新列，
|
||
# 所以只回代码是安全的；数据库本身绝不会被本脚本动到。
|
||
set -eu
|
||
|
||
HOST=${1:-}
|
||
USER_ARG=${2:-root}
|
||
DRY_RUN=0
|
||
for a in "$@"; do
|
||
if [ "$a" = "--dry-run" ]; then DRY_RUN=1; fi
|
||
done
|
||
if [ -z "$HOST" ] || [ "$HOST" = "--dry-run" ]; then
|
||
echo "用法：deploy/deploy.sh <host> [user] [--dry-run]" >&2
|
||
exit 2
|
||
fi
|
||
# user 位置可能是 --dry-run，处理一下
|
||
if [ "$USER_ARG" = "--dry-run" ]; then USER_ARG=root; fi
|
||
|
||
SCRIPT_DIR=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)
|
||
REPO_ROOT=$(CDPATH= cd -- "$SCRIPT_DIR/.." && pwd)
|
||
cd "$REPO_ROOT"
|
||
|
||
COMMIT=$(git rev-parse HEAD 2>/dev/null || echo "unknown")
|
||
TARBALL="$REPO_ROOT/work/external-install/codex-app-server-code.tar.gz"
|
||
REMOTE_TARBALL="/tmp/codex-app-server-code.tar.gz"
|
||
|
||
# ssh/scp 命令构造：有 SSH_PROXY_CMD 时走代理，否则直连。
|
||
ssh_cmd() {
|
||
if [ -n "${SSH_PROXY_CMD:-}" ]; then
|
||
ssh -o "ProxyCommand $SSH_PROXY_CMD" -o StrictHostKeyChecking=no \
|
||
-o UserKnownHostsFile=/dev/null "$@"
|
||
else
|
||
ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null "$@"
|
||
fi
|
||
}
|
||
scp_cmd() {
|
||
if [ -n "${SSH_PROXY_CMD:-}" ]; then
|
||
scp -o "ProxyCommand $SSH_PROXY_CMD" -o StrictHostKeyChecking=no \
|
||
-o UserKnownHostsFile=/dev/null "$@"
|
||
else
|
||
scp -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null "$@"
|
||
fi
|
||
}
|
||
|
||
echo "部署目标：$USER_ARG@$HOST，代码 commit：$COMMIT"
|
||
if [ "$DRY_RUN" = "1" ]; then
|
||
echo "[dry-run] 将打 tar 包：$TARBALL（排除 bin/ work/ .git __pycache__ state host-config-real.json）"
|
||
echo "[dry-run] 将 scp 到 $USER_ARG@$HOST:$REMOTE_TARBALL"
|
||
echo "[dry-run] 将在主机上备份 /opt/codex-app-server 到 /opt/codex-app-server-backups/（留 3 份），解包、chown codexsvc、重启 server+worker、验 health 60 秒，失败自动回滚"
|
||
exit 0
|
||
fi
|
||
|
||
mkdir -p "$REPO_ROOT/work/external-install"
|
||
tar --exclude='./bin' --exclude='./work' --exclude='./.git' \
|
||
--exclude='__pycache__' --exclude='*.pyc' \
|
||
--exclude='host-state*.json*' --exclude='host-daemon.*' \
|
||
--exclude='host-config-real.json' --exclude='host-run.log' \
|
||
-czf "$TARBALL" .
|
||
echo "tar 包已生成：$TARBALL"
|
||
|
||
scp_cmd "$TARBALL" "$USER_ARG@$HOST:$REMOTE_TARBALL"
|
||
echo "tar 包已传到主机"
|
||
|
||
# 主机侧部署脚本：备份 -> 解包 -> 重启 -> 验 health -> 失败回滚。
|
||
# 注意：这里用单引号 heredoc 传过去，主机侧变量不被本机展开；
|
||
# 健康判断用 python3 解析 JSON，不依赖 jq（主机未必装 jq）。
|
||
ssh_cmd "$USER_ARG@$HOST" 'sh -s' <<'HOST_SCRIPT'
|
||
set -eu
|
||
APP_DIR=/opt/codex-app-server
|
||
BACKUP_DIR=/opt/codex-app-server-backups
|
||
STAMP=$(date +%Y%m%d%H%M%S)
|
||
mkdir -p "$BACKUP_DIR"
|
||
BACKUP_FILE="$BACKUP_DIR/codex-app-server-$STAMP.tar.gz"
|
||
|
||
check_health() {
|
||
# 要求 ok=true 且 backend=postgres 且 db_ok=true，满足返回 0。
|
||
OUT=$(curl -s --max-time 5 http://127.0.0.1:8765/health || true)
|
||
[ -n "$OUT" ] && printf '%s' "$OUT" | python3 -c '
|
||
import json, sys
|
||
try:
|
||
d = json.load(sys.stdin)
|
||
except Exception:
|
||
sys.exit(1)
|
||
sys.exit(0 if (d.get("ok") is True and d.get("backend") == "postgres"
|
||
and d.get("db_ok") is True) else 1)
|
||
'
|
||
}
|
||
|
||
wait_health() {
|
||
i=0
|
||
while [ "$i" -lt 12 ]; do
|
||
if check_health; then return 0; fi
|
||
sleep 5
|
||
i=$((i + 1))
|
||
done
|
||
return 1
|
||
}
|
||
|
||
# 1. 备份当前代码（排除 work/，只留最近 3 份）
|
||
tar --exclude='codex-app-server/work' --exclude='codex-app-server/worker-heartbeat.json' -czf "$BACKUP_FILE" -C /opt codex-app-server
|
||
echo "已备份当前代码到：$BACKUP_FILE"
|
||
ls -t "$BACKUP_DIR"/codex-app-server-*.tar.gz 2>/dev/null \
|
||
| tail -n +4 | xargs -r rm -f
|
||
|
||
# 2. 先清掉旧代码（保留 work/ 工作区，里面是任务 worktree），再解包新代码。
|
||
# 只做覆盖式解包会留下新版新增的文件，回滚时旧版文件还在，造成“回滚不干净”
|
||
# （2026-10-26 实测：回滚后 deploy/ 新脚本仍残留）。
|
||
find "$APP_DIR" -mindepth 1 -maxdepth 1 ! -name work -exec rm -rf {} +
|
||
tar -xzf /tmp/codex-app-server-code.tar.gz -C "$APP_DIR"
|
||
chown -R codexsvc:codexsvc "$APP_DIR"
|
||
|
||
# 3. 重启并验 health
|
||
# autoscaler 接管后，worker 由它拉起（autoscale-* 实例），
|
||
# 静态 codex-cloud-worker 必须保持停用，否则两边同时派任务。
|
||
# 这里按 autoscaler 是否在跑决定重启哪一边；回滚路径同理。
|
||
if systemctl is-active -q codex-autoscaler; then
|
||
WORKER_UNIT=codex-autoscaler
|
||
else
|
||
WORKER_UNIT=codex-cloud-worker
|
||
fi
|
||
systemctl restart codex-cloud-server $WORKER_UNIT
|
||
if wait_health; then
|
||
echo "部署成功，health 通过"
|
||
exit 0
|
||
fi
|
||
|
||
# 4. 失败：恢复备份并重启，再验一次
|
||
echo "新代码 health 未通过，开始回滚到备份：$BACKUP_FILE"
|
||
systemctl stop $WORKER_UNIT codex-cloud-server || true
|
||
find "$APP_DIR" -mindepth 1 -maxdepth 1 ! -name work -exec rm -rf {} +
|
||
tar -xzf "$BACKUP_FILE" -C /opt
|
||
chown -R codexsvc:codexsvc "$APP_DIR"
|
||
systemctl start codex-cloud-server $WORKER_UNIT
|
||
if wait_health; then
|
||
echo "部署失败，已回滚到备份，health 已恢复（新代码未上线）" >&2
|
||
exit 1
|
||
fi
|
||
echo "已回滚但 health 仍失败，需人工介入" >&2
|
||
exit 1
|
||
HOST_SCRIPT
|
||
|
||
echo "部署完成，代码 commit：$COMMIT"
|