openqjandhost
fc8e337d0f
* 部署自动化：唯一发布链路 deploy.py + host_deploy.sh，sha/commit 双重验真，失败回滚 旧链路三次栽过的坑：通用 tar 路径导致旧包误部署、传输不验 sha、 主机脚本多份漂移、health 通过不等于新代码上线。本单把四点变硬门禁： - 产物唯一命名 codex-app-server-<commit>.tar.gz + .sha256，远端唯一路径 - git archive 产物 + DEPLOYED_COMMIT 清单，默认拒绝脏工作区 - 传输链 scp -> scp_run -> chunk 分片，每路传完远端验 sha，不符删掉重来 - 主机侧唯一脚本：先验 sha/清单，staging 解包 + 编译检查，换目录上线， work/ 原样保留，health + 上线 commit 不符即回滚并退出非零 - deploy.py 成功后独立 ssh 再读一次 live 清单二次核对 - CI：host-verify 加 deploy/tests；release.yml 只在手动/tag + production environment 审批时部署，主干推送绝不自动部署生产 - deploy.sh 标记为旧链路弃用 - health 的 commit 字段已在父提交 HA 阶段 0 里随 server.py 落地， 本分支整体包含它；commit 为空时只靠文件核对，门禁同样成立 测试：host 32、cloud 487（4 跳过）、deploy 9，全部通过； host_deploy.sh 仿真覆盖成功/坏 sha/清单不符/health commit 不符回滚。 生产实机部署未做，等确认后再用新链路推。 * HA stage 1: worktree_host + queued cross-host PR jobs --------- Co-authored-by: host <host@test.local>
215 lines
8.4 KiB
Bash
215 lines
8.4 KiB
Bash
#!/bin/sh
|
||
# 主机侧唯一部署脚本（取代 deploy.sh 内嵌 HOST_SCRIPT 与
|
||
# ~/workspace/ssh-helpers/deploy_*_host.sh 等多份手工脚本）。
|
||
#
|
||
# 设计目标：旧 tar 绝不能被误部署，health 通过绝不能代替“新代码上线”。
|
||
#
|
||
# 调用（VM 侧 deploy.py 传过来，绝不由人工在主机上拼路径）：
|
||
# sh host_deploy.sh <tarball> <expected_sha256> <expected_commit>
|
||
# tarball：远端唯一路径，例如 /tmp/codex-app-server-releases/<commit>.tar.gz
|
||
# 绝不再用 /tmp/codex-app-server-code.tar.gz 这种通用路径。
|
||
# expected_sha256：VM 侧算好的 tar sha256，主机侧先验再动任何东西。
|
||
# expected_commit：git 完整 commit，tar 根目录的 DEPLOYED_COMMIT
|
||
# 必须与它一致，否则直接退出，旧代码原地不动。
|
||
#
|
||
# 流程（任何一步失败都走回滚，退出非零）：
|
||
# 1. 验 tar sha256，不符直接退出，不备份、不解包、不重启。
|
||
# 2. 验 tar 内 DEPLOYED_COMMIT，不符直接退出。
|
||
# 3. 解包到 staging 目录（与 APP_DIR 同一文件系统，便于 mv 换目录），
|
||
# 并对 staging 做 python 编译检查，语法坏的包绝不上线。
|
||
# 4. 备份当前 live 代码（备份名含旧 commit + 时间戳，留最近 5 份；
|
||
# 备份 tar 遇运行中文件变化返回 1 时不直接 abort，只要备份文件
|
||
# 非空就继续，见 PR #46 教训）。
|
||
# 5. 停 server + worker/autoscaler，再换目录：live -> old，
|
||
# staging -> live，work/ 与 host-config-real.json（如有）原样搬回。
|
||
# 换目录代替“原地 rm + 解包”，中途失败不会留半解包的 live。
|
||
# 6. 启动并验 health 60 秒：ok=true 且 backend=postgres 且 db_ok=true，
|
||
# 且 live 的 DEPLOYED_COMMIT 等于 expected_commit；health 里
|
||
# 若有 commit 字段且非空，也必须等于 expected_commit。
|
||
# 7. 成功打印 RELEASE_OK commit=... sha=... backup=...，删掉 old 目录
|
||
# （备份 tar 已留档）；失败则把 old 换回、重启、再验 health，
|
||
# 回滚后 health 恢复也算部署失败（新代码未上线），退出非零。
|
||
#
|
||
# 可覆盖的环境变量（仅用于本地仿真测试，生产用默认值）：
|
||
# CODEX_DEPLOY_APP_DIR、CODEX_DEPLOY_BACKUP_DIR、
|
||
# CODEX_DEPLOY_HEALTH_INTERVAL、CODEX_DEPLOY_HEALTH_ATTEMPTS
|
||
# 回滚边界：只回代码，不回数据库（迁移只加列、幂等，见 deploy.sh 注释）。
|
||
# 密钥：本脚本不读 env 文件、不打印任何密钥。
|
||
set -eu
|
||
|
||
TARBALL=${1:-}
|
||
EXPECTED_SHA=${2:-}
|
||
EXPECTED_COMMIT=${3:-}
|
||
if [ -z "$TARBALL" ] || [ -z "$EXPECTED_SHA" ] || [ -z "$EXPECTED_COMMIT" ]; then
|
||
echo "用法：sh host_deploy.sh <tarball> <expected_sha256> <expected_commit>" >&2
|
||
exit 2
|
||
fi
|
||
if [ ! -f "$TARBALL" ]; then
|
||
echo "部署包不存在，拒绝部署（旧代码不动）：$TARBALL" >&2
|
||
exit 2
|
||
fi
|
||
|
||
APP_DIR=${CODEX_DEPLOY_APP_DIR:-/opt/codex-app-server}
|
||
APP_BASE=$(basename "$APP_DIR")
|
||
BACKUP_DIR=${CODEX_DEPLOY_BACKUP_DIR:-/opt/codex-app-server-backups}
|
||
STAGING_DIR=${CODEX_DEPLOY_STAGING_DIR:-/opt/codex-app-server-staging}
|
||
OLD_DIR=${CODEX_DEPLOY_OLD_DIR:-/opt/codex-app-server-old}
|
||
FAILED_DIR=${CODEX_DEPLOY_FAILED_DIR:-/opt/codex-app-server-failed}
|
||
PRESERVE_DIR=${CODEX_DEPLOY_PRESERVE_DIR:-/opt/codex-app-server-preserve}
|
||
HEALTH_INTERVAL=${CODEX_DEPLOY_HEALTH_INTERVAL:-5}
|
||
HEALTH_ATTEMPTS=${CODEX_DEPLOY_HEALTH_ATTEMPTS:-12}
|
||
|
||
# --- 1. 验 sha：不符直接退出，什么都不动 -------------------------------
|
||
ACTUAL_SHA=$(sha256sum "$TARBALL" | awk '{print $1}')
|
||
if [ "$ACTUAL_SHA" != "$EXPECTED_SHA" ]; then
|
||
echo "部署包 sha256 不符，拒绝部署（旧代码不动）：" >&2
|
||
echo " expected=$EXPECTED_SHA" >&2
|
||
echo " actual=$ACTUAL_SHA tarball=$TARBALL" >&2
|
||
exit 2
|
||
fi
|
||
|
||
# --- 2. 验 tar 内清单：DEPLOYED_COMMIT 必须等于 expected -------------
|
||
TARBALL_COMMIT=$(tar -xzf "$TARBALL" -O DEPLOYED_COMMIT 2>/dev/null | tr -d ' \n\r' || true)
|
||
if [ "$TARBALL_COMMIT" != "$EXPECTED_COMMIT" ]; then
|
||
echo "部署包内 DEPLOYED_COMMIT 与期望 commit 不符，拒绝部署（旧代码不动）：" >&2
|
||
echo " expected=$EXPECTED_COMMIT tarball_commit=$TARBALL_COMMIT" >&2
|
||
exit 2
|
||
fi
|
||
|
||
# --- 3. staging 解包 + 编译检查 ----------------------------------------
|
||
rm -rf "$STAGING_DIR"
|
||
mkdir -p "$STAGING_DIR"
|
||
tar -xzf "$TARBALL" -C "$STAGING_DIR"
|
||
python3 -m compileall -q "$STAGING_DIR" || {
|
||
echo "staging 编译检查失败，拒绝部署（旧代码不动）" >&2
|
||
rm -rf "$STAGING_DIR"
|
||
exit 2
|
||
}
|
||
|
||
check_health() {
|
||
OUT=$(curl -s --max-time 5 http://127.0.0.1:8765/health || true)
|
||
[ -n "$OUT" ] && printf '%s' "$OUT" | EXPECTED_COMMIT="$EXPECTED_COMMIT" python3 -c '
|
||
import json, os, sys
|
||
try:
|
||
d = json.load(sys.stdin)
|
||
except Exception:
|
||
sys.exit(1)
|
||
if not (d.get("ok") is True and d.get("backend") == "postgres"
|
||
and d.get("db_ok") is True):
|
||
sys.exit(1)
|
||
commit = d.get("commit")
|
||
if commit is not None and commit != os.environ.get("EXPECTED_COMMIT"):
|
||
sys.exit(1)
|
||
sys.exit(0)
|
||
'
|
||
}
|
||
|
||
wait_health() {
|
||
i=0
|
||
while [ "$i" -lt "$HEALTH_ATTEMPTS" ]; do
|
||
if check_health; then return 0; fi
|
||
sleep "$HEALTH_INTERVAL"
|
||
i=$((i + 1))
|
||
done
|
||
return 1
|
||
}
|
||
|
||
worker_unit() {
|
||
if systemctl is-active -q codex-autoscaler 2>/dev/null; then
|
||
echo codex-autoscaler
|
||
else
|
||
echo codex-cloud-worker
|
||
fi
|
||
}
|
||
|
||
# --- 4. 备份 live（名含旧 commit，留 5 份） ------------------------------
|
||
mkdir -p "$BACKUP_DIR"
|
||
PREV_COMMIT=unknown
|
||
if [ -f "$APP_DIR/DEPLOYED_COMMIT" ]; then
|
||
PREV_COMMIT=$(tr -d ' \n\r' < "$APP_DIR/DEPLOYED_COMMIT" || echo unknown)
|
||
[ -n "$PREV_COMMIT" ] || PREV_COMMIT=unknown
|
||
fi
|
||
STAMP=$(date +%Y%m%d%H%M%S)
|
||
BACKUP_FILE="$BACKUP_DIR/codex-app-server-${PREV_COMMIT}-${STAMP}.tar.gz"
|
||
# PR #46 教训：备份 tar 遇运行中文件变化返回 1 时不 abort，文件非空即继续。
|
||
tar --exclude="$APP_BASE/work" --exclude="$APP_BASE/worker-heartbeat.json" \
|
||
-czf "$BACKUP_FILE" -C "$(dirname "$APP_DIR")" "$APP_BASE" || true
|
||
if [ ! -s "$BACKUP_FILE" ]; then
|
||
echo "备份失败（备份文件为空），拒绝部署（旧代码不动）" >&2
|
||
rm -rf "$STAGING_DIR"
|
||
exit 2
|
||
fi
|
||
echo "已备份当前代码到：$BACKUP_FILE"
|
||
ls -t "$BACKUP_DIR"/codex-app-server-*.tar.gz 2>/dev/null \
|
||
| tail -n +6 | xargs -r rm -f
|
||
|
||
WORKER_UNIT=$(worker_unit)
|
||
echo "停服务并换目录：worker_unit=$WORKER_UNIT"
|
||
systemctl stop "$WORKER_UNIT" codex-cloud-server || true
|
||
|
||
# --- 5. 换目录：live -> old，staging -> live，保留文件搬回 --------------
|
||
rm -rf "$OLD_DIR" "$PRESERVE_DIR"
|
||
mkdir -p "$PRESERVE_DIR"
|
||
for name in work host-config-real.json; do
|
||
if [ -e "$APP_DIR/$name" ]; then
|
||
mv "$APP_DIR/$name" "$PRESERVE_DIR/$name"
|
||
fi
|
||
done
|
||
mv "$APP_DIR" "$OLD_DIR"
|
||
mv "$STAGING_DIR" "$APP_DIR"
|
||
for name in work host-config-real.json; do
|
||
if [ -e "$PRESERVE_DIR/$name" ]; then
|
||
mv "$PRESERVE_DIR/$name" "$APP_DIR/$name"
|
||
fi
|
||
done
|
||
rmdir "$PRESERVE_DIR" 2>/dev/null || true
|
||
if id codexsvc >/dev/null 2>&1; then
|
||
chown -R codexsvc:codexsvc "$APP_DIR"
|
||
fi
|
||
|
||
rollback() {
|
||
echo "新代码 health 未通过，开始回滚到备份：$BACKUP_FILE" >&2
|
||
systemctl stop "$WORKER_UNIT" codex-cloud-server || true
|
||
rm -rf "$FAILED_DIR" "$PRESERVE_DIR"
|
||
mkdir -p "$PRESERVE_DIR"
|
||
for name in work host-config-real.json; do
|
||
if [ -e "$APP_DIR/$name" ]; then
|
||
mv "$APP_DIR/$name" "$PRESERVE_DIR/$name"
|
||
fi
|
||
done
|
||
mv "$APP_DIR" "$FAILED_DIR"
|
||
mv "$OLD_DIR" "$APP_DIR"
|
||
for name in work host-config-real.json; do
|
||
if [ -e "$PRESERVE_DIR/$name" ]; then
|
||
mv "$PRESERVE_DIR/$name" "$APP_DIR/$name"
|
||
fi
|
||
done
|
||
rmdir "$PRESERVE_DIR" 2>/dev/null || true
|
||
if id codexsvc >/dev/null 2>&1; then
|
||
chown -R codexsvc:codexsvc "$APP_DIR"
|
||
fi
|
||
systemctl start codex-cloud-server "$WORKER_UNIT" || true
|
||
if wait_health; then
|
||
echo "部署失败，已回滚到备份，health 已恢复（新代码未上线）" >&2
|
||
else
|
||
echo "已回滚但 health 仍失败，需人工介入" >&2
|
||
fi
|
||
exit 1
|
||
}
|
||
|
||
# --- 6. 启动 + 验 health + 验上线 commit -------------------------------
|
||
systemctl start codex-cloud-server "$WORKER_UNIT" || true
|
||
if ! wait_health; then
|
||
rollback
|
||
fi
|
||
LIVE_COMMIT=$(tr -d ' \n\r' < "$APP_DIR/DEPLOYED_COMMIT" 2>/dev/null || true)
|
||
if [ "$LIVE_COMMIT" != "$EXPECTED_COMMIT" ]; then
|
||
echo "上线 commit 不符（health 通过但代码不是新版），开始回滚：" >&2
|
||
echo " expected=$EXPECTED_COMMIT live=$LIVE_COMMIT" >&2
|
||
rollback
|
||
fi
|
||
|
||
rm -rf "$OLD_DIR"
|
||
echo "RELEASE_OK commit=$EXPECTED_COMMIT sha=$EXPECTED_SHA backup=$BACKUP_FILE"
|
||
exit 0
|