diff --git a/deploy/README.md b/deploy/README.md index 01ca454..796e9ea 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -85,7 +85,8 @@ deploy/deploy.sh [user] [--dry-run] - 备份保留策略：每次部署前在主机上把当前 `/opt/codex-app-server` 打包到 `/opt/codex-app-server-backups/codex-app-server-<时间戳>.tar.gz`（排除 work/ 目录以免过大），只留最近 3 份，多余的自动删掉。 -- 健康门：重启后循环 curl `localhost:8765/health` 最多 60 秒，要求 +- 健康门：重启后循环 curl `localhost:8765/health` 与 + `localhost:8766/health`（server-2 存在时）最多 60 秒，要求 `ok=true` 且 `backend=postgres` 且 `db_ok=true`；失败时自动恢复刚打的 备份包并重启，再验一次。回滚后 health 恢复也算部署失败（退出非零）， 因为新代码没上去；回滚后 health 仍失败则退出非零并明说 diff --git a/deploy/host_deploy.sh b/deploy/host_deploy.sh index d5d1dae..8264dc2 100644 --- a/deploy/host_deploy.sh +++ b/deploy/host_deploy.sh @@ -20,12 +20,14 @@ # 4. 备份当前 live 代码（备份名含旧 commit + 时间戳，留最近 5 份； # 备份 tar 遇运行中文件变化返回 1 时不直接 abort，只要备份文件 # 非空就继续，见 PR #46 教训）。 -# 5. 停 server + worker/autoscaler，再换目录：live -> old， +# 5. 停 server(s) + worker/autoscaler，再换目录：live -> old， # staging -> live，work/ 与 host-config-real.json（如有）原样搬回。 # 换目录代替“原地 rm + 解包”，中途失败不会留半解包的 live。 # 6. 启动并验 health 60 秒：ok=true 且 backend=postgres 且 db_ok=true， # 且 live 的 DEPLOYED_COMMIT 等于 expected_commit；health 里 # 若有 commit 字段且非空，也必须等于 expected_commit。 +# server-2（HA 阶段 2 加的 8766）若存在必须同批停/起、同批验 +# health，否则 LB 会混跑新旧内存代码（2026-10-27 首推教训）。 # 7. 成功打印 RELEASE_OK commit=... sha=... backup=...，删掉 old 目录 # （备份 tar 已留档）；失败则把 old 换回、重启、再验 health， # 回滚后 health 恢复也算部署失败（新代码未上线），退出非零。 @@ -87,8 +89,10 @@ python3 -m compileall -q "$STAGING_DIR" || { } check_health() { - OUT=$(curl -s --max-time 5 http://127.0.0.1:8765/health || true) - [ -n "$OUT" ] && printf '%s' "$OUT" | EXPECTED_COMMIT="$EXPECTED_COMMIT" python3 -c ' + for port in $HEALTH_PORTS; do + OUT=$(curl -s --max-time 5 "http://127.0.0.1:$port/health" || true) + if [ -z "$OUT" ]; then return 1; fi + if ! printf '%s' "$OUT" | EXPECTED_COMMIT="$EXPECTED_COMMIT" python3 -c ' import json, os, sys try: d = json.load(sys.stdin) @@ -101,7 +105,11 @@ commit = d.get("commit") if commit is not None and commit != os.environ.get("EXPECTED_COMMIT"): sys.exit(1) sys.exit(0) -' +'; then + return 1 + fi + done + return 0 } wait_health() { @@ -122,6 +130,16 @@ worker_unit() { fi } +# server-2（8766）是 HA 阶段 2 加的；unit 存在才纳入停/起与 health +# 检查，老主机没有它时退回单机行为。2026-10-27 首推教训：脚本只动 +# server-1 时，LB 上游的 server-2 仍跑旧内存代码，新旧混跑。 +SERVER_UNITS="codex-cloud-server" +HEALTH_PORTS="8765" +if systemctl cat codex-cloud-server-2 >/dev/null 2>&1; then + SERVER_UNITS="$SERVER_UNITS codex-cloud-server-2" + HEALTH_PORTS="$HEALTH_PORTS 8766" +fi + # --- 4. 备份 live（名含旧 commit，留 5 份） ------------------------------ mkdir -p "$BACKUP_DIR" PREV_COMMIT=unknown @@ -144,8 +162,8 @@ ls -t "$BACKUP_DIR"/codex-app-server-*.tar.gz 2>/dev/null \ | tail -n +6 | xargs -r rm -f WORKER_UNIT=$(worker_unit) -echo "停服务并换目录：worker_unit=$WORKER_UNIT" -systemctl stop "$WORKER_UNIT" codex-cloud-server || true +echo "停服务并换目录：worker_unit=$WORKER_UNIT server_units=$SERVER_UNITS" +systemctl stop "$WORKER_UNIT" $SERVER_UNITS || true # --- 5. 换目录：live -> old，staging -> live，保留文件搬回 -------------- rm -rf "$OLD_DIR" "$PRESERVE_DIR" @@ -169,7 +187,7 @@ fi rollback() { echo "新代码 health 未通过，开始回滚到备份：$BACKUP_FILE" >&2 - systemctl stop "$WORKER_UNIT" codex-cloud-server || true + systemctl stop "$WORKER_UNIT" $SERVER_UNITS || true rm -rf "$FAILED_DIR" "$PRESERVE_DIR" mkdir -p "$PRESERVE_DIR" for name in work host-config-real.json; do @@ -188,7 +206,7 @@ rollback() { if id codexsvc >/dev/null 2>&1; then chown -R codexsvc:codexsvc "$APP_DIR" fi - systemctl start codex-cloud-server "$WORKER_UNIT" || true + systemctl start $SERVER_UNITS "$WORKER_UNIT" || true if wait_health; then echo "部署失败，已回滚到备份，health 已恢复（新代码未上线）" >&2 else @@ -198,7 +216,7 @@ rollback() { } # --- 6. 启动 + 验 health + 验上线 commit ------------------------------- -systemctl start codex-cloud-server "$WORKER_UNIT" || true +systemctl start $SERVER_UNITS "$WORKER_UNIT" || true if ! wait_health; then rollback fi diff --git a/deploy/tests/test_host_deploy.py b/deploy/tests/test_host_deploy.py index b87b1d0..1509062 100644 --- a/deploy/tests/test_host_deploy.py +++ b/deploy/tests/test_host_deploy.py @@ -39,12 +39,17 @@ class HostDeploySim(unittest.TestCase): f.write("keep\n") with open(os.path.join(self.app, "old.py"), "w") as f: f.write("x = 0\n") - # 假 systemctl：is-active 一律失败（走 cloud-worker 分支），其余成功。 + # 假 systemctl：is-active 一律失败（走 cloud-worker 分支），其余成功； + # cat codex-cloud-server-2 按 FAKE_SERVER2_PRESENT 决定 unit 是否存在。 fake_bin = os.path.join(base, "fakebin") os.makedirs(fake_bin) with open(os.path.join(fake_bin, "systemctl"), "w") as f: f.write("#!/bin/sh\n" "echo \"$@\" >> \"$FAKE_SYSTEMCTL_LOG\"\n" + "if [ \"$1\" = \"cat\" ] && [ \"$2\" = \"codex-cloud-server-2\" ]; then\n" + " if [ \"$FAKE_SERVER2_PRESENT\" = \"0\" ]; then exit 1; fi\n" + " exit 0\n" + "fi\n" "if [ \"$1\" = \"is-active\" ]; then exit 1; fi\n" "exit 0\n") with open(os.path.join(fake_bin, "curl"), "w") as f: @@ -67,6 +72,7 @@ class HostDeploySim(unittest.TestCase): self.env["CODEX_DEPLOY_HEALTH_ATTEMPTS"] = "3" self.env["FAKE_HEALTH_JSON"] = self.health_path self.env["FAKE_SYSTEMCTL_LOG"] = self.log_path + self.env["FAKE_SERVER2_PRESENT"] = "1" def tearDown(self): self.tmp.cleanup() @@ -86,10 +92,17 @@ class HostDeploySim(unittest.TestCase): sha = make_tarball(src, tarball) return tarball, sha - def run_host(self, tarball, sha, commit): + def run_host(self, tarball, sha, commit, extra_env=None): + env = dict(self.env) + if extra_env: + env.update(extra_env) return subprocess.run( ["sh", HOST_SCRIPT, tarball, sha, commit], - capture_output=True, text=True, env=self.env) + capture_output=True, text=True, env=env) + + def systemctl_log(self): + with open(self.log_path) as f: + return f.read() def live_commit(self): with open(os.path.join(self.app, "DEPLOYED_COMMIT")) as f: @@ -142,6 +155,36 @@ class HostDeploySim(unittest.TestCase): with open(os.path.join(self.app, "work", "keep.txt")) as f: self.assertEqual(f.read().strip(), "keep") + def test_server2_stopped_started_when_present(self): + # 2026-10-27 首推教训：server-2 存在时必须同批停/起， + # 否则 LB 混跑新旧内存代码。 + self.set_health({"ok": True, "backend": "postgres", "db_ok": True, + "commit": "newcommit"}) + tarball, sha = self.build_new_tarball() + r = self.run_host(tarball, sha, "newcommit") + self.assertEqual(r.returncode, 0, r.stdout + r.stderr) + log = self.systemctl_log() + self.assertIn( + "stop codex-cloud-worker codex-cloud-server codex-cloud-server-2", + log) + self.assertIn( + "start codex-cloud-server codex-cloud-server-2 codex-cloud-worker", + log) + + def test_server2_skipped_when_unit_absent(self): + # 老主机没有 server-2 unit 时退回单机行为，不报错、不提它。 + self.set_health({"ok": True, "backend": "postgres", "db_ok": True, + "commit": "newcommit"}) + tarball, sha = self.build_new_tarball() + r = self.run_host(tarball, sha, "newcommit", + extra_env={"FAKE_SERVER2_PRESENT": "0"}) + self.assertEqual(r.returncode, 0, r.stdout + r.stderr) + # 探测行 cat 本身会提到 server-2（探测失败才判定不存在）， + # 所以断言停/起行里没有它，而不是全日志没有这个词。 + log = self.systemctl_log() + self.assertIn("stop codex-cloud-worker codex-cloud-server\n", log) + self.assertIn("start codex-cloud-server codex-cloud-worker\n", log) + if __name__ == "__main__": unittest.main()