fix: host_deploy stops/starts and health-checks server-2 too (#56)

Co-authored-by: host <host@test.local>

deploy/README.md

@@ -85,7 +85,8 @@ deploy/deploy.sh <host> [user] [--dry-run]
 - 备份保留策略：每次部署前在主机上把当前 `/opt/codex-app-server` 打包到
   `/opt/codex-app-server-backups/codex-app-server-<时间戳>.tar.gz`（排除
   work/ 目录以免过大），只留最近 3 份，多余的自动删掉。
-- 健康门：重启后循环 curl `localhost:8765/health` 最多 60 秒，要求
+- 健康门：重启后循环 curl `localhost:8765/health` 与
+  `localhost:8766/health`（server-2 存在时）最多 60 秒，要求
   `ok=true` 且 `backend=postgres` 且 `db_ok=true`；失败时自动恢复刚打的
   备份包并重启，再验一次。回滚后 health 恢复也算部署失败（退出非零），
   因为新代码没上去；回滚后 health 仍失败则退出非零并明说

deploy/host_deploy.sh

@@ -20,12 +20,14 @@
 #   4. 备份当前 live 代码（备份名含旧 commit + 时间戳，留最近 5 份；
 #      备份 tar 遇运行中文件变化返回 1 时不直接 abort，只要备份文件
 #      非空就继续，见 PR #46 教训）。
-#   5. 停 server + worker/autoscaler，再换目录：live -> old，
+#   5. 停 server(s) + worker/autoscaler，再换目录：live -> old，
 #      staging -> live，work/ 与 host-config-real.json（如有）原样搬回。
 #      换目录代替“原地 rm + 解包”，中途失败不会留半解包的 live。
 #   6. 启动并验 health 60 秒：ok=true 且 backend=postgres 且 db_ok=true，
 #      且 live 的 DEPLOYED_COMMIT 等于 expected_commit；health 里
 #      若有 commit 字段且非空，也必须等于 expected_commit。
+#      server-2（HA 阶段 2 加的 8766）若存在必须同批停/起、同批验
+#      health，否则 LB 会混跑新旧内存代码（2026-10-27 首推教训）。
 #   7. 成功打印 RELEASE_OK commit=... sha=... backup=...，删掉 old 目录
 #      （备份 tar 已留档）；失败则把 old 换回、重启、再验 health，
 #      回滚后 health 恢复也算部署失败（新代码未上线），退出非零。
@@ -87,8 +89,10 @@ python3 -m compileall -q "$STAGING_DIR" || {
 }
 
 check_health() {
-  OUT=$(curl -s --max-time 5 http://127.0.0.1:8765/health || true)
-  [ -n "$OUT" ] && printf '%s' "$OUT" | EXPECTED_COMMIT="$EXPECTED_COMMIT" python3 -c '
+  for port in $HEALTH_PORTS; do
+    OUT=$(curl -s --max-time 5 "http://127.0.0.1:$port/health" || true)
+    if [ -z "$OUT" ]; then return 1; fi
+    if ! printf '%s' "$OUT" | EXPECTED_COMMIT="$EXPECTED_COMMIT" python3 -c '
 import json, os, sys
 try:
     d = json.load(sys.stdin)
@@ -101,7 +105,11 @@ commit = d.get("commit")
 if commit is not None and commit != os.environ.get("EXPECTED_COMMIT"):
     sys.exit(1)
 sys.exit(0)
-'
+'; then
+      return 1
+    fi
+  done
+  return 0
 }
 
 wait_health() {
@@ -122,6 +130,16 @@ worker_unit() {
   fi
 }
 
+# server-2（8766）是 HA 阶段 2 加的；unit 存在才纳入停/起与 health
+# 检查，老主机没有它时退回单机行为。2026-10-27 首推教训：脚本只动
+# server-1 时，LB 上游的 server-2 仍跑旧内存代码，新旧混跑。
+SERVER_UNITS="codex-cloud-server"
+HEALTH_PORTS="8765"
+if systemctl cat codex-cloud-server-2 >/dev/null 2>&1; then
+  SERVER_UNITS="$SERVER_UNITS codex-cloud-server-2"
+  HEALTH_PORTS="$HEALTH_PORTS 8766"
+fi
+
 # --- 4. 备份 live（名含旧 commit，留 5 份） ------------------------------
 mkdir -p "$BACKUP_DIR"
 PREV_COMMIT=unknown
@@ -144,8 +162,8 @@ ls -t "$BACKUP_DIR"/codex-app-server-*.tar.gz 2>/dev/null \
   | tail -n +6 | xargs -r rm -f
 
 WORKER_UNIT=$(worker_unit)
-echo "停服务并换目录：worker_unit=$WORKER_UNIT"
-systemctl stop "$WORKER_UNIT" codex-cloud-server || true
+echo "停服务并换目录：worker_unit=$WORKER_UNIT server_units=$SERVER_UNITS"
+systemctl stop "$WORKER_UNIT" $SERVER_UNITS || true
 
 # --- 5. 换目录：live -> old，staging -> live，保留文件搬回 --------------
 rm -rf "$OLD_DIR" "$PRESERVE_DIR"
@@ -169,7 +187,7 @@ fi
 
 rollback() {
   echo "新代码 health 未通过，开始回滚到备份：$BACKUP_FILE" >&2
-  systemctl stop "$WORKER_UNIT" codex-cloud-server || true
+  systemctl stop "$WORKER_UNIT" $SERVER_UNITS || true
   rm -rf "$FAILED_DIR" "$PRESERVE_DIR"
   mkdir -p "$PRESERVE_DIR"
   for name in work host-config-real.json; do
@@ -188,7 +206,7 @@ rollback() {
   if id codexsvc >/dev/null 2>&1; then
     chown -R codexsvc:codexsvc "$APP_DIR"
   fi
-  systemctl start codex-cloud-server "$WORKER_UNIT" || true
+  systemctl start $SERVER_UNITS "$WORKER_UNIT" || true
   if wait_health; then
     echo "部署失败，已回滚到备份，health 已恢复（新代码未上线）" >&2
   else
@@ -198,7 +216,7 @@ rollback() {
 }
 
 # --- 6. 启动 + 验 health + 验上线 commit -------------------------------
-systemctl start codex-cloud-server "$WORKER_UNIT" || true
+systemctl start $SERVER_UNITS "$WORKER_UNIT" || true
 if ! wait_health; then
   rollback
 fi

deploy/tests/test_host_deploy.py

@@ -39,12 +39,17 @@ class HostDeploySim(unittest.TestCase):
             f.write("keep\n")
         with open(os.path.join(self.app, "old.py"), "w") as f:
             f.write("x = 0\n")
-        # 假 systemctl：is-active 一律失败（走 cloud-worker 分支），其余成功。
+        # 假 systemctl：is-active 一律失败（走 cloud-worker 分支），其余成功；
+        # cat codex-cloud-server-2 按 FAKE_SERVER2_PRESENT 决定 unit 是否存在。
         fake_bin = os.path.join(base, "fakebin")
         os.makedirs(fake_bin)
         with open(os.path.join(fake_bin, "systemctl"), "w") as f:
             f.write("#!/bin/sh\n"
                     "echo \"$@\" >> \"$FAKE_SYSTEMCTL_LOG\"\n"
+                    "if [ \"$1\" = \"cat\" ] && [ \"$2\" = \"codex-cloud-server-2\" ]; then\n"
+                    "  if [ \"$FAKE_SERVER2_PRESENT\" = \"0\" ]; then exit 1; fi\n"
+                    "  exit 0\n"
+                    "fi\n"
                     "if [ \"$1\" = \"is-active\" ]; then exit 1; fi\n"
                     "exit 0\n")
         with open(os.path.join(fake_bin, "curl"), "w") as f:
@@ -67,6 +72,7 @@ class HostDeploySim(unittest.TestCase):
         self.env["CODEX_DEPLOY_HEALTH_ATTEMPTS"] = "3"
         self.env["FAKE_HEALTH_JSON"] = self.health_path
         self.env["FAKE_SYSTEMCTL_LOG"] = self.log_path
+        self.env["FAKE_SERVER2_PRESENT"] = "1"
 
     def tearDown(self):
         self.tmp.cleanup()
@@ -86,10 +92,17 @@ class HostDeploySim(unittest.TestCase):
         sha = make_tarball(src, tarball)
         return tarball, sha
 
-    def run_host(self, tarball, sha, commit):
+    def run_host(self, tarball, sha, commit, extra_env=None):
+        env = dict(self.env)
+        if extra_env:
+            env.update(extra_env)
         return subprocess.run(
             ["sh", HOST_SCRIPT, tarball, sha, commit],
-            capture_output=True, text=True, env=self.env)
+            capture_output=True, text=True, env=env)
+
+    def systemctl_log(self):
+        with open(self.log_path) as f:
+            return f.read()
 
     def live_commit(self):
         with open(os.path.join(self.app, "DEPLOYED_COMMIT")) as f:
@@ -142,6 +155,36 @@ class HostDeploySim(unittest.TestCase):
         with open(os.path.join(self.app, "work", "keep.txt")) as f:
             self.assertEqual(f.read().strip(), "keep")
 
+    def test_server2_stopped_started_when_present(self):
+        # 2026-10-27 首推教训：server-2 存在时必须同批停/起，
+        # 否则 LB 混跑新旧内存代码。
+        self.set_health({"ok": True, "backend": "postgres", "db_ok": True,
+                         "commit": "newcommit"})
+        tarball, sha = self.build_new_tarball()
+        r = self.run_host(tarball, sha, "newcommit")
+        self.assertEqual(r.returncode, 0, r.stdout + r.stderr)
+        log = self.systemctl_log()
+        self.assertIn(
+            "stop codex-cloud-worker codex-cloud-server codex-cloud-server-2",
+            log)
+        self.assertIn(
+            "start codex-cloud-server codex-cloud-server-2 codex-cloud-worker",
+            log)
+
+    def test_server2_skipped_when_unit_absent(self):
+        # 老主机没有 server-2 unit 时退回单机行为，不报错、不提它。
+        self.set_health({"ok": True, "backend": "postgres", "db_ok": True,
+                         "commit": "newcommit"})
+        tarball, sha = self.build_new_tarball()
+        r = self.run_host(tarball, sha, "newcommit",
+                          extra_env={"FAKE_SERVER2_PRESENT": "0"})
+        self.assertEqual(r.returncode, 0, r.stdout + r.stderr)
+        # 探测行 cat 本身会提到 server-2（探测失败才判定不存在），
+        # 所以断言停/起行里没有它，而不是全日志没有这个词。
+        log = self.systemctl_log()
+        self.assertIn("stop codex-cloud-worker codex-cloud-server\n", log)
+        self.assertIn("start codex-cloud-server codex-cloud-worker\n", log)
+
 
 if __name__ == "__main__":
     unittest.main()