From 29dcae4513fa9670ff841df7b4659bcc6338686b Mon Sep 17 00:00:00 2001 From: host Date: Mon, 26 Oct 2026 03:59:54 +0800 Subject: [PATCH] Hotfix: stale-container cleanup must not delete infra containers Worker startup cleanup previously deleted every container whose name started with codex-, including the codex-postgres database container. On the external host the first worker restart after the prod-grade deploy deleted codex-postgres, leaving the control plane unable to connect to Postgres (connection refused on 127.0.0.1:5432, server auto-restart loop). Fix: cleanup only deletes codex-<32-hex task id> containers, which is exactly the name DockerExecutor gives task containers; infrastructure names like codex-postgres never match and are never deleted. Regression test covers postgres + non-hex names. --- cloud/sandbox.py | 13 ++++++++++++- cloud/tests/test_prod_worker_timeout.py | 20 ++++++++++++++------ 2 files changed, 26 insertions(+), 7 deletions(-) diff --git a/cloud/sandbox.py b/cloud/sandbox.py index ab2b367..b7df802 100644 --- a/cloud/sandbox.py +++ b/cloud/sandbox.py @@ -425,14 +425,25 @@ class DockerExecutor(Executor): names.append(name) return names + # 任务容器名是 codex-<32 位 hex 任务 id>（见 _container_name）。 + # 启动清理只能删这种容器，绝不能按 "codex-" 前缀全删：基础设施 + # 容器也叫 codex-*（例如 codex-postgres），按前缀删会把数据库 + # 容器当成残留任务容器杀掉（2026-10-26 实测：worker 重启时 + # codex-postgres 就是这样被删掉，导致控制面连不上 Postgres）。 + _TASK_CONTAINER_RE = re.compile(r"^codex-[0-9a-f]{32}$") + def cleanup_stale_containers(self, prefix="codex-"): - """清理上一轮残留的 codex- 容器：列出并逐个 kill+rm。 + """清理上一轮残留的任务容器：只删 codex-<32 位 hex 任务 id>。 返回已清理的容器名列表；单个容器清理失败只跳过它（kill 本身尽力而为），列出失败时返回 []，绝不抛异常。 + 基础设施容器（如 codex-postgres）名字不匹配任务 id 正则， + 永远不会被这里删掉。 """ cleaned = [] for name in self.list_containers(prefix=prefix): + if not self._TASK_CONTAINER_RE.match(name): + continue try: if self.kill_container(name): cleaned.append(name) diff --git a/cloud/tests/test_prod_worker_timeout.py b/cloud/tests/test_prod_worker_timeout.py index a79f066..f0fe161 100644 --- a/cloud/tests/test_prod_worker_timeout.py +++ b/cloud/tests/test_prod_worker_timeout.py @@ -212,18 +212,26 @@ class ContainerCleanupTestCase(unittest.TestCase): executor = DockerExecutor(runner=runner) return executor, calls - def test_cleanup_lists_by_prefix_and_removes(self): + def test_cleanup_only_task_containers_never_infra(self): + # 回归：2026-10-26 实测，按 codex- 前缀全删会把数据库容器 + # codex-postgres 当成残留任务容器杀掉，导致控制面连不上 + # Postgres。清理只能删 codex-<32 位 hex 任务 id>。 executor, calls = self._docker_executor( - ps_stdout="codex-abc\nother\ncodex-def\n") + ps_stdout="codex-postgres\n" + "codex-3689cf40d671492992c6bfedd18b004c\n" + "codex-abc\nother\ncodex-app-server\n") cleaned = executor.cleanup_stale_containers() - self.assertEqual(cleaned, ["codex-abc", "codex-def"]) + self.assertEqual( + cleaned, ["codex-3689cf40d671492992c6bfedd18b004c"]) flat = [c for call in calls for c in call] + self.assertNotIn("codex-postgres", flat) + self.assertNotIn("codex-abc", flat) self.assertNotIn("other", flat) - # kill + rm 各调一次（每个容器）。 + # kill + rm 各调一次（仅那个任务容器）。 self.assertEqual( - sum(1 for call in calls if call[1] == "kill"), 2) + sum(1 for call in calls if call[1] == "kill"), 1) self.assertEqual( - sum(1 for call in calls if call[1] == "rm"), 2) + sum(1 for call in calls if call[1] == "rm"), 1) def test_cleanup_failure_does_not_crash_worker(self): executor, _ = self._docker_executor(fail_ps=True)