From 6e78449994c95a4eee95a0ea7ce7ce38dc6ffa3c Mon Sep 17 00:00:00 2001 From: host Date: Fri, 30 Oct 2026 02:00:06 +0800 Subject: [PATCH] =?UTF-8?q?feat(docker):=20=E8=B6=85=E6=97=B6=E4=B8=8D?= =?UTF-8?q?=E5=86=8D=E5=8F=AA=E5=89=A9=E4=B8=80=E5=8F=A5=20docker=20run=20?= =?UTF-8?q?timeout=EF=BC=8C=E7=95=99=E5=AD=98=20partial=20=E4=B8=8E?= =?UTF-8?q?=E5=AE=B9=E5=99=A8=E6=97=A5=E5=BF=97?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 2026-10-29 任务 A 教训的收尾：超时路径直接 kill+rm， --rm 容器一删日志就没了，事后只剩一句 timeout，复发无法 定位（#157 只补了 worker 终态审计，没补容器侧证据）。 修法：_docker_timeout_evidence 先 kill 不 rm，抓 docker logs 尾 200 行，再 rm；同时取 TimeoutExpired 自带的 partial stdout/stderr（旧代码直接丢了）。三步都 best-effort， 任一步失败都不影响返回 failed。dispatch 与三条 goal 路径 （goal/goal-resume/goal-resume-breakpoint）四处超时点同款 处理：stdout = timeout + partial（优先）或 logs，另存 container_log_tail 与 timeout_partial_stdout 字段；--rm 命令本身不动，不留僵尸容器。 单测 test_docker_sandbox 31 项（含新增超时证据 1 项， 在两个测试类里各跑一次）。 --- cloud/sandbox.py | 112 +++++++++++++++++++++++++---- cloud/tests/test_docker_sandbox.py | 35 +++++++++ 2 files changed, 132 insertions(+), 15 deletions(-) diff --git a/cloud/sandbox.py b/cloud/sandbox.py index 55ac0f4..a2fab57 100644 --- a/cloud/sandbox.py +++ b/cloud/sandbox.py @@ -257,6 +257,15 @@ def _result_text(parsed, stdout_full): return text +def _decode_output(value): + """subprocess 输出转 str：bytes 按 utf-8 容错解码，str 原样。""" + if value is None: + return "" + if isinstance(value, bytes): + return value.decode("utf-8", errors="replace") + return str(value) + + def _default_runner(cmd, env=None): """subprocess.run 的薄封装：捕获输出、文本模式。 @@ -1252,6 +1261,38 @@ class DockerExecutor(Executor): continue return ok + def _docker_timeout_evidence(self, task, exc): + """超时证据留存（2026-10-30 任务 A 教训的收尾）。 + + 旧超时路径直接 kill+rm：--rm 容器一删，容器日志就没了， + 事后只剩一句 'docker run timeout'，复发无法定位。修法： + 先 kill 不 rm，抓 docker logs 尾 200 行，再 rm；同时 + 取 TimeoutExpired 自带的 partial stdout/stderr（subprocess + 超时时仍保留已产出部分，旧代码直接丢了）。三步都 + best-effort，任一步失败都不影响返回 failed 本身。 + 返回 (partial_stdout, partial_stderr, log_tail) 均为 str。 + """ + name = self.container_name(task) + partial_out = _decode_output(getattr(exc, "stdout", "")) + partial_err = _decode_output(getattr(exc, "stderr", "")) + try: + self._call_runner(["docker", "kill", name]) + except Exception: + pass + log_tail = "" + try: + res = self._call_runner( + ["docker", "logs", "--tail", "200", name]) + log_tail = (_decode_output(getattr(res, "stdout", "")) + + _decode_output(getattr(res, "stderr", ""))) + except Exception: + pass + try: + self._call_runner(["docker", "rm", "-f", name]) + except Exception: + pass + return partial_out, partial_err, log_tail + def _run_in_container(self, task, inner_cmd, timeout=None, extra_mounts=None, network=None, extra_env=None): @@ -1277,12 +1318,20 @@ class DockerExecutor(Executor): env=self._docker_cli_env(extra_env)) except Exception as exc: if exc.__class__.__name__ == "TimeoutExpired": - self.kill_container(task) + p_out, p_err, log_tail = self._docker_timeout_evidence( + task, exc) + evidence = (p_out or log_tail or "") return { "executor": "docker", "task_id": task_id, - "status": "failed", "stdout": "docker run timeout", - "returncode": None, "host_status": None, "stderr": "", + "status": "failed", + "stdout": _truncate_stdout( + "docker run timeout\n" + evidence) + if evidence else "docker run timeout", + "returncode": None, "host_status": None, + "stderr": _truncate_stdout(p_err), "container": self.container_name(task), + "container_log_tail": log_tail[-2000:], + "timeout_partial_stdout": p_out[-2000:], } return { "executor": "docker", "task_id": task_id, @@ -1421,13 +1470,24 @@ class DockerExecutor(Executor): env=self._docker_cli_env(extra_env)) except Exception as exc: if exc.__class__.__name__ == "TimeoutExpired": - self.kill_container(task) + p_out, p_err, log_tail = self._docker_timeout_evidence( + task, exc) + evidence = (p_out or log_tail or "") return { "executor": "docker", "task_id": task["id"], - "status": "failed", "stdout": "docker run timeout", - "returncode": None, "host_status": None, "stderr": "", + "status": "failed", + "stdout": _truncate_stdout( + "docker run timeout\n" + evidence) + if evidence else "docker run timeout", + "returncode": None, "host_status": None, + "stderr": _truncate_stdout(p_err), "container": self.container_name(task), - "goal_met": None, "goal_error": "docker run timeout", + "container_log_tail": log_tail[-2000:], + "timeout_partial_stdout": p_out[-2000:], + "goal_met": None, + "goal_error": ("docker run timeout: " + + evidence[-400:]) + if evidence else "docker run timeout", } return { "executor": "docker", "task_id": task["id"], @@ -1536,13 +1596,24 @@ class DockerExecutor(Executor): env=self._docker_cli_env(extra_env)) except Exception as exc: if exc.__class__.__name__ == "TimeoutExpired": - self.kill_container(task) + p_out, p_err, log_tail = self._docker_timeout_evidence( + task, exc) + evidence = (p_out or log_tail or "") return { "executor": "docker", "task_id": task["id"], - "status": "failed", "stdout": "docker run timeout", - "returncode": None, "host_status": None, "stderr": "", + "status": "failed", + "stdout": _truncate_stdout( + "docker run timeout\n" + evidence) + if evidence else "docker run timeout", + "returncode": None, "host_status": None, + "stderr": _truncate_stdout(p_err), "container": self.container_name(task), - "goal_met": None, "goal_error": "docker run timeout", + "container_log_tail": log_tail[-2000:], + "timeout_partial_stdout": p_out[-2000:], + "goal_met": None, + "goal_error": ("docker run timeout: " + + evidence[-400:]) + if evidence else "docker run timeout", } return { "executor": "docker", "task_id": task["id"], @@ -1646,13 +1717,24 @@ class DockerExecutor(Executor): env=self._docker_cli_env(extra_env)) except Exception as exc: if exc.__class__.__name__ == "TimeoutExpired": - self.kill_container(task) + p_out, p_err, log_tail = self._docker_timeout_evidence( + task, exc) + evidence = (p_out or log_tail or "") return { "executor": "docker", "task_id": task["id"], - "status": "failed", "stdout": "docker run timeout", - "returncode": None, "host_status": None, "stderr": "", + "status": "failed", + "stdout": _truncate_stdout( + "docker run timeout\n" + evidence) + if evidence else "docker run timeout", + "returncode": None, "host_status": None, + "stderr": _truncate_stdout(p_err), "container": self.container_name(task), - "goal_met": None, "goal_error": "docker run timeout", + "container_log_tail": log_tail[-2000:], + "timeout_partial_stdout": p_out[-2000:], + "goal_met": None, + "goal_error": ("docker run timeout: " + + evidence[-400:]) + if evidence else "docker run timeout", } return { "executor": "docker", "task_id": task["id"], diff --git a/cloud/tests/test_docker_sandbox.py b/cloud/tests/test_docker_sandbox.py index f8e5a8e..97a3ae7 100644 --- a/cloud/tests/test_docker_sandbox.py +++ b/cloud/tests/test_docker_sandbox.py @@ -218,6 +218,41 @@ class DockerCommandTestCase(unittest.TestCase): self.assertIn(("rm", "-f"), [(c[1], c[2]) for c in seen if len(c) > 2]) + def test_timeout_keeps_partial_stdout_and_container_logs(self): + # 旧超时路径只剩一句 'docker run timeout'：TimeoutExpired + # 自带的 partial stdout 与 kill 后抓的 docker logs 尾都 + # 被丢了。新路径：先 kill 不 rm，抓 logs，再 rm。 + seen = [] + + def fake(cmd, timeout=None): + seen.append(list(cmd)) + if len(cmd) > 1 and cmd[1] == "run": + exc = subprocess.TimeoutExpired(cmd, timeout) + exc.stdout = "partial-host-output" + exc.stderr = "partial-err" + raise exc + if len(cmd) > 1 and cmd[1] == "logs": + return SimpleNamespace(returncode=0, + stdout="container-tail-log", + stderr="") + return SimpleNamespace(returncode=0, stdout="", stderr="") + + with mock.patch("cloud.sandbox.shutil.which", + return_value="/usr/bin/docker"): + ex = DockerExecutor("/fake/host.py", runner=fake, + env={"XCM_API_KEY": "placeholder"}, + timeout=5) + result = ex.dispatch_task( + {"id": "t10", "prompt": "slow", "worktree": "/tmp/wt"}) + self.assertEqual(result["status"], "failed") + self.assertIn("partial-host-output", result["stdout"]) + # partial 已有内容时 stdout 优先用 partial，logs 另存字段。 + self.assertEqual(result["container_log_tail"], + "container-tail-log") + # 顺序：kill → logs → rm，logs 必须在 rm 之前抓到。 + kinds = [c[1] for c in seen if len(c) > 1] + self.assertLess(kinds.index("logs"), kinds.index("rm")) + def test_kill_container_calls_kill_and_rm(self): seen = [] with mock.patch("cloud.sandbox.shutil.which",