feat(docker): 超时不再只剩一句 docker run timeout，留存 partial 与容器日志

2026-10-29 任务 A 教训的收尾：超时路径直接 kill+rm， --rm 容器一删日志就没了，事后只剩一句 timeout，复发无法定位（#157 只补了 worker 终态审计，没补容器侧证据）。修法：_docker_timeout_evidence 先 kill 不 rm，抓 docker logs 尾 200 行，再 rm；同时取 TimeoutExpired 自带的 partial stdout/stderr（旧代码直接丢了）。三步都 best-effort，任一步失败都不影响返回 failed。dispatch 与三条 goal 路径（goal/goal-resume/goal-resume-breakpoint）四处超时点同款处理：stdout = timeout + partial（优先）或 logs，另存 container_log_tail 与 timeout_partial_stdout 字段；--rm 命令本身不动，不留僵尸容器。单测 test_docker_sandbox 31 项（含新增超时证据 1 项，在两个测试类里各跑一次）。
2026-10-30 02:47:11 +08:00
1 parent 700641fa37
commit 6e78449994
@@ -257,6 +257,15 @@ def _result_text(parsed, stdout_full):
    return text


+def _decode_output(value):
+    """subprocess 输出转 str：bytes 按 utf-8 容错解码，str 原样。"""
+    if value is None:
+        return ""
+    if isinstance(value, bytes):
+        return value.decode("utf-8", errors="replace")
+    return str(value)
+
+
 def _default_runner(cmd, env=None):
    """subprocess.run 的薄封装：捕获输出、文本模式。

@@ -1252,6 +1261,38 @@ class DockerExecutor(Executor):
                continue
        return ok

+    def _docker_timeout_evidence(self, task, exc):
+        """超时证据留存（2026-10-30 任务 A 教训的收尾）。
+
+        旧超时路径直接 kill+rm：--rm 容器一删，容器日志就没了，
+        事后只剩一句 'docker run timeout'，复发无法定位。修法：
+        先 kill 不 rm，抓 docker logs 尾 200 行，再 rm；同时
+        取 TimeoutExpired 自带的 partial stdout/stderr（subprocess
+        超时时仍保留已产出部分，旧代码直接丢了）。三步都
+        best-effort，任一步失败都不影响返回 failed 本身。
+        返回 (partial_stdout, partial_stderr, log_tail) 均为 str。
+        """
+        name = self.container_name(task)
+        partial_out = _decode_output(getattr(exc, "stdout", ""))
+        partial_err = _decode_output(getattr(exc, "stderr", ""))
+        try:
+            self._call_runner(["docker", "kill", name])
+        except Exception:
+            pass
+        log_tail = ""
+        try:
+            res = self._call_runner(
+                ["docker", "logs", "--tail", "200", name])
+            log_tail = (_decode_output(getattr(res, "stdout", "")) +
+                        _decode_output(getattr(res, "stderr", "")))
+        except Exception:
+            pass
+        try:
+            self._call_runner(["docker", "rm", "-f", name])
+        except Exception:
+            pass
+        return partial_out, partial_err, log_tail
+
    def _run_in_container(self, task, inner_cmd, timeout=None,
                          extra_mounts=None, network=None,
                          extra_env=None):
@@ -1277,12 +1318,20 @@ class DockerExecutor(Executor):
                env=self._docker_cli_env(extra_env))
        except Exception as exc:
            if exc.__class__.__name__ == "TimeoutExpired":
-                self.kill_container(task)
+                p_out, p_err, log_tail = self._docker_timeout_evidence(
+                    task, exc)
+                evidence = (p_out or log_tail or "")
                return {
                    "executor": "docker", "task_id": task_id,
-                    "status": "failed", "stdout": "docker run timeout",
-                    "returncode": None, "host_status": None, "stderr": "",
+                    "status": "failed",
+                    "stdout": _truncate_stdout(
+                        "docker run timeout\n" + evidence)
+                    if evidence else "docker run timeout",
+                    "returncode": None, "host_status": None,
+                    "stderr": _truncate_stdout(p_err),
                    "container": self.container_name(task),
+                    "container_log_tail": log_tail[-2000:],
+                    "timeout_partial_stdout": p_out[-2000:],
                }
            return {
                "executor": "docker", "task_id": task_id,
@@ -1421,13 +1470,24 @@ class DockerExecutor(Executor):
                env=self._docker_cli_env(extra_env))
        except Exception as exc:
            if exc.__class__.__name__ == "TimeoutExpired":
-                self.kill_container(task)
+                p_out, p_err, log_tail = self._docker_timeout_evidence(
+                    task, exc)
+                evidence = (p_out or log_tail or "")
                return {
                    "executor": "docker", "task_id": task["id"],
-                    "status": "failed", "stdout": "docker run timeout",
-                    "returncode": None, "host_status": None, "stderr": "",
+                    "status": "failed",
+                    "stdout": _truncate_stdout(
+                        "docker run timeout\n" + evidence)
+                    if evidence else "docker run timeout",
+                    "returncode": None, "host_status": None,
+                    "stderr": _truncate_stdout(p_err),
                    "container": self.container_name(task),
-                    "goal_met": None, "goal_error": "docker run timeout",
+                    "container_log_tail": log_tail[-2000:],
+                    "timeout_partial_stdout": p_out[-2000:],
+                    "goal_met": None,
+                    "goal_error": ("docker run timeout: " +
+                                   evidence[-400:])
+                    if evidence else "docker run timeout",
                }
            return {
                "executor": "docker", "task_id": task["id"],
@@ -1536,13 +1596,24 @@ class DockerExecutor(Executor):
                env=self._docker_cli_env(extra_env))
        except Exception as exc:
            if exc.__class__.__name__ == "TimeoutExpired":
-                self.kill_container(task)
+                p_out, p_err, log_tail = self._docker_timeout_evidence(
+                    task, exc)
+                evidence = (p_out or log_tail or "")
                return {
                    "executor": "docker", "task_id": task["id"],
-                    "status": "failed", "stdout": "docker run timeout",
-                    "returncode": None, "host_status": None, "stderr": "",
+                    "status": "failed",
+                    "stdout": _truncate_stdout(
+                        "docker run timeout\n" + evidence)
+                    if evidence else "docker run timeout",
+                    "returncode": None, "host_status": None,
+                    "stderr": _truncate_stdout(p_err),
                    "container": self.container_name(task),
-                    "goal_met": None, "goal_error": "docker run timeout",
+                    "container_log_tail": log_tail[-2000:],
+                    "timeout_partial_stdout": p_out[-2000:],
+                    "goal_met": None,
+                    "goal_error": ("docker run timeout: " +
+                                   evidence[-400:])
+                    if evidence else "docker run timeout",
                }
            return {
                "executor": "docker", "task_id": task["id"],
@@ -1646,13 +1717,24 @@ class DockerExecutor(Executor):
                env=self._docker_cli_env(extra_env))
        except Exception as exc:
            if exc.__class__.__name__ == "TimeoutExpired":
-                self.kill_container(task)
+                p_out, p_err, log_tail = self._docker_timeout_evidence(
+                    task, exc)
+                evidence = (p_out or log_tail or "")
                return {
                    "executor": "docker", "task_id": task["id"],
-                    "status": "failed", "stdout": "docker run timeout",
-                    "returncode": None, "host_status": None, "stderr": "",
+                    "status": "failed",
+                    "stdout": _truncate_stdout(
+                        "docker run timeout\n" + evidence)
+                    if evidence else "docker run timeout",
+                    "returncode": None, "host_status": None,
+                    "stderr": _truncate_stdout(p_err),
                    "container": self.container_name(task),
-                    "goal_met": None, "goal_error": "docker run timeout",
+                    "container_log_tail": log_tail[-2000:],
+                    "timeout_partial_stdout": p_out[-2000:],
+                    "goal_met": None,
+                    "goal_error": ("docker run timeout: " +
+                                   evidence[-400:])
+                    if evidence else "docker run timeout",
                }
            return {
                "executor": "docker", "task_id": task["id"],
@@ -218,6 +218,41 @@ class DockerCommandTestCase(unittest.TestCase):
        self.assertIn(("rm", "-f"), [(c[1], c[2]) for c in seen
                                      if len(c) > 2])

+    def test_timeout_keeps_partial_stdout_and_container_logs(self):
+        # 旧超时路径只剩一句 'docker run timeout'：TimeoutExpired
+        # 自带的 partial stdout 与 kill 后抓的 docker logs 尾都
+        # 被丢了。新路径：先 kill 不 rm，抓 logs，再 rm。
+        seen = []
+
+        def fake(cmd, timeout=None):
+            seen.append(list(cmd))
+            if len(cmd) > 1 and cmd[1] == "run":
+                exc = subprocess.TimeoutExpired(cmd, timeout)
+                exc.stdout = "partial-host-output"
+                exc.stderr = "partial-err"
+                raise exc
+            if len(cmd) > 1 and cmd[1] == "logs":
+                return SimpleNamespace(returncode=0,
+                                       stdout="container-tail-log",
+                                       stderr="")
+            return SimpleNamespace(returncode=0, stdout="", stderr="")
+
+        with mock.patch("cloud.sandbox.shutil.which",
+                        return_value="/usr/bin/docker"):
+            ex = DockerExecutor("/fake/host.py", runner=fake,
+                                env={"XCM_API_KEY": "placeholder"},
+                                timeout=5)
+            result = ex.dispatch_task(
+                {"id": "t10", "prompt": "slow", "worktree": "/tmp/wt"})
+        self.assertEqual(result["status"], "failed")
+        self.assertIn("partial-host-output", result["stdout"])
+        # partial 已有内容时 stdout 优先用 partial，logs 另存字段。
+        self.assertEqual(result["container_log_tail"],
+                         "container-tail-log")
+        # 顺序：kill → logs → rm，logs 必须在 rm 之前抓到。
+        kinds = [c[1] for c in seen if len(c) > 1]
+        self.assertLess(kinds.index("logs"), kinds.index("rm"))
+
    def test_kill_container_calls_kill_and_rm(self):
        seen = []
        with mock.patch("cloud.sandbox.shutil.which",