diff --git a/cloud/sandbox.py b/cloud/sandbox.py index 55ac0f4..a2fab57 100644 --- a/cloud/sandbox.py +++ b/cloud/sandbox.py @@ -257,6 +257,15 @@ def _result_text(parsed, stdout_full): return text +def _decode_output(value): + """subprocess 输出转 str：bytes 按 utf-8 容错解码，str 原样。""" + if value is None: + return "" + if isinstance(value, bytes): + return value.decode("utf-8", errors="replace") + return str(value) + + def _default_runner(cmd, env=None): """subprocess.run 的薄封装：捕获输出、文本模式。 @@ -1252,6 +1261,38 @@ class DockerExecutor(Executor): continue return ok + def _docker_timeout_evidence(self, task, exc): + """超时证据留存（2026-10-30 任务 A 教训的收尾）。 + + 旧超时路径直接 kill+rm：--rm 容器一删，容器日志就没了， + 事后只剩一句 'docker run timeout'，复发无法定位。修法： + 先 kill 不 rm，抓 docker logs 尾 200 行，再 rm；同时 + 取 TimeoutExpired 自带的 partial stdout/stderr（subprocess + 超时时仍保留已产出部分，旧代码直接丢了）。三步都 + best-effort，任一步失败都不影响返回 failed 本身。 + 返回 (partial_stdout, partial_stderr, log_tail) 均为 str。 + """ + name = self.container_name(task) + partial_out = _decode_output(getattr(exc, "stdout", "")) + partial_err = _decode_output(getattr(exc, "stderr", "")) + try: + self._call_runner(["docker", "kill", name]) + except Exception: + pass + log_tail = "" + try: + res = self._call_runner( + ["docker", "logs", "--tail", "200", name]) + log_tail = (_decode_output(getattr(res, "stdout", "")) + + _decode_output(getattr(res, "stderr", ""))) + except Exception: + pass + try: + self._call_runner(["docker", "rm", "-f", name]) + except Exception: + pass + return partial_out, partial_err, log_tail + def _run_in_container(self, task, inner_cmd, timeout=None, extra_mounts=None, network=None, extra_env=None): @@ -1277,12 +1318,20 @@ class DockerExecutor(Executor): env=self._docker_cli_env(extra_env)) except Exception as exc: if exc.__class__.__name__ == "TimeoutExpired": - self.kill_container(task) + p_out, p_err, log_tail = self._docker_timeout_evidence( + task, exc) + evidence = (p_out or log_tail or "") return { "executor": "docker", "task_id": task_id, - "status": "failed", "stdout": "docker run timeout", - "returncode": None, "host_status": None, "stderr": "", + "status": "failed", + "stdout": _truncate_stdout( + "docker run timeout\n" + evidence) + if evidence else "docker run timeout", + "returncode": None, "host_status": None, + "stderr": _truncate_stdout(p_err), "container": self.container_name(task), + "container_log_tail": log_tail[-2000:], + "timeout_partial_stdout": p_out[-2000:], } return { "executor": "docker", "task_id": task_id, @@ -1421,13 +1470,24 @@ class DockerExecutor(Executor): env=self._docker_cli_env(extra_env)) except Exception as exc: if exc.__class__.__name__ == "TimeoutExpired": - self.kill_container(task) + p_out, p_err, log_tail = self._docker_timeout_evidence( + task, exc) + evidence = (p_out or log_tail or "") return { "executor": "docker", "task_id": task["id"], - "status": "failed", "stdout": "docker run timeout", - "returncode": None, "host_status": None, "stderr": "", + "status": "failed", + "stdout": _truncate_stdout( + "docker run timeout\n" + evidence) + if evidence else "docker run timeout", + "returncode": None, "host_status": None, + "stderr": _truncate_stdout(p_err), "container": self.container_name(task), - "goal_met": None, "goal_error": "docker run timeout", + "container_log_tail": log_tail[-2000:], + "timeout_partial_stdout": p_out[-2000:], + "goal_met": None, + "goal_error": ("docker run timeout: " + + evidence[-400:]) + if evidence else "docker run timeout", } return { "executor": "docker", "task_id": task["id"], @@ -1536,13 +1596,24 @@ class DockerExecutor(Executor): env=self._docker_cli_env(extra_env)) except Exception as exc: if exc.__class__.__name__ == "TimeoutExpired": - self.kill_container(task) + p_out, p_err, log_tail = self._docker_timeout_evidence( + task, exc) + evidence = (p_out or log_tail or "") return { "executor": "docker", "task_id": task["id"], - "status": "failed", "stdout": "docker run timeout", - "returncode": None, "host_status": None, "stderr": "", + "status": "failed", + "stdout": _truncate_stdout( + "docker run timeout\n" + evidence) + if evidence else "docker run timeout", + "returncode": None, "host_status": None, + "stderr": _truncate_stdout(p_err), "container": self.container_name(task), - "goal_met": None, "goal_error": "docker run timeout", + "container_log_tail": log_tail[-2000:], + "timeout_partial_stdout": p_out[-2000:], + "goal_met": None, + "goal_error": ("docker run timeout: " + + evidence[-400:]) + if evidence else "docker run timeout", } return { "executor": "docker", "task_id": task["id"], @@ -1646,13 +1717,24 @@ class DockerExecutor(Executor): env=self._docker_cli_env(extra_env)) except Exception as exc: if exc.__class__.__name__ == "TimeoutExpired": - self.kill_container(task) + p_out, p_err, log_tail = self._docker_timeout_evidence( + task, exc) + evidence = (p_out or log_tail or "") return { "executor": "docker", "task_id": task["id"], - "status": "failed", "stdout": "docker run timeout", - "returncode": None, "host_status": None, "stderr": "", + "status": "failed", + "stdout": _truncate_stdout( + "docker run timeout\n" + evidence) + if evidence else "docker run timeout", + "returncode": None, "host_status": None, + "stderr": _truncate_stdout(p_err), "container": self.container_name(task), - "goal_met": None, "goal_error": "docker run timeout", + "container_log_tail": log_tail[-2000:], + "timeout_partial_stdout": p_out[-2000:], + "goal_met": None, + "goal_error": ("docker run timeout: " + + evidence[-400:]) + if evidence else "docker run timeout", } return { "executor": "docker", "task_id": task["id"], diff --git a/cloud/tests/test_docker_sandbox.py b/cloud/tests/test_docker_sandbox.py index f8e5a8e..97a3ae7 100644 --- a/cloud/tests/test_docker_sandbox.py +++ b/cloud/tests/test_docker_sandbox.py @@ -218,6 +218,41 @@ class DockerCommandTestCase(unittest.TestCase): self.assertIn(("rm", "-f"), [(c[1], c[2]) for c in seen if len(c) > 2]) + def test_timeout_keeps_partial_stdout_and_container_logs(self): + # 旧超时路径只剩一句 'docker run timeout'：TimeoutExpired + # 自带的 partial stdout 与 kill 后抓的 docker logs 尾都 + # 被丢了。新路径：先 kill 不 rm，抓 logs，再 rm。 + seen = [] + + def fake(cmd, timeout=None): + seen.append(list(cmd)) + if len(cmd) > 1 and cmd[1] == "run": + exc = subprocess.TimeoutExpired(cmd, timeout) + exc.stdout = "partial-host-output" + exc.stderr = "partial-err" + raise exc + if len(cmd) > 1 and cmd[1] == "logs": + return SimpleNamespace(returncode=0, + stdout="container-tail-log", + stderr="") + return SimpleNamespace(returncode=0, stdout="", stderr="") + + with mock.patch("cloud.sandbox.shutil.which", + return_value="/usr/bin/docker"): + ex = DockerExecutor("/fake/host.py", runner=fake, + env={"XCM_API_KEY": "placeholder"}, + timeout=5) + result = ex.dispatch_task( + {"id": "t10", "prompt": "slow", "worktree": "/tmp/wt"}) + self.assertEqual(result["status"], "failed") + self.assertIn("partial-host-output", result["stdout"]) + # partial 已有内容时 stdout 优先用 partial，logs 另存字段。 + self.assertEqual(result["container_log_tail"], + "container-tail-log") + # 顺序：kill → logs → rm，logs 必须在 rm 之前抓到。 + kinds = [c[1] for c in seen if len(c) > 1] + self.assertLess(kinds.index("logs"), kinds.index("rm")) + def test_kill_container_calls_kill_and_rm(self): seen = [] with mock.patch("cloud.sandbox.shutil.which",