feat(docker): 超时不再只剩一句 docker run timeout,留存 partial 与容器日志
2026-10-29 任务 A 教训的收尾:超时路径直接 kill+rm, --rm 容器一删日志就没了,事后只剩一句 timeout,复发无法 定位(#157 只补了 worker 终态审计,没补容器侧证据)。 修法:_docker_timeout_evidence 先 kill 不 rm,抓 docker logs 尾 200 行,再 rm;同时取 TimeoutExpired 自带的 partial stdout/stderr(旧代码直接丢了)。三步都 best-effort, 任一步失败都不影响返回 failed。dispatch 与三条 goal 路径 (goal/goal-resume/goal-resume-breakpoint)四处超时点同款 处理:stdout = timeout + partial(优先)或 logs,另存 container_log_tail 与 timeout_partial_stdout 字段;--rm 命令本身不动,不留僵尸容器。 单测 test_docker_sandbox 31 项(含新增超时证据 1 项, 在两个测试类里各跑一次)。
This commit is contained in:
1 parent
700641fa37
commit
6e78449994
2 files changed
+132
-15
No files matched your search
+97
-15
@@ -257,6 +257,15 @@ def _result_text(parsed, stdout_full):
|
|||||||
return text
|
return text
|
||||||
|
|
||||||
|
|
||||||
|
def _decode_output(value):
|
||||||
|
"""subprocess 输出转 str:bytes 按 utf-8 容错解码,str 原样。"""
|
||||||
|
if value is None:
|
||||||
|
return ""
|
||||||
|
if isinstance(value, bytes):
|
||||||
|
return value.decode("utf-8", errors="replace")
|
||||||
|
return str(value)
|
||||||
|
|
||||||
|
|
||||||
def _default_runner(cmd, env=None):
|
def _default_runner(cmd, env=None):
|
||||||
"""subprocess.run 的薄封装:捕获输出、文本模式。
|
"""subprocess.run 的薄封装:捕获输出、文本模式。
|
||||||
|
|
||||||
@@ -1252,6 +1261,38 @@ class DockerExecutor(Executor):
|
|||||||
continue
|
continue
|
||||||
return ok
|
return ok
|
||||||
|
|
||||||
|
def _docker_timeout_evidence(self, task, exc):
|
||||||
|
"""超时证据留存(2026-10-30 任务 A 教训的收尾)。
|
||||||
|
|
||||||
|
旧超时路径直接 kill+rm:--rm 容器一删,容器日志就没了,
|
||||||
|
事后只剩一句 'docker run timeout',复发无法定位。修法:
|
||||||
|
先 kill 不 rm,抓 docker logs 尾 200 行,再 rm;同时
|
||||||
|
取 TimeoutExpired 自带的 partial stdout/stderr(subprocess
|
||||||
|
超时时仍保留已产出部分,旧代码直接丢了)。三步都
|
||||||
|
best-effort,任一步失败都不影响返回 failed 本身。
|
||||||
|
返回 (partial_stdout, partial_stderr, log_tail) 均为 str。
|
||||||
|
"""
|
||||||
|
name = self.container_name(task)
|
||||||
|
partial_out = _decode_output(getattr(exc, "stdout", ""))
|
||||||
|
partial_err = _decode_output(getattr(exc, "stderr", ""))
|
||||||
|
try:
|
||||||
|
self._call_runner(["docker", "kill", name])
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
log_tail = ""
|
||||||
|
try:
|
||||||
|
res = self._call_runner(
|
||||||
|
["docker", "logs", "--tail", "200", name])
|
||||||
|
log_tail = (_decode_output(getattr(res, "stdout", "")) +
|
||||||
|
_decode_output(getattr(res, "stderr", "")))
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
try:
|
||||||
|
self._call_runner(["docker", "rm", "-f", name])
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
return partial_out, partial_err, log_tail
|
||||||
|
|
||||||
def _run_in_container(self, task, inner_cmd, timeout=None,
|
def _run_in_container(self, task, inner_cmd, timeout=None,
|
||||||
extra_mounts=None, network=None,
|
extra_mounts=None, network=None,
|
||||||
extra_env=None):
|
extra_env=None):
|
||||||
@@ -1277,12 +1318,20 @@ class DockerExecutor(Executor):
|
|||||||
env=self._docker_cli_env(extra_env))
|
env=self._docker_cli_env(extra_env))
|
||||||
except Exception as exc:
|
except Exception as exc:
|
||||||
if exc.__class__.__name__ == "TimeoutExpired":
|
if exc.__class__.__name__ == "TimeoutExpired":
|
||||||
self.kill_container(task)
|
p_out, p_err, log_tail = self._docker_timeout_evidence(
|
||||||
|
task, exc)
|
||||||
|
evidence = (p_out or log_tail or "")
|
||||||
return {
|
return {
|
||||||
"executor": "docker", "task_id": task_id,
|
"executor": "docker", "task_id": task_id,
|
||||||
"status": "failed", "stdout": "docker run timeout",
|
"status": "failed",
|
||||||
"returncode": None, "host_status": None, "stderr": "",
|
"stdout": _truncate_stdout(
|
||||||
|
"docker run timeout\n" + evidence)
|
||||||
|
if evidence else "docker run timeout",
|
||||||
|
"returncode": None, "host_status": None,
|
||||||
|
"stderr": _truncate_stdout(p_err),
|
||||||
"container": self.container_name(task),
|
"container": self.container_name(task),
|
||||||
|
"container_log_tail": log_tail[-2000:],
|
||||||
|
"timeout_partial_stdout": p_out[-2000:],
|
||||||
}
|
}
|
||||||
return {
|
return {
|
||||||
"executor": "docker", "task_id": task_id,
|
"executor": "docker", "task_id": task_id,
|
||||||
@@ -1421,13 +1470,24 @@ class DockerExecutor(Executor):
|
|||||||
env=self._docker_cli_env(extra_env))
|
env=self._docker_cli_env(extra_env))
|
||||||
except Exception as exc:
|
except Exception as exc:
|
||||||
if exc.__class__.__name__ == "TimeoutExpired":
|
if exc.__class__.__name__ == "TimeoutExpired":
|
||||||
self.kill_container(task)
|
p_out, p_err, log_tail = self._docker_timeout_evidence(
|
||||||
|
task, exc)
|
||||||
|
evidence = (p_out or log_tail or "")
|
||||||
return {
|
return {
|
||||||
"executor": "docker", "task_id": task["id"],
|
"executor": "docker", "task_id": task["id"],
|
||||||
"status": "failed", "stdout": "docker run timeout",
|
"status": "failed",
|
||||||
"returncode": None, "host_status": None, "stderr": "",
|
"stdout": _truncate_stdout(
|
||||||
|
"docker run timeout\n" + evidence)
|
||||||
|
if evidence else "docker run timeout",
|
||||||
|
"returncode": None, "host_status": None,
|
||||||
|
"stderr": _truncate_stdout(p_err),
|
||||||
"container": self.container_name(task),
|
"container": self.container_name(task),
|
||||||
"goal_met": None, "goal_error": "docker run timeout",
|
"container_log_tail": log_tail[-2000:],
|
||||||
|
"timeout_partial_stdout": p_out[-2000:],
|
||||||
|
"goal_met": None,
|
||||||
|
"goal_error": ("docker run timeout: " +
|
||||||
|
evidence[-400:])
|
||||||
|
if evidence else "docker run timeout",
|
||||||
}
|
}
|
||||||
return {
|
return {
|
||||||
"executor": "docker", "task_id": task["id"],
|
"executor": "docker", "task_id": task["id"],
|
||||||
@@ -1536,13 +1596,24 @@ class DockerExecutor(Executor):
|
|||||||
env=self._docker_cli_env(extra_env))
|
env=self._docker_cli_env(extra_env))
|
||||||
except Exception as exc:
|
except Exception as exc:
|
||||||
if exc.__class__.__name__ == "TimeoutExpired":
|
if exc.__class__.__name__ == "TimeoutExpired":
|
||||||
self.kill_container(task)
|
p_out, p_err, log_tail = self._docker_timeout_evidence(
|
||||||
|
task, exc)
|
||||||
|
evidence = (p_out or log_tail or "")
|
||||||
return {
|
return {
|
||||||
"executor": "docker", "task_id": task["id"],
|
"executor": "docker", "task_id": task["id"],
|
||||||
"status": "failed", "stdout": "docker run timeout",
|
"status": "failed",
|
||||||
"returncode": None, "host_status": None, "stderr": "",
|
"stdout": _truncate_stdout(
|
||||||
|
"docker run timeout\n" + evidence)
|
||||||
|
if evidence else "docker run timeout",
|
||||||
|
"returncode": None, "host_status": None,
|
||||||
|
"stderr": _truncate_stdout(p_err),
|
||||||
"container": self.container_name(task),
|
"container": self.container_name(task),
|
||||||
"goal_met": None, "goal_error": "docker run timeout",
|
"container_log_tail": log_tail[-2000:],
|
||||||
|
"timeout_partial_stdout": p_out[-2000:],
|
||||||
|
"goal_met": None,
|
||||||
|
"goal_error": ("docker run timeout: " +
|
||||||
|
evidence[-400:])
|
||||||
|
if evidence else "docker run timeout",
|
||||||
}
|
}
|
||||||
return {
|
return {
|
||||||
"executor": "docker", "task_id": task["id"],
|
"executor": "docker", "task_id": task["id"],
|
||||||
@@ -1646,13 +1717,24 @@ class DockerExecutor(Executor):
|
|||||||
env=self._docker_cli_env(extra_env))
|
env=self._docker_cli_env(extra_env))
|
||||||
except Exception as exc:
|
except Exception as exc:
|
||||||
if exc.__class__.__name__ == "TimeoutExpired":
|
if exc.__class__.__name__ == "TimeoutExpired":
|
||||||
self.kill_container(task)
|
p_out, p_err, log_tail = self._docker_timeout_evidence(
|
||||||
|
task, exc)
|
||||||
|
evidence = (p_out or log_tail or "")
|
||||||
return {
|
return {
|
||||||
"executor": "docker", "task_id": task["id"],
|
"executor": "docker", "task_id": task["id"],
|
||||||
"status": "failed", "stdout": "docker run timeout",
|
"status": "failed",
|
||||||
"returncode": None, "host_status": None, "stderr": "",
|
"stdout": _truncate_stdout(
|
||||||
|
"docker run timeout\n" + evidence)
|
||||||
|
if evidence else "docker run timeout",
|
||||||
|
"returncode": None, "host_status": None,
|
||||||
|
"stderr": _truncate_stdout(p_err),
|
||||||
"container": self.container_name(task),
|
"container": self.container_name(task),
|
||||||
"goal_met": None, "goal_error": "docker run timeout",
|
"container_log_tail": log_tail[-2000:],
|
||||||
|
"timeout_partial_stdout": p_out[-2000:],
|
||||||
|
"goal_met": None,
|
||||||
|
"goal_error": ("docker run timeout: " +
|
||||||
|
evidence[-400:])
|
||||||
|
if evidence else "docker run timeout",
|
||||||
}
|
}
|
||||||
return {
|
return {
|
||||||
"executor": "docker", "task_id": task["id"],
|
"executor": "docker", "task_id": task["id"],
|
||||||
|
|||||||
@@ -218,6 +218,41 @@ class DockerCommandTestCase(unittest.TestCase):
|
|||||||
self.assertIn(("rm", "-f"), [(c[1], c[2]) for c in seen
|
self.assertIn(("rm", "-f"), [(c[1], c[2]) for c in seen
|
||||||
if len(c) > 2])
|
if len(c) > 2])
|
||||||
|
|
||||||
|
def test_timeout_keeps_partial_stdout_and_container_logs(self):
|
||||||
|
# 旧超时路径只剩一句 'docker run timeout':TimeoutExpired
|
||||||
|
# 自带的 partial stdout 与 kill 后抓的 docker logs 尾都
|
||||||
|
# 被丢了。新路径:先 kill 不 rm,抓 logs,再 rm。
|
||||||
|
seen = []
|
||||||
|
|
||||||
|
def fake(cmd, timeout=None):
|
||||||
|
seen.append(list(cmd))
|
||||||
|
if len(cmd) > 1 and cmd[1] == "run":
|
||||||
|
exc = subprocess.TimeoutExpired(cmd, timeout)
|
||||||
|
exc.stdout = "partial-host-output"
|
||||||
|
exc.stderr = "partial-err"
|
||||||
|
raise exc
|
||||||
|
if len(cmd) > 1 and cmd[1] == "logs":
|
||||||
|
return SimpleNamespace(returncode=0,
|
||||||
|
stdout="container-tail-log",
|
||||||
|
stderr="")
|
||||||
|
return SimpleNamespace(returncode=0, stdout="", stderr="")
|
||||||
|
|
||||||
|
with mock.patch("cloud.sandbox.shutil.which",
|
||||||
|
return_value="/usr/bin/docker"):
|
||||||
|
ex = DockerExecutor("/fake/host.py", runner=fake,
|
||||||
|
env={"XCM_API_KEY": "placeholder"},
|
||||||
|
timeout=5)
|
||||||
|
result = ex.dispatch_task(
|
||||||
|
{"id": "t10", "prompt": "slow", "worktree": "/tmp/wt"})
|
||||||
|
self.assertEqual(result["status"], "failed")
|
||||||
|
self.assertIn("partial-host-output", result["stdout"])
|
||||||
|
# partial 已有内容时 stdout 优先用 partial,logs 另存字段。
|
||||||
|
self.assertEqual(result["container_log_tail"],
|
||||||
|
"container-tail-log")
|
||||||
|
# 顺序:kill → logs → rm,logs 必须在 rm 之前抓到。
|
||||||
|
kinds = [c[1] for c in seen if len(c) > 1]
|
||||||
|
self.assertLess(kinds.index("logs"), kinds.index("rm"))
|
||||||
|
|
||||||
def test_kill_container_calls_kill_and_rm(self):
|
def test_kill_container_calls_kill_and_rm(self):
|
||||||
seen = []
|
seen = []
|
||||||
with mock.patch("cloud.sandbox.shutil.which",
|
with mock.patch("cloud.sandbox.shutil.which",
|
||||||
|
|||||||
Reference in new issue
Block a user