feat(docker): 超时不再只剩一句 docker run timeout,留存 partial 与容器日志
2026-10-29 任务 A 教训的收尾:超时路径直接 kill+rm, --rm 容器一删日志就没了,事后只剩一句 timeout,复发无法 定位(#157 只补了 worker 终态审计,没补容器侧证据)。 修法:_docker_timeout_evidence 先 kill 不 rm,抓 docker logs 尾 200 行,再 rm;同时取 TimeoutExpired 自带的 partial stdout/stderr(旧代码直接丢了)。三步都 best-effort, 任一步失败都不影响返回 failed。dispatch 与三条 goal 路径 (goal/goal-resume/goal-resume-breakpoint)四处超时点同款 处理:stdout = timeout + partial(优先)或 logs,另存 container_log_tail 与 timeout_partial_stdout 字段;--rm 命令本身不动,不留僵尸容器。 单测 test_docker_sandbox 31 项(含新增超时证据 1 项, 在两个测试类里各跑一次)。
This commit is contained in:
1 parent
700641fa37
commit
6e78449994
2 files changed
+132
-15
No files matched your search
+97
-15
@@ -257,6 +257,15 @@ def _result_text(parsed, stdout_full):
|
||||
return text
|
||||
|
||||
|
||||
def _decode_output(value):
|
||||
"""subprocess 输出转 str:bytes 按 utf-8 容错解码,str 原样。"""
|
||||
if value is None:
|
||||
return ""
|
||||
if isinstance(value, bytes):
|
||||
return value.decode("utf-8", errors="replace")
|
||||
return str(value)
|
||||
|
||||
|
||||
def _default_runner(cmd, env=None):
|
||||
"""subprocess.run 的薄封装:捕获输出、文本模式。
|
||||
|
||||
@@ -1252,6 +1261,38 @@ class DockerExecutor(Executor):
|
||||
continue
|
||||
return ok
|
||||
|
||||
def _docker_timeout_evidence(self, task, exc):
|
||||
"""超时证据留存(2026-10-30 任务 A 教训的收尾)。
|
||||
|
||||
旧超时路径直接 kill+rm:--rm 容器一删,容器日志就没了,
|
||||
事后只剩一句 'docker run timeout',复发无法定位。修法:
|
||||
先 kill 不 rm,抓 docker logs 尾 200 行,再 rm;同时
|
||||
取 TimeoutExpired 自带的 partial stdout/stderr(subprocess
|
||||
超时时仍保留已产出部分,旧代码直接丢了)。三步都
|
||||
best-effort,任一步失败都不影响返回 failed 本身。
|
||||
返回 (partial_stdout, partial_stderr, log_tail) 均为 str。
|
||||
"""
|
||||
name = self.container_name(task)
|
||||
partial_out = _decode_output(getattr(exc, "stdout", ""))
|
||||
partial_err = _decode_output(getattr(exc, "stderr", ""))
|
||||
try:
|
||||
self._call_runner(["docker", "kill", name])
|
||||
except Exception:
|
||||
pass
|
||||
log_tail = ""
|
||||
try:
|
||||
res = self._call_runner(
|
||||
["docker", "logs", "--tail", "200", name])
|
||||
log_tail = (_decode_output(getattr(res, "stdout", "")) +
|
||||
_decode_output(getattr(res, "stderr", "")))
|
||||
except Exception:
|
||||
pass
|
||||
try:
|
||||
self._call_runner(["docker", "rm", "-f", name])
|
||||
except Exception:
|
||||
pass
|
||||
return partial_out, partial_err, log_tail
|
||||
|
||||
def _run_in_container(self, task, inner_cmd, timeout=None,
|
||||
extra_mounts=None, network=None,
|
||||
extra_env=None):
|
||||
@@ -1277,12 +1318,20 @@ class DockerExecutor(Executor):
|
||||
env=self._docker_cli_env(extra_env))
|
||||
except Exception as exc:
|
||||
if exc.__class__.__name__ == "TimeoutExpired":
|
||||
self.kill_container(task)
|
||||
p_out, p_err, log_tail = self._docker_timeout_evidence(
|
||||
task, exc)
|
||||
evidence = (p_out or log_tail or "")
|
||||
return {
|
||||
"executor": "docker", "task_id": task_id,
|
||||
"status": "failed", "stdout": "docker run timeout",
|
||||
"returncode": None, "host_status": None, "stderr": "",
|
||||
"status": "failed",
|
||||
"stdout": _truncate_stdout(
|
||||
"docker run timeout\n" + evidence)
|
||||
if evidence else "docker run timeout",
|
||||
"returncode": None, "host_status": None,
|
||||
"stderr": _truncate_stdout(p_err),
|
||||
"container": self.container_name(task),
|
||||
"container_log_tail": log_tail[-2000:],
|
||||
"timeout_partial_stdout": p_out[-2000:],
|
||||
}
|
||||
return {
|
||||
"executor": "docker", "task_id": task_id,
|
||||
@@ -1421,13 +1470,24 @@ class DockerExecutor(Executor):
|
||||
env=self._docker_cli_env(extra_env))
|
||||
except Exception as exc:
|
||||
if exc.__class__.__name__ == "TimeoutExpired":
|
||||
self.kill_container(task)
|
||||
p_out, p_err, log_tail = self._docker_timeout_evidence(
|
||||
task, exc)
|
||||
evidence = (p_out or log_tail or "")
|
||||
return {
|
||||
"executor": "docker", "task_id": task["id"],
|
||||
"status": "failed", "stdout": "docker run timeout",
|
||||
"returncode": None, "host_status": None, "stderr": "",
|
||||
"status": "failed",
|
||||
"stdout": _truncate_stdout(
|
||||
"docker run timeout\n" + evidence)
|
||||
if evidence else "docker run timeout",
|
||||
"returncode": None, "host_status": None,
|
||||
"stderr": _truncate_stdout(p_err),
|
||||
"container": self.container_name(task),
|
||||
"goal_met": None, "goal_error": "docker run timeout",
|
||||
"container_log_tail": log_tail[-2000:],
|
||||
"timeout_partial_stdout": p_out[-2000:],
|
||||
"goal_met": None,
|
||||
"goal_error": ("docker run timeout: " +
|
||||
evidence[-400:])
|
||||
if evidence else "docker run timeout",
|
||||
}
|
||||
return {
|
||||
"executor": "docker", "task_id": task["id"],
|
||||
@@ -1536,13 +1596,24 @@ class DockerExecutor(Executor):
|
||||
env=self._docker_cli_env(extra_env))
|
||||
except Exception as exc:
|
||||
if exc.__class__.__name__ == "TimeoutExpired":
|
||||
self.kill_container(task)
|
||||
p_out, p_err, log_tail = self._docker_timeout_evidence(
|
||||
task, exc)
|
||||
evidence = (p_out or log_tail or "")
|
||||
return {
|
||||
"executor": "docker", "task_id": task["id"],
|
||||
"status": "failed", "stdout": "docker run timeout",
|
||||
"returncode": None, "host_status": None, "stderr": "",
|
||||
"status": "failed",
|
||||
"stdout": _truncate_stdout(
|
||||
"docker run timeout\n" + evidence)
|
||||
if evidence else "docker run timeout",
|
||||
"returncode": None, "host_status": None,
|
||||
"stderr": _truncate_stdout(p_err),
|
||||
"container": self.container_name(task),
|
||||
"goal_met": None, "goal_error": "docker run timeout",
|
||||
"container_log_tail": log_tail[-2000:],
|
||||
"timeout_partial_stdout": p_out[-2000:],
|
||||
"goal_met": None,
|
||||
"goal_error": ("docker run timeout: " +
|
||||
evidence[-400:])
|
||||
if evidence else "docker run timeout",
|
||||
}
|
||||
return {
|
||||
"executor": "docker", "task_id": task["id"],
|
||||
@@ -1646,13 +1717,24 @@ class DockerExecutor(Executor):
|
||||
env=self._docker_cli_env(extra_env))
|
||||
except Exception as exc:
|
||||
if exc.__class__.__name__ == "TimeoutExpired":
|
||||
self.kill_container(task)
|
||||
p_out, p_err, log_tail = self._docker_timeout_evidence(
|
||||
task, exc)
|
||||
evidence = (p_out or log_tail or "")
|
||||
return {
|
||||
"executor": "docker", "task_id": task["id"],
|
||||
"status": "failed", "stdout": "docker run timeout",
|
||||
"returncode": None, "host_status": None, "stderr": "",
|
||||
"status": "failed",
|
||||
"stdout": _truncate_stdout(
|
||||
"docker run timeout\n" + evidence)
|
||||
if evidence else "docker run timeout",
|
||||
"returncode": None, "host_status": None,
|
||||
"stderr": _truncate_stdout(p_err),
|
||||
"container": self.container_name(task),
|
||||
"goal_met": None, "goal_error": "docker run timeout",
|
||||
"container_log_tail": log_tail[-2000:],
|
||||
"timeout_partial_stdout": p_out[-2000:],
|
||||
"goal_met": None,
|
||||
"goal_error": ("docker run timeout: " +
|
||||
evidence[-400:])
|
||||
if evidence else "docker run timeout",
|
||||
}
|
||||
return {
|
||||
"executor": "docker", "task_id": task["id"],
|
||||
|
||||
Reference in new issue
Block a user