worker: prestart stage logging for dispatch stalls

Claim-to-dispatch stalls (cf. 2026-10-29 task A: 61s to interrupted
with zero detail) were invisible because docker run blocks with
captured output and watchdog kills were silent. Add three layers:
- worker_exec_start/done/error audit rows in _run_attempt (PG,
  survives --rm), start/done pairing shows whether an attempt died
  inside the executor call;
- docker_run_start/done/error log lines in DockerExecutor (container
  + image only, never the inner command/prompt);
- stdout tail (last 2000 chars) to the worker log on failed or
  interrupted results, and a log line on both watchdog kill paths.
Tests: cloud/tests/test_prestart_stages.py, 6 cases incl. a
no-prompt-in-logs assertion.
This commit is contained in:
host committed 2026-10-29 17:07:53 +08:00
1 parent 0129b0149a
commit 9d90d97046
3 files changed
+251

No files matched your search

+22
View File
@@ -14,11 +14,13 @@
"""
import json
import logging
import os
import re
import shutil
import subprocess
import sys
import time
from cloud.marker_tools import extract_task_cancels as _extract_task_cancels
from cloud.marker_tools import extract_task_creates as _extract_task_creates
@@ -29,6 +31,12 @@ from cloud.peer_messages import extract_peer_messages as _extract_peer_messages
MAX_STDOUT_CHARS = 2000
# 启动前分段日志(2026-10-29 任务 A):docker run 是阻塞调用,
# 容器输出要等返回才回传,中途卡死时 worker 侧没有任何标记;
# 这里的开始/结束/异常行进 worker 日志(stderr,不缓冲丢行),
# 只记容器名与镜像,绝不记内层命令(内含任务 prompt)。
_DOCKER_LOGGER = logging.getLogger("cloud.docker")
class Executor:
"""执行器基类。
@@ -1190,11 +1198,20 @@ class DockerExecutor(Executor):
extra_mounts=extra_mounts,
network=network,
extra_env=extra_env)
_docker_t0 = time.time()
_DOCKER_LOGGER.info(
"docker_run_start container=%s image=%s",
self.container_name(task), self.image)
try:
result = self._call_runner(
cmd, timeout=timeout,
env=self._docker_cli_env(extra_env))
except Exception as exc:
_DOCKER_LOGGER.warning(
"docker_run_error container=%s elapsed_ms=%d error=%s",
self.container_name(task),
int((time.time() - _docker_t0) * 1000),
exc.__class__.__name__)
if exc.__class__.__name__ == "TimeoutExpired":
self.kill_container(task)
return {
@@ -1214,6 +1231,11 @@ class DockerExecutor(Executor):
parsed = _parse_host_result(task_id, fake)
parsed["executor"] = "docker"
parsed["container"] = self.container_name(task)
_DOCKER_LOGGER.info(
"docker_run_done container=%s elapsed_ms=%d returncode=%s",
self.container_name(task),
int((time.time() - _docker_t0) * 1000),
parsed.get("returncode"))
return parsed
def dispatch_task(self, task, timeout=None, extra_mounts=None,