worker: prestart stage logging for dispatch stalls
Claim-to-dispatch stalls (cf. 2026-10-29 task A: 61s to interrupted with zero detail) were invisible because docker run blocks with captured output and watchdog kills were silent. Add three layers: - worker_exec_start/done/error audit rows in _run_attempt (PG, survives --rm), start/done pairing shows whether an attempt died inside the executor call; - docker_run_start/done/error log lines in DockerExecutor (container + image only, never the inner command/prompt); - stdout tail (last 2000 chars) to the worker log on failed or interrupted results, and a log line on both watchdog kill paths. Tests: cloud/tests/test_prestart_stages.py, 6 cases incl. a no-prompt-in-logs assertion.
This commit is contained in:
1 parent
0129b0149a
commit
9d90d97046
3 files changed
+251
No files matched your search
@@ -14,11 +14,13 @@
|
||||
"""
|
||||
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
|
||||
from cloud.marker_tools import extract_task_cancels as _extract_task_cancels
|
||||
from cloud.marker_tools import extract_task_creates as _extract_task_creates
|
||||
@@ -29,6 +31,12 @@ from cloud.peer_messages import extract_peer_messages as _extract_peer_messages
|
||||
|
||||
MAX_STDOUT_CHARS = 2000
|
||||
|
||||
# 启动前分段日志(2026-10-29 任务 A):docker run 是阻塞调用,
|
||||
# 容器输出要等返回才回传,中途卡死时 worker 侧没有任何标记;
|
||||
# 这里的开始/结束/异常行进 worker 日志(stderr,不缓冲丢行),
|
||||
# 只记容器名与镜像,绝不记内层命令(内含任务 prompt)。
|
||||
_DOCKER_LOGGER = logging.getLogger("cloud.docker")
|
||||
|
||||
|
||||
class Executor:
|
||||
"""执行器基类。
|
||||
@@ -1190,11 +1198,20 @@ class DockerExecutor(Executor):
|
||||
extra_mounts=extra_mounts,
|
||||
network=network,
|
||||
extra_env=extra_env)
|
||||
_docker_t0 = time.time()
|
||||
_DOCKER_LOGGER.info(
|
||||
"docker_run_start container=%s image=%s",
|
||||
self.container_name(task), self.image)
|
||||
try:
|
||||
result = self._call_runner(
|
||||
cmd, timeout=timeout,
|
||||
env=self._docker_cli_env(extra_env))
|
||||
except Exception as exc:
|
||||
_DOCKER_LOGGER.warning(
|
||||
"docker_run_error container=%s elapsed_ms=%d error=%s",
|
||||
self.container_name(task),
|
||||
int((time.time() - _docker_t0) * 1000),
|
||||
exc.__class__.__name__)
|
||||
if exc.__class__.__name__ == "TimeoutExpired":
|
||||
self.kill_container(task)
|
||||
return {
|
||||
@@ -1214,6 +1231,11 @@ class DockerExecutor(Executor):
|
||||
parsed = _parse_host_result(task_id, fake)
|
||||
parsed["executor"] = "docker"
|
||||
parsed["container"] = self.container_name(task)
|
||||
_DOCKER_LOGGER.info(
|
||||
"docker_run_done container=%s elapsed_ms=%d returncode=%s",
|
||||
self.container_name(task),
|
||||
int((time.time() - _docker_t0) * 1000),
|
||||
parsed.get("returncode"))
|
||||
return parsed
|
||||
|
||||
def dispatch_task(self, task, timeout=None, extra_mounts=None,
|
||||
|
||||
Reference in new issue
Block a user