worker: prestart stage logging for dispatch stalls

Claim-to-dispatch stalls (cf. 2026-10-29 task A: 61s to interrupted
with zero detail) were invisible because docker run blocks with
captured output and watchdog kills were silent. Add three layers:
- worker_exec_start/done/error audit rows in _run_attempt (PG,
  survives --rm), start/done pairing shows whether an attempt died
  inside the executor call;
- docker_run_start/done/error log lines in DockerExecutor (container
  + image only, never the inner command/prompt);
- stdout tail (last 2000 chars) to the worker log on failed or
  interrupted results, and a log line on both watchdog kill paths.
Tests: cloud/tests/test_prestart_stages.py, 6 cases incl. a
no-prompt-in-logs assertion.
This commit is contained in:
host committed 2026-10-29 17:07:53 +08:00
1 parent 0129b0149a
commit 9d90d97046
3 files changed
+251

No files matched your search

+164
View File
@@ -0,0 +1,164 @@
"""启动前分段日志(2026-10-29 任务 A)。
任务 A 认领后 61 秒 interrupted,但阻塞的 docker run 中途
没有任何标记、容器 --rm 后输出也没了,事后无法定位卡在哪
一段。本文件断言三层标记:
- worker 派发开始/结束进审计(PG 落盘,容器没了也丢不了);
- docker run 开始/结束/异常进 worker 日志,只记容器名与镜像;
- 失败/中断时容器输出尾巴记进 worker 日志;
- 看门狗杀容器时记一笔(之前是静默杀)。
标记绝不含任务 prompt 与密钥值。
"""
import json
import os
import sqlite3
import sys
import tempfile
import threading
import unittest
from pathlib import Path
from types import SimpleNamespace
try:
from cloud.sandbox import DockerExecutor
from cloud.server import init_db
from cloud.worker import Worker
except ImportError:
sys.path.insert(0, str(Path(__file__).resolve().parents[2]))
from cloud.sandbox import DockerExecutor
from cloud.server import init_db
from cloud.worker import Worker
def _audit_actions(db_path):
conn = sqlite3.connect(db_path)
try:
rows = conn.execute(
"SELECT action FROM audit_log ORDER BY id").fetchall()
finally:
conn.close()
return [r[0] for r in rows]
class FakeExecutor:
"""非 Docker/非 Local 的假执行器:走 _run_attempt 同步分支。"""
def __init__(self, result=None, error=None):
self._result = result
self._error = error
self.killed = []
def dispatch_task(self, task):
if self._error is not None:
raise self._error
return dict(self._result)
def kill_container(self, name):
self.killed.append(name)
class PrestartStagesTestCase(unittest.TestCase):
def setUp(self):
self.tmp = tempfile.TemporaryDirectory()
self.db_path = os.path.join(self.tmp.name, "test.db")
init_db(self.db_path)
self.worker = Worker(
self.db_path, "/fake/host.py", env={"XCM_API_KEY": "x"},
task_timeout=1800)
def tearDown(self):
self.tmp.cleanup()
def _task(self):
return {"id": "T1", "project_id": "p1", "prompt": "SECRET_PROMPT"}
def test_exec_start_and_done_audited(self):
self.worker.executor = FakeExecutor(result={
"executor": "fake", "task_id": "T1", "status": "completed",
"stdout": "", "returncode": 0, "host_status": None,
"stderr": ""})
result, error = self.worker._run_attempt(self._task())
self.assertIsNone(error)
self.assertEqual(result["status"], "completed")
actions = _audit_actions(self.db_path)
self.assertIn("worker_exec_start", actions)
self.assertIn("worker_exec_done", actions)
self.assertNotIn("worker_exec_error", actions)
def test_exec_error_audited_not_done(self):
self.worker.executor = FakeExecutor(
error=RuntimeError("boom"))
result, error = self.worker._run_attempt(self._task())
self.assertEqual(result["status"], "failed")
self.assertEqual(error, "boom")
actions = _audit_actions(self.db_path)
self.assertIn("worker_exec_start", actions)
self.assertIn("worker_exec_error", actions)
self.assertNotIn("worker_exec_done", actions)
def test_failed_stdout_tail_logged_not_audited(self):
self.worker.executor = FakeExecutor(result={
"executor": "fake", "task_id": "T1", "status": "failed",
"stdout": "partial container output before death",
"returncode": 1, "host_status": None, "stderr": ""})
with self.assertLogs("cloud.worker", level="INFO") as logs:
self.worker._run_attempt(self._task())
tail_lines = [r for r in logs.output
if "worker_exec_stdout_tail" in r]
self.assertEqual(len(tail_lines), 1)
self.assertIn("partial container output before death",
tail_lines[0])
self.assertNotIn("SECRET_PROMPT", "\n".join(logs.output))
def test_cancel_watchdog_kill_logged(self):
self.worker.executor = FakeExecutor()
self.worker._cancel_requested = {"T1"}
with self.assertLogs("cloud.worker", level="INFO") as logs:
self.worker._docker_cancel_watchdog(
"T1", "codex-T1", threading.Event())
self.assertEqual(self.worker.executor.killed, ["codex-T1"])
# log_event 把事件名放在记录的 event 属性里,不在消息文本里。
self.assertTrue(any(getattr(r, "event", None)
== "worker_docker_killed"
for r in logs.records))
class DockerRunStagesTestCase(unittest.TestCase):
def _executor(self, runner):
ex = DockerExecutor(runner=runner, env={"XCM_API_KEY": "x"})
ex._check_docker = lambda: None
return ex
def _task(self):
return {"id": "a" * 32, "project_id": "p1",
"prompt": "SECRET_PROMPT"}
def test_docker_run_start_and_done_logged(self):
def runner(cmd, timeout=None, env=None):
return SimpleNamespace(
returncode=0, stdout=json.dumps({"status": "completed"}),
stderr="")
ex = self._executor(runner)
with self.assertLogs("cloud.docker", level="INFO") as logs:
result = ex._run_in_container(self._task(), ["python3", "x"])
self.assertEqual(result["status"], "completed")
joined = "\n".join(logs.output)
self.assertIn("docker_run_start", joined)
self.assertIn("docker_run_done", joined)
self.assertIn("codex-" + "a" * 32, joined)
self.assertNotIn("SECRET_PROMPT", joined)
def test_docker_run_error_logged_and_failed(self):
def runner(cmd, timeout=None, env=None):
raise RuntimeError("docker daemon down")
ex = self._executor(runner)
with self.assertLogs("cloud.docker", level="WARNING") as logs:
result = ex._run_in_container(self._task(), ["python3", "x"])
self.assertEqual(result["status"], "failed")
self.assertTrue(any("docker_run_error" in r
for r in logs.output))
if __name__ == "__main__":
unittest.main()