worker: prestart stage logging for dispatch stalls
Claim-to-dispatch stalls (cf. 2026-10-29 task A: 61s to interrupted with zero detail) were invisible because docker run blocks with captured output and watchdog kills were silent. Add three layers: - worker_exec_start/done/error audit rows in _run_attempt (PG, survives --rm), start/done pairing shows whether an attempt died inside the executor call; - docker_run_start/done/error log lines in DockerExecutor (container + image only, never the inner command/prompt); - stdout tail (last 2000 chars) to the worker log on failed or interrupted results, and a log line on both watchdog kill paths. Tests: cloud/tests/test_prestart_stages.py, 6 cases incl. a no-prompt-in-logs assertion.
This commit is contained in:
1 parent
0129b0149a
commit
9d90d97046
3 files changed
+251
No files matched your search
@@ -0,0 +1,164 @@
|
||||
"""启动前分段日志(2026-10-29 任务 A)。
|
||||
|
||||
任务 A 认领后 61 秒 interrupted,但阻塞的 docker run 中途
|
||||
没有任何标记、容器 --rm 后输出也没了,事后无法定位卡在哪
|
||||
一段。本文件断言三层标记:
|
||||
- worker 派发开始/结束进审计(PG 落盘,容器没了也丢不了);
|
||||
- docker run 开始/结束/异常进 worker 日志,只记容器名与镜像;
|
||||
- 失败/中断时容器输出尾巴记进 worker 日志;
|
||||
- 看门狗杀容器时记一笔(之前是静默杀)。
|
||||
标记绝不含任务 prompt 与密钥值。
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
import sqlite3
|
||||
import sys
|
||||
import tempfile
|
||||
import threading
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
|
||||
try:
|
||||
from cloud.sandbox import DockerExecutor
|
||||
from cloud.server import init_db
|
||||
from cloud.worker import Worker
|
||||
except ImportError:
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[2]))
|
||||
from cloud.sandbox import DockerExecutor
|
||||
from cloud.server import init_db
|
||||
from cloud.worker import Worker
|
||||
|
||||
|
||||
def _audit_actions(db_path):
|
||||
conn = sqlite3.connect(db_path)
|
||||
try:
|
||||
rows = conn.execute(
|
||||
"SELECT action FROM audit_log ORDER BY id").fetchall()
|
||||
finally:
|
||||
conn.close()
|
||||
return [r[0] for r in rows]
|
||||
|
||||
|
||||
class FakeExecutor:
|
||||
"""非 Docker/非 Local 的假执行器:走 _run_attempt 同步分支。"""
|
||||
|
||||
def __init__(self, result=None, error=None):
|
||||
self._result = result
|
||||
self._error = error
|
||||
self.killed = []
|
||||
|
||||
def dispatch_task(self, task):
|
||||
if self._error is not None:
|
||||
raise self._error
|
||||
return dict(self._result)
|
||||
|
||||
def kill_container(self, name):
|
||||
self.killed.append(name)
|
||||
|
||||
|
||||
class PrestartStagesTestCase(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.tmp = tempfile.TemporaryDirectory()
|
||||
self.db_path = os.path.join(self.tmp.name, "test.db")
|
||||
init_db(self.db_path)
|
||||
self.worker = Worker(
|
||||
self.db_path, "/fake/host.py", env={"XCM_API_KEY": "x"},
|
||||
task_timeout=1800)
|
||||
|
||||
def tearDown(self):
|
||||
self.tmp.cleanup()
|
||||
|
||||
def _task(self):
|
||||
return {"id": "T1", "project_id": "p1", "prompt": "SECRET_PROMPT"}
|
||||
|
||||
def test_exec_start_and_done_audited(self):
|
||||
self.worker.executor = FakeExecutor(result={
|
||||
"executor": "fake", "task_id": "T1", "status": "completed",
|
||||
"stdout": "", "returncode": 0, "host_status": None,
|
||||
"stderr": ""})
|
||||
result, error = self.worker._run_attempt(self._task())
|
||||
self.assertIsNone(error)
|
||||
self.assertEqual(result["status"], "completed")
|
||||
actions = _audit_actions(self.db_path)
|
||||
self.assertIn("worker_exec_start", actions)
|
||||
self.assertIn("worker_exec_done", actions)
|
||||
self.assertNotIn("worker_exec_error", actions)
|
||||
|
||||
def test_exec_error_audited_not_done(self):
|
||||
self.worker.executor = FakeExecutor(
|
||||
error=RuntimeError("boom"))
|
||||
result, error = self.worker._run_attempt(self._task())
|
||||
self.assertEqual(result["status"], "failed")
|
||||
self.assertEqual(error, "boom")
|
||||
actions = _audit_actions(self.db_path)
|
||||
self.assertIn("worker_exec_start", actions)
|
||||
self.assertIn("worker_exec_error", actions)
|
||||
self.assertNotIn("worker_exec_done", actions)
|
||||
|
||||
def test_failed_stdout_tail_logged_not_audited(self):
|
||||
self.worker.executor = FakeExecutor(result={
|
||||
"executor": "fake", "task_id": "T1", "status": "failed",
|
||||
"stdout": "partial container output before death",
|
||||
"returncode": 1, "host_status": None, "stderr": ""})
|
||||
with self.assertLogs("cloud.worker", level="INFO") as logs:
|
||||
self.worker._run_attempt(self._task())
|
||||
tail_lines = [r for r in logs.output
|
||||
if "worker_exec_stdout_tail" in r]
|
||||
self.assertEqual(len(tail_lines), 1)
|
||||
self.assertIn("partial container output before death",
|
||||
tail_lines[0])
|
||||
self.assertNotIn("SECRET_PROMPT", "\n".join(logs.output))
|
||||
|
||||
def test_cancel_watchdog_kill_logged(self):
|
||||
self.worker.executor = FakeExecutor()
|
||||
self.worker._cancel_requested = {"T1"}
|
||||
with self.assertLogs("cloud.worker", level="INFO") as logs:
|
||||
self.worker._docker_cancel_watchdog(
|
||||
"T1", "codex-T1", threading.Event())
|
||||
self.assertEqual(self.worker.executor.killed, ["codex-T1"])
|
||||
# log_event 把事件名放在记录的 event 属性里,不在消息文本里。
|
||||
self.assertTrue(any(getattr(r, "event", None)
|
||||
== "worker_docker_killed"
|
||||
for r in logs.records))
|
||||
|
||||
|
||||
class DockerRunStagesTestCase(unittest.TestCase):
|
||||
def _executor(self, runner):
|
||||
ex = DockerExecutor(runner=runner, env={"XCM_API_KEY": "x"})
|
||||
ex._check_docker = lambda: None
|
||||
return ex
|
||||
|
||||
def _task(self):
|
||||
return {"id": "a" * 32, "project_id": "p1",
|
||||
"prompt": "SECRET_PROMPT"}
|
||||
|
||||
def test_docker_run_start_and_done_logged(self):
|
||||
def runner(cmd, timeout=None, env=None):
|
||||
return SimpleNamespace(
|
||||
returncode=0, stdout=json.dumps({"status": "completed"}),
|
||||
stderr="")
|
||||
ex = self._executor(runner)
|
||||
with self.assertLogs("cloud.docker", level="INFO") as logs:
|
||||
result = ex._run_in_container(self._task(), ["python3", "x"])
|
||||
self.assertEqual(result["status"], "completed")
|
||||
joined = "\n".join(logs.output)
|
||||
self.assertIn("docker_run_start", joined)
|
||||
self.assertIn("docker_run_done", joined)
|
||||
self.assertIn("codex-" + "a" * 32, joined)
|
||||
self.assertNotIn("SECRET_PROMPT", joined)
|
||||
|
||||
def test_docker_run_error_logged_and_failed(self):
|
||||
def runner(cmd, timeout=None, env=None):
|
||||
raise RuntimeError("docker daemon down")
|
||||
ex = self._executor(runner)
|
||||
with self.assertLogs("cloud.docker", level="WARNING") as logs:
|
||||
result = ex._run_in_container(self._task(), ["python3", "x"])
|
||||
self.assertEqual(result["status"], "failed")
|
||||
self.assertTrue(any("docker_run_error" in r
|
||||
for r in logs.output))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in new issue
Block a user