host
f39ed3654e
基底：#251 头 174b631（堆叠于 #242 feat-ci-expand 之上）， 基线 origin/main aebc6238；本轨不改 #251 已定语义、不动 workflow 门槛、不删检查、不加 skip。 lint（ruff E,F@120）：109 -> 10 - F401 43 -> 0：删未用导入；try/except 双绑定的名字出现 “打地鼠”（删 except 分支后 try 分支又被标出），两分支 均已核对正文无引用后清掉；MCP 三模块的 Error 名不删， 改走 importlib 属性绑定（见下），两路径绑定都保留。 - E702 40 -> 0：全在 host.py main() 的 argparse 段，纯分行， tokenize 按分号 token 拆（含 2377 行多行语句只断首个分号）； 一次 5 空格缩进失误已回滚重做并以 ast + 全量 host 测试验证。 - E501 13 -> 0：host.py 5 项随 E702 分行连带清掉；mcp_server 3 项把描述拆成相邻字面量（AST 比对 24 条工具描述与原值 完全一致，注意 ruff 按东亚宽度计长，CJK 算 2）；test_host 5 项断开 git commit 参数表。 - E402 8 -> 10：生产侧 2 项清掉（db.py _re、github_app _base64 移到顶部，之前无引用、无重名）；剩余 10 项全是 tests/ 的 sys.path 前置 import（6 项旧 + 4 个新测试文件 沿用同一既定模式），移上去会破坏单文件直跑，有意保留。 - F841 4 -> 0：只去绑定、不删调用（run_task、DB 入队、 tarfile 上下文、git 提交），各有注释与原测试/新测试覆盖； host dispatch_from_branch 新增真实 git 仓库回归测试。 - F811 1 -> 0：server 本地 _row_get 与 peer_messages.row_get 逐行相同，删本地定义，全部调用点解析到同一 import； 新增 test_server_row_get（同一对象 + 三路行为）。 mypy（生产范围）：24 -> 0，零新增 type: ignore - no-redef 9 -> 0：MCP 三模块 try/except 两次 from-import 改 importlib.import_module + 属性绑定（cloud 优先、flat copy 回落、内层 ImportError 同样回落，语义与原来一致； flat-copy 子进程测试证实回落后 __module__ 为顶层名）； server _row_get 随 F811 消除。 - arg-type 10 -> 0：harness2_tasks shared 改 TypedDict 逐键精确类型；CodexProbeAdapter 补 no-op validate_tree （_adapter_validate 本就跳过缺失方法，行为一致）。 - var-annotated 3 -> 0：sub2api _SHARED_ADMIN/_login_slots、 verify-multiuser RESULTS 加精确注解。 - assignment/misc 2 -> 0：marker_tools ZoneInfo 先声明 Any 再 try import（草稿先验 mypy 通过）。 bandit：总数 286 不变，High 仍 4 项，全在 host.py，本轨不改 （属第 3 项白名单轨范围），只做 triage： - 1319 run_verify：无插值，整串验测命令，管道/&& 须 shell 语义（新增 test_host_run_verify 断言管道与 && 真实生效）。 - 1497 notify：模板来自算子配置，三处插值全部 shlex.quote， 证据为既有 test_notify_failure_quotes_shell_metacharacters （; touch 与 $(touch ) 不执行）。 - 2099/2103 goal verify：无插值，任务配置验测串，同 run_verify。 - 无新增 High；pip-audit 0 漏洞。 覆盖率（同口径三套件 coverage 计量）：75.96% -> 77.87% （22392 行/缺 5385 -> 22437 行/缺 4963，显示 76% -> 78%） - client.py 0% -> 90%（pipe 假子进程 + main 分支） - run_parallel.py 0% -> 97%、migrate 包装 0% -> 95% - playwright_mcp 37% -> 62%、imagegen_mcp 47% -> 79%、 websearch_mcp 51% -> 86%、deploy.py 56% -> 77% - verify-multiuser 0% -> 27%（仅 check/req helper；main() 是主机侧真实多用户实弹验证，按门禁不建生产任务、 不以 mock 冒充，仍为缺口） - host.py 55% -> 57%（文件 2020 -> 2060 行系 E702 分行 的计量口径变化，非代码膨胀） - 未到 85%：门槛保持 75% 不动，不删低覆盖模块出统计口径； 缺口主力仍是 server.py / worker.py / host.py 大文件， 下一阶段须针对性深测，非本轨快赢可及。 验证：三套件 1576 -> 1629 项全过（tests 127、cloud/tests 1449 skipped=4、deploy/tests 53）；ruff E9 通过；mypy 0； bandit High 4（同上）；pip-audit 0 漏洞；试跑产物已删， 提交后工作树干净。
320 lines
13 KiB
Python
320 lines
13 KiB
Python
"""生产级 worker 超时 / 重试 / 容器清理兜底 + Postgres 备份单测。
|
||
|
||
只用标准库与本地 fake runner / 真实 sleep 进程，不跑模型、不联网；
|
||
Postgres 备份用可注入的 fake pg_dump runner，不调真实 pg_dump，
|
||
密码只经 PGPASSWORD 环境变量传给子进程（单测里不放真实密码）。
|
||
"""
|
||
|
||
import os
|
||
import sqlite3
|
||
import subprocess
|
||
import sys
|
||
import tempfile
|
||
import unittest
|
||
from pathlib import Path
|
||
from types import SimpleNamespace
|
||
from unittest import mock
|
||
|
||
try:
|
||
from cloud import backup as backup_mod
|
||
from cloud.sandbox import DockerExecutor
|
||
from cloud.worker import Worker
|
||
except ImportError:
|
||
sys.path.insert(0, str(Path(__file__).resolve().parents[2]))
|
||
from cloud import backup as backup_mod
|
||
from cloud.sandbox import DockerExecutor
|
||
from cloud.worker import Worker
|
||
|
||
|
||
class _FakeRunner:
|
||
"""同步 fake 执行器 runner：按 results 顺序返回结果。
|
||
|
||
LocalExecutor 的 runner 收的是命令列表、返回 SimpleNamespace
|
||
（returncode/stdout/stderr），与真实 subprocess 同款接口；
|
||
这里按顺序返回，不按任务 id 查表。
|
||
"""
|
||
|
||
def __init__(self, results):
|
||
self.results = list(results)
|
||
self.calls = []
|
||
|
||
def __call__(self, cmd):
|
||
self.calls.append(list(cmd))
|
||
spec = self.results[min(len(self.calls) - 1,
|
||
len(self.results) - 1)]
|
||
if isinstance(spec, Exception):
|
||
raise spec
|
||
return SimpleNamespace(
|
||
returncode=0 if spec[0] == "completed" else 1,
|
||
stdout=spec[1], stderr="")
|
||
|
||
|
||
def _make_worker(tmp, runner=None, process_factory=None,
|
||
task_timeout=1800, max_retries=1,
|
||
retry_backoff_seconds=0):
|
||
db_path = os.path.join(tmp, "control.db")
|
||
# worker 本身不查项目成员关系：init_db + 直接插任务即可，
|
||
# 不用走 AuthStore（其 create_project 只写成员表、不返项目）。
|
||
from cloud.server import init_db
|
||
init_db(db_path)
|
||
project_id = "proj"
|
||
task_id = "t1"
|
||
conn = sqlite3.connect(db_path)
|
||
try:
|
||
conn.execute(
|
||
"INSERT INTO tasks(id, project_id, prompt, status,"
|
||
" depends_on, priority, created_by, created_at, updated_at,"
|
||
" last_heartbeat) VALUES(?, ?, ?, 'queued', '[]', 0,"
|
||
" 'system', 0, 0, NULL)",
|
||
(task_id, project_id, "do it"))
|
||
conn.commit()
|
||
finally:
|
||
conn.close()
|
||
worker = Worker(db_path, "host.py", config_path=None, repo=None,
|
||
runner=runner,
|
||
env={"XCM_API_KEY": "test-placeholder"},
|
||
env_key=None,
|
||
process_factory=process_factory,
|
||
task_timeout=task_timeout, max_retries=max_retries,
|
||
retry_backoff_seconds=retry_backoff_seconds)
|
||
task = {"id": task_id, "project_id": project_id,
|
||
"prompt": "do it", "depends_on": "[]", "priority": 0}
|
||
return worker, db_path, task
|
||
|
||
|
||
def _task_row(db_path):
|
||
conn = sqlite3.connect(db_path)
|
||
try:
|
||
return conn.execute(
|
||
"SELECT status, error_text FROM tasks WHERE id='t1'").fetchone()
|
||
finally:
|
||
conn.close()
|
||
|
||
|
||
def _audit_actions(db_path):
|
||
conn = sqlite3.connect(db_path)
|
||
try:
|
||
return [r[0] for r in conn.execute(
|
||
"SELECT action FROM audit_log").fetchall()]
|
||
finally:
|
||
conn.close()
|
||
|
||
|
||
class WorkerTimeoutTestCase(unittest.TestCase):
|
||
def test_popen_timeout_kills_process_group_and_fails(self):
|
||
tmp = tempfile.TemporaryDirectory()
|
||
|
||
def factory(task):
|
||
return subprocess.Popen(
|
||
["sleep", "30"], stdout=subprocess.PIPE,
|
||
stderr=subprocess.PIPE, text=True, start_new_session=True)
|
||
|
||
worker, db_path, task = _make_worker(
|
||
tmp.name, process_factory=factory, task_timeout=0.2,
|
||
max_retries=0, retry_backoff_seconds=0)
|
||
result = worker.run_one(task)
|
||
self.assertEqual(result["status"], "failed")
|
||
# 超时是失败，不是取消：库里必须是 failed，绝不能是 interrupted。
|
||
status, error_text = _task_row(db_path)
|
||
self.assertEqual(status, "failed")
|
||
self.assertIn("timeout", error_text)
|
||
self.assertIn("worker_timeout", _audit_actions(db_path))
|
||
tmp.cleanup()
|
||
|
||
def test_task_timeout_zero_means_unlimited(self):
|
||
# 对齐 host turn_timeout：0 归一化为 None（不限时），
|
||
# 而不是在 popen 轮询第一轮就立即超时。sleep 0.5 的
|
||
# 任务在 task_timeout=0 下应正常完成，库里是 completed。
|
||
tmp = tempfile.TemporaryDirectory()
|
||
|
||
def factory(task):
|
||
return subprocess.Popen(
|
||
["sleep", "0.5"], stdout=subprocess.PIPE,
|
||
stderr=subprocess.PIPE, text=True, start_new_session=True)
|
||
|
||
worker, db_path, task = _make_worker(
|
||
tmp.name, process_factory=factory, task_timeout=0,
|
||
max_retries=0, retry_backoff_seconds=0)
|
||
self.assertIsNone(worker.task_timeout)
|
||
result = worker.run_one(task)
|
||
self.assertEqual(result["status"], "completed")
|
||
status, _ = _task_row(db_path)
|
||
self.assertEqual(status, "completed")
|
||
tmp.cleanup()
|
||
|
||
def test_docker_timeout_converts_to_failed_and_audit(self):
|
||
tmp = tempfile.TemporaryDirectory()
|
||
|
||
def docker_runner(cmd, timeout=None):
|
||
if cmd[1] in ("kill", "rm"):
|
||
return SimpleNamespace(returncode=0, stdout="", stderr="")
|
||
raise subprocess.TimeoutExpired(cmd, timeout)
|
||
|
||
worker, db_path, task = _make_worker(
|
||
tmp.name, runner=_FakeRunner([]), max_retries=0,
|
||
retry_backoff_seconds=0)
|
||
# 测试占位 key（非真实密钥），只为通过执行器的“key 存在性”
|
||
# 前置检查，让 runner 走到 TimeoutExpired 分支。
|
||
worker.executor = DockerExecutor(
|
||
runner=docker_runner, env={"XCM_API_KEY": "test"},
|
||
timeout=5)
|
||
# VM 上无 docker 二进制：只 mock 存在性检查（_check_docker），
|
||
# 真正的 run 仍走 fake runner 的 TimeoutExpired 分支。
|
||
with mock.patch("cloud.sandbox.shutil.which",
|
||
return_value="/usr/bin/docker"):
|
||
result = worker.run_one(task)
|
||
self.assertEqual(result["status"], "failed")
|
||
status, error_text = _task_row(db_path)
|
||
self.assertEqual(status, "failed")
|
||
self.assertIn("timeout", error_text)
|
||
self.assertIn("worker_timeout", _audit_actions(db_path))
|
||
tmp.cleanup()
|
||
|
||
|
||
class WorkerRetryTestCase(unittest.TestCase):
|
||
def test_generic_failure_retries_once_then_failed(self):
|
||
tmp = tempfile.TemporaryDirectory()
|
||
runner = _FakeRunner([("failed", "boom"), ("failed", "boom")])
|
||
worker, db_path, task = _make_worker(
|
||
tmp.name, runner=runner, max_retries=1,
|
||
retry_backoff_seconds=0)
|
||
result = worker.run_one(task)
|
||
self.assertEqual(result["status"], "failed")
|
||
self.assertEqual(len(runner.calls), 2)
|
||
status, error_text = _task_row(db_path)
|
||
self.assertEqual(status, "failed")
|
||
self.assertIn("[retries=1]", error_text)
|
||
self.assertIn("boom", error_text)
|
||
self.assertIn("worker_retry", _audit_actions(db_path))
|
||
tmp.cleanup()
|
||
|
||
def test_retry_success_recovers(self):
|
||
tmp = tempfile.TemporaryDirectory()
|
||
runner = _FakeRunner([("failed", "boom"), ("completed", "ok")])
|
||
worker, db_path, task = _make_worker(
|
||
tmp.name, runner=runner, max_retries=1,
|
||
retry_backoff_seconds=0)
|
||
result = worker.run_one(task)
|
||
self.assertEqual(result["status"], "completed")
|
||
self.assertEqual(len(runner.calls), 2)
|
||
status, _ = _task_row(db_path)
|
||
self.assertEqual(status, "completed")
|
||
tmp.cleanup()
|
||
|
||
def test_max_retries_zero_means_no_retry(self):
|
||
tmp = tempfile.TemporaryDirectory()
|
||
runner = _FakeRunner([("failed", "boom")])
|
||
worker, db_path, task = _make_worker(
|
||
tmp.name, runner=runner, max_retries=0,
|
||
retry_backoff_seconds=0)
|
||
result = worker.run_one(task)
|
||
self.assertEqual(result["status"], "failed")
|
||
self.assertEqual(len(runner.calls), 1)
|
||
_, error_text = _task_row(db_path)
|
||
self.assertNotIn("[retries=", error_text)
|
||
tmp.cleanup()
|
||
|
||
|
||
class ContainerCleanupTestCase(unittest.TestCase):
|
||
def _docker_executor(self, ps_stdout=None, fail_ps=False):
|
||
calls = []
|
||
|
||
def runner(cmd, timeout=None):
|
||
calls.append(list(cmd))
|
||
if cmd[1] == "ps":
|
||
if fail_ps:
|
||
raise RuntimeError("docker down")
|
||
return SimpleNamespace(
|
||
returncode=0, stdout=ps_stdout or "", stderr="")
|
||
return SimpleNamespace(returncode=0, stdout="", stderr="")
|
||
|
||
executor = DockerExecutor(runner=runner)
|
||
return executor, calls
|
||
|
||
def test_cleanup_only_task_containers_never_infra(self):
|
||
# 回归：2026-10-26 实测，按 codex- 前缀全删会把数据库容器
|
||
# codex-postgres 当成残留任务容器杀掉，导致控制面连不上
|
||
# Postgres。清理只能删 codex-<32 位 hex 任务 id>。
|
||
executor, calls = self._docker_executor(
|
||
ps_stdout="codex-postgres\n"
|
||
"codex-3689cf40d671492992c6bfedd18b004c\n"
|
||
"codex-abc\nother\ncodex-app-server\n")
|
||
cleaned = executor.cleanup_stale_containers()
|
||
self.assertEqual(
|
||
cleaned, ["codex-3689cf40d671492992c6bfedd18b004c"])
|
||
flat = [c for call in calls for c in call]
|
||
self.assertNotIn("codex-postgres", flat)
|
||
self.assertNotIn("codex-abc", flat)
|
||
self.assertNotIn("other", flat)
|
||
# kill + rm 各调一次（仅那个任务容器）。
|
||
self.assertEqual(
|
||
sum(1 for call in calls if call[1] == "kill"), 1)
|
||
self.assertEqual(
|
||
sum(1 for call in calls if call[1] == "rm"), 1)
|
||
|
||
def test_cleanup_failure_does_not_crash_worker(self):
|
||
executor, _ = self._docker_executor(fail_ps=True)
|
||
self.assertEqual(executor.cleanup_stale_containers(), [])
|
||
tmp = tempfile.TemporaryDirectory()
|
||
worker, _, _ = _make_worker(tmp.name, runner=_FakeRunner([]))
|
||
worker.executor = executor
|
||
# worker 层兜底同样不抛异常。
|
||
self.assertEqual(worker.cleanup_stale_containers(), [])
|
||
tmp.cleanup()
|
||
|
||
def test_list_containers_unparsable_or_failed_returns_empty(self):
|
||
executor, _ = self._docker_executor(ps_stdout="")
|
||
self.assertEqual(executor.list_containers(), [])
|
||
|
||
|
||
class PostgresBackupTestCase(unittest.TestCase):
|
||
def test_backup_postgres_fake_runner_success(self):
|
||
tmp = tempfile.TemporaryDirectory()
|
||
created = {}
|
||
|
||
def fake_runner(cmd, env=None):
|
||
self.assertNotIn("secret-pw", " ".join(cmd))
|
||
self.assertEqual(env.get("PGPASSWORD"), "secret-pw")
|
||
dest = cmd[cmd.index("-f") + 1]
|
||
Path(dest).write_text("-- dummy dump\n")
|
||
created["cmd"] = cmd
|
||
return SimpleNamespace(returncode=0, stdout="", stderr="")
|
||
|
||
env = {"CLOUD_DB_NAME": "codexdb", "CLOUD_DB_USER": "codexsvc",
|
||
"CLOUD_DB_PASSWORD": "secret-pw"}
|
||
with mock.patch.dict(os.environ, env):
|
||
dest = backup_mod.backup_postgres(
|
||
tmp.name, runner=fake_runner)
|
||
self.assertTrue(os.path.exists(dest))
|
||
self.assertGreater(os.path.getsize(dest), 0)
|
||
self.assertNotIn("secret-pw", dest)
|
||
self.assertIn("pg_dump", created["cmd"][0])
|
||
tmp.cleanup()
|
||
|
||
def test_backup_postgres_failure_raises_clear_error(self):
|
||
tmp = tempfile.TemporaryDirectory()
|
||
|
||
def failing_runner(cmd, env=None):
|
||
return SimpleNamespace(
|
||
returncode=1, stdout="", stderr="pg_dump: boom")
|
||
|
||
env = {"CLOUD_DB_NAME": "codexdb", "CLOUD_DB_USER": "codexsvc"}
|
||
with mock.patch.dict(os.environ, env):
|
||
with self.assertRaises(RuntimeError) as ctx:
|
||
backup_mod.backup_postgres(tmp.name, runner=failing_runner)
|
||
self.assertIn("pg_dump", str(ctx.exception))
|
||
tmp.cleanup()
|
||
|
||
def test_backup_postgres_missing_binary_raises(self):
|
||
tmp = tempfile.TemporaryDirectory()
|
||
env = {"CLOUD_DB_NAME": "codexdb"}
|
||
with mock.patch.dict(os.environ, env):
|
||
with self.assertRaises(RuntimeError):
|
||
backup_mod.backup_postgres(
|
||
tmp.name, pg_dump_cmd="nonexistent-pg-dump-xyz")
|
||
tmp.cleanup()
|
||
|
||
|
||
if __name__ == "__main__":
|
||
unittest.main()
|