Files
codex-app-server/cloud/tests/test_v2cov_gaps_d2.py
T
host 12e72dab9b code-to10 v4 coverage thickening: 219 new assertion tests (batches A/B1/C/D1-D6)
Baseline merge head e5ff26a: 28144 stmts, 2807 missing, exact 90.0263%, margin +7.
After thickening (full run): 2350 missing, exact 91.6501%, margin +464.
Full suite 3516 items: 3509 pass, 1 failure (pre-existing November date
failure, untouched), 6 skipped. No production code changed.
2026-11-03 02:49:26 +00:00

577 lines
21 KiB
Python

"""Coverage batch v4-D2: cloud/worker.py tree/quota/cancel/watchdog/
cleanup/gateway/process branches.
Stubbing discipline: the method under test is never stubbed. Patched
collaborators are: _tenant_safe_name / _tenant_dirs / _project_repo_url
/ _quota_limit_and_used / _list_gateway_models / _gateway_name /
_gateway_origin / ready_tasks / orphaned_containers / CostStore /
_db_connect / dispatch sinks / instance-level _kill_proc_group /
_set_child_pid / get_task / heartbeat (data, sink and predicate
collaborators). Every assertion reads a real return value or a real
DB row, except watchdog/cleanup tests where the assertion is the
returned list or the absence of a raise plus a real side-effect set.
"""
import os
import sqlite3
import sys
import tempfile
import time
import unittest
from pathlib import Path
from types import SimpleNamespace
from unittest import mock
REPO_ROOT = Path(__file__).resolve().parents[2]
sys.path.insert(0, str(REPO_ROOT))
from cloud.server import init_db
from cloud.worker import Worker
FAKE_ENV = {"XCM_API_KEY": "test-placeholder"}
def _worker(db_path, **kwargs):
return Worker(db_path, "/fake/host.py", env=FAKE_ENV, **kwargs)
class _Base(unittest.TestCase):
def setUp(self):
self.tmp = tempfile.TemporaryDirectory()
self.db_path = os.path.join(self.tmp.name, "control.db")
init_db(self.db_path)
def tearDown(self):
self.tmp.cleanup()
def _insert_task(self, task_id="t1", status="queued", **cols):
conn = sqlite3.connect(self.db_path)
try:
conn.execute(
"INSERT INTO tasks(id, project_id, prompt, status,"
" created_by, created_at, updated_at)"
" VALUES(?, ?, ?, ?, ?, ?, ?)",
(task_id, "p1", "do work", status, "u1", time.time(),
time.time()))
for key, value in cols.items():
conn.execute(
"UPDATE tasks SET %s = ? WHERE id = ?" % key,
(value, task_id))
conn.commit()
finally:
conn.close()
def _row(self, task_id="t1"):
conn = sqlite3.connect(self.db_path)
try:
conn.row_factory = sqlite3.Row
return conn.execute(
"SELECT * FROM tasks WHERE id = ?", (task_id,)).fetchone()
finally:
conn.close()
class _StopOnce:
"""wait() returns False once, then True."""
def __init__(self):
self.n = 0
def wait(self, timeout=None):
self.n += 1
return self.n > 1
class TreeSnapshotTestCase(_Base):
def _docker_worker(self):
# Subclass of the real DockerExecutor (no-op __init__) so the
# isinstance check in _tree_snapshot_source passes without
# patching the module-level class.
from cloud.worker import DockerExecutor as _RealDocker
class _FakeDocker(_RealDocker):
def __init__(self):
pass
w = _worker(self.db_path, executor=_FakeDocker())
w.tenants_root = self.tmp.name
w.tenant_isolation = True
return w
def test_safe_name_failure_returns_none(self):
# 2003-2004
w = self._docker_worker()
w._tenant_safe_name = mock.Mock(side_effect=RuntimeError("bad"))
self.assertIsNone(w._tree_snapshot_source({"project_id": "p1"}))
def test_checkout_exists_returns_checkout(self):
# 2007
w = self._docker_worker()
w._tenant_safe_name = mock.Mock(return_value="safe")
os.makedirs(os.path.join(self.tmp.name, "safe", "safe"))
self.assertEqual(
w._tree_snapshot_source({"project_id": "p1"}),
os.path.join(self.tmp.name, "safe", "safe"))
def test_tenant_dirs_failure_returns_none(self):
# 2017-2018
w = self._docker_worker()
w._tenant_safe_name = mock.Mock(return_value="safe")
w._project_repo_url = mock.Mock(return_value=None)
w._tenant_dirs = mock.Mock(side_effect=RuntimeError("dirs down"))
self.assertIsNone(w._tree_snapshot_source({"project_id": "p1"}))
def test_test_repo_missing_returns_none(self):
# 2023
w = self._docker_worker()
w._tenant_safe_name = mock.Mock(return_value="safe")
w._project_repo_url = mock.Mock(return_value=None)
w._tenant_dirs = mock.Mock(
return_value={"test_repo": "/nonexistent/repo-xyz"})
self.assertIsNone(w._tree_snapshot_source({"project_id": "p1"}))
def test_injection_source_failure_swallowed(self):
# 2051-2053, 2056
w = _worker(self.db_path)
w._tree_snapshot_source = mock.Mock(
side_effect=RuntimeError("snap down"))
task = {"id": "t1", "project_id": "p1", "prompt": "do work"}
self.assertIs(w._apply_tree_injection(task), task)
def test_injection_logger_failure_swallowed(self):
# 2054-2055
w = _worker(self.db_path)
w._tree_snapshot_source = mock.Mock(
side_effect=RuntimeError("snap down"))
w.logger = mock.Mock()
w.logger.warning.side_effect = RuntimeError("logger down")
task = {"id": "t1", "project_id": "p1", "prompt": "do work"}
self.assertIs(w._apply_tree_injection(task), task)
class QuotaTestCase(_Base):
def test_limit_commit_failure_swallowed_unlimited(self):
# 2131-2132: commit raising, explicit-NULL quota row -> None.
conn = sqlite3.connect(self.db_path)
try:
conn.execute(
"INSERT INTO user_quotas(user_id, quota_usd_micros,"
" updated_at) VALUES(?, NULL, ?)", ("u1", time.time()))
conn.commit()
finally:
conn.close()
w = _worker(self.db_path)
real = sqlite3.connect(self.db_path)
class _Proxy:
def __init__(self, conn):
self._conn = conn
def __getattr__(self, name):
return getattr(self._conn, name)
def commit(self):
raise RuntimeError("commit down")
def close(self):
self._conn.close()
with mock.patch("cloud.worker._db_connect",
return_value=_Proxy(real)):
quota, used, month, unpriced = w._quota_limit_and_used("u1")
self.assertIsNone(quota)
self.assertEqual((used, unpriced), (0, 0))
def test_limit_coststore_close_failure_swallowed(self):
# 2142-2143
from cloud.cost import CostStore
CostStore(self.db_path).close() if hasattr(
CostStore(self.db_path), "close") else None
store = CostStore(self.db_path)
try:
store._conn.close()
except Exception:
pass
w = _worker(self.db_path)
class _BadStore:
def __init__(self, db_path):
self._conn = mock.Mock()
self._conn.close.side_effect = RuntimeError("close down")
with mock.patch("cloud.worker.CostStore", _BadStore):
quota, used, month, unpriced = w._quota_limit_and_used("u9")
self.assertIsNotNone(quota)
self.assertEqual((used, unpriced), (0, 0))
def test_quota_parent_row_type_error(self):
# 2196-2197
w = _worker(self.db_path)
class _Cursor:
def fetchone(self):
return 5
class _Conn:
def execute(self, *a, **k):
return _Cursor()
def close(self):
pass
with mock.patch("cloud.worker._db_connect", return_value=_Conn()):
msg = w._quota_exceeded({"parent_task_id": "p0"}, None)
self.assertIn("缺少属主", msg)
def test_quota_parent_lookup_failure(self):
# 2198-2199
w = _worker(self.db_path)
with mock.patch("cloud.worker._db_connect",
side_effect=RuntimeError("db down")):
msg = w._quota_exceeded({"parent_task_id": "p0"}, None)
self.assertIn("缺少属主", msg)
def test_quota_limit_failure_logger_failure(self):
# 2213-2214, both on_error flavours
w = _worker(self.db_path)
w._quota_limit_and_used = mock.Mock(
side_effect=RuntimeError("quota db down"))
w.logger = mock.Mock()
w.logger.warning.side_effect = RuntimeError("logger down")
self.assertIn("quota_check_unavailable",
w._quota_exceeded({"created_by": "u1"}, None,
on_error="closed"))
self.assertIn("quota_check_failed",
w._quota_exceeded({"created_by": "u1"}, None))
def test_ready_budget_exceeded_empty(self):
# 2252
w = _worker(self.db_path)
w.usage_turns = 5
w.max_turns = 1
self.assertEqual(w._ready_tasks(), [])
def test_ready_value_error_empty(self):
# 2258-2259
self._insert_task()
w = _worker(self.db_path)
with mock.patch("cloud.worker.ready_tasks",
side_effect=ValueError("cycle")):
self.assertEqual(w._ready_tasks(), [])
class HeartbeatCancelTestCase(_Base):
def test_heartbeat_task_failure_swallowed(self):
# 2330-2331
w = _worker(self.db_path)
w._running_set("t1")
w.heartbeat = mock.Mock(side_effect=RuntimeError("beat down"))
w._heartbeat_running_tasks()
def test_cancel_proc_wait_failure_swallowed(self):
# 2349-2350
self._insert_task(status="running")
w = _worker(self.db_path)
proc = mock.Mock()
proc.wait.side_effect = RuntimeError("wait down")
w._running_set("t1", proc=proc)
w._kill_proc_group = lambda p: None
self.assertTrue(w.cancel("t1"))
self.assertEqual(self._row()["status"], "interrupted")
def test_cancel_container_kill_failure_swallowed(self):
# 2359-2360
self._insert_task(status="running")
w = _worker(self.db_path, executor=SimpleNamespace(
kill_container=mock.Mock(
side_effect=RuntimeError("kill down"))))
w._running_set("t1", container="c1")
self.assertTrue(w.cancel("t1"))
self.assertEqual(self._row()["status"], "interrupted")
class WatchdogTestCase(_Base):
def test_cancel_watchdog_lookup_failure_continues(self):
# 2407-2408
w = _worker(self.db_path)
w.get_task = mock.Mock(side_effect=RuntimeError("db down"))
w._docker_cancel_watchdog("t1", "c1", _StopOnce())
def test_cancel_watchdog_kill_failure_swallowed(self):
# 2420-2421
w = _worker(self.db_path, executor=SimpleNamespace(
kill_container=mock.Mock(
side_effect=RuntimeError("kill down"))))
w._cancel_requested.add("t1")
w._docker_cancel_watchdog("t1", "c1", _StopOnce())
def test_quota_watchdog_check_failure_continues(self):
# 2446-2447
w = _worker(self.db_path)
w._docker_quota_watchdog(
"t1", "c1", _StopOnce(),
lambda: (_ for _ in ()).throw(RuntimeError("check down")))
def test_quota_watchdog_kill_failure_swallowed(self):
# 2459-2460
w = _worker(self.db_path, executor=SimpleNamespace(
kill_container=mock.Mock(
side_effect=RuntimeError("kill down"))))
w._docker_quota_watchdog("t1", "c1", _StopOnce(), lambda: True)
self.assertIn("t1", w._quota_hit_tasks)
class CleanupTestCase(_Base):
def test_orphan_no_executor_caps_returns_empty(self):
# 2485
w = _worker(self.db_path)
self.assertEqual(w.cleanup_orphaned_containers(), [])
def test_orphan_list_failure_returns_empty(self):
# 2488-2491
w = _worker(self.db_path, executor=SimpleNamespace(
list_containers=mock.Mock(
side_effect=RuntimeError("list down")),
kill_container=mock.Mock()))
self.assertEqual(w.cleanup_orphaned_containers(), [])
def test_orphan_identify_failure_returns_empty(self):
# 2496-2499
w = _worker(self.db_path, executor=SimpleNamespace(
list_containers=mock.Mock(return_value=["c1"]),
kill_container=mock.Mock()))
with mock.patch("cloud.worker.orphaned_containers",
side_effect=RuntimeError("identify down")):
self.assertEqual(w.cleanup_orphaned_containers(), [])
def test_orphan_kill_failure_skipped(self):
# 2509-2510
kill = mock.Mock(side_effect=RuntimeError("kill down"))
w = _worker(self.db_path, executor=SimpleNamespace(
list_containers=mock.Mock(return_value=["c1"]),
kill_container=kill))
with mock.patch("cloud.worker.orphaned_containers",
return_value=[{"container": "c1",
"pid_verified": True}]):
self.assertEqual(w.cleanup_orphaned_containers(), [])
def test_stale_cleanup_failure_returns_empty(self):
# 2534-2537
w = _worker(self.db_path, executor=SimpleNamespace(
cleanup_stale_containers=mock.Mock(
side_effect=RuntimeError("cleanup down"))))
self.assertEqual(w.cleanup_stale_containers(), [])
def test_stale_cleanup_nonempty_returned(self):
# 2539
w = _worker(self.db_path, executor=SimpleNamespace(
cleanup_stale_containers=mock.Mock(return_value=["c1"])))
self.assertEqual(w.cleanup_stale_containers(), ["c1"])
class GatewayTestCase(_Base):
def test_key_available_env_failure_false(self):
# 2564-2565
class _BadEnv:
def get(self, *a, **k):
raise RuntimeError("env down")
w = Worker(self.db_path, "/fake/host.py", env=_BadEnv())
self.assertFalse(w.key_available())
def _sub2api_env(self):
return (mock.patch("cloud.worker._gateway_name",
return_value="sub2api"),
mock.patch("cloud.worker._gateway_origin",
return_value="http://gw.example"))
def test_gateway_parent_row_type_error(self):
# 2607-2608 -> owner missing -> RuntimeError
w = _worker(self.db_path)
name_p, origin_p = self._sub2api_env()
class _Cursor:
def fetchone(self):
return 5
class _Conn:
def execute(self, *a, **k):
return _Cursor()
def close(self):
pass
with name_p, origin_p, mock.patch(
"cloud.worker._db_connect", return_value=_Conn()):
with self.assertRaises(RuntimeError):
w._gateway_context({"parent_task_id": "p0"})
def test_gateway_parent_lookup_failure(self):
# 2609-2610 -> owner missing -> RuntimeError
w = _worker(self.db_path)
name_p, origin_p = self._sub2api_env()
with name_p, origin_p, mock.patch(
"cloud.worker._db_connect",
side_effect=RuntimeError("db down")):
with self.assertRaises(RuntimeError):
w._gateway_context({"parent_task_id": "p0"})
def test_sub2api_default_model_none_available(self):
# 2660-2662
w = _worker(self.db_path)
with mock.patch.dict(os.environ,
{"CLOUD_SUB2API_DEFAULT_MODEL": ""}):
with mock.patch("cloud.worker._gateway_origin",
return_value="http://gw.example"):
with mock.patch("cloud.worker._list_gateway_models",
return_value=[]):
with self.assertRaises(RuntimeError):
w._sub2api_default_model(
{"SUB2API_API_KEY": "k"})
def test_sub2api_default_model_first(self):
# 2663
w = _worker(self.db_path)
with mock.patch.dict(os.environ,
{"CLOUD_SUB2API_DEFAULT_MODEL": ""}):
with mock.patch("cloud.worker._gateway_origin",
return_value="http://gw.example"):
with mock.patch("cloud.worker._list_gateway_models",
return_value=["m1", "m2"]):
self.assertEqual(
w._sub2api_default_model(
{"SUB2API_API_KEY": "k"}), "m1")
def test_task_seed_dir_db_failure_none(self):
# 2680-2681
w = _worker(self.db_path)
with mock.patch("cloud.worker._db_connect",
side_effect=RuntimeError("db down")):
self.assertIsNone(w._task_seed_dir({"id": "t1"}))
def test_cleanup_seed_failure_swallowed(self):
# 2701-2702
w = _worker(self.db_path)
w._task_seed_dir = mock.Mock(
side_effect=RuntimeError("seed down"))
w._cleanup_task_seed({"id": "t1"})
class ProcessTestCase(_Base):
def _finished_proc(self, returncode=0):
proc = mock.Mock()
proc.poll.return_value = returncode
proc.communicate.return_value = ("out", "err")
proc.returncode = returncode
proc.pid = 4242
return proc
def _proc_worker(self, proc):
return _worker(self.db_path,
executor=SimpleNamespace(
popen_task=mock.Mock(return_value=proc)))
def test_executor_popen_path(self):
# 2730 + 2820 (no _result_from_runner on the executor)
self._insert_task(status="running")
proc = self._finished_proc()
w = self._proc_worker(proc)
out = w._execute_via_process({"id": "t1", "project_id": "p1"})
self.assertEqual(out["status"], "completed")
self.assertEqual(out["stdout"], "out")
def test_set_child_pid_failure_swallowed(self):
# 2736-2737
self._insert_task(status="running")
w = _worker(self.db_path,
process_factory=lambda task: self._finished_proc())
w._set_child_pid = mock.Mock(
side_effect=RuntimeError("pid down"))
out = w._execute_via_process({"id": "t1", "project_id": "p1"})
self.assertEqual(out["status"], "completed")
def test_quota_interval_none_defaults(self):
# 2747
self._insert_task(status="running")
w = _worker(self.db_path,
process_factory=lambda task: self._finished_proc())
w.quota_poll_interval_seconds = None
out = w._execute_via_process({"id": "t1", "project_id": "p1"})
self.assertEqual(out["status"], "completed")
def test_loop_db_lookup_failure_continues(self):
# 2763-2764
self._insert_task(status="running")
proc = mock.Mock()
proc.poll.side_effect = [None, 0]
proc.communicate.return_value = ("out", "err")
proc.returncode = 0
proc.pid = 4242
w = _worker(self.db_path,
process_factory=lambda task: proc)
w.get_task = mock.Mock(side_effect=RuntimeError("db down"))
out = w._execute_via_process({"id": "t1", "project_id": "p1"})
self.assertEqual(out["status"], "completed")
def test_loop_db_interrupted_cancels(self):
# 2767-2769
self._insert_task(status="running")
proc = mock.Mock()
proc.poll.side_effect = [None, 0]
proc.communicate.return_value = ("out", "err")
proc.returncode = 0
proc.pid = 4242
w = _worker(self.db_path,
process_factory=lambda task: proc)
w.get_task = mock.Mock(return_value={"status": "interrupted"})
w._kill_proc_group = lambda p: None
out = w._execute_via_process({"id": "t1", "project_id": "p1"})
self.assertEqual(out["status"], "cancelled")
def test_loop_quota_check_failure_continues(self):
# 2780-2781
self._insert_task(status="running")
proc = mock.Mock()
proc.poll.side_effect = [None, 0]
proc.communicate.return_value = ("out", "err")
proc.returncode = 0
proc.pid = 4242
w = _worker(self.db_path,
process_factory=lambda task: proc)
w.quota_poll_interval_seconds = 0
out = w._execute_via_process(
{"id": "t1", "project_id": "p1"},
quota_check=mock.Mock(side_effect=RuntimeError("quota down")))
self.assertEqual(out["status"], "completed")
def test_communicate_failure_falls_back(self):
# 2789-2794
self._insert_task(status="running")
proc = mock.Mock()
proc.poll.return_value = 0
proc.communicate.side_effect = [RuntimeError("comm down"),
("out2", "err2")]
proc.returncode = 0
proc.pid = 4242
w = _worker(self.db_path,
process_factory=lambda task: proc)
out = w._execute_via_process({"id": "t1", "project_id": "p1"})
self.assertEqual(out["stdout"], "out2")
proc.kill.assert_called_once()
def test_goal_result_from_runner(self):
# 2817
self._insert_task(status="running")
proc = self._finished_proc()
marker = {"status": "goal-met", "task_id": "t1"}
w = _worker(self.db_path, executor=SimpleNamespace(
popen_task=mock.Mock(return_value=proc),
_goal_result_from_runner=mock.Mock(return_value=marker)))
out = w._execute_via_process({"id": "t1", "project_id": "p1"},
kind="goal")
self.assertEqual(out, marker)
if __name__ == "__main__":
unittest.main()