Files
codex-app-server/cloud/tests/test_gapfill2_worker_gaps.py
T
host aba06d9a24 gapfill2: worker/sub90 coverage tests, G7 rollback branch, G5 caller-contract docs
H1 worker.py 92%->94% (277->210 miss): 35 real-assertion tests
covering goal-signal, tenant/env, followup, poll/wait,
process/watchdog and injected defensive branches; prod code 0 changes.
H2 sub-90 sweep: approval_timeout 100%, merge_gate_hard 100%,
worker_sweep 98% (only __main__ line 174 left); 14 tests.
H3 G7 rollback except:pass (server.py:6615-6616) covered by an
injected rollback-raising connection; original error still propagates,
status stays completed, retry succeeds; no semantics change.
H4 caller contract documented only (docstrings in _audit_scorecard
endpoint, check_baseline_stale, build_scorecard, baseline store):
current_main_head caller-supplied, any string as-is, no format checks,
missing/empty treated as not passed with no stale determination at the
endpoint, always 200, overall never rewritten; no new validation or
rejection, no response-structure change. 3 doc tests pin the wording.
H5 design notes (no decisions, no code changes) live in
verification/code-to10-gapfill2-design-notes.md, referenced by the
batch summary.
No bug found by the new tests; no existing assertion touched.
2026-11-04 16:20:10 +00:00

611 lines
24 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
"""gapfill2 H1:worker.py 覆盖缺口补测试(生产代码 0 改动)。
基线(本批自己跑的 CI 同款覆盖率,基底 5d0651e):
cloud/worker.py 3493 语句、277 缺失、92%。本文件只加真实断言
测试,瞄准基线缺失行里可真实驱动的分支;每组测试头上注明
它覆盖的基底行号。需要 Docker / 协调员 / run_one 主循环等重
型 harness 的缺失行(run_one、_run_attempt、_docker_task_context、
run_loop、run_all 等)不在本文件硬凑,见总结残留清单。
真实模型 0 轮,不联网;不启动真实 Docker / 子进程,凡需注入
处均用假连接、假 runner、假 proc 或方法补丁,且断言均为行为
断言(返回值、落盘行、调用记录),不是“跑过即过”。
"""
import json
import logging
import os
import sqlite3
import sys
import tempfile
import threading
import time
import unittest
from pathlib import Path
from types import SimpleNamespace
from unittest import mock
REPO_ROOT = Path(__file__).resolve().parents[2]
sys.path.insert(0, str(REPO_ROOT))
import cloud.worker as worker_mod # noqa: E402
from cloud.server import init_db # noqa: E402
from cloud.worker import DockerExecutor, Worker # noqa: E402
def _worker(db_path, **attrs):
w = Worker.__new__(Worker)
w.db_path = db_path
w.tenant_isolation = False
w.executor = SimpleNamespace()
w.logger = logging.getLogger("test_gapfill2_worker")
w._cancel_requested = set()
w._running = {}
w._running_lock = threading.Lock()
w._loop_stop = False
w.quota_poll_interval_seconds = 30
w.process_factory = None
w.repos_runner = None
for key, value in attrs.items():
setattr(w, key, value)
return w
def _insert_task(db_path, task_id="T1", status="running", goal="old goal",
goal_version=0, applied=0):
now = time.time()
conn = sqlite3.connect(db_path)
try:
conn.execute(
"INSERT INTO tasks(id, project_id, prompt, status,"
" idempotency_key, depends_on, priority, created_by,"
" created_at, updated_at, last_heartbeat, goal, goal_mode,"
" verify_command, goal_max_rounds, goal_version,"
" applied_goal_version) VALUES(?, 'p1', ?, ?, NULL, '[]',"
" 0, 'tester', ?, ?, NULL, ?, 1, NULL, 0, ?, ?)",
(task_id, goal, status, now, now, goal, goal_version,
applied))
conn.commit()
finally:
conn.close()
def _task_row(db_path, task_id="T1"):
conn = sqlite3.connect(db_path)
try:
conn.row_factory = sqlite3.Row
return dict(conn.execute(
"SELECT * FROM tasks WHERE id = ?", (task_id,)).fetchone())
finally:
conn.close()
def _write_state(state_path, tasks):
with open(state_path, "w", encoding="utf-8") as fh:
json.dump({"tasks": tasks}, fh)
class GoalSignalBranchTestCase(unittest.TestCase):
"""goal 信号写侧 / 镜像分支(基底缺失 1294 / 1300 / 1306 /
1313-1314 / 1339-1340 / 1381-1382 / 1397-1398)。"""
def setUp(self):
self.tmp = tempfile.TemporaryDirectory()
self.db_path = os.path.join(self.tmp.name, "test.db")
init_db(self.db_path)
self.state_path = os.path.join(self.tmp.name, "host-state.json")
def tearDown(self):
self.tmp.cleanup()
def _signal_worker(self):
w = _worker(self.db_path)
w._host_state_path_for_task = lambda task: self.state_path
return w
def test_bad_goal_rejected(self):
# 1294:db_goal 为空或非字符串时返回 False,不写信号。
w = self._signal_worker()
task = {"id": "T1", "project_id": "p1"}
self.assertFalse(w._write_goal_signal_if_newer(task, None, 2))
self.assertFalse(w._write_goal_signal_if_newer(task, "", 2))
self.assertFalse(w._write_goal_signal_if_newer(task, 123, 2))
self.assertFalse(os.path.exists(
os.path.join(self.tmp.name, "goal-updates")))
def test_no_state_path_returns_false(self):
# 1300:解析不出 state 路径时返回 False,不猜路径。
w = _worker(self.db_path)
w._host_state_path_for_task = lambda task: None
self.assertFalse(w._write_goal_signal_if_newer(
{"id": "T1", "project_id": "p1"}, "new goal", 2))
def test_entry_not_dict_returns_false(self):
# 1306:state 里本任务条目不是 dict 时返回 False。
_write_state(self.state_path, {"T1": "not-a-dict"})
w = self._signal_worker()
self.assertFalse(w._write_goal_signal_if_newer(
{"id": "T1", "project_id": "p1"}, "new goal", 2))
def test_corrupt_state_swallowed(self):
# 1313-1314:state 文件损坏(json 解析失败)时异常被吞掉,
# 返回 False,绝不炸掉调用方。
with open(self.state_path, "w", encoding="utf-8") as fh:
fh.write("{not json")
w = self._signal_worker()
self.assertFalse(w._write_goal_signal_if_newer(
{"id": "T1", "project_id": "p1"}, "new goal", 2))
def test_write_signal_success_control(self):
# 对照项(非缺失行):版本大于已应用版本时写信号成功,
# 证明上面几组失败测试的 harness 本身能走通成功路径。
_write_state(self.state_path, {
"T1": {"applied_goal_version": 0, "goal_rounds_done": 1}})
w = self._signal_worker()
self.assertTrue(w._write_goal_signal_if_newer(
{"id": "T1", "project_id": "p1"}, "new goal", 2))
signal_dir = os.path.join(self.tmp.name, "goal-updates")
self.assertTrue(os.path.isdir(signal_dir))
self.assertEqual(len(os.listdir(signal_dir)), 1)
def test_sync_resume_missing_id_swallowed(self):
# 1339-1340:续跑认领钩子读库时任务缺 id(KeyError)时
# 异常被吞掉,返回 False,续跑本身不受影响。
w = _worker(self.db_path)
self.assertFalse(w._sync_resume_goal_signal({}))
def test_mirror_bad_applied_ignored(self):
# 1381-1382:state 里 applied_goal_version 不是数字时,
# 转换失败置 None、跳过回写,不改任务行、不炸镜像。
_insert_task(self.db_path, applied=0)
_write_state(self.state_path, {"T1": {
"goal_rounds_done": 3, "status": "goal-running",
"applied_goal_version": "not-a-number",
"thread_id": "th-1"}})
w = self._signal_worker()
self.assertEqual(
w._mirror_checkpoint_once({"id": "T1", "project_id": "p1"}), 3)
self.assertEqual(_task_row(self.db_path)["applied_goal_version"],
0)
def test_mirror_db_read_failure_skips_signal_only(self):
# 1397-1398:镜像内读库(goal/goal_version)失败时只跳过
# 反向信号,检查点镜像本身继续、返回轮次,不炸、不回 -1。
_insert_task(self.db_path, applied=0)
_write_state(self.state_path, {"T1": {
"goal_rounds_done": 2, "status": "goal-running",
"applied_goal_version": 0, "thread_id": "th-1"}})
w = self._signal_worker()
real_connect = worker_mod._db_connect
class _FailingGoalReadConn:
def __init__(self, real):
self._real = real
def execute(self, sql, params=()):
if "SELECT goal, goal_version" in sql:
raise sqlite3.OperationalError(
"injected goal read failure")
return self._real.execute(sql, params)
def __getattr__(self, name):
return getattr(self._real, name)
def failing_connect(db_path):
return _FailingGoalReadConn(real_connect(db_path))
with mock.patch.object(worker_mod, "_db_connect",
failing_connect):
self.assertEqual(
w._mirror_checkpoint_once(
{"id": "T1", "project_id": "p1"}), 2)
conn = sqlite3.connect(self.db_path)
try:
row = conn.execute(
"SELECT round_no FROM task_checkpoints"
" WHERE task_id = 'T1'").fetchone()
finally:
conn.close()
self.assertEqual(row, (2,))
class TenantEnvBranchTestCase(unittest.TestCase):
"""租户目录 / 环境分支(基底缺失 3062 / 3091 / 3121-3122 /
3148-3149 / 3156-3157 / 3181-3182 / 3196)。"""
def setUp(self):
self.tmp = tempfile.TemporaryDirectory()
def tearDown(self):
self.tmp.cleanup()
def _tenant_worker(self, **extra):
w = _worker(os.path.join(self.tmp.name, "test.db"))
w.tenant_isolation = True
w.tenants_root = os.path.join(self.tmp.name, "tenants")
w.tenant_template_home = os.path.join(self.tmp.name, "tmpl-home")
w.tenant_template_repo = os.path.join(self.tmp.name, "tmpl-repo")
w.seed_skills = False
w.websearch_mcp = False
w.imagegen_mcp = False
w.playwright_mcp = False
for key, value in extra.items():
setattr(w, key, value)
return w
def test_tenant_dirs_missing_test_repo_template(self):
# 3062:test-repo 为空目录、模板不存在时直接抛错,
# 不静默播种一个空 test-repo。
os.makedirs(self._tenant_worker().tenant_template_home)
with open(os.path.join(self.tmp.name, "tmpl-home",
"config.toml"), "w") as fh:
fh.write("[x]\n")
w = self._tenant_worker()
with self.assertRaises(RuntimeError) as ctx:
w._tenant_dirs("p1")
self.assertIn("tmpl-repo", str(ctx.exception))
def test_tenant_mounts_includes_repo_checkout(self):
# 3091:repo_checkout 目录已存在时,挂载列表里追加
# repos/<安全项目名> 挂载;不存在时不硬挂载。
tmpl_home = os.path.join(self.tmp.name, "tmpl-home")
tmpl_repo = os.path.join(self.tmp.name, "tmpl-repo")
os.makedirs(tmpl_home)
os.makedirs(tmpl_repo)
with open(os.path.join(tmpl_home, "config.toml"), "w") as fh:
fh.write("[x]\n")
with open(os.path.join(tmpl_repo, "sample.py"), "w") as fh:
fh.write("# sample\n")
w = self._tenant_worker()
dirs = w._tenant_dirs("p1")
os.makedirs(dirs["repo_checkout"], exist_ok=True)
mounts = w._tenant_mounts("p1")
self.assertIn(
"%s:/var/lib/codex-app-server/repos/p1:rw"
% dirs["repo_checkout"], mounts)
def test_env_host_dir_tenant_docker(self):
# 3121-3122:租户 + Docker 时环境宿主目录走
# tenants_root/<项目>/envs/<env_id>。
w = self._tenant_worker()
w.executor = DockerExecutor.__new__(DockerExecutor)
result = w._env_host_dir({"project_id": "p1", "id": "env1"})
self.assertTrue(result.startswith(w.tenants_root))
self.assertTrue(result.endswith(
os.path.join("envs", "env1")))
def test_fetch_environment_db_failure_none(self):
# 3148-3149:库连接失败时返回 None,不炸 worker。
w = _worker(os.path.join(self.tmp.name, "no-such-dir",
"test.db"))
self.assertIsNone(w._fetch_environment("e1"))
def test_fetch_environment_narrow_schema_none_fields(self):
# 3156-3157:老库 environments 表缺可选列时,_get 兜底
# 回 None,不炸整行;必备列仍正常读出。
db_path = os.path.join(self.tmp.name, "test.db")
conn = sqlite3.connect(db_path)
try:
conn.execute(
"CREATE TABLE environments(id TEXT, project_id TEXT,"
" name TEXT, status TEXT)")
conn.execute(
"INSERT INTO environments VALUES('e1', 'p1', 'E',"
" 'ready')")
conn.commit()
finally:
conn.close()
w = _worker(db_path)
env = w._fetch_environment("e1")
self.assertEqual(env["id"], "e1")
self.assertEqual(env["project_id"], "p1")
self.assertEqual(env["status"], "ready")
self.assertIsNone(env["branch"])
self.assertIsNone(env["repo_url"])
def test_fetch_environment_missing_required_column_none(self):
# 3170-3171:environments 行缺必备列(project_id)时,
# 外层兜底回 None,不炸 worker。
db_path = os.path.join(self.tmp.name, "test.db")
conn = sqlite3.connect(db_path)
try:
conn.execute("CREATE TABLE environments(id TEXT)")
conn.execute("INSERT INTO environments VALUES('e1')")
conn.commit()
finally:
conn.close()
w = _worker(db_path)
self.assertIsNone(w._fetch_environment("e1"))
def test_run_git_typeerror_fallback(self):
# 3181-3182:repos_runner 单参调用抛 TypeError 时,
# 回落到 (cmd, timeout=None) 双参调用。
calls = []
_unset = object()
def runner(*args, timeout=_unset):
calls.append((args, timeout))
if timeout is _unset:
raise TypeError("single-arg not supported")
return SimpleNamespace(returncode=0, stdout="ok",
stderr="")
w = _worker(os.path.join(self.tmp.name, "test.db"))
w.repos_runner = runner
result = w._run_git(["git", "status"])
self.assertEqual(result.returncode, 0)
self.assertEqual(result.stdout, "ok")
self.assertEqual(len(calls), 2)
self.assertIs(calls[1][1], None)
def test_prepare_environment_no_repo_url(self):
# 3196:环境无 repo_url 时直接返回错误文本,不建目录。
w = _worker(os.path.join(self.tmp.name, "test.db"))
self.assertEqual(w._prepare_environment({}),
"environment has no repo_url")
class DbFailureBranchTestCase(unittest.TestCase):
"""查库失败即回落的分支(基底缺失 3333-3334 / 3339-3340 /
3583-3584 / 3604-3605 / 3620-3621 / 3650-3651 / 3665-3666 /
3694-3695 / 5343-5344 / 5798-5799)。"""
def setUp(self):
self.tmp = tempfile.TemporaryDirectory()
self.db_path = os.path.join(self.tmp.name, "test.db")
init_db(self.db_path)
def tearDown(self):
self.tmp.cleanup()
def _bad_db_worker(self):
return _worker(os.path.join(self.tmp.name, "no-such-dir",
"test.db"))
def test_project_repo_url_db_failure_none(self):
# 3333-3334:projects 表查库失败时回 None,不炸派发。
self.assertIsNone(self._bad_db_worker()._project_repo_url("p1"))
def test_project_repo_url_row_missing_key_none(self):
# 3339-3340:行里缺 repo_url 键(非 Row 行)时回 None。
class _Cursor:
def fetchone(self):
return {}
class _Conn:
def execute(self, sql, params=()):
return _Cursor()
def close(self):
pass
w = _worker(self.db_path)
with mock.patch.object(worker_mod, "_db_connect",
lambda path: _Conn()):
self.assertIsNone(w._project_repo_url("p1"))
def test_claim_followup_db_connect_failure_none(self):
# 3583-3584:followup 认领时库连接失败回 None。
self.assertIsNone(
self._bad_db_worker()._claim_next_followup("t1"))
def test_claim_followup_missing_table_none(self):
# 3604-3605:task_followups 表不存在时回 None,不炸 run_one。
fresh = os.path.join(self.tmp.name, "fresh.db")
sqlite3.connect(fresh).close()
w = _worker(fresh)
self.assertIsNone(w._claim_next_followup("t1"))
def test_finish_followup_db_failure_silent(self):
# 3620-3621:结束 followup 时库失败静默,不抛。
self._bad_db_worker()._finish_followup("f1", "done")
def test_convert_to_goal_db_failure_false(self):
# 3650-3651:转 goal 时库失败回 False,不悄悄改状态。
self.assertFalse(
self._bad_db_worker()._convert_running_task_to_goal(
"t1", "new goal"))
def test_restore_followup_db_failure_silent(self):
# 3665-3666:放回 followup 时库失败静默,不抛。
self._bad_db_worker()._restore_followup("f1")
def test_queue_self_followup_db_failure_false(self):
# 3694-3695:排自有 followup 时库失败回 False。
w = self._bad_db_worker()
self.assertFalse(w._queue_self_followup(
{"id": "t1", "project_id": "p1"}, "body"))
def test_coord_child_rows_missing_key_skipped(self):
# 5343-5344:子任务行缺列(非 Row 行)时跳过该行,
# 不炸等齐判断。
class _Cursor:
def fetchone(self):
return {"id": "c1"}
class _Conn:
def execute(self, sql, params=()):
return _Cursor()
def close(self):
pass
w = _worker(self.db_path)
with mock.patch.object(worker_mod, "_db_connect",
lambda path: _Conn()):
self.assertEqual(w._coord_child_rows(["c1"]), {})
def test_record_ledger_attempt_outbox_failure_swallowed(self):
# 5798-5799:台账写失败后 outbox 追加本身也失败时,
# 仍只吞掉、返回 None,不影响任务状态与返回值。
w = self._bad_db_worker()
w.attempt_ledger_hook = True
import cloud.ledger_outbox as outbox_mod # noqa: E402
with mock.patch.object(
outbox_mod, "append_failure",
side_effect=RuntimeError("outbox down")):
self.assertIsNone(
w._record_ledger_attempt("t1", "fail", True))
class ProcessWatchdogBranchTestCase(unittest.TestCase):
"""进程 / 看门狗分支(基底缺失 2505-2506 / 2892-2893 /
3853-3854 / 3874-3875 / 3881-3882)。"""
def setUp(self):
self.tmp = tempfile.TemporaryDirectory()
self.db_path = os.path.join(self.tmp.name, "test.db")
init_db(self.db_path)
def tearDown(self):
self.tmp.cleanup()
def test_docker_cancel_watchdog_db_interrupted(self):
# 2505-2506:看门狗查库发现任务已 interrupted 时杀容器。
killed = []
w = _worker(self.db_path)
w.executor = SimpleNamespace(
kill_container=lambda name: killed.append(name))
w.get_task = lambda task_id: {"status": "interrupted"}
stop_event = threading.Event()
w._docker_cancel_watchdog("t1", "c1", stop_event)
self.assertEqual(killed, ["c1"])
def test_execute_via_process_kill_failure_swallowed(self):
# 2892-2893:communicate 超时后 proc.kill() 本身抛错时
# 被吞掉,第二次 communicate 的结果仍正常返回 completed。
calls = {"kill": 0}
class _FakeProc:
pid = 999
returncode = 0
def poll(self):
return 0
def communicate(self, timeout=None):
if timeout is not None:
raise RuntimeError("communicate timeout")
return ("out", "err")
def kill(self):
calls["kill"] += 1
raise RuntimeError("kill failed")
w = _worker(self.db_path)
w.process_factory = lambda task: _FakeProc()
with mock.patch.object(Worker, "_ensure_gateway",
return_value=(None, None)):
result = w._execute_via_process({"id": "t1"})
self.assertEqual(calls["kill"], 1)
self.assertEqual(result["status"], "completed")
self.assertEqual(result["stdout"], "out")
self.assertEqual(result["stderr"], "err")
self.assertEqual(result["returncode"], 0)
def test_marker_auth_close_failure_swallowed(self):
# 3853-3854:权限查完后 store.conn.close() 抛错时被吞掉,
# 权限结论(True)不受影响。
class _FakeConn:
def close(self):
raise RuntimeError("close failed")
class _FakeStore:
def __init__(self, db_path):
self.conn = _FakeConn()
def check_permission(self, creator, project_id, action):
return True
w = _worker(self.db_path)
with mock.patch.object(worker_mod, "AuthStore", _FakeStore):
self.assertEqual(
w._marker_auth({"created_by": "u1",
"project_id": "p1"}, "read"),
(True, ""))
def test_task_files_worktree_remote_host(self):
# 3874-3875:worktree 在别机时直接拒绝,不指错目录。
w = _worker(self.db_path)
path, reason = w._task_files_worktree(
{"executor": "local", "worktree_host": "other-host-xyz",
"id": "t1"})
self.assertIsNone(path)
self.assertIn("other-host-xyz", reason)
def test_task_files_worktree_bad_project_falls_through(self):
# 3881-3882:project_id 缺失致安全名解析失败时,
# safe_project 置 None、走本地 worktree 分支,不炸。
w = _worker(self.db_path)
w.host_py_path = os.path.join(self.tmp.name, "host.py")
path, reason = w._task_files_worktree(
{"executor": "local", "project_id": None, "id": "t1"})
self.assertIsNone(path)
self.assertIn("没有本地 worktree", reason)
class PollWaitBranchTestCase(unittest.TestCase):
"""轮询 / 等待分支(基底缺失 3711-3712 / 3720-3721 /
3728-3729 / 3747 / 3768-3769 / 3774-3775 / 3804-3805 / 3825)。"""
def setUp(self):
self.tmp = tempfile.TemporaryDirectory()
self.db_path = os.path.join(self.tmp.name, "test.db")
init_db(self.db_path)
def tearDown(self):
self.tmp.cleanup()
def _poll(self, timeout=10):
return {"task_id": "T2", "interval_seconds": 1,
"timeout_seconds": timeout, "chars": 100}
def test_poll_cancelled_before_snapshot(self):
# 3711-3712 + 3747:父任务已取消时不等快照、直接回
# “读取失败”,不挂起。
w = _worker(self.db_path)
w._cancel_requested = {"P1"}
result = w._poll_task_snapshot(
{"id": "P1", "project_id": "p1"}, self._poll())
self.assertEqual(result, "轮询任务 T2:读取失败。")
def test_poll_db_failure_returns_missing(self):
# 3720-3721 + 3728-3729:心跳与快照查库都失败时,
# 异常被吞掉、回“不存在”,不炸等待循环。
w = _worker(os.path.join(self.tmp.name, "no-such-dir",
"test.db"))
w.get_task = lambda task_id: None
result = w._poll_task_snapshot(
{"id": "P1", "project_id": "p1"}, self._poll())
self.assertEqual(result, "轮询任务 T2:不存在(同项目内也查不到)。")
def test_wait_cancelled(self):
# 3768-3769 + 3825 + 3831:父任务已取消时等待中止,
# 未读到的目标记“读取失败”,末尾附取消说明。
w = _worker(self.db_path)
w._cancel_requested = {"P1"}
lines = w._wait_task_targets(
{"id": "P1", "project_id": "p1"},
{"task_ids": ["T2"], "interval_seconds": 1,
"timeout_seconds": 10, "chars": 100})
self.assertEqual(lines[0], "[等待 1 个任务]")
self.assertIn("等待任务 T2:读取失败。", lines)
self.assertIn("说明:等待期间父任务被取消,等待已中止。", lines)
def test_wait_db_failure_silent(self):
# 3774-3775 + 3804-3805:心跳与批量快照查库都失败时,
# 异常被吞掉、超时 0 立即收尾,不炸、不挂起。
w = _worker(os.path.join(self.tmp.name, "no-such-dir",
"test.db"))
w.get_task = lambda task_id: None
lines = w._wait_task_targets(
{"id": "P1", "project_id": "p1"},
{"task_ids": ["T2"], "interval_seconds": 1,
"timeout_seconds": 0, "chars": 100})
self.assertEqual(lines[0], "[等待 1 个任务]")
self.assertIn("等待任务 T2:读取失败。", lines)
if __name__ == "__main__":
unittest.main()