439 lines
17 KiB
Python
439 lines
17 KiB
Python
"""code-to10 v6 A8：worker 缺口补测。
|
||
|
||
补测基线（v5 头 10023fb 全量覆盖率，cloud/worker.py 3402
|
||
语句缺 338 行，90%）：本文件按“假执行器/假派发”补下列
|
||
区间，其余缺口行以全量复测为准：
|
||
- 2092-2095：process_auto_memories 的 extracted 项审计。
|
||
- 2936-2944：_tenant_dirs 技能播种失败只记日志不阻塞，
|
||
且日志本身抛错也被吞掉。
|
||
- 3092-3111（及 checkout 失败行）：_prepare_environment
|
||
的 clone/非 git/get-url/fetch/checkout 失败分支。
|
||
- 6626-6652：_run_pr_jobs 的 PrSkipped/PrStepError/通用
|
||
异常三路失败写回。
|
||
- 6977-7010：run_loop 退出排空（杀 proc、杀容器、标
|
||
interrupted、审计），含 project_id 为 NULL 与审计查询
|
||
抛错两路 _pid 回落。
|
||
|
||
绝不碰（任务口径不碰项，单列）：
|
||
- claim/goal_resume_pending 的 except-pass 分支。
|
||
- MCP handler-error 通知分支。
|
||
只加测试，不改生产代码；假的是执行器/派发/播种等外部
|
||
依赖，不是被测代码本身；不加 no-coverage pragma、不 skip。
|
||
"""
|
||
|
||
import os
|
||
import sqlite3
|
||
import sys
|
||
import tempfile
|
||
import threading
|
||
import unittest
|
||
from pathlib import Path
|
||
from types import SimpleNamespace
|
||
|
||
try:
|
||
import cloud.worker as worker_mod
|
||
from cloud.db import connect as _db_connect
|
||
from cloud.pr import PrSkipped, PrStepError
|
||
from cloud.pr_exec import enqueue_pr_job
|
||
from cloud.server import init_db
|
||
from cloud.worker import Worker
|
||
except ImportError: # discover -s cloud/tests 时把项目根目录补进 sys.path
|
||
sys.path.insert(0, str(Path(__file__).resolve().parents[2]))
|
||
import cloud.worker as worker_mod
|
||
from cloud.db import connect as _db_connect
|
||
from cloud.pr import PrSkipped, PrStepError
|
||
from cloud.pr_exec import enqueue_pr_job
|
||
from cloud.server import init_db
|
||
from cloud.worker import Worker
|
||
|
||
|
||
class _FakeSync:
|
||
"""同步假执行器：不走 Local/Docker/Cloud，只给 run_loop
|
||
的排空路径一个带 kill_container 的执行器替身。"""
|
||
|
||
def dispatch_task(self, task):
|
||
return {"status": "completed"}
|
||
|
||
def run_task(self, task):
|
||
return {"status": "completed"}
|
||
|
||
def kill_container(self, name):
|
||
raise RuntimeError("kill boom")
|
||
|
||
|
||
class _FakeProc:
|
||
"""pid 不存在的假进程：_kill_proc_group 在 getpgid 处即
|
||
返回（不真杀），wait 抛错走排空的 except 分支。"""
|
||
|
||
pid = 999999
|
||
|
||
def wait(self, timeout=None):
|
||
raise RuntimeError("wait boom")
|
||
|
||
|
||
class _RaisingLogger:
|
||
"""warning 一律抛错的假 logger：测 _tenant_dirs 里
|
||
“日志失败也被吞掉”的内层 except。"""
|
||
|
||
def warning(self, *args, **kwargs):
|
||
raise RuntimeError("log boom")
|
||
|
||
def info(self, *args, **kwargs):
|
||
pass
|
||
|
||
def debug(self, *args, **kwargs):
|
||
pass
|
||
|
||
def error(self, *args, **kwargs):
|
||
pass
|
||
|
||
|
||
def _insert_task(db_path, task_id, project_id="p1", status="queued"):
|
||
conn = sqlite3.connect(db_path)
|
||
try:
|
||
conn.execute(
|
||
"INSERT INTO tasks(id, project_id, prompt, status,"
|
||
" idempotency_key, depends_on, priority, created_by,"
|
||
" created_at, updated_at, last_heartbeat)"
|
||
" VALUES(?, ?, ?, ?, NULL, '[]', 0, 'tester', 0, 0, NULL)",
|
||
(task_id, project_id, "do work", status),
|
||
)
|
||
conn.commit()
|
||
finally:
|
||
conn.close()
|
||
|
||
|
||
def _row(db_path, sql):
|
||
conn = sqlite3.connect(db_path)
|
||
try:
|
||
return conn.execute(sql).fetchone()
|
||
finally:
|
||
conn.close()
|
||
|
||
|
||
class WorkerGapsBase(unittest.TestCase):
|
||
def setUp(self):
|
||
self.tmp = tempfile.TemporaryDirectory()
|
||
self.addCleanup(self.tmp.cleanup)
|
||
self.db_path = os.path.join(self.tmp.name, "control.db")
|
||
init_db(self.db_path)
|
||
self.hb = os.path.join(self.tmp.name, "hb.json")
|
||
|
||
def _worker(self, **kwargs):
|
||
kwargs.setdefault("env", {"XCM_API_KEY": "test-placeholder"})
|
||
kwargs.setdefault("heartbeat_file", self.hb)
|
||
kwargs.setdefault("max_retries", 0)
|
||
kwargs.setdefault("retry_backoff_seconds", 0)
|
||
return Worker(self.db_path, "/fake/host.py", **kwargs)
|
||
|
||
|
||
class ProcessAutoMemoriesTestCase(WorkerGapsBase):
|
||
def test_extracted_item_audited(self):
|
||
worker = self._worker()
|
||
orig = worker_mod._auto_process_due
|
||
worker_mod._auto_process_due = lambda db_path, **kw: [
|
||
{"status": "extracted", "task_id": "T1"},
|
||
{"status": "other", "task_id": "T2"},
|
||
]
|
||
try:
|
||
results = worker.process_auto_memories()
|
||
finally:
|
||
worker_mod._auto_process_due = orig
|
||
self.assertEqual(len(results), 2)
|
||
row = _row(self.db_path,
|
||
"SELECT action, detail FROM audit_log"
|
||
" WHERE action = 'worker_memory_extracted'")
|
||
self.assertEqual(row, ("worker_memory_extracted", "T1"))
|
||
|
||
def test_sweep_failure_returns_empty(self):
|
||
worker = self._worker()
|
||
orig = worker_mod._auto_process_due
|
||
worker_mod._auto_process_due = lambda db_path, **kw: (_ for _ in ()).throw(
|
||
RuntimeError("sweep boom"))
|
||
try:
|
||
self.assertEqual(worker.process_auto_memories(), [])
|
||
finally:
|
||
worker_mod._auto_process_due = orig
|
||
|
||
def test_sweep_failure_and_logger_failure_returns_empty(self):
|
||
# 扫失败 + 日志本身抛错：内层 except-pass 分支（2088-2089）
|
||
# 同样只回 []，不炸循环。
|
||
worker = self._worker()
|
||
worker.logger = _RaisingLogger()
|
||
orig = worker_mod._auto_process_due
|
||
worker_mod._auto_process_due = lambda db_path, **kw: (_ for _ in ()).throw(
|
||
RuntimeError("sweep boom"))
|
||
try:
|
||
self.assertEqual(worker.process_auto_memories(), [])
|
||
finally:
|
||
worker_mod._auto_process_due = orig
|
||
|
||
|
||
class TenantDirsSkillFailureTestCase(WorkerGapsBase):
|
||
def _seeded_worker(self):
|
||
tenants = os.path.join(self.tmp.name, "tenants")
|
||
template_home = os.path.join(self.tmp.name, "template-home")
|
||
os.makedirs(template_home, exist_ok=True)
|
||
with open(os.path.join(template_home, "config.toml"), "w") as f:
|
||
f.write("# template\n")
|
||
worker = self._worker(tenants_root=tenants,
|
||
tenant_template_home=template_home,
|
||
seed_skills=True)
|
||
# 预置 codex-home 配置与非空 test-repo：播种分支只测
|
||
# 技能失败本身，不走到 test-repo 模板复制/报错。
|
||
base = os.path.join(tenants, "p1")
|
||
os.makedirs(os.path.join(base, "codex-home"), exist_ok=True)
|
||
with open(os.path.join(base, "codex-home", "config.toml"),
|
||
"w") as f:
|
||
f.write("# existing\n")
|
||
os.makedirs(os.path.join(base, "test-repo"), exist_ok=True)
|
||
with open(os.path.join(base, "test-repo", "sample.py"),
|
||
"w") as f:
|
||
f.write("# sample\n")
|
||
return worker
|
||
|
||
def test_skill_failure_only_logged_not_blocking(self):
|
||
worker = self._seeded_worker()
|
||
orig = worker_mod._ensure_skills
|
||
worker_mod._ensure_skills = lambda home: (_ for _ in ()).throw(
|
||
RuntimeError("skill boom"))
|
||
try:
|
||
dirs = worker._tenant_dirs("p1")
|
||
finally:
|
||
worker_mod._ensure_skills = orig
|
||
self.assertTrue(os.path.isdir(dirs["test_repo"]))
|
||
self.assertTrue(os.path.exists(
|
||
os.path.join(dirs["test_repo"], "sample.py")))
|
||
|
||
def test_skill_failure_and_logger_failure_swallowed(self):
|
||
worker = self._seeded_worker()
|
||
worker.logger = _RaisingLogger()
|
||
orig = worker_mod._ensure_skills
|
||
worker_mod._ensure_skills = lambda home: (_ for _ in ()).throw(
|
||
RuntimeError("skill boom"))
|
||
try:
|
||
dirs = worker._tenant_dirs("p1")
|
||
finally:
|
||
worker_mod._ensure_skills = orig
|
||
self.assertTrue(os.path.isdir(dirs["work"]))
|
||
|
||
|
||
class PrepareEnvironmentTestCase(WorkerGapsBase):
|
||
def _env_worker(self, runner):
|
||
repos = os.path.join(self.tmp.name, "repos")
|
||
return self._worker(repos_dir=repos, repos_runner=runner), repos
|
||
|
||
def test_clone_failure(self):
|
||
def runner(cmd):
|
||
return SimpleNamespace(returncode=1, stdout="",
|
||
stderr="clone boom")
|
||
worker, _repos = self._env_worker(runner)
|
||
err = worker._prepare_environment(
|
||
{"id": "env1", "repo_url": "https://example.invalid/x.git"})
|
||
self.assertIn("git clone 失败", err)
|
||
self.assertIn("clone boom", err)
|
||
|
||
def test_existing_dir_not_a_git_repo(self):
|
||
worker, repos = self._env_worker(lambda cmd: SimpleNamespace(
|
||
returncode=0, stdout="", stderr=""))
|
||
target = os.path.join(repos, "envs", "env2")
|
||
os.makedirs(target, exist_ok=True)
|
||
err = worker._prepare_environment(
|
||
{"id": "env2", "repo_url": "https://example.invalid/x.git"})
|
||
self.assertIn("不是 git 仓库", err)
|
||
|
||
def _git_repo_worker(self, behaviour):
|
||
worker, repos = self._env_worker(behaviour)
|
||
target = os.path.join(repos, "envs", "env3")
|
||
os.makedirs(os.path.join(target, ".git"), exist_ok=True)
|
||
return worker
|
||
|
||
def test_get_url_failure(self):
|
||
def runner(cmd):
|
||
return SimpleNamespace(returncode=1, stdout="",
|
||
stderr="no remote")
|
||
worker = self._git_repo_worker(runner)
|
||
err = worker._prepare_environment(
|
||
{"id": "env3", "repo_url": "https://example.invalid/x.git"})
|
||
self.assertIn("取不到 remote origin", err)
|
||
|
||
def test_origin_mismatch(self):
|
||
def runner(cmd):
|
||
if "get-url" in cmd:
|
||
return SimpleNamespace(
|
||
returncode=0, stdout="https://other.invalid/y.git\n",
|
||
stderr="")
|
||
return SimpleNamespace(returncode=0, stdout="", stderr="")
|
||
worker = self._git_repo_worker(runner)
|
||
err = worker._prepare_environment(
|
||
{"id": "env3", "repo_url": "https://example.invalid/x.git"})
|
||
self.assertIn("origin 与环境 repo_url 不一致", err)
|
||
|
||
def test_fetch_failure(self):
|
||
def runner(cmd):
|
||
if "get-url" in cmd:
|
||
return SimpleNamespace(
|
||
returncode=0,
|
||
stdout="https://example.invalid/x.git\n", stderr="")
|
||
if "fetch" in cmd:
|
||
return SimpleNamespace(returncode=1, stdout="",
|
||
stderr="fetch boom")
|
||
return SimpleNamespace(returncode=0, stdout="", stderr="")
|
||
worker = self._git_repo_worker(runner)
|
||
err = worker._prepare_environment(
|
||
{"id": "env3", "repo_url": "https://example.invalid/x.git"})
|
||
self.assertIn("git fetch origin 失败", err)
|
||
self.assertIn("fetch boom", err)
|
||
|
||
def test_checkout_failure_and_success(self):
|
||
def runner(cmd):
|
||
if "get-url" in cmd:
|
||
return SimpleNamespace(
|
||
returncode=0,
|
||
stdout="https://example.invalid/x.git\n", stderr="")
|
||
if "fetch" in cmd:
|
||
return SimpleNamespace(returncode=0, stdout="", stderr="")
|
||
if "checkout" in cmd:
|
||
return SimpleNamespace(returncode=1, stdout="",
|
||
stderr="no such branch")
|
||
return SimpleNamespace(returncode=0, stdout="", stderr="")
|
||
worker = self._git_repo_worker(runner)
|
||
env = {"id": "env3", "repo_url": "https://example.invalid/x.git",
|
||
"branch": "nope"}
|
||
err = worker._prepare_environment(env)
|
||
self.assertIn("git checkout nope 失败", err)
|
||
self.assertIn("no such branch", err)
|
||
# 同一 runner 改走成功 checkout：返回 None（准备完成）。
|
||
def ok_runner(cmd):
|
||
return SimpleNamespace(returncode=0,
|
||
stdout="https://example.invalid/x.git\n",
|
||
stderr="")
|
||
worker.repos_runner = ok_runner
|
||
self.assertIsNone(worker._prepare_environment(env))
|
||
|
||
|
||
class RunPrJobsFailureTestCase(WorkerGapsBase):
|
||
def _pr_worker(self, failure):
|
||
worker = self._worker(env_key="TEST_KEY",
|
||
host_name="elsewhere-host-1")
|
||
conn = _db_connect(self.db_path)
|
||
try:
|
||
worker._ensure_result_columns(conn)
|
||
conn.execute(
|
||
"INSERT INTO tasks(id, project_id, prompt, status,"
|
||
" depends_on, priority, created_by, created_at,"
|
||
" updated_at, last_heartbeat) VALUES('t1', 'p1', 'x',"
|
||
" 'merged', '[]', 0, 'system', 0, 0, NULL)")
|
||
conn.commit()
|
||
enqueue_pr_job(conn, "t1", "p1", "elsewhere-host-1")
|
||
finally:
|
||
conn.close()
|
||
return worker
|
||
|
||
def _run_with_failure(self, failure):
|
||
worker = self._pr_worker(failure)
|
||
orig = worker_mod.execute_task_pr
|
||
|
||
def _fake(db_path, task_id, user_id, **kwargs):
|
||
raise failure
|
||
worker_mod.execute_task_pr = _fake
|
||
try:
|
||
results = worker._run_pr_jobs()
|
||
finally:
|
||
worker_mod.execute_task_pr = orig
|
||
return results
|
||
|
||
def test_pr_skipped_marks_failed(self):
|
||
results = self._run_with_failure(PrSkipped("no longer merged"))
|
||
self.assertEqual(len(results), 1)
|
||
self.assertEqual(results[0]["status"], "failed")
|
||
self.assertIn("skipped: no longer merged", results[0]["error"])
|
||
row = _row(self.db_path,
|
||
"SELECT status FROM pr_jobs WHERE task_id = 't1'")
|
||
self.assertEqual(row[0], "failed")
|
||
|
||
def test_pr_step_error_marks_failed(self):
|
||
results = self._run_with_failure(
|
||
PrStepError("push", "remote denied"))
|
||
self.assertEqual(results[0]["error"], "push: remote denied")
|
||
row = _row(self.db_path,
|
||
"SELECT status FROM pr_jobs WHERE task_id = 't1'")
|
||
self.assertEqual(row[0], "failed")
|
||
|
||
def test_generic_error_marks_failed(self):
|
||
results = self._run_with_failure(RuntimeError("boom"))
|
||
self.assertEqual(results[0]["error"], "boom")
|
||
row = _row(self.db_path,
|
||
"SELECT status FROM pr_jobs WHERE task_id = 't1'")
|
||
self.assertEqual(row[0], "failed")
|
||
|
||
|
||
class RunLoopDrainTestCase(WorkerGapsBase):
|
||
def _drain_worker(self, task_id, project_id):
|
||
_insert_task(self.db_path, task_id, project_id=project_id,
|
||
status="running")
|
||
worker = self._worker(executor=_FakeSync())
|
||
worker._running_set(task_id, proc=_FakeProc(), container="c1")
|
||
worker.executor.kill_container = lambda name: (_ for _ in ()).throw(
|
||
RuntimeError("kill boom"))
|
||
return worker
|
||
|
||
def _run_drain(self, worker):
|
||
stop = threading.Event()
|
||
stop.set()
|
||
results = worker.run_loop(interval=10, drain_timeout=0,
|
||
stop_event=stop)
|
||
self.assertEqual(results, [])
|
||
return worker
|
||
|
||
def test_drain_kills_proc_and_container_marks_interrupted(self):
|
||
worker = self._drain_worker("t-drain", "p1")
|
||
self._run_drain(worker)
|
||
row = _row(self.db_path,
|
||
"SELECT status, interrupted_reason FROM tasks"
|
||
" WHERE id = 't-drain'")
|
||
self.assertEqual(row[0], "interrupted")
|
||
self.assertEqual(row[1], "worker_shutdown")
|
||
audit = _row(self.db_path,
|
||
"SELECT action, detail FROM audit_log"
|
||
" WHERE action = 'worker_shutdown_interrupted'")
|
||
self.assertEqual(audit,
|
||
("worker_shutdown_interrupted", "t-drain"))
|
||
|
||
def test_drain_null_project_id_audits_empty_project(self):
|
||
worker = self._drain_worker("t-null", None)
|
||
self._run_drain(worker)
|
||
audit = _row(self.db_path,
|
||
"SELECT action, detail FROM audit_log"
|
||
" WHERE action = 'worker_shutdown_interrupted'"
|
||
" AND detail = 't-null'")
|
||
self.assertIsNotNone(audit)
|
||
|
||
def test_drain_audit_lookup_failure_still_interrupts(self):
|
||
worker = self._drain_worker("t-dberr", "p1")
|
||
orig_connect = worker_mod._db_connect
|
||
import inspect
|
||
|
||
def _guarded(db_path):
|
||
# 只有 run_loop 本体直接发起的审计查询才抛错：
|
||
# _set_status/_mark_interrupted_reason 调 _db_connect
|
||
# 时直接调用方是各自方法名，不受影响；排空审计的
|
||
# except 分支据此覆盖。
|
||
caller = inspect.currentframe().f_back
|
||
if caller is not None and \
|
||
caller.f_code.co_name == "run_loop":
|
||
raise RuntimeError("db gone")
|
||
return orig_connect(db_path)
|
||
worker_mod._db_connect = _guarded
|
||
try:
|
||
self._run_drain(worker)
|
||
finally:
|
||
worker_mod._db_connect = orig_connect
|
||
row = _row(self.db_path,
|
||
"SELECT status FROM tasks WHERE id = 't-dberr'")
|
||
self.assertEqual(row[0], "interrupted")
|
||
|
||
|
||
if __name__ == "__main__":
|
||
unittest.main()
|