1337 lines
55 KiB
Python
1337 lines
55 KiB
Python
"""CHV 收尾轨（feat-chv-closeup）新增测试。
|
||
|
||
只补上一轨 coverage 下 cloud/chv/executor.py 仍未覆盖的
|
||
深层生命周期分支：唤醒失败（vm.create/vm.boot/API 超时/
|
||
guest 就绪超时/host key 不匹配/provisioned 缺钉死）、
|
||
恢复与快照失败（restore 失败关 VM、快照未落盘/配额/
|
||
总量剪枝）、取消与清理（cancel 真关、recycle、残留
|
||
socket/登记/磁盘/密钥扫描、脏标记）、中途死亡分类
|
||
（异常与 failed 状态改判 interrupted）、只读 execute
|
||
归类、入口校验、preflight 事后改写校验、worker_adapter
|
||
看门狗/调和的剩余分支。
|
||
|
||
全部用标准库与 mock runner/launcher/api_client，绝不启动
|
||
真实 VM（本机无 /dev/kvm，真机验证仍是 §13.2 的 V-1～V-9，
|
||
不在本文件声称已验证）。
|
||
"""
|
||
|
||
import inspect
|
||
import json
|
||
import os
|
||
import sys
|
||
import tempfile
|
||
import threading
|
||
import unittest
|
||
from pathlib import Path
|
||
from unittest import mock
|
||
|
||
try:
|
||
from cloud.chv.executor import CloudHypervisorExecutor
|
||
from cloud.chv.hostkeys import dirty_marker_path
|
||
from cloud.chv.worker_adapter import (
|
||
call_goal_resume_breakpoint,
|
||
cancel_watchdog_loop,
|
||
quota_watchdog_loop,
|
||
reconcile_warm_pool,
|
||
watchdog_vm_name,
|
||
)
|
||
from cloud.sandbox import ChvFailure
|
||
except ImportError: # discover -s cloud/tests 时把项目根目录补进 sys.path
|
||
sys.path.insert(0, str(Path(__file__).resolve().parents[2]))
|
||
from cloud.chv.executor import CloudHypervisorExecutor
|
||
from cloud.chv.hostkeys import dirty_marker_path
|
||
from cloud.chv.worker_adapter import (
|
||
call_goal_resume_breakpoint,
|
||
cancel_watchdog_loop,
|
||
quota_watchdog_loop,
|
||
reconcile_warm_pool,
|
||
watchdog_vm_name,
|
||
)
|
||
from cloud.sandbox import ChvFailure
|
||
|
||
|
||
class FakeResult:
|
||
def __init__(self, returncode=0, stdout="", stderr=""):
|
||
self.returncode = returncode
|
||
self.stdout = stdout
|
||
self.stderr = stderr
|
||
|
||
|
||
class FakeProc:
|
||
def __init__(self, alive=True):
|
||
self._alive = alive
|
||
self.terminated = False
|
||
self.killed = False
|
||
|
||
def poll(self):
|
||
return None if self._alive else 1
|
||
|
||
def terminate(self):
|
||
self.terminated = True
|
||
self._alive = False
|
||
|
||
def wait(self, timeout=None):
|
||
return 0
|
||
|
||
def kill(self):
|
||
self.killed = True
|
||
self._alive = False
|
||
|
||
|
||
def _write(path, content=b"x"):
|
||
with open(path, "wb") as fh:
|
||
fh.write(content)
|
||
os.chmod(path, 0o755)
|
||
return path
|
||
|
||
|
||
class CloseupCase(unittest.TestCase):
|
||
def setUp(self):
|
||
self.tmp = tempfile.TemporaryDirectory()
|
||
self.dir = self.tmp.name
|
||
self.binary = _write(os.path.join(self.dir, "cloud-hypervisor"))
|
||
self.ch_remote = _write(os.path.join(self.dir, "ch-remote"))
|
||
self.kernel = _write(os.path.join(self.dir, "vmlinux"))
|
||
self.rootfs = _write(os.path.join(self.dir, "disk.raw"),
|
||
b"R" * 100)
|
||
self.ssh_key = _write(os.path.join(self.dir, "id_chv"))
|
||
self.kvm = _write(os.path.join(self.dir, "kvm"))
|
||
self.cpuinfo = os.path.join(self.dir, "cpuinfo")
|
||
with open(self.cpuinfo, "w") as fh:
|
||
fh.write("flags : fpu vmx sse\n")
|
||
self.state_dir = os.path.join(self.dir, "state")
|
||
self.calls = []
|
||
self.api_calls = []
|
||
self.launches = []
|
||
self.audit = []
|
||
self.proc = None
|
||
|
||
def tearDown(self):
|
||
self.tmp.cleanup()
|
||
|
||
def make_exec(self, runner=None, api_client=None, launcher=None,
|
||
env_extra=None, **overrides):
|
||
env = {"CHV_EGRESS_PROXY": "http://10.0.0.1:8888",
|
||
"CHV_HOST_KEY_MODE": "keyscan",
|
||
"CHV_SSH_FORWARD": "1"}
|
||
if env_extra:
|
||
env.update(env_extra)
|
||
kwargs = dict(
|
||
binary=self.binary, ch_remote=self.ch_remote,
|
||
kernel=self.kernel, rootfs=self.rootfs,
|
||
ssh_key=self.ssh_key, kvm_path=self.kvm,
|
||
cpuinfo_path=self.cpuinfo, state_dir=self.state_dir,
|
||
runner=runner or self._default_runner,
|
||
launcher=launcher or self._default_launcher,
|
||
api_client=api_client or self._default_api,
|
||
sleeper=lambda s: None, env=env)
|
||
kwargs.update(overrides)
|
||
ex = CloudHypervisorExecutor(**kwargs)
|
||
ex.audit_callback = lambda e, t, d, k=None: self.audit.append(
|
||
(e, t.get("id") if isinstance(t, dict) else t, d, k))
|
||
return ex
|
||
|
||
def _default_runner(self, cmd, timeout=None):
|
||
self.calls.append(list(cmd))
|
||
if cmd[0] == "ssh":
|
||
if cmd[-1] == "true":
|
||
return FakeResult(0, "", "")
|
||
if "dispatch" in cmd:
|
||
return FakeResult(0, json.dumps(
|
||
{"status": "completed", "text": "ok-dispatch"}), "")
|
||
if "goal" in cmd or "goal-resume" in cmd:
|
||
return FakeResult(0, "goal met after rounds\n", "")
|
||
if "status" in cmd:
|
||
return FakeResult(0, json.dumps(
|
||
{"status": "completed", "text": "ok-status"}), "")
|
||
return FakeResult(0, "", "")
|
||
return FakeResult(0, "", "")
|
||
|
||
def _default_api(self, socket_path, method, path, payload=None):
|
||
self.api_calls.append((method, path, payload))
|
||
return (200, {})
|
||
|
||
def _default_launcher(self, cmd):
|
||
self.launches.append(list(cmd))
|
||
self.proc = FakeProc(alive=True)
|
||
return self.proc
|
||
|
||
def _task(self, task_id="abc123"):
|
||
return {"id": task_id, "prompt": "做一个加法函数",
|
||
"project_id": "proj1"}
|
||
|
||
def _audit_events(self):
|
||
return [e for e, _t, _d, _k in self.audit]
|
||
|
||
def _wake_ok(self, ex, task=None):
|
||
return ex.wake_vm(task or self._task())
|
||
|
||
def _known_hosts(self, task_id="abc123"):
|
||
vm_dir = os.path.join(self.state_dir, "chv-" + task_id)
|
||
os.makedirs(vm_dir, exist_ok=True)
|
||
path = os.path.join(vm_dir, "known_hosts")
|
||
with open(path, "w") as fh:
|
||
fh.write("127.0.0.1 ssh-ed25519 AAAAC3\n")
|
||
return path
|
||
|
||
|
||
# -- 唤醒失败 ------------------------------------------------------------
|
||
class WakeFailureTest(CloseupCase):
|
||
def test_wake_prechecks_secret_keyscan_fail_closed(self):
|
||
ex = self.make_exec()
|
||
# 密钥子集的判定名是 SUB2API_API_KEY（见 ssh_transport
|
||
# 的 _CHV_SECRET_ENV_NAMES），其它名走普通 env 前缀。
|
||
with self.assertRaises(ChvFailure) as ctx:
|
||
ex.wake_vm(self._task(),
|
||
extra_env={"SUB2API_API_KEY": "secret-value"})
|
||
self.assertEqual(ctx.exception.kind,
|
||
"secret_env_requires_provisioned")
|
||
self.assertIn("vm_failed", self._audit_events())
|
||
self.assertEqual(self.launches, [])
|
||
|
||
def test_wake_vm_create_failed_closes_vm(self):
|
||
def api(socket_path, method, path, payload=None):
|
||
self.api_calls.append((method, path))
|
||
if method == "PUT" and path == "/vm.create":
|
||
return (500, {})
|
||
return (200, {})
|
||
ex = self.make_exec(api_client=api)
|
||
with self.assertRaises(ChvFailure) as ctx:
|
||
ex.wake_vm(self._task())
|
||
self.assertEqual(ctx.exception.kind, "vm_create_failed")
|
||
self.assertIn("vm_failed", self._audit_events())
|
||
self.assertEqual(ex._vms, {})
|
||
self.assertTrue(any(c[-1] == "shutdown" for c in self.calls))
|
||
|
||
def test_wake_vm_boot_failed_closes_vm(self):
|
||
def api(socket_path, method, path, payload=None):
|
||
self.api_calls.append((method, path))
|
||
if method == "PUT" and path == "/vm.boot":
|
||
return (500, {})
|
||
return (200, {})
|
||
ex = self.make_exec(api_client=api)
|
||
with self.assertRaises(ChvFailure) as ctx:
|
||
ex.wake_vm(self._task())
|
||
self.assertEqual(ctx.exception.kind, "vm_boot_failed")
|
||
self.assertIn("vm_failed", self._audit_events())
|
||
self.assertEqual(ex._vms, {})
|
||
|
||
def test_wake_api_timeout(self):
|
||
def api(socket_path, method, path, payload=None):
|
||
return (404, {})
|
||
ex = self.make_exec(api_client=api, api_timeout=2)
|
||
with self.assertRaises(ChvFailure) as ctx:
|
||
ex.wake_vm(self._task())
|
||
self.assertEqual(ctx.exception.kind, "api_timeout")
|
||
self.assertIn("vm_failed", self._audit_events())
|
||
|
||
def test_wake_failure_stop_exception_swallowed(self):
|
||
def api(socket_path, method, path, payload=None):
|
||
if method == "PUT" and path == "/vm.create":
|
||
return (500, {})
|
||
return (200, {})
|
||
|
||
def runner(cmd, timeout=None):
|
||
self.calls.append(list(cmd))
|
||
if cmd[-1] == "shutdown":
|
||
raise RuntimeError("shutdown boom")
|
||
return FakeResult(0, "", "")
|
||
ex = self.make_exec(api_client=api, runner=runner)
|
||
with self.assertRaises(ChvFailure) as ctx:
|
||
ex.wake_vm(self._task())
|
||
# 原始失败仍是 vm_create_failed，不被 stop 的异常掩盖。
|
||
self.assertEqual(ctx.exception.kind, "vm_create_failed")
|
||
|
||
def test_wait_guest_ready_host_key_mismatch(self):
|
||
self._known_hosts()
|
||
ex = self.make_exec()
|
||
|
||
def runner(cmd, timeout=None):
|
||
self.calls.append(list(cmd))
|
||
if cmd[0] == "ssh" and cmd[-1] == "true":
|
||
return FakeResult(1, "",
|
||
"Host key verification failed")
|
||
return FakeResult(0, "", "")
|
||
ex.runner = runner
|
||
with self.assertRaises(ChvFailure) as ctx:
|
||
ex.wait_guest_ready(self._task())
|
||
self.assertEqual(ctx.exception.kind, "host_key_mismatch")
|
||
self.assertIn("host_key_mismatch", self._audit_events())
|
||
|
||
def test_wait_guest_ready_timeout(self):
|
||
self._known_hosts()
|
||
ex = self.make_exec(ready_timeout=2, ready_interval=1)
|
||
|
||
def runner(cmd, timeout=None):
|
||
self.calls.append(list(cmd))
|
||
if cmd[0] == "ssh":
|
||
return FakeResult(1, "", "")
|
||
return FakeResult(0, "", "")
|
||
ex.runner = runner
|
||
with self.assertRaises(ChvFailure) as ctx:
|
||
ex.wait_guest_ready(self._task())
|
||
self.assertEqual(ctx.exception.kind, "guest_not_ready")
|
||
|
||
def test_wait_guest_ready_provisioned_missing_known_hosts(self):
|
||
ex = self.make_exec(env_extra={"CHV_HOST_KEY_MODE": "provisioned"})
|
||
with self.assertRaises(ChvFailure) as ctx:
|
||
ex.wait_guest_ready(self._task())
|
||
self.assertEqual(ctx.exception.kind, "host_key_not_pinned")
|
||
|
||
|
||
# -- 恢复与快照失败 ------------------------------------------------------
|
||
class RestoreSnapshotTest(CloseupCase):
|
||
def _snap_url(self, name="snap1.raw"):
|
||
path = os.path.join(self.state_dir, name)
|
||
os.makedirs(self.state_dir, exist_ok=True)
|
||
with open(path, "wb") as fh:
|
||
fh.write(b"S" * 60)
|
||
return "file://" + path, path
|
||
|
||
def test_restore_failure_closes_vm(self):
|
||
url, _ = self._snap_url()
|
||
ex = self.make_exec()
|
||
|
||
def runner(cmd, timeout=None):
|
||
self.calls.append(list(cmd))
|
||
if cmd[0] == self.ch_remote and "restore" in cmd:
|
||
return FakeResult(1, "", "")
|
||
return self._default_runner(cmd, timeout)
|
||
ex.runner = runner
|
||
with self.assertRaises(RuntimeError):
|
||
ex.restore_vm(self._task(), url)
|
||
self.assertEqual(ex._vms, {})
|
||
self.assertTrue(any(c[-1] == "shutdown" for c in self.calls))
|
||
|
||
def test_restore_success_and_wake_from_hibernate(self):
|
||
url, _ = self._snap_url()
|
||
ex = self.make_exec()
|
||
out = ex.restore_vm(self._task(), url)
|
||
self.assertEqual(out["mode"], "restored")
|
||
self.assertEqual(out["state"], "running")
|
||
out2 = ex.wake_from_hibernate(self._task(), url)
|
||
self.assertEqual(out2["mode"], "restored")
|
||
|
||
def test_restore_provisioned_missing_known_hosts(self):
|
||
url, _ = self._snap_url()
|
||
ex = self.make_exec(env_extra={"CHV_HOST_KEY_MODE": "provisioned"})
|
||
with self.assertRaises(ChvFailure) as ctx:
|
||
ex.restore_vm(self._task(), url)
|
||
self.assertEqual(ctx.exception.kind, "host_key_not_pinned")
|
||
self.assertEqual(self.launches, [])
|
||
|
||
def test_snapshot_not_persisted_missing_and_empty(self):
|
||
ex = self.make_exec()
|
||
missing = "file://" + os.path.join(self.state_dir, "nope.raw")
|
||
os.makedirs(self.state_dir, exist_ok=True)
|
||
with self.assertRaises(ChvFailure) as ctx:
|
||
ex.snapshot_vm(self._task(), missing)
|
||
self.assertEqual(ctx.exception.kind, "snapshot_not_persisted")
|
||
empty_path = os.path.join(self.state_dir, "empty.raw")
|
||
open(empty_path, "wb").close()
|
||
with self.assertRaises(ChvFailure) as ctx:
|
||
ex.snapshot_vm(self._task(), "file://" + empty_path)
|
||
self.assertEqual(ctx.exception.kind, "snapshot_not_persisted")
|
||
|
||
def test_snapshot_prunes_oldest_on_total_cap(self):
|
||
vm = "chv-abc123"
|
||
vm_dir = os.path.join(self.state_dir, vm)
|
||
os.makedirs(vm_dir, exist_ok=True)
|
||
old_path = os.path.join(self.state_dir, "old.raw")
|
||
with open(old_path, "wb") as fh:
|
||
fh.write(b"O" * 100)
|
||
import time
|
||
with open(os.path.join(vm_dir, "snapshots.json"), "w") as fh:
|
||
json.dump([{"url": "file://" + old_path, "path": old_path,
|
||
"size": 100, "created_at": time.time()}], fh)
|
||
# 上限 150 字节：预检时旧 100 字节通过，新 60 字节记入后
|
||
# 总量 160 超帽，最旧一条被剪枝并审计。
|
||
cap_gb = 150 / (1024 ** 3)
|
||
ex = self.make_exec(env_extra={
|
||
"CHV_SNAPSHOT_MAX_TOTAL_GB": repr(cap_gb)})
|
||
new_path = os.path.join(self.state_dir, "new.raw")
|
||
with open(new_path, "wb") as fh:
|
||
fh.write(b"N" * 60)
|
||
out = ex.snapshot_vm(self._task(), "file://" + new_path)
|
||
self.assertEqual(out, "file://" + new_path)
|
||
self.assertFalse(os.path.exists(old_path))
|
||
self.assertIn("vm_snapshot_pruned", self._audit_events())
|
||
|
||
def test_snapshot_quota_count(self):
|
||
vm_dir = os.path.join(self.state_dir, "chv-abc123")
|
||
os.makedirs(vm_dir, exist_ok=True)
|
||
import time
|
||
with open(os.path.join(vm_dir, "snapshots.json"), "w") as fh:
|
||
json.dump([{"url": "file://x", "path": None, "size": 0,
|
||
"created_at": time.time()}], fh)
|
||
ex = self.make_exec(env_extra={"CHV_SNAPSHOT_MAX_COUNT": "1"})
|
||
url, _ = self._snap_url("snap2.raw")
|
||
with self.assertRaises(ChvFailure) as ctx:
|
||
ex.snapshot_vm(self._task(), url)
|
||
self.assertEqual(ctx.exception.kind, "snapshot_quota_exceeded")
|
||
|
||
def test_hibernate_snapshot_failure_does_not_shutdown(self):
|
||
ex = self.make_exec()
|
||
with self.assertRaises(ValueError):
|
||
ex.hibernate_vm(self._task(), "")
|
||
self.assertFalse(any(c[-1] == "shutdown" for c in self.calls))
|
||
|
||
|
||
# -- 取消与清理 ----------------------------------------------------------
|
||
class CancelCleanupTest(CloseupCase):
|
||
def test_cancel_vm_really_shuts_down(self):
|
||
ex = self.make_exec()
|
||
self._wake_ok(ex)
|
||
proc = self.proc
|
||
self.assertTrue(ex.cancel_vm(self._task()))
|
||
self.assertTrue(proc.terminated)
|
||
self.assertEqual(ex._vms, {})
|
||
self.assertIn("vm_shutdown", self._audit_events())
|
||
self.assertIn("vm_destroy", self._audit_events())
|
||
|
||
def test_shutdown_vm_socket_residual(self):
|
||
ex = self.make_exec()
|
||
os.makedirs(self.state_dir, exist_ok=True)
|
||
sock = os.path.join(self.state_dir, "chv-abc123.sock")
|
||
with open(sock, "w") as fh:
|
||
fh.write("")
|
||
self.assertTrue(ex.shutdown_vm(self._task()))
|
||
self.assertFalse(os.path.exists(sock))
|
||
|
||
def test_recycle_vm_success_and_missing_id(self):
|
||
ex = self.make_exec()
|
||
self._wake_ok(ex)
|
||
out = ex.recycle_vm(self._task())
|
||
self.assertEqual(out["state"], "recycled")
|
||
self.assertEqual(out["preserved"],
|
||
["task_checkpoints", "task_attempts"])
|
||
self.assertIn("vm_recycled", self._audit_events())
|
||
with self.assertRaises(ValueError):
|
||
ex.recycle_vm({})
|
||
|
||
def test_cleanup_stale_vms_dead_socket_and_live_skip(self):
|
||
ex = self.make_exec()
|
||
os.makedirs(self.state_dir, exist_ok=True)
|
||
for name in ("chv-dead.sock", "chv-live.sock", "other.sock"):
|
||
with open(os.path.join(self.state_dir, name), "w") as fh:
|
||
fh.write("")
|
||
ex._vms["chv-live"] = FakeProc(alive=True)
|
||
cleaned = ex.cleanup_stale_vms()
|
||
self.assertEqual(cleaned, ["chv-dead"])
|
||
self.assertFalse(
|
||
os.path.exists(os.path.join(self.state_dir, "chv-dead.sock")))
|
||
self.assertTrue(
|
||
os.path.exists(os.path.join(self.state_dir, "other.sock")))
|
||
|
||
def test_cleanup_stale_vms_purges_dead_registry(self):
|
||
ex = self.make_exec()
|
||
os.makedirs(self.state_dir, exist_ok=True)
|
||
ex._vms["chv-ghost"] = FakeProc(alive=False)
|
||
self.assertEqual(ex.cleanup_stale_vms(), [])
|
||
self.assertEqual(ex._vms, {})
|
||
|
||
def test_cleanup_stale_vms_missing_state_dir(self):
|
||
ex = self.make_exec()
|
||
self.assertFalse(os.path.exists(self.state_dir))
|
||
self.assertEqual(ex.cleanup_stale_vms(), [])
|
||
|
||
def test_gc_stale_disks_deletes_stale_keeps_live(self):
|
||
ex = self.make_exec()
|
||
stale_dir = os.path.join(self.state_dir, "chv-stale")
|
||
live_dir = os.path.join(self.state_dir, "chv-live")
|
||
os.makedirs(stale_dir, exist_ok=True)
|
||
os.makedirs(live_dir, exist_ok=True)
|
||
with open(os.path.join(stale_dir, "disk.raw"), "wb") as fh:
|
||
fh.write(b"D")
|
||
with open(os.path.join(live_dir, "disk.raw"), "wb") as fh:
|
||
fh.write(b"D")
|
||
with open(os.path.join(self.state_dir, "chv-live.sock"),
|
||
"w") as fh:
|
||
fh.write("")
|
||
out = ex.gc_stale_disks()
|
||
self.assertEqual(out["deleted"], ["chv-stale"])
|
||
self.assertFalse(
|
||
os.path.exists(os.path.join(stale_dir, "disk.raw")))
|
||
self.assertTrue(os.path.exists(os.path.join(live_dir, "disk.raw")))
|
||
|
||
def test_cleanup_stale_secrets_live_keeps_seed_dead_removes(self):
|
||
ex = self.make_exec()
|
||
live_dir = os.path.join(self.state_dir, "chv-live")
|
||
dead_dir = os.path.join(self.state_dir, "chv-dead")
|
||
os.makedirs(live_dir, exist_ok=True)
|
||
os.makedirs(dead_dir, exist_ok=True)
|
||
for vm_dir in (live_dir, dead_dir):
|
||
with open(os.path.join(vm_dir, "ssh_host_ed25519_key"),
|
||
"w") as fh:
|
||
fh.write("priv")
|
||
with open(os.path.join(vm_dir, "seed.iso"), "wb") as fh:
|
||
fh.write(b"S")
|
||
with open(os.path.join(self.state_dir, "chv-live.sock"),
|
||
"w") as fh:
|
||
fh.write("")
|
||
cleaned = ex.cleanup_stale_secrets()
|
||
self.assertEqual(sorted(cleaned), ["chv-dead", "chv-live"])
|
||
self.assertFalse(
|
||
os.path.exists(os.path.join(live_dir, "ssh_host_ed25519_key")))
|
||
self.assertTrue(os.path.exists(os.path.join(live_dir, "seed.iso")))
|
||
self.assertFalse(
|
||
os.path.exists(os.path.join(dead_dir, "ssh_host_ed25519_key")))
|
||
self.assertFalse(os.path.exists(os.path.join(dead_dir, "seed.iso")))
|
||
|
||
def test_stop_dirty_marker_on_cleanup_failure(self):
|
||
ex = self.make_exec()
|
||
vm_dir = os.path.join(self.state_dir, "chv-abc123")
|
||
seed_dir = os.path.join(vm_dir, "seed")
|
||
os.makedirs(seed_dir, exist_ok=True)
|
||
with open(os.path.join(seed_dir, "user-data"), "w") as fh:
|
||
fh.write("seed")
|
||
with open(os.path.join(self.state_dir, "chv-abc123.sock"),
|
||
"w") as fh:
|
||
fh.write("")
|
||
# 本环境以 root 运行，chmod 000/0555 会被 root 绕过，
|
||
# 权限失败分支改用 rmtree 抛错模拟（只包 stop_vm 一次
|
||
# 调用，上下文退出即恢复，不影响 tearDown 清理）。
|
||
import shutil
|
||
real_rmtree = shutil.rmtree
|
||
|
||
def _failing_rmtree(path, *args, **kwargs):
|
||
if path == seed_dir:
|
||
raise OSError("mocked rmtree failure")
|
||
return real_rmtree(path, *args, **kwargs)
|
||
with mock.patch("shutil.rmtree", side_effect=_failing_rmtree):
|
||
self.assertTrue(ex.stop_vm(self._task()))
|
||
self.assertIn("vm_key_cleanup_failed", self._audit_events())
|
||
marker = dirty_marker_path(self.state_dir, "chv-abc123")
|
||
self.assertTrue(os.path.exists(marker))
|
||
|
||
def test_stop_clears_dirty_marker_on_success(self):
|
||
ex = self.make_exec()
|
||
vm_dir = os.path.join(self.state_dir, "chv-abc123")
|
||
os.makedirs(vm_dir, exist_ok=True)
|
||
marker = dirty_marker_path(self.state_dir, "chv-abc123")
|
||
os.makedirs(os.path.dirname(marker), exist_ok=True)
|
||
with open(marker, "w") as fh:
|
||
fh.write("cleanup failed\n")
|
||
with open(os.path.join(self.state_dir, "chv-abc123.sock"),
|
||
"w") as fh:
|
||
fh.write("")
|
||
self.assertTrue(ex.stop_vm(self._task()))
|
||
self.assertFalse(os.path.exists(marker))
|
||
|
||
|
||
# -- 中途死亡分类与只读 execute ------------------------------------------
|
||
class VmDeathTest(CloseupCase):
|
||
def _wake_with_dead_on_dispatch(self, mode):
|
||
ex = self.make_exec()
|
||
|
||
def runner(cmd, timeout=None):
|
||
self.calls.append(list(cmd))
|
||
if cmd[0] == "ssh" and "dispatch" in cmd:
|
||
proc = ex._vms.get("chv-abc123")
|
||
if proc is not None:
|
||
proc._alive = False
|
||
if mode == "raise":
|
||
raise RuntimeError("ssh broken")
|
||
return FakeResult(0, json.dumps(
|
||
{"status": "failed", "text": "boom"}), "")
|
||
return self._default_runner(cmd, timeout)
|
||
ex.runner = runner
|
||
return ex
|
||
|
||
def test_run_with_vm_exception_vm_dead_interrupted(self):
|
||
ex = self._wake_with_dead_on_dispatch("raise")
|
||
out = ex.dispatch_task(self._task())
|
||
self.assertEqual(out["status"], "interrupted")
|
||
self.assertTrue(out["vm_dead"])
|
||
|
||
def test_run_with_vm_failed_status_vm_dead_reclassified(self):
|
||
ex = self._wake_with_dead_on_dispatch("failed")
|
||
out = ex.dispatch_task(self._task())
|
||
self.assertEqual(out["status"], "interrupted")
|
||
self.assertTrue(out["vm_dead"])
|
||
self.assertIn("VM 中途死亡", out["interrupted_reason"])
|
||
|
||
def test_run_with_vm_exception_vm_alive_reraises(self):
|
||
ex = self.make_exec()
|
||
|
||
def runner(cmd, timeout=None):
|
||
self.calls.append(list(cmd))
|
||
if cmd[0] == "ssh" and "dispatch" in cmd:
|
||
raise RuntimeError("ssh broken")
|
||
return self._default_runner(cmd, timeout)
|
||
ex.runner = runner
|
||
with self.assertRaises(RuntimeError):
|
||
ex.dispatch_task(self._task())
|
||
|
||
def test_execute_dead_vm_reclassified(self):
|
||
ex = self.make_exec()
|
||
ex._vms["chv-abc123"] = FakeProc(alive=False)
|
||
os.makedirs(self.state_dir, exist_ok=True)
|
||
with open(os.path.join(self.state_dir, "chv-abc123.sock"),
|
||
"w") as fh:
|
||
fh.write("")
|
||
|
||
def runner(cmd, timeout=None):
|
||
self.calls.append(list(cmd))
|
||
raise RuntimeError("ssh broken")
|
||
ex.runner = runner
|
||
with self.assertRaises(ChvFailure) as ctx:
|
||
ex.execute(self._task())
|
||
self.assertEqual(ctx.exception.kind, "vm_not_running")
|
||
|
||
def test_execute_not_running_no_runner_call(self):
|
||
ex = self.make_exec()
|
||
with self.assertRaises(ChvFailure) as ctx:
|
||
ex.execute(self._task())
|
||
self.assertEqual(ctx.exception.kind, "vm_not_running")
|
||
self.assertEqual(self.calls, [])
|
||
|
||
|
||
# -- 入口校验 ------------------------------------------------------------
|
||
class EntryGuardTest(CloseupCase):
|
||
def test_dispatch_run_goal_guards(self):
|
||
ex = self.make_exec()
|
||
with self.assertRaises(ValueError):
|
||
ex.dispatch_task({})
|
||
with self.assertRaises(ValueError):
|
||
ex.dispatch_task({"id": "x"})
|
||
with self.assertRaises(ValueError):
|
||
ex.run_task({"id": "x"})
|
||
with self.assertRaises(ValueError):
|
||
ex.goal_task({"id": "x", "goal": "g",
|
||
"goal_max_rounds": 101})
|
||
with self.assertRaises(ValueError):
|
||
ex.goal_task({"id": "x", "goal": "g", "verify_command": "v",
|
||
"goal_max_rounds": 0})
|
||
with self.assertRaises(ValueError):
|
||
ex.goal_resume_task({})
|
||
with self.assertRaises(ValueError):
|
||
ex.goal_resume_breakpoint_task({})
|
||
with self.assertRaises(ValueError):
|
||
ex.wake_vm({})
|
||
|
||
def test_dispatch_wake_failure_does_not_run_guest(self):
|
||
ex = self.make_exec(binary="/nonexistent/cloud-hypervisor")
|
||
with self.assertRaises(ChvFailure):
|
||
ex.dispatch_task(self._task())
|
||
self.assertFalse(any("dispatch" in c for c in self.calls))
|
||
|
||
|
||
# -- preflight 事后改写校验 ------------------------------------------------
|
||
class PreflightMutationTest(CloseupCase):
|
||
def test_reuse_blocked_and_tenant_undeclared(self):
|
||
ex = self.make_exec()
|
||
ex.disk_mode = "reuse"
|
||
ex.reuse_experiment = False
|
||
with self.assertRaises(ChvFailure) as ctx:
|
||
ex.preflight()
|
||
self.assertEqual(ctx.exception.kind, "disk_mode_reuse_blocked")
|
||
ex.reuse_experiment = True
|
||
ex.single_tenant_project = None
|
||
with self.assertRaises(ChvFailure) as ctx:
|
||
ex.preflight()
|
||
self.assertEqual(ctx.exception.kind, "reuse_tenant_undeclared")
|
||
|
||
def test_config_and_numeric_mutations(self):
|
||
cases = [
|
||
("cpus", "not-an-int"),
|
||
("memory", "bogus-mem"),
|
||
("ssh_port", 99999),
|
||
("max_vms", -1),
|
||
("disk_gb", 0),
|
||
("api_timeout", 0),
|
||
("warm_pool_size", -1),
|
||
("snapshot_max_count", -1),
|
||
("host_key_mode", "bogus-mode"),
|
||
]
|
||
for attr, value in cases:
|
||
with self.subTest(attr=attr):
|
||
ex = self.make_exec()
|
||
setattr(ex, attr, value)
|
||
with self.assertRaises(ChvFailure) as ctx:
|
||
ex.preflight()
|
||
self.assertEqual(ctx.exception.kind, "config_invalid")
|
||
|
||
def test_cpu_flag_missing_and_kvm_unreadable(self):
|
||
with open(self.cpuinfo, "w") as fh:
|
||
fh.write("flags : fpu sse\n")
|
||
ex = self.make_exec()
|
||
with self.assertRaises(ChvFailure) as ctx:
|
||
ex.preflight()
|
||
self.assertEqual(ctx.exception.kind, "cpu_flag_missing")
|
||
with open(self.cpuinfo, "w") as fh:
|
||
fh.write("flags : fpu vmx sse\n")
|
||
ex = self.make_exec()
|
||
# 本环境以 root 运行，chmod 000 会被 root 绕过，
|
||
# kvm 不可读/写分支改用 os.access 模拟（只包本次
|
||
# preflight，上下文退出即恢复）。
|
||
real_access = os.access
|
||
|
||
def _fake_access(path, mode, *args, **kwargs):
|
||
if path == self.kvm and mode in (os.R_OK, os.W_OK):
|
||
return False
|
||
return real_access(path, mode, *args, **kwargs)
|
||
with mock.patch("os.access", side_effect=_fake_access):
|
||
with self.assertRaises(ChvFailure) as ctx:
|
||
ex.preflight()
|
||
self.assertEqual(ctx.exception.kind, "kvm_missing")
|
||
|
||
def test_net_dual_warns_but_passes(self):
|
||
ex = self.make_exec(env_extra={
|
||
"CHV_NET": "tap0",
|
||
"CHV_NET_JSON": '[{"tap": "tap0"}]'})
|
||
self.assertTrue(ex.preflight())
|
||
self.assertIn("net_config_dual", self._audit_events())
|
||
|
||
def test_net_json_invalid(self):
|
||
ex = self.make_exec(env_extra={"CHV_NET_JSON": "{not json"})
|
||
with self.assertRaises(ChvFailure) as ctx:
|
||
ex.preflight()
|
||
self.assertEqual(ctx.exception.kind, "net_config_invalid")
|
||
|
||
|
||
# -- worker_adapter 剩余分支 ------------------------------------------------
|
||
class WorkerAdapterCloseupTest(unittest.TestCase):
|
||
def test_call_breakpoint_extra_env_forwarded(self):
|
||
seen = {}
|
||
|
||
class _Ex:
|
||
def goal_resume_breakpoint_task(self, task, model_provider=None,
|
||
extra_env=None):
|
||
seen["model_provider"] = model_provider
|
||
seen["extra_env"] = extra_env
|
||
return "ok"
|
||
out = call_goal_resume_breakpoint(
|
||
_Ex(), {"id": "x"}, model_provider="m",
|
||
extra_env={"A": "1"})
|
||
self.assertEqual(out, "ok")
|
||
self.assertEqual(seen["extra_env"], {"A": "1"})
|
||
out = call_goal_resume_breakpoint(_Ex(), {"id": "x"})
|
||
self.assertEqual(seen["extra_env"], None)
|
||
|
||
def test_call_breakpoint_signature_error_fallback(self):
|
||
class _BadSig:
|
||
__signature__ = "not-a-signature"
|
||
|
||
def __call__(self, task):
|
||
return "fallback-ok"
|
||
|
||
class _Ex:
|
||
def __init__(self):
|
||
self.goal_resume_breakpoint_task = _BadSig()
|
||
self.assertEqual(
|
||
call_goal_resume_breakpoint(_Ex(), {"id": "x"}), "fallback-ok")
|
||
|
||
def test_reconcile_non_chv_and_failure_audited(self):
|
||
self.assertEqual(reconcile_warm_pool(object(), lambda *a: None,
|
||
"when"), [])
|
||
audit = []
|
||
# CHV 失败分支：patch orphans 调和函数抛错，执行器
|
||
# 用 spec Mock 并把 __class__ 设为真执行器类，使
|
||
# isinstance 判定进入 CHV 分支。
|
||
from cloud.chv.executor import CloudHypervisorExecutor as _RealEx
|
||
self.assertTrue(inspect.isclass(_RealEx))
|
||
with mock.patch(
|
||
"cloud.chv.orphans.reconcile_orphans_for_executor",
|
||
side_effect=RuntimeError("boom")):
|
||
fake_ex = mock.Mock(spec=_RealEx)
|
||
# spec Mock 的 isinstance 判定走 __class__，需显式
|
||
# 把类设为真执行器类才能进 CHV 分支。
|
||
fake_ex.__class__ = _RealEx
|
||
out = reconcile_warm_pool(
|
||
fake_ex, lambda *a: audit.append(a), "when")
|
||
self.assertEqual(out, [])
|
||
self.assertTrue(any("reconcile_failed" in a[1] for a in audit))
|
||
|
||
def test_cancel_watchdog_cancel_vm_none(self):
|
||
stop = threading.Event()
|
||
requested = {"t1"}
|
||
cancel_watchdog_loop("t1", "chv-t1", stop, requested,
|
||
lambda tid: None, None, object())
|
||
# 命中取消但 cancel_vm 为 None 时不炸、直接退出。
|
||
|
||
def test_quota_watchdog_not_over_then_stop(self):
|
||
stop = threading.Event()
|
||
|
||
def check():
|
||
stop.set()
|
||
return False
|
||
calls = []
|
||
quota_watchdog_loop("t1", "chv-t1", stop, check,
|
||
lambda tid: calls.append(tid), None,
|
||
object(), interval=0.01)
|
||
self.assertEqual(calls, [])
|
||
|
||
def test_watchdog_vm_name_non_callable(self):
|
||
self.assertIsNone(watchdog_vm_name(object(), {"id": "x"}))
|
||
|
||
|
||
if __name__ == "__main__":
|
||
unittest.main()
|
||
|
||
|
||
# -- 收尾第二批：防御性 except 与文件/入口变体 --------------------------
|
||
class RaisingProc:
|
||
def poll(self):
|
||
raise RuntimeError("poll boom")
|
||
|
||
def terminate(self):
|
||
raise RuntimeError("terminate boom")
|
||
|
||
def wait(self, timeout=None):
|
||
raise RuntimeError("wait boom")
|
||
|
||
def kill(self):
|
||
raise RuntimeError("kill boom")
|
||
|
||
|
||
class DefensiveBranchTest(CloseupCase):
|
||
def test_default_popen_launcher_runs_true(self):
|
||
from cloud.chv.executor import _default_popen_launcher
|
||
proc = _default_popen_launcher(["true"])
|
||
self.assertEqual(proc.wait(), 0)
|
||
|
||
def test_binary_ok_bare_name_via_which(self):
|
||
ex = self.make_exec(binary="true", ch_remote="true")
|
||
self.assertTrue(ex.preflight())
|
||
|
||
def test_kill_proc_none_and_raising(self):
|
||
ex = self.make_exec()
|
||
ex._kill_proc(None)
|
||
ex._kill_proc(RaisingProc()) # terminate/wait/kill 全抛也不炸
|
||
|
||
def test_active_and_persistent_count_poll_exception(self):
|
||
ex = self.make_exec()
|
||
self.assertEqual(ex._active_count(), 0)
|
||
ex._vms["chv-bad"] = RaisingProc()
|
||
self.assertEqual(ex._active_count(), 1)
|
||
self.assertEqual(ex._persistent_vm_count(), 0)
|
||
|
||
def test_preflight_cpuinfo_missing_and_unset_files(self):
|
||
ex = self.make_exec(cpuinfo_path="/nonexistent/cpuinfo")
|
||
with self.assertRaises(ChvFailure) as ctx:
|
||
ex.preflight()
|
||
self.assertEqual(ctx.exception.kind, "cpu_flag_missing")
|
||
for kwargs in ({"kernel": None}, {"rootfs": None},
|
||
{"ssh_key": None}):
|
||
with self.subTest(kwargs=kwargs):
|
||
ex = self.make_exec(**kwargs)
|
||
with self.assertRaises(ChvFailure):
|
||
ex.preflight()
|
||
|
||
def test_preflight_unreadable_files_via_access_mock(self):
|
||
ex = self.make_exec()
|
||
real_access = os.access
|
||
|
||
def _fake_access(path, mode, *args, **kwargs):
|
||
if path in (self.kernel, self.rootfs, self.ssh_key) \
|
||
and mode == os.R_OK:
|
||
return False
|
||
return real_access(path, mode, *args, **kwargs)
|
||
with mock.patch("os.access", side_effect=_fake_access):
|
||
with self.assertRaises(ChvFailure) as ctx:
|
||
ex.preflight()
|
||
self.assertIn(ctx.exception.kind,
|
||
("kernel_missing", "image_missing",
|
||
"ssh_key_missing"))
|
||
|
||
def test_preflight_state_dir_is_file(self):
|
||
blocker = os.path.join(self.dir, "blocker")
|
||
with open(blocker, "w") as fh:
|
||
fh.write("x")
|
||
ex = self.make_exec(state_dir=blocker)
|
||
with self.assertRaises(ChvFailure) as ctx:
|
||
ex.preflight()
|
||
self.assertEqual(ctx.exception.kind, "state_dir_insecure")
|
||
|
||
def test_build_vm_command_with_net_and_config_errors(self):
|
||
ex = self.make_exec(net="tap0")
|
||
cmd = ex._build_vm_command(self._task())
|
||
self.assertIn("--net", cmd)
|
||
self.assertIn("tap0", cmd)
|
||
ex.cpus = "bogus"
|
||
with self.assertRaises(ValueError):
|
||
ex.build_vm_config()
|
||
ex.cpus = 2
|
||
ex.net_json = "{bad"
|
||
with self.assertRaises(ValueError):
|
||
ex.build_vm_config()
|
||
|
||
def test_net_guest_ip_invalid_and_ssh_target_undeclared(self):
|
||
ex = self.make_exec()
|
||
ex.net_json = "{bad"
|
||
self.assertIsNone(ex._net_guest_ip())
|
||
ex2 = self.make_exec(env_extra={"CHV_SSH_FORWARD": "0"})
|
||
with self.assertRaises(ChvFailure) as ctx:
|
||
ex2.resolve_ssh_host()
|
||
self.assertEqual(ctx.exception.kind, "ssh_target_undeclared")
|
||
|
||
def test_capture_host_key_resolve_and_runner_and_write_failures(self):
|
||
ex = self.make_exec(env_extra={"CHV_SSH_FORWARD": "0"})
|
||
self.assertFalse(ex.capture_host_key(self._task()))
|
||
ex = self.make_exec()
|
||
|
||
def raising_runner(cmd, timeout=None):
|
||
raise RuntimeError("keyscan boom")
|
||
ex.runner = raising_runner
|
||
self.assertFalse(ex.capture_host_key(self._task()))
|
||
ex = self.make_exec()
|
||
blocker = os.path.join(self.state_dir, "chv-abc123")
|
||
os.makedirs(self.state_dir, exist_ok=True)
|
||
with open(blocker, "w") as fh:
|
||
fh.write("x")
|
||
|
||
def keyscan_runner(cmd, timeout=None):
|
||
self.calls.append(list(cmd))
|
||
if cmd[0] == "ssh-keyscan":
|
||
return FakeResult(0, "[127.0.0.1]:2222 ssh-ed25519 AAAA-good", "")
|
||
return FakeResult(0, "", "")
|
||
ex.runner = keyscan_runner
|
||
self.assertFalse(ex.capture_host_key(self._task()))
|
||
|
||
def test_build_ssh_command_undeclared_raises_and_secret_not_delivered(self):
|
||
# feat-chv-p1fix（P1-04）：删除回环回落——目标未声明
|
||
# 时构造即抛 ssh_target_undeclared，不再悄悄连回环。
|
||
ex = self.make_exec(env_extra={"CHV_SSH_FORWARD": "0"})
|
||
with self.assertRaises(ChvFailure) as ctx:
|
||
ex.build_ssh_command(self._task(), ["true"])
|
||
self.assertEqual(ctx.exception.kind, "ssh_target_undeclared")
|
||
ex = self.make_exec()
|
||
with self.assertRaises(ChvFailure) as ctx:
|
||
ex.build_ssh_command(self._task(), ["true"],
|
||
extra_env={"SUB2API_API_KEY": "v"})
|
||
self.assertEqual(ctx.exception.kind, "secret_env_not_delivered")
|
||
|
||
def test_write_provisioned_known_hosts_undeclared_raises(self):
|
||
# feat-chv-p1fix（P1-04 同款）：预置 known_hosts 解析
|
||
# 失败即抛，不再回落回环写钉死。
|
||
ex = self.make_exec(env_extra={"CHV_SSH_FORWARD": "0",
|
||
"CHV_HOST_KEY_MODE": "provisioned"})
|
||
vm_dir = os.path.join(self.state_dir, "chv-abc123")
|
||
os.makedirs(vm_dir, exist_ok=True)
|
||
paths = {"known_hosts": os.path.join(vm_dir, "known_hosts")}
|
||
with self.assertRaises(ChvFailure) as ctx:
|
||
ex._write_provisioned_known_hosts(paths, "ssh-ed25519 AAAA")
|
||
self.assertEqual(ctx.exception.kind, "ssh_target_undeclared")
|
||
self.assertFalse(os.path.exists(paths["known_hosts"]))
|
||
|
||
def test_provision_host_key_dirty_refused(self):
|
||
ex = self.make_exec()
|
||
marker = dirty_marker_path(self.state_dir, "chv-abc123")
|
||
os.makedirs(os.path.dirname(marker), exist_ok=True)
|
||
with open(marker, "w") as fh:
|
||
fh.write("cleanup failed\n")
|
||
with self.assertRaises(ChvFailure) as ctx:
|
||
ex.provision_host_key(self._task())
|
||
self.assertEqual(ctx.exception.kind, "vm_dirty_refused")
|
||
|
||
def test_api_call_raises_then_api_timeout(self):
|
||
def api(socket_path, method, path, payload=None):
|
||
raise ConnectionError("socket gone")
|
||
ex = self.make_exec(api_client=api, api_timeout=2)
|
||
with self.assertRaises(ChvFailure) as ctx:
|
||
ex.wait_api_ready(self._task())
|
||
self.assertEqual(ctx.exception.kind, "api_timeout")
|
||
|
||
def test_wait_guest_runner_exception_retried_then_timeout(self):
|
||
self._known_hosts()
|
||
ex = self.make_exec(ready_timeout=2, ready_interval=1)
|
||
|
||
def runner(cmd, timeout=None):
|
||
self.calls.append(list(cmd))
|
||
if cmd[0] == "ssh":
|
||
raise RuntimeError("ssh boom")
|
||
return FakeResult(0, "", "")
|
||
ex.runner = runner
|
||
with self.assertRaises(ChvFailure) as ctx:
|
||
ex.wait_guest_ready(self._task())
|
||
self.assertEqual(ctx.exception.kind, "guest_not_ready")
|
||
|
||
def test_stop_exceptions_swallowed_around_stop_vm(self):
|
||
# wait_guest_ready 的 provisioned 守卫、mismatch 分支、
|
||
# wake_vm 与 restore_vm 的 stop_vm 外层 except：stop_vm
|
||
# 本身不抛，改用 mock 让它抛，验证原始失败不被掩盖。
|
||
ex = self.make_exec(env_extra={"CHV_HOST_KEY_MODE": "provisioned"})
|
||
ex.stop_vm = mock.Mock(side_effect=RuntimeError("stop boom"))
|
||
with self.assertRaises(ChvFailure) as ctx:
|
||
ex.wait_guest_ready(self._task())
|
||
self.assertEqual(ctx.exception.kind, "host_key_not_pinned")
|
||
|
||
def api(socket_path, method, path, payload=None):
|
||
if method == "PUT" and path == "/vm.create":
|
||
return (500, {})
|
||
return (200, {})
|
||
ex = self.make_exec(api_client=api)
|
||
ex.stop_vm = mock.Mock(side_effect=RuntimeError("stop boom"))
|
||
with self.assertRaises(ChvFailure) as ctx:
|
||
ex.wake_vm(self._task())
|
||
self.assertEqual(ctx.exception.kind, "vm_create_failed")
|
||
|
||
url, _ = self._snap_url_for(ex)
|
||
ex = self.make_exec()
|
||
ex.stop_vm = mock.Mock(side_effect=RuntimeError("stop boom"))
|
||
|
||
def runner(cmd, timeout=None):
|
||
self.calls.append(list(cmd))
|
||
if cmd[0] == self.ch_remote and "restore" in cmd:
|
||
return FakeResult(1, "", "")
|
||
return self._default_runner(cmd, timeout)
|
||
ex.runner = runner
|
||
with self.assertRaises(RuntimeError):
|
||
ex.restore_vm(self._task(), url)
|
||
|
||
def _snap_url_for(self, ex):
|
||
path = os.path.join(self.state_dir, "snapx.raw")
|
||
os.makedirs(self.state_dir, exist_ok=True)
|
||
with open(path, "wb") as fh:
|
||
fh.write(b"S" * 10)
|
||
return "file://" + path, path
|
||
|
||
def test_registered_net_entries_variants(self):
|
||
ex = self.make_exec()
|
||
os.makedirs(os.path.join(self.state_dir, "chv-skip"),
|
||
exist_ok=True)
|
||
with open(os.path.join(self.state_dir, "chv-skip", "net.json"),
|
||
"w") as fh:
|
||
json.dump([{"tap": "tap9", "ip": "10.0.0.9"}], fh)
|
||
os.makedirs(os.path.join(self.state_dir, "chv-bad"),
|
||
exist_ok=True)
|
||
with open(os.path.join(self.state_dir, "chv-bad", "net.json"),
|
||
"w") as fh:
|
||
fh.write('{"not": "a list"}')
|
||
os.makedirs(os.path.join(self.state_dir, "chv-empty"),
|
||
exist_ok=True)
|
||
out = ex._registered_net_entries(exclude_vm="chv-skip")
|
||
self.assertEqual(out, [])
|
||
ex2 = self.make_exec(state_dir="/nonexistent-state-dir-x")
|
||
self.assertEqual(ex2._registered_net_entries(), [])
|
||
|
||
def test_socket_permissions_stat_failure(self):
|
||
ex = self.make_exec()
|
||
sock = os.path.join(self.state_dir, "chv-abc123.sock")
|
||
os.makedirs(self.state_dir, exist_ok=True)
|
||
with open(sock, "w") as fh:
|
||
fh.write("")
|
||
real_stat = os.stat
|
||
with mock.patch("os.stat", side_effect=OSError("stat boom")):
|
||
ex._check_api_socket_permissions(sock) # 不抛，只跳过
|
||
self.assertTrue(real_stat(sock) is not None)
|
||
|
||
def test_rewake_kills_old_proc_and_start_vm_alias(self):
|
||
ex = self.make_exec()
|
||
self._wake_ok(ex)
|
||
old = self.proc
|
||
out = ex.start_vm(self._task())
|
||
self.assertEqual(out["mode"], "cold")
|
||
self.assertTrue(old.terminated)
|
||
|
||
def test_validate_restore_guards(self):
|
||
ex = self.make_exec()
|
||
with self.assertRaises(ValueError):
|
||
ex._validate_restore({}, "file://x")
|
||
with self.assertRaises(ValueError):
|
||
ex._validate_restore(self._task(), "")
|
||
with self.assertRaises(ChvFailure) as ctx:
|
||
ex._validate_restore(self._task(), "snap.raw")
|
||
self.assertEqual(ctx.exception.kind, "snapshot_path_not_allowed")
|
||
|
||
def test_stop_registered_proc_poll_exception(self):
|
||
ex = self.make_exec()
|
||
ex._vms["chv-bad"] = RaisingProc()
|
||
ex._stop_registered_proc("chv-bad", RaisingProc())
|
||
self.assertEqual(ex._vms, {})
|
||
|
||
def test_cleanup_net_remove_failure_marks_dirty(self):
|
||
ex = self.make_exec()
|
||
vm_dir = os.path.join(self.state_dir, "chv-abc123")
|
||
os.makedirs(vm_dir, exist_ok=True)
|
||
os.makedirs(os.path.join(vm_dir, "net.json"), exist_ok=True)
|
||
with open(os.path.join(self.state_dir, "chv-abc123.sock"),
|
||
"w") as fh:
|
||
fh.write("")
|
||
self.assertTrue(ex.stop_vm(self._task()))
|
||
marker = dirty_marker_path(self.state_dir, "chv-abc123")
|
||
self.assertTrue(os.path.exists(marker))
|
||
|
||
def test_write_dirty_marker_failure_swallowed(self):
|
||
blocker = os.path.join(self.dir, "blocker2")
|
||
with open(blocker, "w") as fh:
|
||
fh.write("x")
|
||
ex = self.make_exec(state_dir=blocker)
|
||
ex._write_dirty_marker("chv-abc123", True) # 不抛
|
||
|
||
def test_stop_socket_remove_failure_swallowed(self):
|
||
ex = self.make_exec()
|
||
os.makedirs(os.path.join(self.state_dir, "chv-abc123.sock"),
|
||
exist_ok=True)
|
||
self.assertTrue(ex.stop_vm(self._task()))
|
||
|
||
def test_gc_poll_exception_and_delete_failure(self):
|
||
ex = self.make_exec()
|
||
vm_dir = os.path.join(self.state_dir, "chv-bad")
|
||
os.makedirs(vm_dir, exist_ok=True)
|
||
with open(os.path.join(vm_dir, "disk.raw"), "wb") as fh:
|
||
fh.write(b"D")
|
||
ex._vms["chv-bad"] = RaisingProc()
|
||
out = ex.gc_stale_disks()
|
||
self.assertEqual(out["deleted"], ["chv-bad"])
|
||
vm2 = os.path.join(self.state_dir, "chv-dir")
|
||
os.makedirs(os.path.join(vm2, "disk.raw"), exist_ok=True)
|
||
out = ex.gc_stale_disks()
|
||
self.assertEqual(out["failed"], ["chv-dir"])
|
||
self.assertIn("vm_disk_delete_failed", self._audit_events())
|
||
|
||
def test_is_live_and_proc_dead_poll_exception(self):
|
||
ex = self.make_exec()
|
||
ex._vms["chv-bad"] = RaisingProc()
|
||
self.assertFalse(ex._is_live_vm("chv-bad"))
|
||
self.assertFalse(ex._vm_proc_dead("chv-bad"))
|
||
self.assertFalse(ex._vm_is_running("chv-bad"))
|
||
|
||
def test_clean_one_vm_secrets_remove_failures(self):
|
||
# feat-chv-p1fix（P1-03）：返回 3 元组，删失败以
|
||
# failed=True 区分，不再静默 pass。
|
||
ex = self.make_exec()
|
||
vm_dir = os.path.join(self.state_dir, "chv-x")
|
||
os.makedirs(os.path.join(vm_dir, "ssh_host_ed25519_key"),
|
||
exist_ok=True)
|
||
removed_priv, removed_seed, failed = ex._clean_one_vm_secrets(
|
||
"chv-x", vm_dir)
|
||
self.assertFalse(removed_priv)
|
||
self.assertTrue(failed)
|
||
vm2 = os.path.join(self.state_dir, "chv-y")
|
||
os.makedirs(os.path.join(vm2, "seed"), exist_ok=True)
|
||
import shutil
|
||
real_rmtree = shutil.rmtree
|
||
with mock.patch("shutil.rmtree",
|
||
side_effect=OSError("rmtree boom")):
|
||
removed_priv, removed_seed, failed = ex._clean_one_vm_secrets(
|
||
"chv-y", vm2)
|
||
self.assertFalse(removed_seed)
|
||
self.assertTrue(failed)
|
||
self.assertTrue(real_rmtree is not None)
|
||
|
||
def test_cleanup_stale_vms_subcall_failures_and_poll_branches(self):
|
||
ex = self.make_exec()
|
||
os.makedirs(self.state_dir, exist_ok=True)
|
||
with open(os.path.join(self.state_dir, "notes.txt"),
|
||
"w") as fh:
|
||
fh.write("not a socket")
|
||
with open(os.path.join(self.state_dir, "chv-dead.sock"),
|
||
"w") as fh:
|
||
fh.write("")
|
||
ex._vms["chv-dead"] = RaisingProc()
|
||
ex.cleanup_stale_secrets = mock.Mock(
|
||
side_effect=RuntimeError("secrets boom"))
|
||
ex.gc_stale_disks = mock.Mock(
|
||
side_effect=RuntimeError("gc boom"))
|
||
cleaned = ex.cleanup_stale_vms()
|
||
self.assertEqual(cleaned, ["chv-dead"])
|
||
ex2 = self.make_exec()
|
||
ex2._vms["chv-ghost"] = RaisingProc()
|
||
ex2._purge_dead_registry()
|
||
self.assertEqual(ex2._vms, {})
|
||
|
||
def test_run_with_vm_finally_shutdown_exception_swallowed(self):
|
||
ex = self.make_exec()
|
||
real_shutdown = ex.shutdown_vm
|
||
ex.shutdown_vm = mock.Mock(side_effect=RuntimeError("sd boom"))
|
||
# _run_with_vm 的 finally 里 shutdown 抛错应被吞掉，
|
||
# dispatch 仍返回 completed（finally 不改返回值）。
|
||
out = ex.dispatch_task(self._task())
|
||
self.assertEqual(out["status"], "completed")
|
||
ex.shutdown_vm = real_shutdown
|
||
|
||
def test_execute_guards_and_alive_reraise(self):
|
||
ex = self.make_exec()
|
||
with self.assertRaises(ValueError):
|
||
ex.execute({})
|
||
ex._vms["chv-abc123"] = FakeProc(alive=True)
|
||
os.makedirs(self.state_dir, exist_ok=True)
|
||
with open(os.path.join(self.state_dir, "chv-abc123.sock"),
|
||
"w") as fh:
|
||
fh.write("")
|
||
|
||
def runner(cmd, timeout=None):
|
||
raise RuntimeError("ssh boom")
|
||
ex.runner = runner
|
||
with self.assertRaises(RuntimeError):
|
||
ex.execute(self._task())
|
||
|
||
def test_entry_success_branches(self):
|
||
ex = self.make_exec()
|
||
out = ex.dispatch_task({**self._task(),
|
||
"repo": "https://example.com/r.git",
|
||
"plan": True})
|
||
self.assertEqual(out["status"], "completed")
|
||
ssh_cmds = [c for c in self.calls if c and c[0] == "ssh"]
|
||
self.assertTrue(any("https://example.com/r.git" in " ".join(c)
|
||
for c in ssh_cmds))
|
||
self.calls.clear()
|
||
out = ex.run_task(self._task())
|
||
self.assertEqual(out["status"], "completed")
|
||
with self.assertRaises(ValueError):
|
||
ex.run_task({})
|
||
with self.assertRaises(ValueError):
|
||
ex.goal_task({"id": "x"})
|
||
with self.assertRaises(ValueError):
|
||
ex.goal_task({})
|
||
out = ex.goal_task({**self._task(), "goal": "g",
|
||
"verify_command": "make check",
|
||
"repo": "https://example.com/r.git"})
|
||
self.assertEqual(out["status"], "completed")
|
||
goal_cmds = [c for c in self.calls
|
||
if c and c[0] == "ssh" and "goal" in c]
|
||
self.assertTrue(any("make check" in " ".join(c)
|
||
for c in goal_cmds))
|
||
self.calls.clear()
|
||
out = ex.goal_resume_task(self._task(), goal_text="new goal",
|
||
max_rounds=2)
|
||
self.assertEqual(out["status"], "completed")
|
||
resume_cmds = [c for c in self.calls
|
||
if c and c[0] == "ssh" and "goal-resume" in c]
|
||
self.assertTrue(any("--goal-text" in c for c in resume_cmds))
|
||
self.calls.clear()
|
||
out = ex.goal_resume_task(self._task(), max_rounds=5)
|
||
self.assertEqual(out["status"], "completed")
|
||
resume_cmds = [c for c in self.calls
|
||
if c and c[0] == "ssh" and "goal-resume" in c]
|
||
self.assertTrue(any("--max-rounds" in c for c in resume_cmds))
|
||
|
||
|
||
class WorkerAdapterCloseup2Test(unittest.TestCase):
|
||
def test_is_chv_executor(self):
|
||
from cloud.chv.worker_adapter import is_chv_executor
|
||
self.assertFalse(is_chv_executor(object()))
|
||
|
||
def test_reconcile_audit_fn_failures_swallowed(self):
|
||
from cloud.chv.executor import CloudHypervisorExecutor as _RealEx
|
||
from cloud.chv.worker_adapter import reconcile_warm_pool
|
||
with mock.patch(
|
||
"cloud.chv.orphans.reconcile_orphans_for_executor",
|
||
side_effect=RuntimeError("boom")):
|
||
fake_ex = mock.Mock(spec=_RealEx)
|
||
fake_ex.__class__ = _RealEx
|
||
|
||
def raising_audit(*args):
|
||
raise RuntimeError("audit boom")
|
||
out = reconcile_warm_pool(fake_ex, raising_audit, "when")
|
||
self.assertEqual(out, [])
|
||
with mock.patch(
|
||
"cloud.chv.orphans.reconcile_orphans_for_executor",
|
||
return_value=["chv-x"]):
|
||
fake_ex = mock.Mock(spec=_RealEx)
|
||
fake_ex.__class__ = _RealEx
|
||
out = reconcile_warm_pool(fake_ex, raising_audit, "when")
|
||
self.assertEqual(out, ["chv-x"])
|
||
|
||
def test_cancel_watchdog_get_task_variants(self):
|
||
from cloud.chv.worker_adapter import cancel_watchdog_loop
|
||
stop = threading.Event()
|
||
|
||
def get_task_raising(tid):
|
||
stop.set()
|
||
raise RuntimeError("db boom")
|
||
cancel_watchdog_loop("t1", "chv-t1", stop, set(),
|
||
get_task_raising, None, object())
|
||
stop2 = threading.Event()
|
||
|
||
def get_task_running(tid):
|
||
stop2.set()
|
||
return {"status": "running"}
|
||
cancel_watchdog_loop("t1", "chv-t1", stop2, set(),
|
||
get_task_running, None, object())
|
||
|
||
def test_cancel_watchdog_cancel_vm_raises_swallowed(self):
|
||
from cloud.chv.worker_adapter import cancel_watchdog_loop
|
||
stop = threading.Event()
|
||
|
||
def bad_cancel(vm):
|
||
raise RuntimeError("cancel boom")
|
||
cancel_watchdog_loop("t1", "chv-t1", stop, {"t1"},
|
||
lambda tid: None, bad_cancel, object())
|
||
|
||
def test_quota_watchdog_check_raises_swallowed(self):
|
||
from cloud.chv.worker_adapter import quota_watchdog_loop
|
||
stop = threading.Event()
|
||
|
||
def bad_check():
|
||
stop.set()
|
||
raise RuntimeError("quota boom")
|
||
quota_watchdog_loop("t1", "chv-t1", stop, bad_check,
|
||
lambda tid: None, None, object(),
|
||
interval=0.01)
|
||
|
||
|
||
class ExecutorLastLinesTest(CloseupCase):
|
||
def test_net_guest_ip_no_ip_entry_returns_none(self):
|
||
ex = self.make_exec()
|
||
ex.net_json = '[{"tap": "tap0"}]'
|
||
self.assertIsNone(ex._net_guest_ip())
|
||
|
||
def test_wait_guest_mismatch_stop_exception_swallowed(self):
|
||
self._known_hosts()
|
||
ex = self.make_exec()
|
||
ex.stop_vm = mock.Mock(side_effect=RuntimeError("stop boom"))
|
||
|
||
def runner(cmd, timeout=None):
|
||
self.calls.append(list(cmd))
|
||
if cmd[0] == "ssh" and cmd[-1] == "true":
|
||
return FakeResult(1, "",
|
||
"Host key verification failed")
|
||
return FakeResult(0, "", "")
|
||
ex.runner = runner
|
||
with self.assertRaises(ChvFailure) as ctx:
|
||
ex.wait_guest_ready(self._task())
|
||
self.assertEqual(ctx.exception.kind, "host_key_mismatch")
|
||
|
||
def test_wait_guest_timeout_stop_exception_swallowed(self):
|
||
self._known_hosts()
|
||
ex = self.make_exec(ready_timeout=2, ready_interval=1)
|
||
ex.stop_vm = mock.Mock(side_effect=RuntimeError("stop boom"))
|
||
|
||
def runner(cmd, timeout=None):
|
||
self.calls.append(list(cmd))
|
||
if cmd[0] == "ssh":
|
||
return FakeResult(1, "", "")
|
||
return FakeResult(0, "", "")
|
||
ex.runner = runner
|
||
with self.assertRaises(ChvFailure) as ctx:
|
||
ex.wait_guest_ready(self._task())
|
||
self.assertEqual(ctx.exception.kind, "guest_not_ready")
|
||
|
||
def test_record_net_entries_write_failure_only_warns(self):
|
||
ex = self.make_exec()
|
||
blocker = os.path.join(self.state_dir, "chv-x")
|
||
os.makedirs(self.state_dir, exist_ok=True)
|
||
with open(blocker, "w") as fh:
|
||
fh.write("x")
|
||
with self.assertLogs("cloud.cloud_hypervisor",
|
||
level="WARNING") as logs:
|
||
ex._record_net_entries("chv-x", [{"tap": "tap0"}])
|
||
self.assertTrue(any("net_registry_write_failed" in line
|
||
for line in logs.output))
|
||
|
||
def test_socket_permissions_stat_failure_after_exists(self):
|
||
ex = self.make_exec()
|
||
sock = os.path.join(self.state_dir, "chv-abc123.sock")
|
||
os.makedirs(self.state_dir, exist_ok=True)
|
||
with open(sock, "w") as fh:
|
||
fh.write("")
|
||
real_stat = os.stat
|
||
calls = {"n": 0}
|
||
|
||
def _stat_then_raise(path, *args, **kwargs):
|
||
calls["n"] += 1
|
||
if calls["n"] == 1:
|
||
return real_stat(path, *args, **kwargs)
|
||
raise OSError("stat boom")
|
||
# 第 1 次 stat 是 exists() 的探查（须成功才进到显式
|
||
# stat），第 2 次显式 stat 抛错 → except 分支 return。
|
||
with mock.patch("os.stat", side_effect=_stat_then_raise):
|
||
ex._check_api_socket_permissions(sock)
|
||
|
||
def test_vm_proc_dead_unregistered_returns_false(self):
|
||
ex = self.make_exec()
|
||
self.assertFalse(ex._vm_proc_dead(self._task()))
|