feat: Cloud Hypervisor 执行器生产级 v1（cloud-hypervisor）
单任务单 VM：API 唤醒（preflight→vmm.ping→vm.create→vm.boot→SSH 就绪）、 暂停/快照/休眠等价/关闭/回收、预热快照池、每任务磁盘 copy、容量与磁盘配额门、 受控 egress fail-closed、VM 事件审计与分类失败；worker/server/autoscaler 接线 （executor=cloud-hypervisor，任务内 LLM 不可自选）；默认关闭，不进生产路由。 未做真实 VM 验证：生产主机与本 VM 均无 /dev/kvm、无嵌套虚拟化，验证为 mock + 干跑命令 + 单元测试；真 VM、快照恢复、压测、安全审计见 docs/chv-production-readiness.md。CI 账单故障仍是硬门禁，本提交只到 PR OPEN。
This commit is contained in:
host committed
1 parent
aebc6238a6
commit
c371bbdf1e
9 files changed
+2364
-11
No files matched your search
@@ -0,0 +1,665 @@
|
||||
"""Cloud Hypervisor 执行器 v1 + 生产级唤醒/生命周期测试。
|
||||
|
||||
只用标准库与 mock runner/launcher/api_client，绝不启动真实
|
||||
VM：当前环境没有可跑真 VM 的宿主（生产主机是 CVM、无
|
||||
/dev/kvm、无嵌套虚拟化；本 VM 容器内同样无 /dev/kvm），
|
||||
所以本文件验证的是：preflight 分类、API 唤醒顺序、VmConfig、
|
||||
SSH 就绪、暂停/快照/休眠/关闭/清理、预热池、单任务单 VM
|
||||
隔离、容量门、磁盘门、worker/server/autoscaler 接线。
|
||||
真实 VM 启动、快照恢复、压测一律不在本文件声称已验证。
|
||||
"""
|
||||
|
||||
import http.client
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import tempfile
|
||||
import threading
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
from unittest import mock
|
||||
|
||||
try:
|
||||
from cloud.sandbox import (
|
||||
ChvFailure,
|
||||
CloudHypervisorExecutor,
|
||||
WarmPool,
|
||||
_parse_memory_bytes,
|
||||
)
|
||||
except ImportError: # discover -s cloud/tests 时把项目根目录补进 sys.path
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[2]))
|
||||
from cloud.sandbox import (
|
||||
ChvFailure,
|
||||
CloudHypervisorExecutor,
|
||||
WarmPool,
|
||||
_parse_memory_bytes,
|
||||
)
|
||||
|
||||
|
||||
class FakeResult:
|
||||
def __init__(self, returncode=0, stdout="", stderr=""):
|
||||
self.returncode = returncode
|
||||
self.stdout = stdout
|
||||
self.stderr = stderr
|
||||
|
||||
|
||||
class FakeProc:
|
||||
"""假 VMM 进程：poll() 返回 None=活着，数字=已死。"""
|
||||
|
||||
def __init__(self, alive=True):
|
||||
self._alive = alive
|
||||
self.terminated = False
|
||||
self.killed = False
|
||||
|
||||
def poll(self):
|
||||
return None if self._alive else 1
|
||||
|
||||
def terminate(self):
|
||||
self.terminated = True
|
||||
self._alive = False
|
||||
|
||||
def wait(self, timeout=None):
|
||||
return 0
|
||||
|
||||
def kill(self):
|
||||
self.killed = True
|
||||
self._alive = False
|
||||
|
||||
|
||||
def _write(path, content=b"x"):
|
||||
with open(path, "wb") as fh:
|
||||
fh.write(content)
|
||||
os.chmod(path, 0o755)
|
||||
return path
|
||||
|
||||
|
||||
class ChvCase(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.tmp = tempfile.TemporaryDirectory()
|
||||
self.dir = self.tmp.name
|
||||
self.binary = _write(os.path.join(self.dir, "cloud-hypervisor"))
|
||||
self.ch_remote = _write(os.path.join(self.dir, "ch-remote"))
|
||||
self.kernel = _write(os.path.join(self.dir, "vmlinux"))
|
||||
self.rootfs = _write(os.path.join(self.dir, "disk.raw"),
|
||||
b"R" * 100)
|
||||
self.ssh_key = _write(os.path.join(self.dir, "id_chv"))
|
||||
self.kvm = _write(os.path.join(self.dir, "kvm"))
|
||||
self.cpuinfo = os.path.join(self.dir, "cpuinfo")
|
||||
with open(self.cpuinfo, "w") as fh:
|
||||
fh.write("flags : fpu vmx sse\n")
|
||||
self.state_dir = os.path.join(self.dir, "state")
|
||||
self.calls = [] # runner 调用记录
|
||||
self.api_calls = [] # api_client 调用记录
|
||||
self.launches = [] # launcher 调用记录
|
||||
self.audit = [] # audit 事件记录
|
||||
|
||||
def tearDown(self):
|
||||
self.tmp.cleanup()
|
||||
|
||||
def make_exec(self, runner=None, api_client=None, launcher=None,
|
||||
env_extra=None, **overrides):
|
||||
env = {"CHV_EGRESS_PROXY": "http://10.0.0.1:8888"}
|
||||
if env_extra:
|
||||
env.update(env_extra)
|
||||
if runner is None:
|
||||
runner = self._default_runner
|
||||
if api_client is None:
|
||||
api_client = self._default_api
|
||||
if launcher is None:
|
||||
launcher = self._default_launcher
|
||||
kwargs = dict(
|
||||
binary=self.binary, ch_remote=self.ch_remote,
|
||||
kernel=self.kernel, rootfs=self.rootfs,
|
||||
ssh_key=self.ssh_key, kvm_path=self.kvm,
|
||||
cpuinfo_path=self.cpuinfo, state_dir=self.state_dir,
|
||||
runner=runner, launcher=launcher, api_client=api_client,
|
||||
sleeper=lambda s: None, env=env)
|
||||
kwargs.update(overrides)
|
||||
ex = CloudHypervisorExecutor(**kwargs)
|
||||
ex.audit_callback = lambda e, t, d, k=None: self.audit.append(
|
||||
(e, t.get("id") if isinstance(t, dict) else t, d, k))
|
||||
return ex
|
||||
|
||||
def _default_runner(self, cmd, timeout=None):
|
||||
self.calls.append(list(cmd))
|
||||
if cmd[0] == "ssh":
|
||||
if cmd[-1] == "true":
|
||||
return FakeResult(0, "", "")
|
||||
if "dispatch" in cmd:
|
||||
return FakeResult(0, json.dumps(
|
||||
{"status": "completed", "text": "ok-dispatch"}), "")
|
||||
if "goal" in cmd or "goal-resume" in cmd:
|
||||
return FakeResult(0, "goal met after rounds\n", "")
|
||||
if "status" in cmd:
|
||||
return FakeResult(0, json.dumps(
|
||||
{"status": "completed", "text": "ok-status"}), "")
|
||||
return FakeResult(0, "", "")
|
||||
return FakeResult(0, "", "")
|
||||
|
||||
def _default_api(self, socket_path, method, path, payload=None):
|
||||
self.api_calls.append((method, path, payload))
|
||||
return (200, {})
|
||||
|
||||
def _default_launcher(self, cmd):
|
||||
self.launches.append(list(cmd))
|
||||
return FakeProc(alive=True)
|
||||
|
||||
def _task(self, task_id="abc123"):
|
||||
return {"id": task_id, "prompt": "做一个加法函数",
|
||||
"project_id": "proj1"}
|
||||
|
||||
|
||||
# -- preflight ---------------------------------------------------------
|
||||
class PreflightTest(ChvCase):
|
||||
def test_all_present_passes(self):
|
||||
self.assertTrue(self.make_exec().preflight())
|
||||
|
||||
def test_missing_binary_reports_it(self):
|
||||
ex = self.make_exec(binary="/tmp/no-such-chv-binary")
|
||||
with self.assertRaises(ChvFailure) as ctx:
|
||||
ex.preflight()
|
||||
self.assertEqual(ctx.exception.kind, "binary_missing")
|
||||
self.assertIn("cloud-hypervisor", str(ctx.exception))
|
||||
|
||||
def test_missing_ch_remote_reports_it(self):
|
||||
ex = self.make_exec(ch_remote="/tmp/no-such-ch-remote")
|
||||
with self.assertRaises(ChvFailure) as ctx:
|
||||
ex.preflight()
|
||||
self.assertEqual(ctx.exception.kind, "binary_missing")
|
||||
self.assertIn("ch-remote", str(ctx.exception))
|
||||
|
||||
def test_missing_kvm_reports_it(self):
|
||||
ex = self.make_exec(kvm_path="/tmp/no-such-kvm")
|
||||
with self.assertRaises(ChvFailure) as ctx:
|
||||
ex.preflight()
|
||||
self.assertEqual(ctx.exception.kind, "kvm_missing")
|
||||
self.assertIn("/tmp/no-such-kvm", str(ctx.exception))
|
||||
|
||||
def test_missing_cpu_flag_reports_it(self):
|
||||
with open(self.cpuinfo, "w") as fh:
|
||||
fh.write("flags : fpu sse\n")
|
||||
with self.assertRaises(ChvFailure) as ctx:
|
||||
self.make_exec().preflight()
|
||||
self.assertEqual(ctx.exception.kind, "cpu_flag_missing")
|
||||
self.assertIn("vmx", str(ctx.exception))
|
||||
|
||||
def test_missing_kernel_reports_it(self):
|
||||
ex = self.make_exec(kernel="/tmp/no-such-vmlinux")
|
||||
with self.assertRaises(ChvFailure) as ctx:
|
||||
ex.preflight()
|
||||
self.assertEqual(ctx.exception.kind, "kernel_missing")
|
||||
|
||||
def test_missing_image_reports_it(self):
|
||||
ex = self.make_exec(rootfs="/tmp/no-such-disk.raw")
|
||||
with self.assertRaises(ChvFailure) as ctx:
|
||||
ex.preflight()
|
||||
self.assertEqual(ctx.exception.kind, "image_missing")
|
||||
|
||||
def test_missing_ssh_key_reports_it(self):
|
||||
ex = self.make_exec(ssh_key="/tmp/no-such-key")
|
||||
with self.assertRaises(ChvFailure) as ctx:
|
||||
ex.preflight()
|
||||
self.assertEqual(ctx.exception.kind, "ssh_key_missing")
|
||||
|
||||
def test_missing_egress_is_fail_closed(self):
|
||||
ex = self.make_exec(env_extra={"CHV_EGRESS_PROXY": ""})
|
||||
# env_extra 空值会被 __init__ 的 or 链忽略，故改用显式 env：
|
||||
ex.egress_proxy = None
|
||||
ex.allow_direct_egress = False
|
||||
with self.assertRaises(ChvFailure) as ctx:
|
||||
ex.preflight()
|
||||
self.assertEqual(ctx.exception.kind, "egress_not_configured")
|
||||
|
||||
def test_multiple_missing_lists_all(self):
|
||||
ex = self.make_exec(binary="/tmp/no-chv", kernel="/tmp/no-kern")
|
||||
with self.assertRaises(ChvFailure) as ctx:
|
||||
ex.preflight()
|
||||
text = str(ctx.exception)
|
||||
self.assertIn("cloud-hypervisor", text)
|
||||
self.assertIn("内核", text)
|
||||
|
||||
def test_dispatch_missing_dependency_does_not_call_runner(self):
|
||||
ex = self.make_exec(binary="/tmp/no-chv")
|
||||
with self.assertRaises(ChvFailure):
|
||||
ex.dispatch_task(self._task())
|
||||
self.assertEqual(self.calls, [])
|
||||
|
||||
|
||||
# -- 配置与命令构造 -----------------------------------------------------
|
||||
class ConfigCommandTest(ChvCase):
|
||||
def test_vm_config_shape_and_memory_bytes(self):
|
||||
ex = self.make_exec()
|
||||
cfg = ex.build_vm_config("/tmp/x/disk.raw")
|
||||
self.assertEqual(cfg["cpus"], {"boot_vcpus": 2,
|
||||
"max_vcpus": 2})
|
||||
self.assertEqual(cfg["memory"]["size"], 2 * 1024 ** 3)
|
||||
self.assertEqual(cfg["kernel"]["path"], self.kernel)
|
||||
self.assertEqual(cfg["disks"], [{"path": "/tmp/x/disk.raw"}])
|
||||
self.assertEqual(cfg["cmdline"]["args"],
|
||||
"console=hvc0 root=/dev/vda1 rw")
|
||||
|
||||
def test_memory_parsing(self):
|
||||
self.assertEqual(_parse_memory_bytes("512M"), 512 * 1024 ** 2)
|
||||
self.assertEqual(_parse_memory_bytes("1024"), 1024)
|
||||
self.assertEqual(_parse_memory_bytes("1K"), 1024)
|
||||
with self.assertRaises(ValueError):
|
||||
_parse_memory_bytes("abc")
|
||||
|
||||
def test_rootfs_image_alias(self):
|
||||
ex = self.make_exec(rootfs=None, env_extra={})
|
||||
# rootfs=None 时 __init__ 仍从 env 取不到（env 无 CHV_*），
|
||||
# 故用 env 含 CHV_IMAGE 的构造验证别名：
|
||||
env = {"CHV_EGRESS_PROXY": "http://10.0.0.1:8888",
|
||||
"CHV_IMAGE": self.rootfs}
|
||||
ex2 = CloudHypervisorExecutor(
|
||||
binary=self.binary, ch_remote=self.ch_remote,
|
||||
kernel=self.kernel, ssh_key=self.ssh_key,
|
||||
kvm_path=self.kvm, cpuinfo_path=self.cpuinfo,
|
||||
state_dir=self.state_dir, env=env,
|
||||
sleeper=lambda s: None)
|
||||
self.assertEqual(ex2.rootfs, self.rootfs)
|
||||
|
||||
def test_manual_vm_command_shape(self):
|
||||
ex = self.make_exec()
|
||||
cmd = ex.build_vm_command(self._task())
|
||||
self.assertEqual(cmd[0], self.binary)
|
||||
self.assertIn("--api-socket", cmd)
|
||||
self.assertIn("--cpus", cmd)
|
||||
self.assertIn("boot=2", cmd)
|
||||
self.assertIn("size=2G", cmd)
|
||||
self.assertIn("--kernel", cmd)
|
||||
self.assertIn("--disk", cmd)
|
||||
|
||||
def test_ssh_command_has_key_path_not_secret_value(self):
|
||||
ex = self.make_exec()
|
||||
cmd = ex.build_ssh_command(self._task(), ["true"])
|
||||
joined = " ".join(cmd)
|
||||
self.assertIn(self.ssh_key, joined)
|
||||
self.assertIn("BatchMode=yes", joined)
|
||||
self.assertNotIn("SECRET", joined)
|
||||
# egress 代理地址在 env 前缀里（不是密钥），且只在代理已设时出现：
|
||||
self.assertIn("HTTPS_PROXY=http://10.0.0.1:8888", joined)
|
||||
|
||||
def test_isolation_names_and_disks_differ(self):
|
||||
ex = self.make_exec()
|
||||
a, b = {"id": "task-a"}, {"id": "task-b"}
|
||||
self.assertNotEqual(ex.vm_name(a), ex.vm_name(b))
|
||||
self.assertNotEqual(ex.api_socket(a), ex.api_socket(b))
|
||||
self.assertNotEqual(ex._provision_task_disk(a),
|
||||
ex._provision_task_disk(b))
|
||||
# copy 模式：每任务磁盘是基础镜像的真实副本，内容一致、
|
||||
# 路径不同（两个任务绝不写同一文件）：
|
||||
with open(ex._provision_task_disk(a), "rb") as fh:
|
||||
self.assertEqual(fh.read(), b"R" * 100)
|
||||
|
||||
def test_disk_quota_gate(self):
|
||||
ex = self.make_exec(env_extra={"CHV_DISK_GB": "0.00000001"})
|
||||
with self.assertRaises(ChvFailure) as ctx:
|
||||
ex.wake_vm(self._task())
|
||||
self.assertEqual(ctx.exception.kind, "disk_quota_exceeded")
|
||||
|
||||
def test_capacity_gate_rejects_not_fallback(self):
|
||||
ex = self.make_exec(env_extra={"CHV_MAX_VMS": "1"})
|
||||
ex._vms["chv-other"] = FakeProc(alive=True)
|
||||
with self.assertRaises(ChvFailure) as ctx:
|
||||
ex.wake_vm(self._task())
|
||||
self.assertEqual(ctx.exception.kind, "host_capacity_exceeded")
|
||||
self.assertEqual(self.launches, [])
|
||||
|
||||
|
||||
# -- API 唤醒 -----------------------------------------------------------
|
||||
class WakeApiTest(ChvCase):
|
||||
def test_wake_uses_api_mode_not_direct_command(self):
|
||||
ex = self.make_exec()
|
||||
out = ex.wake_vm(self._task())
|
||||
self.assertEqual(out["state"], "running")
|
||||
self.assertTrue(out["ready"])
|
||||
self.assertEqual(out["mode"], "cold")
|
||||
# launcher 只带 --api-socket，不带内核/磁盘（API 模式证明）：
|
||||
self.assertEqual(len(self.launches), 1)
|
||||
launch = self.launches[0]
|
||||
self.assertIn("--api-socket", launch)
|
||||
self.assertNotIn("--kernel", launch)
|
||||
self.assertNotIn("--disk", launch)
|
||||
# API 调用顺序：ping → create → boot：
|
||||
kinds = [(m, p) for m, p, _ in self.api_calls]
|
||||
self.assertEqual(kinds[0], ("GET", "/vmm.ping"))
|
||||
self.assertEqual(kinds[1], ("PUT", "/vm.create"))
|
||||
self.assertEqual(kinds[2], ("PUT", "/vm.boot"))
|
||||
create_payload = self.api_calls[1][2]
|
||||
self.assertEqual(create_payload["kernel"]["path"], self.kernel)
|
||||
self.assertTrue(create_payload["disks"][0]["path"].endswith(
|
||||
"disk.raw"))
|
||||
events = [e for e, _, _, _ in self.audit]
|
||||
self.assertIn("vm_create", events)
|
||||
self.assertIn("vm_boot", events)
|
||||
self.assertIn("vm_ready", events)
|
||||
|
||||
def test_wake_api_timeout_kills_proc(self):
|
||||
def never_ok(socket_path, method, path, payload=None):
|
||||
self.api_calls.append((method, path, payload))
|
||||
return (500, {})
|
||||
ex = self.make_exec(api_client=never_ok, api_timeout=3)
|
||||
proc_holder = []
|
||||
|
||||
def launcher(cmd):
|
||||
proc = FakeProc(alive=True)
|
||||
proc_holder.append(proc)
|
||||
self.launches.append(list(cmd))
|
||||
return proc
|
||||
ex.launcher = launcher
|
||||
with self.assertRaises(ChvFailure) as ctx:
|
||||
ex.wake_vm(self._task())
|
||||
self.assertEqual(ctx.exception.kind, "api_timeout")
|
||||
self.assertTrue(proc_holder[0].terminated)
|
||||
self.assertNotIn("chv-abc123", ex._vms)
|
||||
|
||||
def test_guest_ready_timeout_shuts_down(self):
|
||||
def ssh_never(cmd, timeout=None):
|
||||
self.calls.append(list(cmd))
|
||||
if cmd[0] == "ssh":
|
||||
return FakeResult(1, "", "")
|
||||
return FakeResult(0, "", "")
|
||||
ex = self.make_exec(runner=ssh_never, ready_timeout=10,
|
||||
ready_interval=5)
|
||||
with self.assertRaises(ChvFailure) as ctx:
|
||||
ex.wake_vm(self._task())
|
||||
self.assertEqual(ctx.exception.kind, "guest_not_ready")
|
||||
# 超时后关 VM：runner 里有 ch-remote shutdown：
|
||||
self.assertTrue(any("shutdown" in c for c in self.calls))
|
||||
|
||||
|
||||
# -- 暂停 / 快照 / 休眠 / 关闭 / 清理 -----------------------------------
|
||||
class LifecycleTest(ChvCase):
|
||||
def test_pause_resume(self):
|
||||
ex = self.make_exec()
|
||||
self.assertTrue(ex.pause_vm(self._task()))
|
||||
self.assertTrue(ex.resume_vm(self._task()))
|
||||
kinds = [c[-1] for c in self.calls if c[0] == self.ch_remote]
|
||||
self.assertEqual(kinds, ["pause", "resume"])
|
||||
|
||||
def test_restore_then_explicit_resume_order(self):
|
||||
ex = self.make_exec()
|
||||
out = ex.restore_vm(self._task(), "file:///snap/1")
|
||||
self.assertEqual(out["mode"], "restored")
|
||||
self.assertTrue(out["ready"])
|
||||
kinds = [c[-1] if c[-1] != "restore" else c[-1]
|
||||
for c in self.calls if c[0] == self.ch_remote]
|
||||
# ch-remote 调用里 restore 必须在 resume 之前：
|
||||
flat = []
|
||||
for c in self.calls:
|
||||
if c[0] == self.ch_remote:
|
||||
flat.append(c[-1] if c[-1].startswith("source_url")
|
||||
else c[-1])
|
||||
self.assertIn("resume", flat)
|
||||
restore_idx = next(i for i, c in enumerate(self.calls)
|
||||
if c[0] == self.ch_remote
|
||||
and any(a.startswith("source_url")
|
||||
for a in c))
|
||||
resume_idx = next(i for i, c in enumerate(self.calls)
|
||||
if c[0] == self.ch_remote and c[-1] == "resume")
|
||||
self.assertLess(restore_idx, resume_idx)
|
||||
|
||||
def test_hibernate_is_pause_snapshot_shutdown(self):
|
||||
ex = self.make_exec()
|
||||
out = ex.hibernate_vm(self._task(), "file:///snap/h1")
|
||||
self.assertEqual(out["state"], "hibernated")
|
||||
self.assertEqual(out["snapshot"], "file:///snap/h1")
|
||||
kinds = [c[-1] for c in self.calls if c[0] == self.ch_remote]
|
||||
self.assertEqual(kinds[0], "pause")
|
||||
self.assertTrue(any(a.startswith("source_url")
|
||||
for c in self.calls for a in c))
|
||||
self.assertIn("shutdown", kinds)
|
||||
# 从休眠唤醒 = restore + resume：
|
||||
ex2 = self.make_exec()
|
||||
out2 = ex2.wake_from_hibernate(self._task(), "file:///snap/h1")
|
||||
self.assertEqual(out2["mode"], "restored")
|
||||
|
||||
def test_stop_dead_proc(self):
|
||||
ex = self.make_exec()
|
||||
proc = FakeProc(alive=False)
|
||||
ex._vms["chv-abc123"] = proc
|
||||
sock = os.path.join(self.state_dir, "chv-abc123.sock")
|
||||
os.makedirs(self.state_dir, exist_ok=True)
|
||||
with open(sock, "w") as _fh:
|
||||
_fh.write("")
|
||||
self.assertTrue(ex.stop_vm(self._task()))
|
||||
self.assertFalse(os.path.exists(sock))
|
||||
self.assertNotIn("chv-abc123", ex._vms)
|
||||
|
||||
def test_stop_already_closed(self):
|
||||
ex = self.make_exec()
|
||||
self.assertFalse(ex.stop_vm(self._task()))
|
||||
# 已关仍尽力调过一次 ch-remote shutdown：
|
||||
self.assertTrue(any("shutdown" in c for c in self.calls))
|
||||
|
||||
def test_stop_socket_residual(self):
|
||||
ex = self.make_exec()
|
||||
sock = os.path.join(self.state_dir, "chv-abc123.sock")
|
||||
os.makedirs(self.state_dir, exist_ok=True)
|
||||
with open(sock, "w") as _fh:
|
||||
_fh.write("")
|
||||
self.assertTrue(ex.stop_vm(self._task()))
|
||||
self.assertFalse(os.path.exists(sock))
|
||||
|
||||
def test_cleanup_skips_live_and_non_chv(self):
|
||||
ex = self.make_exec()
|
||||
os.makedirs(self.state_dir, exist_ok=True)
|
||||
live = os.path.join(self.state_dir, "chv-live.sock")
|
||||
with open(live, "w") as _fh:
|
||||
_fh.write("")
|
||||
ex._vms["chv-live"] = FakeProc(alive=True)
|
||||
dead = os.path.join(self.state_dir, "chv-dead.sock")
|
||||
with open(dead, "w") as _fh:
|
||||
_fh.write("")
|
||||
other = os.path.join(self.state_dir, "other.sock")
|
||||
with open(other, "w") as _fh:
|
||||
_fh.write("")
|
||||
cleaned = ex.cleanup_stale_vms()
|
||||
self.assertEqual(cleaned, ["chv-dead"])
|
||||
self.assertTrue(os.path.exists(live))
|
||||
self.assertTrue(os.path.exists(other))
|
||||
self.assertFalse(os.path.exists(dead))
|
||||
|
||||
def test_cleanup_missing_state_dir(self):
|
||||
ex = self.make_exec()
|
||||
self.assertEqual(ex.cleanup_stale_vms(), [])
|
||||
|
||||
def test_recycle_preserves_checkpoints_not_temp(self):
|
||||
ex = self.make_exec()
|
||||
out = ex.recycle_vm(self._task())
|
||||
self.assertEqual(out["state"], "recycled")
|
||||
self.assertIn("task_checkpoints", out["preserved"])
|
||||
self.assertIn("task_attempts", out["preserved"])
|
||||
self.assertIn("guest /tmp", out["lost"])
|
||||
|
||||
|
||||
# -- 预热池 --------------------------------------------------------------
|
||||
class WarmPoolTest(ChvCase):
|
||||
def test_prewarm_acquire_release(self):
|
||||
ex = self.make_exec()
|
||||
pool = WarmPool(ex, size=2, snapshot_url="file:///snap/base")
|
||||
added = pool.prewarm()
|
||||
self.assertEqual(len(added), 2)
|
||||
self.assertEqual(pool.warm_count(), 2)
|
||||
first = pool.acquire(self._task("t1"))
|
||||
self.assertEqual(first["mode"], "warm")
|
||||
second = pool.acquire(self._task("t2"))
|
||||
self.assertEqual(second["mode"], "warm")
|
||||
third = pool.acquire(self._task("t3"))
|
||||
self.assertEqual(third["mode"], "cold")
|
||||
# release 一律销毁，不放回池：
|
||||
self.assertTrue(pool.release(self._task("t1"),
|
||||
pool_vm=added[0]))
|
||||
self.assertEqual(pool.warm_count(), 0)
|
||||
|
||||
def test_prewarm_without_snapshot_is_empty(self):
|
||||
ex = self.make_exec()
|
||||
pool = WarmPool(ex, size=2, snapshot_url=None)
|
||||
self.assertEqual(pool.prewarm(), [])
|
||||
self.assertEqual(pool.acquire(self._task())["mode"], "cold")
|
||||
|
||||
|
||||
# -- dispatch / goal（wake → 跑 → finally 关） ---------------------------
|
||||
class DispatchTest(ChvCase):
|
||||
def test_dispatch_wakes_runs_shuts_down(self):
|
||||
ex = self.make_exec()
|
||||
out = ex.dispatch_task(self._task())
|
||||
self.assertEqual(out["executor"], "cloud-hypervisor")
|
||||
self.assertEqual(out["status"], "completed")
|
||||
self.assertIn("ok-dispatch", out["stdout"])
|
||||
self.assertEqual(out["vm"], "chv-abc123")
|
||||
# finally 关 VM：ch-remote shutdown 在 dispatch 之后出现：
|
||||
shutdown_idx = next(i for i, c in enumerate(self.calls)
|
||||
if c[0] == self.ch_remote
|
||||
and c[-1] == "shutdown")
|
||||
ssh_idx = next(i for i, c in enumerate(self.calls)
|
||||
if c[0] == "ssh" and "dispatch" in c)
|
||||
self.assertLess(ssh_idx, shutdown_idx)
|
||||
self.assertNotIn("chv-abc123", ex._vms)
|
||||
|
||||
def test_goal_met(self):
|
||||
ex = self.make_exec()
|
||||
out = ex.goal_task({"id": "g1", "prompt": "做目标",
|
||||
"goal_mode": True})
|
||||
self.assertEqual(out["status"], "completed")
|
||||
self.assertTrue(out["goal_met"])
|
||||
|
||||
def test_goal_resume_breakpoint(self):
|
||||
ex = self.make_exec()
|
||||
out = ex.goal_resume_breakpoint_task({"id": "g1",
|
||||
"prompt": "续跑"})
|
||||
self.assertEqual(out["status"], "completed")
|
||||
self.assertTrue(out["goal_met"])
|
||||
|
||||
def test_execute_is_ssh_status_only(self):
|
||||
ex = self.make_exec()
|
||||
out = ex.execute(self._task())
|
||||
self.assertEqual(out["executor"], "cloud-hypervisor")
|
||||
self.assertIn("ok-status", out["stdout"])
|
||||
# execute 不 wake（launcher 未调用）：
|
||||
self.assertEqual(self.launches, [])
|
||||
|
||||
|
||||
# -- worker / autoscaler / server 接线 -----------------------------------
|
||||
class WiringTest(ChvCase):
|
||||
def test_worker_executor_name_and_cli_builder(self):
|
||||
from cloud import worker as worker_mod
|
||||
from cloud.sandbox import CloudHypervisorExecutor as CHV
|
||||
ex = self.make_exec()
|
||||
w = worker_mod.Worker.__new__(worker_mod.Worker)
|
||||
w.executor = ex
|
||||
self.assertEqual(w._executor_name(), "cloud-hypervisor")
|
||||
|
||||
class Args:
|
||||
executor = "cloud-hypervisor"
|
||||
built = worker_mod._build_cli_executor(Args(), None)
|
||||
self.assertIsInstance(built, CHV)
|
||||
|
||||
def test_autoscaler_passthrough(self):
|
||||
from cloud.autoscaler import Autoscaler
|
||||
scaler = Autoscaler("/tmp/no.db", [], executor="cloud-hypervisor")
|
||||
cmd = scaler._worker_command("w1")
|
||||
self.assertIn("--executor", cmd)
|
||||
self.assertEqual(cmd[cmd.index("--executor") + 1],
|
||||
"cloud-hypervisor")
|
||||
scaler2 = Autoscaler("/tmp/no.db",
|
||||
["--executor", "docker"],
|
||||
executor="cloud-hypervisor")
|
||||
cmd2 = scaler2._worker_command("w1")
|
||||
self.assertEqual(cmd2.count("--executor"), 1)
|
||||
self.assertEqual(cmd2[cmd2.index("--executor") + 1], "docker")
|
||||
|
||||
def test_server_default_executor(self):
|
||||
from cloud.server import _default_executor
|
||||
with mock.patch.dict(os.environ,
|
||||
{"CLOUD_DEFAULT_EXECUTOR":
|
||||
"cloud-hypervisor"}):
|
||||
self.assertEqual(_default_executor(), "cloud-hypervisor")
|
||||
with mock.patch.dict(os.environ,
|
||||
{"CLOUD_DEFAULT_EXECUTOR": "bogus"}):
|
||||
self.assertEqual(_default_executor(), "docker")
|
||||
|
||||
|
||||
class ServerWhitelistTest(unittest.TestCase):
|
||||
"""server 白名单端到端：真实 HTTP + 临时库 + 临时端口。"""
|
||||
|
||||
def setUp(self):
|
||||
self.tmp = tempfile.TemporaryDirectory()
|
||||
self.db_path = os.path.join(self.tmp.name, "control.db")
|
||||
try:
|
||||
from cloud.auth import AuthStore
|
||||
from cloud.server import create_server
|
||||
except ImportError:
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[2]))
|
||||
from cloud.auth import AuthStore
|
||||
from cloud.server import create_server
|
||||
self.auth = AuthStore(self.db_path)
|
||||
self.owner_id, self.owner_key = self.auth.create_user("owner")
|
||||
old = os.environ.get("CODEX_GLOBAL_ADMINS")
|
||||
self._old_admins = old
|
||||
os.environ["CODEX_GLOBAL_ADMINS"] = self.owner_id
|
||||
self.server = create_server(self.db_path, "127.0.0.1", 0)
|
||||
self.port = self.server.server_address[1]
|
||||
self.thread = threading.Thread(
|
||||
target=self.server.serve_forever, daemon=True)
|
||||
self.thread.start()
|
||||
status, self.project = self._req(
|
||||
"POST", "/projects", {"name": "demo"}, self.owner_key)
|
||||
self.assertEqual(status, 201, self.project)
|
||||
|
||||
def tearDown(self):
|
||||
self.server.shutdown()
|
||||
self.server.server_close()
|
||||
self.thread.join(timeout=5)
|
||||
try:
|
||||
self.auth.conn.close()
|
||||
except Exception:
|
||||
pass
|
||||
self.tmp.cleanup()
|
||||
if self._old_admins is None:
|
||||
os.environ.pop("CODEX_GLOBAL_ADMINS", None)
|
||||
else:
|
||||
os.environ["CODEX_GLOBAL_ADMINS"] = self._old_admins
|
||||
|
||||
def _req(self, method, path, body=None, key=None):
|
||||
conn = http.client.HTTPConnection("127.0.0.1", self.port,
|
||||
timeout=10)
|
||||
hdrs = {}
|
||||
payload = None
|
||||
if body is not None:
|
||||
payload = json.dumps(body).encode("utf-8")
|
||||
hdrs["Content-Type"] = "application/json"
|
||||
if key is not None:
|
||||
hdrs["X-API-Key"] = key
|
||||
try:
|
||||
conn.request(method, path, body=payload, headers=hdrs)
|
||||
resp = conn.getresponse()
|
||||
data = resp.read()
|
||||
finally:
|
||||
conn.close()
|
||||
try:
|
||||
parsed = json.loads(data.decode("utf-8")) if data else None
|
||||
except ValueError:
|
||||
parsed = None
|
||||
return resp.status, parsed
|
||||
|
||||
def test_task_executor_cloud_hypervisor_accepted(self):
|
||||
status, task = self._req(
|
||||
"POST", "/tasks",
|
||||
{"project_id": self.project["id"], "prompt": "x",
|
||||
"executor": "cloud-hypervisor"}, self.owner_key)
|
||||
self.assertEqual(status, 201, task)
|
||||
self.assertEqual(task["executor"], "cloud-hypervisor")
|
||||
|
||||
def test_task_executor_bogus_rejected(self):
|
||||
status, body = self._req(
|
||||
"POST", "/tasks",
|
||||
{"project_id": self.project["id"], "prompt": "x",
|
||||
"executor": "bogus"}, self.owner_key)
|
||||
self.assertEqual(status, 400)
|
||||
self.assertIn("cloud-hypervisor", body["error"])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in new issue
Block a user