host
c06ba50983
- 受限推理通道：cloud/chv/inference_channel.py（origin 解析、 allowlist、上限、Sentinel 预审），执行器 _run_with_vm 起 VM 前 必过 _inference_gate（allow 才继续，deny/ask 审计 + fail-closed， 任务级 extra_env 合并进预审）；host.py 第二道防线（_server_command 的 origin 结构校验 + Server.request 超时/重试走 env）。 origin 规则：https 公网网关（只 443）；http 只接受本地中继 （字面环回/私网 IP，如 http://172.17.0.1:28084，端口保留， 无 DNS、无重绑定，allowlist 与单次能力照常）；未知 provider 标签走默认网关（与 _server_command 一致）。 - sentinel.py：从 P5（25bcd35）取的超集版本 + 推理连接器条目 （POST /v1/responses、GET /v1/models；scheme/端口只 https/443， 本地中继不走 evaluate，不把 http 加进表）。 - 遥测最小化：cloud/chv/telemetry.py（字段白名单、确定性采样、 落盘失败不影响任务、token 未回传记 null）+ 训练退出开关 （默认开，本项目决定；关后轨迹不再产生；损坏 fail-closed；变更审计）。 - 访问边界：docs/chv-access-boundary.md（运营访问口径 + 审计钩子， 不冒充技术不可访问；Confidential VM 明确未做）。 - 测试：cloud/tests/test_chv_muse_p9.py 51 组全过；5 个旧文件 的 make_exec 配假 origin + 假解析器（新 fail-closed 门要求的 测试配置）；全量 cloud 套件 1918 OK（skipped=8）。 详见 docs/chv-muse-p9-gap.md（含自定清单与真机待验证清单）。
716 lines
30 KiB
Python
716 lines
30 KiB
Python
"""Cloud Hypervisor 执行器 v1 + 生产级唤醒/生命周期测试。
|
||
|
||
只用标准库与 mock runner/launcher/api_client，绝不启动真实
|
||
VM：当前环境没有可跑真 VM 的宿主（生产主机是 CVM、无
|
||
/dev/kvm、无嵌套虚拟化；本 VM 容器内同样无 /dev/kvm），
|
||
所以本文件验证的是：preflight 分类、API 唤醒顺序、VmConfig、
|
||
SSH 就绪、暂停/快照/休眠/关闭/清理、预热池、单任务单 VM
|
||
隔离、容量门、磁盘门、worker/server/autoscaler 接线。
|
||
真实 VM 启动、快照恢复、压测一律不在本文件声称已验证。
|
||
"""
|
||
|
||
import http.client
|
||
import json
|
||
import os
|
||
import sys
|
||
import tempfile
|
||
import threading
|
||
import unittest
|
||
from pathlib import Path
|
||
from unittest import mock
|
||
|
||
try:
|
||
from cloud.chv.experimental_pool import WarmPool
|
||
from cloud.sandbox import (
|
||
ChvFailure,
|
||
CloudHypervisorExecutor,
|
||
_parse_memory_bytes,
|
||
)
|
||
except ImportError: # discover -s cloud/tests 时把项目根目录补进 sys.path
|
||
sys.path.insert(0, str(Path(__file__).resolve().parents[2]))
|
||
from cloud.chv.experimental_pool import WarmPool
|
||
from cloud.sandbox import (
|
||
ChvFailure,
|
||
CloudHypervisorExecutor,
|
||
_parse_memory_bytes,
|
||
)
|
||
|
||
|
||
class FakeResult:
|
||
def __init__(self, returncode=0, stdout="", stderr=""):
|
||
self.returncode = returncode
|
||
self.stdout = stdout
|
||
self.stderr = stderr
|
||
|
||
|
||
class FakeProc:
|
||
"""假 VMM 进程：poll() 返回 None=活着，数字=已死。"""
|
||
|
||
def __init__(self, alive=True):
|
||
self._alive = alive
|
||
self.terminated = False
|
||
self.killed = False
|
||
|
||
def poll(self):
|
||
return None if self._alive else 1
|
||
|
||
def terminate(self):
|
||
self.terminated = True
|
||
self._alive = False
|
||
|
||
def wait(self, timeout=None):
|
||
return 0
|
||
|
||
def kill(self):
|
||
self.killed = True
|
||
self._alive = False
|
||
|
||
|
||
def _write(path, content=b"x"):
|
||
with open(path, "wb") as fh:
|
||
fh.write(content)
|
||
os.chmod(path, 0o755)
|
||
return path
|
||
|
||
|
||
def _public_resolver(host, port):
|
||
"""feat-chv-muse-p9：假解析器（公网 IP），供推理出口预审用；
|
||
实验室绝不联网，真实解析只在真机发生。"""
|
||
return ["93.184.216.34"]
|
||
|
||
|
||
class ChvCase(unittest.TestCase):
|
||
def setUp(self):
|
||
self.tmp = tempfile.TemporaryDirectory()
|
||
self.dir = self.tmp.name
|
||
self.binary = _write(os.path.join(self.dir, "cloud-hypervisor"))
|
||
self.ch_remote = _write(os.path.join(self.dir, "ch-remote"))
|
||
self.kernel = _write(os.path.join(self.dir, "vmlinux"))
|
||
self.rootfs = _write(os.path.join(self.dir, "disk.raw"),
|
||
b"R" * 100)
|
||
self.ssh_key = _write(os.path.join(self.dir, "id_chv"))
|
||
self.kvm = _write(os.path.join(self.dir, "kvm"))
|
||
self.cpuinfo = os.path.join(self.dir, "cpuinfo")
|
||
with open(self.cpuinfo, "w") as fh:
|
||
fh.write("flags : fpu vmx sse\n")
|
||
self.state_dir = os.path.join(self.dir, "state")
|
||
self.calls = [] # runner 调用记录
|
||
self.api_calls = [] # api_client 调用记录
|
||
self.launches = [] # launcher 调用记录
|
||
self.audit = [] # audit 事件记录
|
||
|
||
def tearDown(self):
|
||
self.tmp.cleanup()
|
||
|
||
def make_exec(self, runner=None, api_client=None, launcher=None,
|
||
env_extra=None, **overrides):
|
||
env = {"CHV_EGRESS_PROXY": "http://10.0.0.1:8888",
|
||
# 实验室宣称：mock runner 假装回环端口已转发到客机
|
||
#（feat-chv-fix 起，执行器不再无条件假设回环能通）。
|
||
# 实验室显式 opt-in keyscan：生产默认已是
|
||
# provisioned；本套件旧 fake 不实现 ssh-keygen/
|
||
# genisoimage，测的不是 host key 模式，故显式
|
||
# 声明实验室路径，绝不把它写成生产默认。
|
||
"CHV_HOST_KEY_MODE": "keyscan",
|
||
"CHV_SSH_FORWARD": "1",
|
||
# feat-chv-muse-p9：推理出口预审的假 origin（公网
|
||
# https，配假解析器；实验室不起真 VM、不联网）。
|
||
"SUB2API_BASE_URL": "https://xgapi.lol",
|
||
"CHV_INFERENCE_ORIGIN": "https://xgapi.lol"}
|
||
if env_extra:
|
||
env.update(env_extra)
|
||
if runner is None:
|
||
runner = self._default_runner
|
||
if api_client is None:
|
||
api_client = self._default_api
|
||
if launcher is None:
|
||
launcher = self._default_launcher
|
||
kwargs = dict(
|
||
binary=self.binary, ch_remote=self.ch_remote,
|
||
kernel=self.kernel, rootfs=self.rootfs,
|
||
ssh_key=self.ssh_key, kvm_path=self.kvm,
|
||
cpuinfo_path=self.cpuinfo, state_dir=self.state_dir,
|
||
runner=runner, launcher=launcher, api_client=api_client,
|
||
sleeper=lambda s: None, env=env)
|
||
kwargs.update(overrides)
|
||
ex = CloudHypervisorExecutor(**kwargs)
|
||
ex.sentinel_resolver = _public_resolver
|
||
ex.audit_callback = lambda e, t, d, k=None: self.audit.append(
|
||
(e, t.get("id") if isinstance(t, dict) else t, d, k))
|
||
return ex
|
||
|
||
def _default_runner(self, cmd, timeout=None):
|
||
self.calls.append(list(cmd))
|
||
if cmd[0] == "ssh":
|
||
if cmd[-1] == "true":
|
||
return FakeResult(0, "", "")
|
||
if "dispatch" in cmd:
|
||
return FakeResult(0, json.dumps(
|
||
{"status": "completed", "text": "ok-dispatch"}), "")
|
||
if "goal" in cmd or "goal-resume" in cmd:
|
||
return FakeResult(0, "goal met after rounds\n", "")
|
||
if "status" in cmd:
|
||
return FakeResult(0, json.dumps(
|
||
{"status": "completed", "text": "ok-status"}), "")
|
||
return FakeResult(0, "", "")
|
||
return FakeResult(0, "", "")
|
||
|
||
def _default_api(self, socket_path, method, path, payload=None):
|
||
self.api_calls.append((method, path, payload))
|
||
return (200, {})
|
||
|
||
def _default_launcher(self, cmd):
|
||
self.launches.append(list(cmd))
|
||
return FakeProc(alive=True)
|
||
|
||
def _task(self, task_id="abc123"):
|
||
return {"id": task_id, "prompt": "做一个加法函数",
|
||
"project_id": "proj1"}
|
||
|
||
|
||
# -- preflight ---------------------------------------------------------
|
||
class PreflightTest(ChvCase):
|
||
def test_all_present_passes(self):
|
||
self.assertTrue(self.make_exec().preflight())
|
||
|
||
def test_missing_binary_reports_it(self):
|
||
ex = self.make_exec(binary="/tmp/no-such-chv-binary")
|
||
with self.assertRaises(ChvFailure) as ctx:
|
||
ex.preflight()
|
||
self.assertEqual(ctx.exception.kind, "binary_missing")
|
||
self.assertIn("cloud-hypervisor", str(ctx.exception))
|
||
|
||
def test_missing_ch_remote_reports_it(self):
|
||
ex = self.make_exec(ch_remote="/tmp/no-such-ch-remote")
|
||
with self.assertRaises(ChvFailure) as ctx:
|
||
ex.preflight()
|
||
self.assertEqual(ctx.exception.kind, "binary_missing")
|
||
self.assertIn("ch-remote", str(ctx.exception))
|
||
|
||
def test_missing_kvm_reports_it(self):
|
||
ex = self.make_exec(kvm_path="/tmp/no-such-kvm")
|
||
with self.assertRaises(ChvFailure) as ctx:
|
||
ex.preflight()
|
||
self.assertEqual(ctx.exception.kind, "kvm_missing")
|
||
self.assertIn("/tmp/no-such-kvm", str(ctx.exception))
|
||
|
||
def test_missing_cpu_flag_reports_it(self):
|
||
with open(self.cpuinfo, "w") as fh:
|
||
fh.write("flags : fpu sse\n")
|
||
with self.assertRaises(ChvFailure) as ctx:
|
||
self.make_exec().preflight()
|
||
self.assertEqual(ctx.exception.kind, "cpu_flag_missing")
|
||
self.assertIn("vmx", str(ctx.exception))
|
||
|
||
def test_missing_kernel_reports_it(self):
|
||
ex = self.make_exec(kernel="/tmp/no-such-vmlinux")
|
||
with self.assertRaises(ChvFailure) as ctx:
|
||
ex.preflight()
|
||
self.assertEqual(ctx.exception.kind, "kernel_missing")
|
||
|
||
def test_missing_image_reports_it(self):
|
||
ex = self.make_exec(rootfs="/tmp/no-such-disk.raw")
|
||
with self.assertRaises(ChvFailure) as ctx:
|
||
ex.preflight()
|
||
self.assertEqual(ctx.exception.kind, "image_missing")
|
||
|
||
def test_missing_ssh_key_reports_it(self):
|
||
ex = self.make_exec(ssh_key="/tmp/no-such-key")
|
||
with self.assertRaises(ChvFailure) as ctx:
|
||
ex.preflight()
|
||
self.assertEqual(ctx.exception.kind, "ssh_key_missing")
|
||
|
||
def test_missing_egress_is_fail_closed(self):
|
||
ex = self.make_exec(env_extra={"CHV_EGRESS_PROXY": ""})
|
||
# env_extra 空值会被 __init__ 的 or 链忽略，故改用显式 env：
|
||
ex.egress_proxy = None
|
||
ex.allow_direct_egress = False
|
||
with self.assertRaises(ChvFailure) as ctx:
|
||
ex.preflight()
|
||
self.assertEqual(ctx.exception.kind, "egress_not_configured")
|
||
|
||
def test_multiple_missing_lists_all(self):
|
||
ex = self.make_exec(binary="/tmp/no-chv", kernel="/tmp/no-kern")
|
||
with self.assertRaises(ChvFailure) as ctx:
|
||
ex.preflight()
|
||
text = str(ctx.exception)
|
||
self.assertIn("cloud-hypervisor", text)
|
||
self.assertIn("内核", text)
|
||
|
||
def test_dispatch_missing_dependency_does_not_call_runner(self):
|
||
ex = self.make_exec(binary="/tmp/no-chv")
|
||
with self.assertRaises(ChvFailure):
|
||
ex.dispatch_task(self._task())
|
||
self.assertEqual(self.calls, [])
|
||
|
||
|
||
# -- 配置与命令构造 -----------------------------------------------------
|
||
class ConfigCommandTest(ChvCase):
|
||
def test_vm_config_shape_and_memory_bytes(self):
|
||
ex = self.make_exec()
|
||
# feat-chv-secfix（L2）：磁盘须在 state_dir 内。
|
||
disk = os.path.join(self.state_dir, "x", "disk.raw")
|
||
cfg = ex.build_vm_config(disk)
|
||
self.assertEqual(cfg["cpus"], {"boot_vcpus": 2,
|
||
"max_vcpus": 2})
|
||
self.assertEqual(cfg["memory"]["size"], 2 * 1024 ** 3)
|
||
self.assertEqual(cfg["payload"]["kernel"], self.kernel)
|
||
self.assertEqual(cfg["disks"],
|
||
[{"path": disk, "image_type": "Raw"}])
|
||
self.assertEqual(cfg["payload"]["cmdline"],
|
||
"console=hvc0 root=/dev/vda1 rw")
|
||
|
||
def test_memory_parsing(self):
|
||
self.assertEqual(_parse_memory_bytes("512M"), 512 * 1024 ** 2)
|
||
self.assertEqual(_parse_memory_bytes("1024"), 1024)
|
||
self.assertEqual(_parse_memory_bytes("1K"), 1024)
|
||
with self.assertRaises(ValueError):
|
||
_parse_memory_bytes("abc")
|
||
|
||
def test_rootfs_image_alias(self):
|
||
ex = self.make_exec(rootfs=None, env_extra={})
|
||
# rootfs=None 时 __init__ 仍从 env 取不到（env 无 CHV_*），
|
||
# 故用 env 含 CHV_IMAGE 的构造验证别名：
|
||
env = {"CHV_EGRESS_PROXY": "http://10.0.0.1:8888",
|
||
"CHV_IMAGE": self.rootfs}
|
||
ex2 = CloudHypervisorExecutor(
|
||
binary=self.binary, ch_remote=self.ch_remote,
|
||
kernel=self.kernel, ssh_key=self.ssh_key,
|
||
kvm_path=self.kvm, cpuinfo_path=self.cpuinfo,
|
||
state_dir=self.state_dir, env=env,
|
||
sleeper=lambda s: None)
|
||
self.assertEqual(ex2.rootfs, self.rootfs)
|
||
|
||
def test_manual_vm_command_shape(self):
|
||
ex = self.make_exec()
|
||
self.assertFalse(hasattr(ex, "build_vm_command"))
|
||
cmd = ex._build_vm_command(self._task())
|
||
self.assertEqual(cmd[0], self.binary)
|
||
self.assertIn("--api-socket", cmd)
|
||
self.assertIn("--cpus", cmd)
|
||
self.assertIn("boot=2", cmd)
|
||
self.assertIn("size=2G", cmd)
|
||
self.assertIn("--kernel", cmd)
|
||
self.assertIn("--disk", cmd)
|
||
|
||
def test_ssh_command_has_key_path_not_secret_value(self):
|
||
ex = self.make_exec()
|
||
cmd = ex.build_ssh_command(self._task(), ["true"])
|
||
joined = " ".join(cmd)
|
||
self.assertIn(self.ssh_key, joined)
|
||
self.assertIn("BatchMode=yes", joined)
|
||
self.assertNotIn("SECRET", joined)
|
||
# egress 代理地址在 env 前缀里（不是密钥），且只在代理已设时出现：
|
||
self.assertIn("HTTPS_PROXY=http://10.0.0.1:8888", joined)
|
||
# 本轨安全默认变化（#216 的 no + /dev/null 已废除）：
|
||
# host key 钉死在每 VM 的 known_hosts 文件上：
|
||
self.assertIn("StrictHostKeyChecking=yes", joined)
|
||
self.assertNotIn("StrictHostKeyChecking=no", joined)
|
||
self.assertNotIn("/dev/null", joined)
|
||
self.assertIn(os.path.join(self.state_dir,
|
||
"chv-abc123", "known_hosts"),
|
||
joined)
|
||
|
||
def test_isolation_names_and_disks_differ(self):
|
||
ex = self.make_exec()
|
||
a, b = {"id": "task-a"}, {"id": "task-b"}
|
||
self.assertNotEqual(ex.vm_name(a), ex.vm_name(b))
|
||
self.assertNotEqual(ex.api_socket(a), ex.api_socket(b))
|
||
self.assertNotEqual(ex._provision_task_disk(a),
|
||
ex._provision_task_disk(b))
|
||
# copy 模式：每任务磁盘是基础镜像的真实副本，内容一致、
|
||
# 路径不同（两个任务绝不写同一文件）：
|
||
with open(ex._provision_task_disk(a), "rb") as fh:
|
||
self.assertEqual(fh.read(), b"R" * 100)
|
||
|
||
def test_disk_quota_gate(self):
|
||
ex = self.make_exec(env_extra={"CHV_DISK_GB": "0.00000001"})
|
||
with self.assertRaises(ChvFailure) as ctx:
|
||
ex.wake_vm(self._task())
|
||
self.assertEqual(ctx.exception.kind, "disk_quota_exceeded")
|
||
|
||
def test_capacity_gate_rejects_not_fallback(self):
|
||
ex = self.make_exec(env_extra={"CHV_MAX_VMS": "1"})
|
||
ex._vms["chv-other"] = FakeProc(alive=True)
|
||
with self.assertRaises(ChvFailure) as ctx:
|
||
ex.wake_vm(self._task())
|
||
self.assertEqual(ctx.exception.kind, "host_capacity_exceeded")
|
||
self.assertEqual(self.launches, [])
|
||
|
||
|
||
# -- API 唤醒 -----------------------------------------------------------
|
||
class WakeApiTest(ChvCase):
|
||
def test_wake_uses_api_mode_not_direct_command(self):
|
||
ex = self.make_exec()
|
||
out = ex.wake_vm(self._task())
|
||
self.assertEqual(out["state"], "running")
|
||
self.assertTrue(out["ready"])
|
||
self.assertEqual(out["mode"], "cold")
|
||
# launcher 只带 --api-socket，不带内核/磁盘（API 模式证明）：
|
||
self.assertEqual(len(self.launches), 1)
|
||
launch = self.launches[0]
|
||
self.assertIn("--api-socket", launch)
|
||
self.assertNotIn("--kernel", launch)
|
||
self.assertNotIn("--disk", launch)
|
||
# API 调用顺序：ping → create → boot：
|
||
kinds = [(m, p) for m, p, _ in self.api_calls]
|
||
self.assertEqual(kinds[0], ("GET", "/vmm.ping"))
|
||
self.assertEqual(kinds[1], ("PUT", "/vm.create"))
|
||
self.assertEqual(kinds[2], ("PUT", "/vm.boot"))
|
||
create_payload = self.api_calls[1][2]
|
||
# CHV 54：内核走 payload.kernel 纯字符串（旧顶层
|
||
# kernel:{path} 会被忽略、vm.boot 400）。
|
||
self.assertEqual(create_payload["payload"]["kernel"], self.kernel)
|
||
self.assertEqual(create_payload["disks"][0]["image_type"], "Raw")
|
||
self.assertTrue(create_payload["disks"][0]["path"].endswith(
|
||
"disk.raw"))
|
||
events = [e for e, _, _, _ in self.audit]
|
||
self.assertIn("vm_create", events)
|
||
self.assertIn("vm_boot", events)
|
||
self.assertIn("vm_ready", events)
|
||
|
||
def test_wake_api_timeout_kills_proc(self):
|
||
def never_ok(socket_path, method, path, payload=None):
|
||
self.api_calls.append((method, path, payload))
|
||
return (500, {})
|
||
ex = self.make_exec(api_client=never_ok, api_timeout=3)
|
||
proc_holder = []
|
||
|
||
def launcher(cmd):
|
||
proc = FakeProc(alive=True)
|
||
proc_holder.append(proc)
|
||
self.launches.append(list(cmd))
|
||
return proc
|
||
ex.launcher = launcher
|
||
with self.assertRaises(ChvFailure) as ctx:
|
||
ex.wake_vm(self._task())
|
||
self.assertEqual(ctx.exception.kind, "api_timeout")
|
||
self.assertTrue(proc_holder[0].terminated)
|
||
self.assertNotIn("chv-abc123", ex._vms)
|
||
|
||
def test_guest_ready_timeout_shuts_down(self):
|
||
def ssh_never(cmd, timeout=None):
|
||
self.calls.append(list(cmd))
|
||
if cmd[0] == "ssh":
|
||
return FakeResult(1, "", "")
|
||
return FakeResult(0, "", "")
|
||
ex = self.make_exec(runner=ssh_never, ready_timeout=10,
|
||
ready_interval=5)
|
||
with self.assertRaises(ChvFailure) as ctx:
|
||
ex.wake_vm(self._task())
|
||
self.assertEqual(ctx.exception.kind, "guest_not_ready")
|
||
# 超时后关 VM：runner 里有 ch-remote shutdown：
|
||
self.assertTrue(any("shutdown" in c for c in self.calls))
|
||
|
||
|
||
# -- 暂停 / 快照 / 休眠 / 关闭 / 清理 -----------------------------------
|
||
class LifecycleTest(ChvCase):
|
||
def test_pause_resume(self):
|
||
ex = self.make_exec()
|
||
self.assertTrue(ex.pause_vm(self._task()))
|
||
self.assertTrue(ex.resume_vm(self._task()))
|
||
kinds = [c[-1] for c in self.calls if c[0] == self.ch_remote]
|
||
self.assertEqual(kinds, ["pause", "resume"])
|
||
|
||
def test_restore_then_explicit_resume_order(self):
|
||
# feat-chv-secfix（L2/H4）：本测试走实验室 keyscan
|
||
# 路径 + 声明快照目录（生产默认 provisioned + 白名单
|
||
# 目录，旧 file:///snap/1 不再直通）。
|
||
ex = self.make_exec(env_extra={"CHV_SNAPSHOT_DIR": "/snap"})
|
||
out = ex.restore_vm(self._task(), "file:///snap/1")
|
||
self.assertEqual(out["mode"], "restored")
|
||
self.assertTrue(out["ready"])
|
||
kinds = [c[-1] if c[-1] != "restore" else c[-1]
|
||
for c in self.calls if c[0] == self.ch_remote]
|
||
# ch-remote 调用里 restore 必须在 resume 之前：
|
||
flat = []
|
||
for c in self.calls:
|
||
if c[0] == self.ch_remote:
|
||
flat.append(c[-1] if c[-1].startswith("source_url")
|
||
else c[-1])
|
||
self.assertIn("resume", flat)
|
||
restore_idx = next(i for i, c in enumerate(self.calls)
|
||
if c[0] == self.ch_remote
|
||
and any(a.startswith("source_url")
|
||
for a in c))
|
||
resume_idx = next(i for i, c in enumerate(self.calls)
|
||
if c[0] == self.ch_remote and c[-1] == "resume")
|
||
self.assertLess(restore_idx, resume_idx)
|
||
|
||
def test_hibernate_is_pause_snapshot_shutdown(self):
|
||
ex = self.make_exec()
|
||
# feat-chv-fix（P1-7）：file:// 快照须真实存在且非空，
|
||
# 否则 hibernate 不 shutdown；用临时文件作快照目标。
|
||
# feat-chv-secfix（L2）：快照须在白名单目录内，放在
|
||
# state_dir 下。
|
||
snap = os.path.join(self.state_dir, "h1.snap")
|
||
os.makedirs(self.state_dir, exist_ok=True)
|
||
with open(snap, "wb") as fh:
|
||
fh.write(b"S" * 64)
|
||
snap_url = "file://" + snap
|
||
out = ex.hibernate_vm(self._task(), snap_url)
|
||
self.assertEqual(out["state"], "hibernated")
|
||
self.assertEqual(out["snapshot"], snap_url)
|
||
kinds = [c[-1] for c in self.calls if c[0] == self.ch_remote]
|
||
self.assertEqual(kinds[0], "pause")
|
||
self.assertTrue(any(a.startswith("source_url")
|
||
for c in self.calls for a in c))
|
||
self.assertIn("shutdown", kinds)
|
||
# 从休眠唤醒 = restore + resume：
|
||
ex2 = self.make_exec()
|
||
out2 = ex2.wake_from_hibernate(self._task(), snap_url)
|
||
self.assertEqual(out2["mode"], "restored")
|
||
|
||
def test_stop_dead_proc(self):
|
||
ex = self.make_exec()
|
||
proc = FakeProc(alive=False)
|
||
ex._vms["chv-abc123"] = proc
|
||
sock = os.path.join(self.state_dir, "chv-abc123.sock")
|
||
os.makedirs(self.state_dir, exist_ok=True)
|
||
with open(sock, "w") as _fh:
|
||
_fh.write("")
|
||
self.assertTrue(ex.stop_vm(self._task()))
|
||
self.assertFalse(os.path.exists(sock))
|
||
self.assertNotIn("chv-abc123", ex._vms)
|
||
|
||
def test_stop_already_closed(self):
|
||
ex = self.make_exec()
|
||
self.assertFalse(ex.stop_vm(self._task()))
|
||
# 已关仍尽力调过一次 ch-remote shutdown：
|
||
self.assertTrue(any("shutdown" in c for c in self.calls))
|
||
|
||
def test_stop_socket_residual(self):
|
||
ex = self.make_exec()
|
||
sock = os.path.join(self.state_dir, "chv-abc123.sock")
|
||
os.makedirs(self.state_dir, exist_ok=True)
|
||
with open(sock, "w") as _fh:
|
||
_fh.write("")
|
||
self.assertTrue(ex.stop_vm(self._task()))
|
||
self.assertFalse(os.path.exists(sock))
|
||
|
||
def test_cleanup_skips_live_and_non_chv(self):
|
||
ex = self.make_exec()
|
||
os.makedirs(self.state_dir, exist_ok=True)
|
||
live = os.path.join(self.state_dir, "chv-live.sock")
|
||
with open(live, "w") as _fh:
|
||
_fh.write("")
|
||
ex._vms["chv-live"] = FakeProc(alive=True)
|
||
dead = os.path.join(self.state_dir, "chv-dead.sock")
|
||
with open(dead, "w") as _fh:
|
||
_fh.write("")
|
||
other = os.path.join(self.state_dir, "other.sock")
|
||
with open(other, "w") as _fh:
|
||
_fh.write("")
|
||
cleaned = ex.cleanup_stale_vms()
|
||
self.assertEqual(cleaned, ["chv-dead"])
|
||
self.assertTrue(os.path.exists(live))
|
||
self.assertTrue(os.path.exists(other))
|
||
self.assertFalse(os.path.exists(dead))
|
||
|
||
def test_cleanup_missing_state_dir(self):
|
||
ex = self.make_exec()
|
||
self.assertEqual(ex.cleanup_stale_vms(), [])
|
||
|
||
def test_recycle_preserves_checkpoints_not_temp(self):
|
||
ex = self.make_exec()
|
||
out = ex.recycle_vm(self._task())
|
||
self.assertEqual(out["state"], "recycled")
|
||
self.assertIn("task_checkpoints", out["preserved"])
|
||
self.assertIn("task_attempts", out["preserved"])
|
||
self.assertIn("guest /tmp", out["lost"])
|
||
|
||
|
||
# -- 预热池 --------------------------------------------------------------
|
||
class WarmPoolTest(ChvCase):
|
||
def test_prewarm_acquire_release(self):
|
||
ex = self.make_exec()
|
||
pool = WarmPool(ex, size=2, snapshot_url="file:///snap/base")
|
||
added = pool.prewarm()
|
||
self.assertEqual(len(added), 2)
|
||
self.assertEqual(pool.warm_count(), 2)
|
||
first = pool.acquire(self._task("t1"))
|
||
self.assertEqual(first["mode"], "warm")
|
||
second = pool.acquire(self._task("t2"))
|
||
self.assertEqual(second["mode"], "warm")
|
||
third = pool.acquire(self._task("t3"))
|
||
self.assertEqual(third["mode"], "cold")
|
||
# release 一律销毁，不放回池：
|
||
self.assertTrue(pool.release(self._task("t1"),
|
||
pool_vm=added[0]))
|
||
self.assertEqual(pool.warm_count(), 0)
|
||
|
||
def test_prewarm_without_snapshot_is_empty(self):
|
||
ex = self.make_exec()
|
||
pool = WarmPool(ex, size=2, snapshot_url=None)
|
||
self.assertEqual(pool.prewarm(), [])
|
||
self.assertEqual(pool.acquire(self._task())["mode"], "cold")
|
||
|
||
|
||
# -- dispatch / goal（wake → 跑 → finally 关） ---------------------------
|
||
class DispatchTest(ChvCase):
|
||
def test_dispatch_wakes_runs_shuts_down(self):
|
||
ex = self.make_exec()
|
||
out = ex.dispatch_task(self._task())
|
||
self.assertEqual(out["executor"], "cloud-hypervisor")
|
||
self.assertEqual(out["status"], "completed")
|
||
self.assertIn("ok-dispatch", out["stdout"])
|
||
self.assertEqual(out["vm"], "chv-abc123")
|
||
# finally 关 VM：ch-remote shutdown 在 dispatch 之后出现：
|
||
shutdown_idx = next(i for i, c in enumerate(self.calls)
|
||
if c[0] == self.ch_remote
|
||
and c[-1] == "shutdown")
|
||
ssh_idx = next(i for i, c in enumerate(self.calls)
|
||
if c[0] == "ssh" and "dispatch" in c)
|
||
self.assertLess(ssh_idx, shutdown_idx)
|
||
self.assertNotIn("chv-abc123", ex._vms)
|
||
|
||
def test_goal_met(self):
|
||
ex = self.make_exec()
|
||
out = ex.goal_task({"id": "g1", "prompt": "做目标",
|
||
"goal_mode": True})
|
||
self.assertEqual(out["status"], "completed")
|
||
self.assertTrue(out["goal_met"])
|
||
|
||
def test_goal_resume_breakpoint(self):
|
||
ex = self.make_exec()
|
||
out = ex.goal_resume_breakpoint_task({"id": "g1",
|
||
"prompt": "续跑"})
|
||
self.assertEqual(out["status"], "completed")
|
||
self.assertTrue(out["goal_met"])
|
||
|
||
def test_execute_is_ssh_status_only(self):
|
||
ex = self.make_exec()
|
||
# 本轨 execute 语义门禁：只读已运行 VM。先登记一个活
|
||
# 的 VMM 进程表示 VM 已醒（未登记时 execute 报
|
||
# vm_not_running，见 test_chv_optimize.py）：
|
||
ex._vms["chv-abc123"] = FakeProc(alive=True)
|
||
out = ex.execute(self._task())
|
||
self.assertEqual(out["executor"], "cloud-hypervisor")
|
||
self.assertIn("ok-status", out["stdout"])
|
||
# execute 不 wake（launcher 未调用）：
|
||
self.assertEqual(self.launches, [])
|
||
|
||
|
||
# -- worker / autoscaler / server 接线 -----------------------------------
|
||
class WiringTest(ChvCase):
|
||
def test_worker_executor_name_and_cli_builder(self):
|
||
from cloud import worker as worker_mod
|
||
from cloud.sandbox import CloudHypervisorExecutor as CHV
|
||
ex = self.make_exec()
|
||
w = worker_mod.Worker.__new__(worker_mod.Worker)
|
||
w.executor = ex
|
||
self.assertEqual(w._executor_name(), "cloud-hypervisor")
|
||
|
||
class Args:
|
||
executor = "cloud-hypervisor"
|
||
built = worker_mod._build_cli_executor(Args(), None)
|
||
self.assertIsInstance(built, CHV)
|
||
|
||
def test_autoscaler_passthrough(self):
|
||
from cloud.autoscaler import Autoscaler
|
||
scaler = Autoscaler("/tmp/no.db", [], executor="cloud-hypervisor")
|
||
cmd = scaler._worker_command("w1")
|
||
self.assertIn("--executor", cmd)
|
||
self.assertEqual(cmd[cmd.index("--executor") + 1],
|
||
"cloud-hypervisor")
|
||
scaler2 = Autoscaler("/tmp/no.db",
|
||
["--executor", "docker"],
|
||
executor="cloud-hypervisor")
|
||
cmd2 = scaler2._worker_command("w1")
|
||
self.assertEqual(cmd2.count("--executor"), 1)
|
||
self.assertEqual(cmd2[cmd2.index("--executor") + 1], "docker")
|
||
|
||
def test_server_default_executor(self):
|
||
from cloud.server import _default_executor
|
||
with mock.patch.dict(os.environ,
|
||
{"CLOUD_DEFAULT_EXECUTOR":
|
||
"cloud-hypervisor"}):
|
||
self.assertEqual(_default_executor(), "cloud-hypervisor")
|
||
with mock.patch.dict(os.environ,
|
||
{"CLOUD_DEFAULT_EXECUTOR": "bogus"}):
|
||
self.assertEqual(_default_executor(), "docker")
|
||
|
||
|
||
class ServerWhitelistTest(unittest.TestCase):
|
||
"""server 白名单端到端：真实 HTTP + 临时库 + 临时端口。"""
|
||
|
||
def setUp(self):
|
||
self.tmp = tempfile.TemporaryDirectory()
|
||
self.db_path = os.path.join(self.tmp.name, "control.db")
|
||
try:
|
||
from cloud.auth import AuthStore
|
||
from cloud.server import create_server
|
||
except ImportError:
|
||
sys.path.insert(0, str(Path(__file__).resolve().parents[2]))
|
||
from cloud.auth import AuthStore
|
||
from cloud.server import create_server
|
||
self.auth = AuthStore(self.db_path)
|
||
self.owner_id, self.owner_key = self.auth.create_user("owner")
|
||
old = os.environ.get("CODEX_GLOBAL_ADMINS")
|
||
self._old_admins = old
|
||
os.environ["CODEX_GLOBAL_ADMINS"] = self.owner_id
|
||
self.server = create_server(self.db_path, "127.0.0.1", 0)
|
||
self.port = self.server.server_address[1]
|
||
self.thread = threading.Thread(
|
||
target=self.server.serve_forever, daemon=True)
|
||
self.thread.start()
|
||
status, self.project = self._req(
|
||
"POST", "/projects", {"name": "demo"}, self.owner_key)
|
||
self.assertEqual(status, 201, self.project)
|
||
|
||
def tearDown(self):
|
||
self.server.shutdown()
|
||
self.server.server_close()
|
||
self.thread.join(timeout=5)
|
||
try:
|
||
self.auth.conn.close()
|
||
except Exception:
|
||
pass
|
||
self.tmp.cleanup()
|
||
if self._old_admins is None:
|
||
os.environ.pop("CODEX_GLOBAL_ADMINS", None)
|
||
else:
|
||
os.environ["CODEX_GLOBAL_ADMINS"] = self._old_admins
|
||
|
||
def _req(self, method, path, body=None, key=None):
|
||
conn = http.client.HTTPConnection("127.0.0.1", self.port,
|
||
timeout=10)
|
||
hdrs = {}
|
||
payload = None
|
||
if body is not None:
|
||
payload = json.dumps(body).encode("utf-8")
|
||
hdrs["Content-Type"] = "application/json"
|
||
if key is not None:
|
||
hdrs["X-API-Key"] = key
|
||
try:
|
||
conn.request(method, path, body=payload, headers=hdrs)
|
||
resp = conn.getresponse()
|
||
data = resp.read()
|
||
finally:
|
||
conn.close()
|
||
try:
|
||
parsed = json.loads(data.decode("utf-8")) if data else None
|
||
except ValueError:
|
||
parsed = None
|
||
return resp.status, parsed
|
||
|
||
def test_task_executor_cloud_hypervisor_accepted(self):
|
||
status, task = self._req(
|
||
"POST", "/tasks",
|
||
{"project_id": self.project["id"], "prompt": "x",
|
||
"executor": "cloud-hypervisor"}, self.owner_key)
|
||
self.assertEqual(status, 201, task)
|
||
self.assertEqual(task["executor"], "cloud-hypervisor")
|
||
|
||
def test_task_executor_bogus_rejected(self):
|
||
status, body = self._req(
|
||
"POST", "/tasks",
|
||
{"project_id": self.project["id"], "prompt": "x",
|
||
"executor": "bogus"}, self.owner_key)
|
||
self.assertEqual(status, 400)
|
||
self.assertIn("cloud-hypervisor", body["error"])
|
||
|
||
|
||
if __name__ == "__main__":
|
||
unittest.main()
|