feat(chv): 阶段 1 持久 VM、overlay 分层与预热快照
- disk.py：overlay 模式（qcow2 backing file 指只读 base， 建盘/复用/重建校验 + qemu-img fail-closed）、数据盘 data.qcow2 独立供给（不进快照/GC/stop）、配额增量口径 （overlay 本次增量 0）、快照登记 tenant 字段、每 VM tenant.json、持久登记 persistent/<tenant>.json、快照 登记按 URL 查找。 - config.py：CHV_DISK_MODE 白名单加 overlay（默认仍 copy； 错误文案保留旧子串，防回归测试钉死）。 - executor.py：VmConfig image_type 按后缀（.qcow2→Qcow2， 其余 Raw）+ 数据盘第三块盘；overlay 供给写 tenant 标记； snapshot_vm 登记带 tenant，_validate_restore 归属校验 （跨用户 snapshot_tenant_mismatch 拒绝 + 审计，legacy 未标记/未登记放行并警告）；stop/GC 在 overlay 模式下 保护持久登记/快照认领的盘；持久 VM（prewarm/acquire/ suspend/dispatch_task_persistent/recycle_persistent_vm， CHV_WARM_POOL_SIZE>=1 启用，=0 退化冷启旧语义）， resume 只认 vm.info=Paused + socket 活性 + 归属一致。 - tests：cloud/tests/test_chv_opt_stage1.py 26 项（overlay、 tenant、持久池、跨用户拒绝、两连任务、显式回收）。 回归：discover test_chv_*.py 与 test_cloud_hypervisor.py 全过（结构契约：函数 ≤50 行、公开 API 全注解、文案钉死）。 不推、不合、不部署；只本地分支。
This commit is contained in:
host committed
1 parent
c904ae9358
commit
7a6fae7993
4 files changed
+1643
-65
No files matched your search
@@ -0,0 +1,657 @@
|
||||
"""Cloud Hypervisor 优化落地阶段 1（feat-chv-opt-landing）测试。
|
||||
|
||||
只用标准库与 mock runner/launcher/api_client，绝不启动真实
|
||||
VM（与 test_chv_optimize.py 同款边界）。覆盖：
|
||||
- overlay 分层：qemu-img backing 建盘/复用/重建、配额增量 0、
|
||||
VmConfig image_type Qcow2、数据盘独立供给与永不 GC/随 stop 删
|
||||
- tenant 归属：快照登记 tenant 字段、overlay tenant.json、
|
||||
持久登记、跨用户恢复拒绝（_validate_restore）
|
||||
- 持久 VM 与预热池：prewarm 幂等、acquire resume/warm_hit、
|
||||
池空回落冷启、warm_pool_size=0 退化旧语义、暂停回池、
|
||||
dispatch_task_persistent 两连任务、显式回收删盘
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
from unittest import mock
|
||||
|
||||
try:
|
||||
from cloud.chv import ChvFailure
|
||||
from cloud.chv import CloudHypervisorExecutor
|
||||
from cloud.chv import disk as chv_disk
|
||||
from cloud.chv.disk import (
|
||||
clear_persistent_vm,
|
||||
create_overlay_disk,
|
||||
delete_data_disk,
|
||||
delete_task_disk,
|
||||
find_snapshot_entry,
|
||||
provision_data_disk,
|
||||
provision_task_disk,
|
||||
read_persistent_vm,
|
||||
read_vm_tenant,
|
||||
record_snapshot,
|
||||
write_persistent_vm,
|
||||
write_vm_tenant,
|
||||
)
|
||||
except ImportError: # discover -s cloud/tests 时把项目根目录补进 sys.path
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[2]))
|
||||
from cloud.chv import ChvFailure
|
||||
from cloud.chv import CloudHypervisorExecutor
|
||||
from cloud.chv import disk as chv_disk
|
||||
from cloud.chv.disk import (
|
||||
clear_persistent_vm,
|
||||
create_overlay_disk,
|
||||
delete_data_disk,
|
||||
delete_task_disk,
|
||||
find_snapshot_entry,
|
||||
provision_data_disk,
|
||||
provision_task_disk,
|
||||
read_persistent_vm,
|
||||
read_vm_tenant,
|
||||
record_snapshot,
|
||||
write_persistent_vm,
|
||||
write_vm_tenant,
|
||||
)
|
||||
|
||||
|
||||
class FakeResult:
|
||||
def __init__(self, returncode=0, stdout="", stderr=""):
|
||||
self.returncode = returncode
|
||||
self.stdout = stdout
|
||||
self.stderr = stderr
|
||||
|
||||
|
||||
class FakeProc:
|
||||
def __init__(self, alive=True):
|
||||
self._alive = alive
|
||||
|
||||
def poll(self):
|
||||
return None if self._alive else 1
|
||||
|
||||
def terminate(self):
|
||||
self._alive = False
|
||||
|
||||
def wait(self, timeout=None):
|
||||
return 0
|
||||
|
||||
def kill(self):
|
||||
self._alive = False
|
||||
|
||||
|
||||
def _write(path, content=b"x"):
|
||||
with open(path, "wb") as fh:
|
||||
fh.write(content)
|
||||
os.chmod(path, 0o755)
|
||||
return path
|
||||
|
||||
|
||||
def _fake_qemu_runner_factory(files_to_create):
|
||||
"""fake qemu-img：create 写空文件，info 回 backing JSON。
|
||||
|
||||
files_to_create：overlay 路径 -> backing 绝对路径（info
|
||||
应答用）；create 时按命令写文件并登记 backing。"""
|
||||
state = {"backing": dict(files_to_create)}
|
||||
|
||||
def runner(cmd, **kwargs):
|
||||
if cmd[1] == "create" and cmd[2] == "-f":
|
||||
# overlay：... -b <base> -F <fmt> <overlay>（目标在末尾）；
|
||||
# 数据盘：... <path> <size>G（目标在倒数第二）。
|
||||
target = cmd[-1] if "-b" in cmd else cmd[4]
|
||||
with open(target, "wb") as fh:
|
||||
fh.write(b"QFI\xfb fake-qcow2")
|
||||
if "-b" in cmd:
|
||||
state["backing"][target] = cmd[cmd.index("-b") + 1]
|
||||
return FakeResult(0, "", "")
|
||||
if cmd[1] == "info":
|
||||
target = cmd[-1]
|
||||
backing = state["backing"].get(target)
|
||||
payload = {"format": "qcow2"}
|
||||
if backing:
|
||||
payload["backing-filename"] = backing
|
||||
return FakeResult(0, json.dumps(payload), "")
|
||||
return FakeResult(0, "", "")
|
||||
|
||||
runner.state = state
|
||||
return runner
|
||||
|
||||
|
||||
class ChvOpt1Case(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.tmp = tempfile.TemporaryDirectory()
|
||||
self.dir = self.tmp.name
|
||||
self.binary = _write(os.path.join(self.dir, "cloud-hypervisor"))
|
||||
self.ch_remote = _write(os.path.join(self.dir, "ch-remote"))
|
||||
self.kernel = _write(os.path.join(self.dir, "vmlinux"))
|
||||
self.rootfs = _write(os.path.join(self.dir, "base.qcow2"),
|
||||
b"R" * 100)
|
||||
self.ssh_key = _write(os.path.join(self.dir, "id_chv"))
|
||||
self.kvm = _write(os.path.join(self.dir, "kvm"))
|
||||
self.cpuinfo = os.path.join(self.dir, "cpuinfo")
|
||||
with open(self.cpuinfo, "w") as fh:
|
||||
fh.write("flags : fpu vmx sse\n")
|
||||
self.state_dir = os.path.join(self.dir, "state")
|
||||
self.calls = []
|
||||
self.api_calls = []
|
||||
self.launches = []
|
||||
self.vm_states = {}
|
||||
|
||||
def tearDown(self):
|
||||
self.tmp.cleanup()
|
||||
|
||||
def make_exec(self, runner=None, api_client=None, launcher=None,
|
||||
env_extra=None, **overrides):
|
||||
env = {"CHV_EGRESS_PROXY": "http://10.0.0.1:8888",
|
||||
"CHV_HOST_KEY_MODE": "keyscan",
|
||||
"CHV_SSH_FORWARD": "1"}
|
||||
if env_extra:
|
||||
env.update(env_extra)
|
||||
if runner is None:
|
||||
runner = self._default_runner
|
||||
if api_client is None:
|
||||
api_client = self._default_api
|
||||
if launcher is None:
|
||||
launcher = self._default_launcher
|
||||
kwargs = dict(
|
||||
binary=self.binary, ch_remote=self.ch_remote,
|
||||
kernel=self.kernel, rootfs=self.rootfs,
|
||||
ssh_key=self.ssh_key, kvm_path=self.kvm,
|
||||
cpuinfo_path=self.cpuinfo, state_dir=self.state_dir,
|
||||
runner=runner, launcher=launcher, api_client=api_client,
|
||||
sleeper=lambda s: None, env=env)
|
||||
kwargs.update(overrides)
|
||||
return CloudHypervisorExecutor(**kwargs)
|
||||
|
||||
def _default_runner(self, cmd, timeout=None):
|
||||
self.calls.append(list(cmd))
|
||||
if cmd[0] == "ssh-keyscan":
|
||||
return FakeResult(
|
||||
0, "[127.0.0.1]:2222 ssh-ed25519 AAAAB3test\n", "")
|
||||
if cmd[0] == "ssh":
|
||||
if cmd[-1] == "true":
|
||||
return FakeResult(0, "", "")
|
||||
if "dispatch" in cmd:
|
||||
return FakeResult(0, json.dumps(
|
||||
{"status": "completed", "text": "ok-dispatch"}), "")
|
||||
return FakeResult(0, "", "")
|
||||
return FakeResult(0, "", "")
|
||||
|
||||
def _default_api(self, socket_path, method, path, payload=None):
|
||||
self.api_calls.append((method, path, payload))
|
||||
if path == "/vm.info":
|
||||
state = self.vm_states.get(
|
||||
os.path.basename(socket_path)[:-len(".sock")])
|
||||
if state:
|
||||
return (200, {"state": state})
|
||||
return (200, {})
|
||||
return (200, {})
|
||||
|
||||
def _default_launcher(self, cmd):
|
||||
self.launches.append(list(cmd))
|
||||
return FakeProc(alive=True)
|
||||
|
||||
def _task(self, task_id="abc123", project="proj1"):
|
||||
return {"id": task_id, "prompt": "做一个加法函数",
|
||||
"project_id": project}
|
||||
|
||||
|
||||
# -- overlay 分层 ------------------------------------------------------------
|
||||
class OverlayDiskTest(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.tmp = tempfile.TemporaryDirectory()
|
||||
self.dir = self.tmp.name
|
||||
self.base = os.path.join(self.dir, "base.qcow2")
|
||||
with open(self.base, "wb") as fh:
|
||||
fh.write(b"R" * 100)
|
||||
self.state = os.path.join(self.dir, "state")
|
||||
|
||||
def tearDown(self):
|
||||
self.tmp.cleanup()
|
||||
|
||||
def test_create_overlay_backing_and_reuse(self):
|
||||
runner = _fake_qemu_runner_factory({})
|
||||
overlay = os.path.join(self.dir, "ov.qcow2")
|
||||
out = create_overlay_disk(self.base, overlay, runner=runner)
|
||||
self.assertEqual(out, overlay)
|
||||
self.assertTrue(os.path.exists(overlay))
|
||||
create_args = runner.state["backing"]
|
||||
self.assertEqual(create_args[overlay],
|
||||
os.path.abspath(self.base))
|
||||
# 第二次：backing 一致即复用，不重建（文件 mtime 不变）。
|
||||
before = os.stat(overlay).st_mtime_ns
|
||||
out2 = create_overlay_disk(self.base, overlay, runner=runner)
|
||||
self.assertEqual(out2, overlay)
|
||||
self.assertEqual(os.stat(overlay).st_mtime_ns, before)
|
||||
|
||||
def test_create_overlay_backing_mismatch_rebuilds(self):
|
||||
other = os.path.join(self.dir, "other.qcow2")
|
||||
with open(other, "wb") as fh:
|
||||
fh.write(b"O" * 100)
|
||||
runner = _fake_qemu_runner_factory({})
|
||||
overlay = os.path.join(self.dir, "ov.qcow2")
|
||||
create_overlay_disk(self.base, overlay, runner=runner)
|
||||
# backing 被换成别的 base：下次建盘须删了重建。
|
||||
runner.state["backing"][overlay] = os.path.abspath(other)
|
||||
out = create_overlay_disk(self.base, overlay, runner=runner)
|
||||
self.assertEqual(out, overlay)
|
||||
self.assertEqual(runner.state["backing"][overlay],
|
||||
os.path.abspath(self.base))
|
||||
|
||||
def test_create_overlay_base_missing_fails(self):
|
||||
runner = _fake_qemu_runner_factory({})
|
||||
with self.assertRaises(ChvFailure) as ctx:
|
||||
create_overlay_disk(os.path.join(self.dir, "nope.qcow2"),
|
||||
os.path.join(self.dir, "ov.qcow2"),
|
||||
runner=runner)
|
||||
self.assertEqual(ctx.exception.kind,
|
||||
"disk_overlay_unsupported")
|
||||
|
||||
def test_create_overlay_unknown_base_suffix_fails(self):
|
||||
weird = os.path.join(self.dir, "base.img")
|
||||
with open(weird, "wb") as fh:
|
||||
fh.write(b"R" * 10)
|
||||
runner = _fake_qemu_runner_factory({})
|
||||
with self.assertRaises(ChvFailure) as ctx:
|
||||
create_overlay_disk(weird,
|
||||
os.path.join(self.dir, "ov.qcow2"),
|
||||
runner=runner)
|
||||
self.assertEqual(ctx.exception.kind,
|
||||
"disk_overlay_unsupported")
|
||||
|
||||
def test_provision_overlay_and_data_disk(self):
|
||||
runner = _fake_qemu_runner_factory({})
|
||||
disk = provision_task_disk(self.state, "chv-a", self.base,
|
||||
"overlay", runner=runner)
|
||||
self.assertEqual(disk,
|
||||
os.path.join(self.state, "chv-a", "disk.qcow2"))
|
||||
self.assertTrue(os.path.exists(disk))
|
||||
data = provision_data_disk(self.state, "chv-a", runner=runner)
|
||||
self.assertEqual(data,
|
||||
os.path.join(self.state, "chv-a", "data.qcow2"))
|
||||
self.assertTrue(os.path.exists(data))
|
||||
# 数据盘已存在即复用，不重建。
|
||||
data2 = provision_data_disk(self.state, "chv-a", runner=runner)
|
||||
self.assertEqual(data2, data)
|
||||
|
||||
def test_delete_task_disk_removes_both_but_not_data(self):
|
||||
runner = _fake_qemu_runner_factory({})
|
||||
provision_task_disk(self.state, "chv-a", self.base,
|
||||
"overlay", runner=runner)
|
||||
provision_data_disk(self.state, "chv-a", runner=runner)
|
||||
raw = os.path.join(self.state, "chv-a", "disk.raw")
|
||||
with open(raw, "wb") as fh:
|
||||
fh.write(b"R")
|
||||
self.assertTrue(delete_task_disk(self.state, "chv-a"))
|
||||
self.assertFalse(os.path.exists(
|
||||
os.path.join(self.state, "chv-a", "disk.qcow2")))
|
||||
self.assertFalse(os.path.exists(raw))
|
||||
# 数据盘还在：删任务盘绝不删用户数据。
|
||||
self.assertTrue(os.path.exists(
|
||||
os.path.join(self.state, "chv-a", "data.qcow2")))
|
||||
self.assertTrue(delete_data_disk(self.state, "chv-a"))
|
||||
self.assertFalse(delete_task_disk(self.state, "chv-a"))
|
||||
|
||||
def test_quota_overlay_increment_zero(self):
|
||||
# state_dir 已用逼近配额：copy 增量（base 大小）超帽，
|
||||
# overlay 增量 0 不超帽。
|
||||
os.makedirs(self.state, exist_ok=True)
|
||||
with open(os.path.join(self.state, "used.bin"), "wb") as fh:
|
||||
fh.write(b"U" * 1000)
|
||||
disk_gb = 1000 / (1024 ** 3) # 1000 字节配额
|
||||
with self.assertRaises(ChvFailure) as ctx:
|
||||
chv_disk.check_disk_quota(self.base, disk_gb,
|
||||
state_dir=self.state)
|
||||
self.assertEqual(ctx.exception.kind, "disk_quota_exceeded")
|
||||
chv_disk.check_disk_quota(self.base, disk_gb,
|
||||
state_dir=self.state,
|
||||
increment_bytes=0)
|
||||
|
||||
def test_disk_mode_overlay_accepted_in_config(self):
|
||||
from cloud.chv.config import build_chv_config
|
||||
cfg = build_chv_config({}, {"CHV_DISK_MODE": "overlay"})
|
||||
self.assertEqual(cfg.disk_mode, "overlay")
|
||||
with self.assertRaises(ValueError):
|
||||
build_chv_config({}, {"CHV_DISK_MODE": "bogus"})
|
||||
|
||||
|
||||
class VmConfigOverlayTest(ChvOpt1Case):
|
||||
def test_qcow2_disks_are_qcow2_image_type(self):
|
||||
ex = self.make_exec(env_extra={"CHV_DISK_MODE": "overlay"})
|
||||
cfg = ex.build_vm_config(
|
||||
os.path.join(self.state_dir, "chv-a", "disk.qcow2"),
|
||||
seed_disk=None,
|
||||
data_disk=os.path.join(self.state_dir, "chv-a",
|
||||
"data.qcow2"))
|
||||
self.assertEqual(cfg["disks"][0]["image_type"], "Qcow2")
|
||||
self.assertEqual(cfg["disks"][1]["image_type"], "Qcow2")
|
||||
self.assertEqual(len(cfg["disks"]), 2)
|
||||
|
||||
def test_raw_disks_stay_raw(self):
|
||||
ex = self.make_exec()
|
||||
cfg = ex.build_vm_config(
|
||||
os.path.join(self.state_dir, "chv-a", "disk.raw"))
|
||||
self.assertEqual(cfg["disks"][0]["image_type"], "Raw")
|
||||
|
||||
|
||||
# -- tenant 归属 --------------------------------------------------------------
|
||||
class TenantMarkingTest(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.tmp = tempfile.TemporaryDirectory()
|
||||
self.state = os.path.join(self.tmp.name, "state")
|
||||
|
||||
def tearDown(self):
|
||||
self.tmp.cleanup()
|
||||
|
||||
def test_snapshot_entry_carries_tenant(self):
|
||||
url = "file:///snap/a"
|
||||
entries, _ = record_snapshot(self.state, "chv-a", url, 20.0,
|
||||
tenant_id="tenant-a")
|
||||
self.assertEqual(entries[0]["tenant"], "tenant-a")
|
||||
entries2, _ = record_snapshot(self.state, "chv-b", url, 20.0)
|
||||
self.assertNotIn("tenant", entries2[0])
|
||||
|
||||
def test_find_snapshot_entry(self):
|
||||
url = "file:///snap/a"
|
||||
record_snapshot(self.state, "chv-a", url, 20.0,
|
||||
tenant_id="tenant-a")
|
||||
vm, entry = find_snapshot_entry(self.state, url)
|
||||
self.assertEqual(vm, "chv-a")
|
||||
self.assertEqual(entry["tenant"], "tenant-a")
|
||||
self.assertEqual(find_snapshot_entry(self.state,
|
||||
"file:///snap/nope"),
|
||||
(None, None))
|
||||
|
||||
def test_vm_tenant_roundtrip_and_corrupt(self):
|
||||
path = write_vm_tenant(self.state, "chv-a", "tenant-a",
|
||||
disk="/x/disk.qcow2")
|
||||
self.assertTrue(os.path.exists(path))
|
||||
self.assertEqual(read_vm_tenant(self.state, "chv-a"),
|
||||
"tenant-a")
|
||||
self.assertIsNone(read_vm_tenant(self.state, "chv-nope"))
|
||||
with open(os.path.join(self.state, "chv-a",
|
||||
"tenant.json"), "w") as fh:
|
||||
fh.write("{broken")
|
||||
self.assertIsNone(read_vm_tenant(self.state, "chv-a"))
|
||||
|
||||
def test_persistent_record_roundtrip(self):
|
||||
write_persistent_vm(self.state, "tenant-a", {
|
||||
"vm": "chv-persist-tenant-a", "tenant_id": "tenant-a",
|
||||
"state": "paused", "origin": "prewarm",
|
||||
"gitea_synced": True})
|
||||
rec = read_persistent_vm(self.state, "tenant-a")
|
||||
self.assertEqual(rec["vm"], "chv-persist-tenant-a")
|
||||
self.assertEqual(rec["state"], "paused")
|
||||
# 租户不符的登记读不到（不把 A 的登记当成 B 的）。
|
||||
self.assertIsNone(read_persistent_vm(self.state, "tenant-b"))
|
||||
self.assertTrue(clear_persistent_vm(self.state, "tenant-a"))
|
||||
self.assertIsNone(read_persistent_vm(self.state, "tenant-a"))
|
||||
self.assertFalse(clear_persistent_vm(self.state, "tenant-a"))
|
||||
|
||||
|
||||
class CrossTenantRestoreTest(ChvOpt1Case):
|
||||
def _record_snapshot(self, vm, url, tenant):
|
||||
os.makedirs(os.path.join(self.state_dir, vm), exist_ok=True)
|
||||
record_snapshot(self.state_dir, vm, url, 20.0,
|
||||
tenant_id=tenant)
|
||||
|
||||
def _snap_url(self, vm, name="snap1"):
|
||||
return "file://" + os.path.join(self.state_dir, vm, name)
|
||||
|
||||
def test_cross_tenant_restore_refused(self):
|
||||
url = self._snap_url("chv-a")
|
||||
self._record_snapshot("chv-a", url, "tenant-a")
|
||||
ex = self.make_exec()
|
||||
events = []
|
||||
ex.audit_callback = lambda e, t, d, k: events.append((e, k))
|
||||
task_b = {"id": "task-b", "project_id": "tenant-b"}
|
||||
with self.assertRaises(ChvFailure) as ctx:
|
||||
ex._validate_restore(task_b, url)
|
||||
self.assertEqual(ctx.exception.kind,
|
||||
"snapshot_tenant_mismatch")
|
||||
self.assertTrue(any(
|
||||
e == "vm_restore_tenant_denied" for e, _ in events))
|
||||
|
||||
def test_same_tenant_restore_passes_validation(self):
|
||||
url = self._snap_url("chv-a")
|
||||
self._record_snapshot("chv-a", url, "tenant-a")
|
||||
ex = self.make_exec()
|
||||
snapshot_url, _net = ex._validate_restore(
|
||||
{"id": "task-a2", "project_id": "tenant-a"}, url)
|
||||
self.assertEqual(snapshot_url, url)
|
||||
|
||||
def test_legacy_unmarked_snapshot_warns_but_passes(self):
|
||||
url = self._snap_url("chv-a", "old")
|
||||
self._record_snapshot("chv-a", url, None)
|
||||
ex = self.make_exec()
|
||||
events = []
|
||||
ex.audit_callback = lambda e, t, d, k: events.append((e, k))
|
||||
snapshot_url, _net = ex._validate_restore(
|
||||
{"id": "task-a2", "project_id": "tenant-a"}, url)
|
||||
self.assertEqual(snapshot_url, url)
|
||||
self.assertTrue(any(
|
||||
e == "vm_restore_tenant_unmarked" for e, _ in events))
|
||||
|
||||
def test_snapshot_vm_records_tenant(self):
|
||||
ex = self.make_exec()
|
||||
vm = "chv-snap1"
|
||||
snap_dir = os.path.join(self.state_dir, vm, "snap1")
|
||||
os.makedirs(snap_dir, exist_ok=True)
|
||||
with open(os.path.join(snap_dir, "config.json"), "w") as fh:
|
||||
fh.write("{}")
|
||||
url = "file://" + snap_dir
|
||||
out = ex.snapshot_vm({"id": "snap1", "project_id": "tenant-a"},
|
||||
url)
|
||||
self.assertEqual(out, url)
|
||||
_vm, entry = find_snapshot_entry(self.state_dir, url)
|
||||
self.assertEqual(entry["tenant"], "tenant-a")
|
||||
|
||||
|
||||
# -- 持久 VM 与预热池 ----------------------------------------------------------
|
||||
class PersistentPoolTest(ChvOpt1Case):
|
||||
def _patch_overlay_provision(self, ex):
|
||||
"""overlay 建盘改走 fake qemu-img（本机无 qemu-img）。"""
|
||||
runner = _fake_qemu_runner_factory({})
|
||||
|
||||
def fake_provision(state_dir, vm_name, rootfs, disk_mode,
|
||||
runner=None):
|
||||
if disk_mode != "overlay":
|
||||
return chv_disk.provision_task_disk(
|
||||
state_dir, vm_name, rootfs, disk_mode)
|
||||
path = os.path.join(state_dir, vm_name, "disk.qcow2")
|
||||
os.makedirs(os.path.dirname(path), exist_ok=True)
|
||||
with open(path, "wb") as fh:
|
||||
fh.write(b"QFI\xfb fake")
|
||||
return path
|
||||
|
||||
def fake_data(state_dir, vm_name, size_gb=10.0, runner=None):
|
||||
path = os.path.join(state_dir, vm_name, "data.qcow2")
|
||||
os.makedirs(os.path.dirname(path), exist_ok=True)
|
||||
if not os.path.exists(path):
|
||||
with open(path, "wb") as fh:
|
||||
fh.write(b"QFI\xfb fake-data")
|
||||
return path
|
||||
|
||||
p1 = mock.patch("cloud.chv.executor.provision_task_disk",
|
||||
side_effect=fake_provision)
|
||||
p2 = mock.patch("cloud.chv.executor.provision_data_disk",
|
||||
side_effect=fake_data)
|
||||
p1.start()
|
||||
p2.start()
|
||||
self.addCleanup(p1.stop)
|
||||
self.addCleanup(p2.stop)
|
||||
return ex
|
||||
|
||||
def _socket(self, vm):
|
||||
os.makedirs(self.state_dir, exist_ok=True)
|
||||
path = os.path.join(self.state_dir, vm + ".sock")
|
||||
with open(path, "w") as fh:
|
||||
fh.write("")
|
||||
return path
|
||||
|
||||
def test_pool_disabled_when_size_zero(self):
|
||||
ex = self.make_exec(env_extra={"CHV_WARM_POOL_SIZE": "0"})
|
||||
self.assertEqual(
|
||||
ex.prewarm_persistent_vm("proj1")["pooled"], False)
|
||||
self.assertEqual(
|
||||
ex.acquire_for_task(self._task())["mode"], "cold")
|
||||
# 退化为旧 dispatch：冷启→跑→关机（_vms 不留登记）。
|
||||
out = ex.dispatch_task_persistent(self._task())
|
||||
self.assertEqual(out["status"], "completed")
|
||||
self.assertEqual(ex._vms, {})
|
||||
|
||||
def test_prewarm_idempotent_and_acquire_resume(self):
|
||||
ex = self.make_exec(
|
||||
env_extra={"CHV_WARM_POOL_SIZE": "1",
|
||||
"CHV_DISK_MODE": "overlay"})
|
||||
self._patch_overlay_provision(ex)
|
||||
first = ex.prewarm_persistent_vm("proj1")
|
||||
self.assertTrue(first["pooled"])
|
||||
self.assertEqual(first["mode"], "prewarmed")
|
||||
self.assertEqual(first["vm"], "chv-persist-proj1")
|
||||
self.assertEqual(len(self.launches), 1)
|
||||
# socket 造出来后第二次 prewarm 即 already-warm，不重建。
|
||||
self._socket("chv-persist-proj1")
|
||||
second = ex.prewarm_persistent_vm("proj1")
|
||||
self.assertEqual(second["mode"], "already-warm")
|
||||
self.assertEqual(len(self.launches), 1)
|
||||
# acquire：vm.info=Paused → resume + SSH 就绪。
|
||||
self.vm_states["chv-persist-proj1"] = "Paused"
|
||||
got = ex.acquire_for_task(self._task())
|
||||
self.assertEqual(got["mode"], "resumed")
|
||||
self.assertTrue(got["warm_hit"])
|
||||
self.assertIn("resume_seconds", got)
|
||||
self.assertEqual(got["vm"], "chv-persist-proj1")
|
||||
self.assertTrue(any("resume" in c for c in self.calls))
|
||||
# 暂停回池后 origin 变 task，第二次 acquire 不再是
|
||||
# warm_hit，但仍是 resumed。
|
||||
self.assertTrue(ex.suspend_after_task(
|
||||
{"id": "chv-persist-proj1"}, tenant_id="proj1"))
|
||||
rec = read_persistent_vm(self.state_dir, "proj1")
|
||||
self.assertEqual(rec["state"], "paused")
|
||||
self.assertEqual(rec["origin"], "task")
|
||||
got2 = ex.acquire_for_task(self._task(task_id="second"))
|
||||
self.assertEqual(got2["mode"], "resumed")
|
||||
self.assertFalse(got2["warm_hit"])
|
||||
|
||||
def test_acquire_falls_back_cold(self):
|
||||
ex = self.make_exec(
|
||||
env_extra={"CHV_WARM_POOL_SIZE": "1",
|
||||
"CHV_DISK_MODE": "overlay"})
|
||||
self._patch_overlay_provision(ex)
|
||||
ex.prewarm_persistent_vm("proj1")
|
||||
self._socket("chv-persist-proj1")
|
||||
# vm.info=Running：不是 paused，不认领，回落冷启。
|
||||
self.vm_states["chv-persist-proj1"] = "Running"
|
||||
self.assertEqual(
|
||||
ex.acquire_for_task(self._task())["mode"], "cold")
|
||||
# socket 消失：登记清掉，回落冷启。
|
||||
os.remove(os.path.join(self.state_dir,
|
||||
"chv-persist-proj1.sock"))
|
||||
self.vm_states["chv-persist-proj1"] = "Paused"
|
||||
self.assertEqual(
|
||||
ex.acquire_for_task(self._task())["mode"], "cold")
|
||||
self.assertIsNone(read_persistent_vm(self.state_dir, "proj1"))
|
||||
|
||||
def test_acquire_tenant_mismatch_falls_back_cold(self):
|
||||
ex = self.make_exec(
|
||||
env_extra={"CHV_WARM_POOL_SIZE": "1",
|
||||
"CHV_DISK_MODE": "overlay"})
|
||||
self._patch_overlay_provision(ex)
|
||||
ex.prewarm_persistent_vm("proj1")
|
||||
self._socket("chv-persist-proj1")
|
||||
self.vm_states["chv-persist-proj1"] = "Paused"
|
||||
# 盘的归属被改成另一租户：绝不 resume，回落冷启。
|
||||
with open(os.path.join(self.state_dir, "chv-persist-proj1",
|
||||
"tenant.json"), "w") as fh:
|
||||
json.dump({"tenant_id": "tenant-b",
|
||||
"vm": "chv-persist-proj1"}, fh)
|
||||
self.assertEqual(
|
||||
ex.acquire_for_task(self._task())["mode"], "cold")
|
||||
self.assertIsNone(read_persistent_vm(self.state_dir, "proj1"))
|
||||
|
||||
def test_dispatch_persistent_two_tasks(self):
|
||||
ex = self.make_exec(
|
||||
env_extra={"CHV_WARM_POOL_SIZE": "1",
|
||||
"CHV_DISK_MODE": "overlay"})
|
||||
self._patch_overlay_provision(ex)
|
||||
ex.prewarm_persistent_vm("proj1")
|
||||
self._socket("chv-persist-proj1")
|
||||
self.vm_states["chv-persist-proj1"] = "Paused"
|
||||
out1 = ex.dispatch_task_persistent(self._task(task_id="t1"))
|
||||
self.assertEqual(out1["status"], "completed")
|
||||
self.assertEqual(out1["vm_mode"], "resumed")
|
||||
self.assertEqual(out1["vm"], "chv-persist-proj1")
|
||||
self.assertIn("resume_seconds", out1)
|
||||
# 任务结束已暂停回池：第二个任务仍是 resumed，且是
|
||||
# 同一台 VM（同一 overlay/数据盘）。
|
||||
out2 = ex.dispatch_task_persistent(
|
||||
self._task(task_id="t2"))
|
||||
self.assertEqual(out2["status"], "completed")
|
||||
self.assertEqual(out2["vm_mode"], "resumed")
|
||||
self.assertEqual(out2["vm"], "chv-persist-proj1")
|
||||
rec = read_persistent_vm(self.state_dir, "proj1")
|
||||
self.assertEqual(rec["state"], "paused")
|
||||
# 任务盘仍在（暂停不删盘、不关机语义）。
|
||||
self.assertTrue(os.path.exists(os.path.join(
|
||||
self.state_dir, "chv-persist-proj1", "disk.qcow2")))
|
||||
self.assertTrue(os.path.exists(os.path.join(
|
||||
self.state_dir, "chv-persist-proj1", "data.qcow2")))
|
||||
|
||||
def test_dispatch_persistent_cold_when_pool_empty(self):
|
||||
ex = self.make_exec(
|
||||
env_extra={"CHV_WARM_POOL_SIZE": "1",
|
||||
"CHV_DISK_MODE": "overlay"})
|
||||
self._patch_overlay_provision(ex)
|
||||
# 池空（未 prewarm）：冷启持久 VM，跑完暂停回池。
|
||||
out = ex.dispatch_task_persistent(self._task(task_id="t1"))
|
||||
self.assertEqual(out["status"], "completed")
|
||||
self.assertEqual(out["vm_mode"], "cold")
|
||||
self.assertEqual(out["vm"], "chv-persist-proj1")
|
||||
rec = read_persistent_vm(self.state_dir, "proj1")
|
||||
self.assertEqual(rec["state"], "paused")
|
||||
self.assertEqual(rec["vm"], "chv-persist-proj1")
|
||||
|
||||
def test_stop_keeps_protected_overlay_and_gc_skips_it(self):
|
||||
ex = self.make_exec(
|
||||
env_extra={"CHV_WARM_POOL_SIZE": "1",
|
||||
"CHV_DISK_MODE": "overlay"})
|
||||
self._patch_overlay_provision(ex)
|
||||
ex.prewarm_persistent_vm("proj1")
|
||||
self._socket("chv-persist-proj1")
|
||||
overlay = os.path.join(self.state_dir, "chv-persist-proj1",
|
||||
"disk.qcow2")
|
||||
self.assertTrue(ex.stop_vm({"id": "chv-persist-proj1"}))
|
||||
# 有持久登记认领：stop 不删 overlay。
|
||||
self.assertTrue(os.path.exists(overlay))
|
||||
# GC 同样跳过认领的盘。
|
||||
self.assertEqual(ex.gc_stale_disks(), {"deleted": [],
|
||||
"failed": []})
|
||||
self.assertTrue(os.path.exists(overlay))
|
||||
# 显式回收才删 overlay + 数据盘 + 登记。
|
||||
out = ex.recycle_persistent_vm({"id": "chv-persist-proj1"},
|
||||
tenant_id="proj1")
|
||||
self.assertEqual(out["state"], "recycled")
|
||||
self.assertIn("task", out["deleted_disks"])
|
||||
self.assertIn("data", out["deleted_disks"])
|
||||
self.assertFalse(os.path.exists(overlay))
|
||||
self.assertFalse(os.path.exists(os.path.join(
|
||||
self.state_dir, "chv-persist-proj1", "data.qcow2")))
|
||||
self.assertIsNone(read_persistent_vm(self.state_dir, "proj1"))
|
||||
|
||||
def test_refill_warm_pool_noop_when_warm(self):
|
||||
ex = self.make_exec(
|
||||
env_extra={"CHV_WARM_POOL_SIZE": "1",
|
||||
"CHV_DISK_MODE": "overlay"})
|
||||
self._patch_overlay_provision(ex)
|
||||
ex.prewarm_persistent_vm("proj1")
|
||||
self._socket("chv-persist-proj1")
|
||||
out = ex.refill_warm_pool("proj1")
|
||||
self.assertEqual(out["mode"], "already-warm")
|
||||
self.assertEqual(len(self.launches), 1)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in new issue
Block a user