feat(chv-muse-p1): 常驻 VM + WarmPool 扶正 + 气球 + vsock，对齐 Muse
每用户一台常驻 VM（暂停态，resume 到 SSH 5 秒内为待验证目标）， 池空/失败回落冷启动；CHV 配置加 virtio-balloon + free-page-reporting 与 vsock notify 最小实现（SSH 仍是命令通道与就绪判定）。 新模块 resident_pool.py / vsock.py + 客内 vsock_notify.py； _config 增加 CHV_RESIDENT_IDLE_MINUTES / CHV_BALLOON / CHV_VSOCK； worker_adapter 接常驻孤儿清理 facade；ADR-17 差距表见 docs/chv-muse-p1-gap.md。 恢复收尾（分离后回归，见 gap.md §4）：VmDeathTest 死亡模拟改杀 实际跑命令的 VM；两处冷路径旧测试显式关常驻；_balloon_vsock_payload 与 listen_notify 拆到 50 行内；resident_pool.py:208 mypy 修掉。 executor.py out 未注解与 marker_tools.py:142 是基线旧错，未动。 测试：全量 cloud 套件 1934 OK（skipped=8）；mypy 在改动文件上 干净（仅基线旧错 4 处）。真机验证待第三棒让出 WSL 测试机。
This commit is contained in:
host committed
1 parent
06c3e172fe
commit
78098c5594
13 files changed
+2527
-36
No files matched your search
@@ -31,6 +31,7 @@ try:
|
||||
from cloud.chv.compat215 import REPLACEMENT_POINTS
|
||||
from cloud.chv.compat215 import resolve_inside, tmpfs_isolated
|
||||
from cloud.chv.config import _assemble_chv_config
|
||||
from cloud.chv.config import _balloon_vsock_fragment
|
||||
from cloud.chv.config import _base_fragment
|
||||
from cloud.chv.config import _disk_fragment
|
||||
from cloud.chv.config import _network_fragment
|
||||
@@ -48,6 +49,7 @@ except ImportError:
|
||||
from cloud.chv.compat215 import REPLACEMENT_POINTS
|
||||
from cloud.chv.compat215 import resolve_inside, tmpfs_isolated
|
||||
from cloud.chv.config import _assemble_chv_config
|
||||
from cloud.chv.config import _balloon_vsock_fragment
|
||||
from cloud.chv.config import _base_fragment
|
||||
from cloud.chv.config import _disk_fragment
|
||||
from cloud.chv.config import _network_fragment
|
||||
@@ -76,6 +78,8 @@ EXPECTED_FIELDS = {
|
||||
"snapshot_max_total_gb", "snapshot_retention_days",
|
||||
"snapshot_dir", "allow_remote_snapshot", "snapshot_url",
|
||||
"task_timeout", "api_timeout", "ready_timeout", "ready_interval",
|
||||
# P1-B（feat-chv-muse-p1）：气球/vsock 三字段。
|
||||
"balloon", "vsock", "vsock_cid",
|
||||
}
|
||||
|
||||
|
||||
@@ -249,13 +253,15 @@ class ConfigFragmentsTest(unittest.TestCase):
|
||||
def test_fragment_keys_disjoint_and_union_is_field_set(self):
|
||||
frags = [_base_fragment({}, {}), _ssh_fragment({}, {}),
|
||||
_network_fragment({}, {}), _quota_fragment({}),
|
||||
_disk_fragment({}), _timeout_fragment({}, {})]
|
||||
_disk_fragment({}), _timeout_fragment({}, {}),
|
||||
# P1-B：新片段加入并集断言（字段 38 → 41）。
|
||||
_balloon_vsock_fragment({})]
|
||||
seen = set()
|
||||
for frag in frags:
|
||||
self.assertFalse(seen & set(frag), "片段键重叠")
|
||||
seen |= set(frag)
|
||||
self.assertEqual(seen, EXPECTED_FIELDS)
|
||||
self.assertEqual(len(EXPECTED_FIELDS), 38)
|
||||
self.assertEqual(len(EXPECTED_FIELDS), 41)
|
||||
|
||||
def test_defaults_and_overrides(self):
|
||||
cfg = build_chv_config({}, {})
|
||||
|
||||
@@ -598,8 +598,16 @@ class VmDeathTest(CloseupCase):
|
||||
def runner(cmd, timeout=None):
|
||||
self.calls.append(list(cmd))
|
||||
if cmd[0] == "ssh" and "dispatch" in cmd:
|
||||
proc = ex._vms.get("chv-abc123")
|
||||
if proc is not None:
|
||||
# 死亡模拟须杀当时已登记的 VM（即实际跑命令
|
||||
# 的那台）：P1 常驻路由下 dispatch 跑在常驻 VM
|
||||
#（chv-resident-*）上，任务自己的 chv-abc123
|
||||
# 从未 wake、登记里根本没有，盯着它杀是空操
|
||||
# 作、模拟落空（探针实证：登记里只有
|
||||
# chv-resident-default 且活着，异常直抛）。
|
||||
# 常驻 VM 真死时 _run_on_resident 的判定已验
|
||||
# 证正确（两态皆 interrupted），这里只修模拟
|
||||
# 器的靶子，不动断言与判定逻辑。
|
||||
for proc in list(ex._vms.values()):
|
||||
proc._alive = False
|
||||
if mode == "raise":
|
||||
raise RuntimeError("ssh broken")
|
||||
|
||||
@@ -0,0 +1,423 @@
|
||||
"""P1-B（feat-chv-muse-p1）：气球/vsock 配置、payload 与 vsock 编解码测试。
|
||||
|
||||
覆盖三件：
|
||||
|
||||
- config：CHV_BALLOON/CHV_VSOCK 开关默认开、只认 0/1，
|
||||
CHV_VSOCK_CID 默认 3 且拒绝保留 CID（0/1/2）与非法值；
|
||||
- executor：build_vm_config 默认带 balloon/vsock 设备，开关
|
||||
关时对应键整个不出现，vsock socket 路径按任务盘派生；
|
||||
- vsock（宿主侧）与客内脚本：JSON 行编解码、监听/超时、
|
||||
客内脚本静默失败（连不上宿主即 exit 0，不影响 sshd/任务）。
|
||||
|
||||
全部用假执行器与本地 Unix socket，绝不连真实网络、不起
|
||||
真实 VM。CHV 54 的 balloon size=0 与 vsock 端到端收发尚未
|
||||
真机验证（本机无 /dev/kvm，第三棒占着 WSL 测试机），见
|
||||
汇报的待验证项——这里只钉住配置层 + payload 形态。
|
||||
"""
|
||||
|
||||
import ast
|
||||
import importlib.util
|
||||
import json
|
||||
import os
|
||||
import socket
|
||||
import sys
|
||||
import tempfile
|
||||
import threading
|
||||
import time
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
from cloud.chv import vsock as chv_vsock
|
||||
from cloud.chv.config import _balloon_vsock_fragment
|
||||
from cloud.chv.config import _base_fragment
|
||||
from cloud.chv.config import _disk_fragment
|
||||
from cloud.chv.config import _network_fragment
|
||||
from cloud.chv.config import _quota_fragment
|
||||
from cloud.chv.config import _ssh_fragment
|
||||
from cloud.chv.config import _timeout_fragment
|
||||
from cloud.chv.config import build_chv_config
|
||||
from cloud.chv.executor import CloudHypervisorExecutor
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
GUEST_SCRIPT = ROOT / "cloud" / "chv" / "guest" / "vsock_notify.py"
|
||||
|
||||
|
||||
def _load_guest_module():
|
||||
spec = importlib.util.spec_from_file_location(
|
||||
"vsock_notify_guest", str(GUEST_SCRIPT))
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(module)
|
||||
return module
|
||||
|
||||
|
||||
def _stdlib_only(path):
|
||||
tree = ast.parse(Path(path).read_text())
|
||||
mods = set()
|
||||
for node in ast.walk(tree):
|
||||
if isinstance(node, ast.Import):
|
||||
mods.update(a.name.split(".")[0] for a in node.names)
|
||||
elif isinstance(node, ast.ImportFrom) and node.module:
|
||||
mods.add(node.module.split(".")[0])
|
||||
return mods <= set(sys.stdlib_module_names)
|
||||
|
||||
|
||||
class BalloonVsockConfigTest(unittest.TestCase):
|
||||
def test_defaults_on(self):
|
||||
cfg = build_chv_config({}, {})
|
||||
self.assertTrue(cfg.balloon)
|
||||
self.assertTrue(cfg.vsock)
|
||||
self.assertEqual(cfg.vsock_cid, 3)
|
||||
|
||||
def test_switches_off(self):
|
||||
cfg = build_chv_config(
|
||||
{}, {"CHV_BALLOON": "0", "CHV_VSOCK": "0"})
|
||||
self.assertFalse(cfg.balloon)
|
||||
self.assertFalse(cfg.vsock)
|
||||
self.assertEqual(cfg.vsock_cid, 3)
|
||||
|
||||
def test_switches_on_explicit(self):
|
||||
cfg = build_chv_config(
|
||||
{}, {"CHV_BALLOON": "1", "CHV_VSOCK": "1"})
|
||||
self.assertTrue(cfg.balloon)
|
||||
self.assertTrue(cfg.vsock)
|
||||
|
||||
def test_illegal_switch_values_raise(self):
|
||||
for env in ({"CHV_BALLOON": "2"}, {"CHV_BALLOON": "yes"},
|
||||
{"CHV_BALLOON": "true"}, {"CHV_VSOCK": "on"},
|
||||
{"CHV_VSOCK": ""}):
|
||||
with self.assertRaises(ValueError):
|
||||
build_chv_config({}, env)
|
||||
|
||||
def test_vsock_cid_override(self):
|
||||
cfg = build_chv_config({}, {"CHV_VSOCK_CID": "7"})
|
||||
self.assertEqual(cfg.vsock_cid, 7)
|
||||
cfg2 = build_chv_config({}, {"CHV_VSOCK_CID": "4294967295"})
|
||||
self.assertEqual(cfg2.vsock_cid, 4294967295)
|
||||
|
||||
def test_vsock_cid_reserved_and_illegal_raise(self):
|
||||
# 0/1/2 是 vsock 保留 CID（Hypervisor/Loopback/Host），
|
||||
# -1 随机 CID 也被 CHV vm.create 400 拒绝。
|
||||
for cid in ("0", "1", "2", "-1", "4294967296", "abc", ""):
|
||||
with self.assertRaises(ValueError, msg=cid):
|
||||
build_chv_config({}, {"CHV_VSOCK_CID": cid})
|
||||
|
||||
def test_fragment_keys_disjoint(self):
|
||||
frag = _balloon_vsock_fragment({})
|
||||
self.assertEqual(set(frag), {"balloon", "vsock", "vsock_cid"})
|
||||
others = [_base_fragment({}, {}), _ssh_fragment({}, {}),
|
||||
_network_fragment({}, {}), _quota_fragment({}),
|
||||
_disk_fragment({}), _timeout_fragment({}, {})]
|
||||
seen = set()
|
||||
for other in others:
|
||||
seen |= set(other)
|
||||
self.assertFalse(seen & set(frag), "新片段键与旧片段冲突")
|
||||
|
||||
|
||||
class _PayloadCase(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.dir = tempfile.mkdtemp(prefix="chv-p1b-")
|
||||
self.state_dir = os.path.join(self.dir, "state")
|
||||
os.makedirs(self.state_dir)
|
||||
|
||||
def tearDown(self):
|
||||
import shutil
|
||||
shutil.rmtree(self.dir, ignore_errors=True)
|
||||
|
||||
def make_exec(self, env_extra=None):
|
||||
env = dict(env_extra or {})
|
||||
return CloudHypervisorExecutor(
|
||||
kernel=os.path.join(self.dir, "vmlinux"),
|
||||
rootfs=os.path.join(self.dir, "root.raw"),
|
||||
ssh_key=os.path.join(self.dir, "id_ed25519"),
|
||||
state_dir=self.state_dir,
|
||||
runner=lambda cmd, timeout=None: None,
|
||||
launcher=lambda cmd: None,
|
||||
api_client=lambda *a: (200, {}),
|
||||
sleeper=lambda s: None, env=env)
|
||||
|
||||
|
||||
class BalloonVsockPayloadTest(_PayloadCase):
|
||||
def test_default_payload_has_balloon_and_vsock(self):
|
||||
ex = self.make_exec()
|
||||
cfg = ex.build_vm_config()
|
||||
# CHV 54 BalloonConfig：size 是目标气球尺寸（字节），
|
||||
# 0 = 启动不预扣，只靠 free_page_reporting 被动回收。
|
||||
self.assertEqual(cfg["balloon"], {
|
||||
"size": 0, "deflate_on_oom": True,
|
||||
"free_page_reporting": True})
|
||||
self.assertEqual(cfg["vsock"], {
|
||||
"cid": 3,
|
||||
"socket": os.path.join(self.state_dir, "vsock.sock")})
|
||||
|
||||
def test_switches_off_drop_keys(self):
|
||||
ex = self.make_exec({"CHV_BALLOON": "0", "CHV_VSOCK": "0"})
|
||||
cfg = ex.build_vm_config()
|
||||
self.assertNotIn("balloon", cfg)
|
||||
self.assertNotIn("vsock", cfg)
|
||||
# 旧键原样保留。
|
||||
self.assertIn("cpus", cfg)
|
||||
self.assertIn("memory", cfg)
|
||||
self.assertIn("payload", cfg)
|
||||
self.assertIn("disks", cfg)
|
||||
self.assertIn("rng", cfg)
|
||||
|
||||
def test_balloon_off_only(self):
|
||||
ex = self.make_exec({"CHV_BALLOON": "0"})
|
||||
cfg = ex.build_vm_config()
|
||||
self.assertNotIn("balloon", cfg)
|
||||
self.assertIn("vsock", cfg)
|
||||
|
||||
def test_vsock_cid_honored(self):
|
||||
ex = self.make_exec({"CHV_VSOCK_CID": "9"})
|
||||
cfg = ex.build_vm_config()
|
||||
self.assertEqual(cfg["vsock"]["cid"], 9)
|
||||
|
||||
def test_vsock_socket_derived_from_task_disk_dir(self):
|
||||
ex = self.make_exec()
|
||||
disk = os.path.join(self.state_dir, "chv-t1", "disk.raw")
|
||||
cfg = ex.build_vm_config(disk)
|
||||
self.assertEqual(
|
||||
cfg["vsock"]["socket"],
|
||||
os.path.join(self.state_dir, "chv-t1", "vsock.sock"))
|
||||
|
||||
def test_vsock_socket_falls_back_when_disk_outside_state(self):
|
||||
# reuse/lab：任务盘即基础镜像（state_dir 外），socket
|
||||
# 落 state_dir/vsock.sock（单 VM 可用，见 helper 注释）。
|
||||
ex = self.make_exec()
|
||||
cfg = ex.build_vm_config()
|
||||
self.assertEqual(
|
||||
cfg["vsock"]["socket"],
|
||||
os.path.join(self.state_dir, "vsock.sock"))
|
||||
|
||||
def test_explicit_vsock_socket_wins(self):
|
||||
ex = self.make_exec()
|
||||
cfg = ex.build_vm_config(
|
||||
vsock_socket="/tmp/per-vm/v.sock")
|
||||
self.assertEqual(cfg["vsock"]["socket"], "/tmp/per-vm/v.sock")
|
||||
|
||||
def test_helper_direct(self):
|
||||
ex = self.make_exec({"CHV_BALLOON": "0"})
|
||||
extra = ex._balloon_vsock_payload(
|
||||
vsock_socket="/tmp/s.sock")
|
||||
self.assertEqual(extra, {"vsock": {"cid": 3,
|
||||
"socket": "/tmp/s.sock"}})
|
||||
|
||||
|
||||
class VsockCodecTest(unittest.TestCase):
|
||||
def test_encode_decode_round_trip(self):
|
||||
raw = chv_vsock.encode_notify("ready", "chv-t1", ts=1234.5)
|
||||
self.assertTrue(raw.endswith(b"\n"))
|
||||
msg = chv_vsock.decode_notify(raw)
|
||||
self.assertEqual(msg, {"type": "ready", "vm": "chv-t1",
|
||||
"ts": 1234.5})
|
||||
|
||||
def test_encode_rejects_bad_type_or_vm(self):
|
||||
with self.assertRaises(ValueError):
|
||||
chv_vsock.encode_notify("bogus", "chv-t1")
|
||||
with self.assertRaises(ValueError):
|
||||
chv_vsock.encode_notify("ready", "")
|
||||
|
||||
def test_decode_rejects_garbage(self):
|
||||
cases = [b"not json\n", b"[1, 2]\n", b"123\n", b"\n",
|
||||
b'{"type": "bogus", "vm": "v", "ts": 1}\n',
|
||||
b'{"type": "ready", "vm": "", "ts": 1}\n',
|
||||
b'{"type": "ready", "vm": 7, "ts": 1}\n',
|
||||
b'{"type": "ready", "vm": "v"}\n',
|
||||
b'{"type": "ready", "vm": "v", "ts": "x"}\n',
|
||||
b'{"type": "ready", "vm": "v", "ts": true}\n',
|
||||
b'\xff\xfe bad utf8\n']
|
||||
for line in cases:
|
||||
self.assertIsNone(chv_vsock.decode_notify(line), line)
|
||||
|
||||
def test_decode_accepts_int_ts(self):
|
||||
msg = chv_vsock.decode_notify(
|
||||
b'{"type": "heartbeat", "vm": "v", "ts": 7}')
|
||||
self.assertEqual(msg["ts"], 7)
|
||||
|
||||
def test_paths(self):
|
||||
self.assertEqual(
|
||||
chv_vsock.vsock_socket_path("/s", "chv-t1"),
|
||||
"/s/chv-t1/vsock.sock")
|
||||
self.assertEqual(
|
||||
chv_vsock.notify_socket_path("/s/chv-t1/vsock.sock"),
|
||||
"/s/chv-t1/vsock.sock_1024")
|
||||
self.assertEqual(chv_vsock.HOST_CID, 2)
|
||||
self.assertEqual(chv_vsock.VSOCK_NOTIFY_PORT, 1024)
|
||||
|
||||
|
||||
def _wait_for_socket(path, timeout=2.0):
|
||||
deadline = time.monotonic() + timeout
|
||||
while time.monotonic() < deadline:
|
||||
if os.path.exists(path):
|
||||
return True
|
||||
time.sleep(0.02)
|
||||
return False
|
||||
|
||||
|
||||
class VsockListenTest(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.dir = tempfile.mkdtemp(prefix="chv-p1b-vsock-")
|
||||
|
||||
def tearDown(self):
|
||||
import shutil
|
||||
shutil.rmtree(self.dir, ignore_errors=True)
|
||||
|
||||
def _client(self, path, lines, delay=0.05):
|
||||
"""后台线程：等 socket 出现，连上发 lines 后关闭。
|
||||
|
||||
connect 带重试：监听方 bind 与 listen 之间、以及删
|
||||
残留文件重 bind 的窗口里，单次 connect 可能被拒；
|
||||
须重试到成功或超时，否则测试会偶发失败。"""
|
||||
def run():
|
||||
if not _wait_for_socket(path):
|
||||
return
|
||||
deadline = time.monotonic() + 5
|
||||
sock = None
|
||||
while time.monotonic() < deadline:
|
||||
sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
|
||||
try:
|
||||
sock.connect(path)
|
||||
break
|
||||
except OSError:
|
||||
try:
|
||||
sock.close()
|
||||
except OSError:
|
||||
pass
|
||||
sock = None
|
||||
time.sleep(0.02)
|
||||
if sock is None:
|
||||
return
|
||||
try:
|
||||
time.sleep(delay)
|
||||
for line in lines:
|
||||
sock.sendall(line)
|
||||
except OSError:
|
||||
pass
|
||||
finally:
|
||||
try:
|
||||
sock.close()
|
||||
except OSError:
|
||||
pass
|
||||
thread = threading.Thread(target=run, daemon=True)
|
||||
thread.start()
|
||||
return thread
|
||||
|
||||
def test_listen_returns_ready(self):
|
||||
path = os.path.join(self.dir, "n.sock_1024")
|
||||
thread = self._client(
|
||||
path, [chv_vsock.encode_notify("ready", "chv-t1")])
|
||||
msg = chv_vsock.listen_notify(path, timeout=5)
|
||||
thread.join(timeout=5)
|
||||
self.assertIsNotNone(msg)
|
||||
self.assertEqual(msg["type"], "ready")
|
||||
self.assertEqual(msg["vm"], "chv-t1")
|
||||
self.assertFalse(os.path.exists(path), "监听完须清 socket 文件")
|
||||
|
||||
def test_listen_timeout_returns_none(self):
|
||||
path = os.path.join(self.dir, "t.sock_1024")
|
||||
start = time.monotonic()
|
||||
self.assertIsNone(chv_vsock.listen_notify(path, timeout=0.3))
|
||||
self.assertLess(time.monotonic() - start, 2)
|
||||
self.assertFalse(os.path.exists(path))
|
||||
|
||||
def test_listen_skips_invalid_lines_and_wrong_type(self):
|
||||
path = os.path.join(self.dir, "w.sock_1024")
|
||||
thread = self._client(path, [
|
||||
b"not json\n",
|
||||
chv_vsock.encode_notify("heartbeat", "chv-t1"),
|
||||
chv_vsock.encode_notify("ready", "chv-t1"),
|
||||
])
|
||||
msg = chv_vsock.listen_notify(path, timeout=5,
|
||||
want_type="ready")
|
||||
thread.join(timeout=5)
|
||||
self.assertIsNotNone(msg)
|
||||
self.assertEqual(msg["type"], "ready")
|
||||
|
||||
def test_listen_survives_stale_socket_file(self):
|
||||
path = os.path.join(self.dir, "s.sock_1024")
|
||||
with open(path, "w") as fh:
|
||||
fh.write("stale")
|
||||
thread = self._client(
|
||||
path, [chv_vsock.encode_notify("ready", "chv-t1")])
|
||||
msg = chv_vsock.listen_notify(path, timeout=5)
|
||||
thread.join(timeout=5)
|
||||
self.assertIsNotNone(msg)
|
||||
self.assertEqual(msg["type"], "ready")
|
||||
|
||||
def test_wait_ready_end_to_end(self):
|
||||
vm_dir = os.path.join(self.dir, "chv-t1")
|
||||
os.makedirs(vm_dir)
|
||||
path = chv_vsock.notify_socket_path(
|
||||
chv_vsock.vsock_socket_path(self.dir, "chv-t1"))
|
||||
thread = self._client(
|
||||
path, [chv_vsock.encode_notify("ready", "chv-t1")])
|
||||
msg = chv_vsock.wait_ready(self.dir, "chv-t1", timeout=5)
|
||||
thread.join(timeout=5)
|
||||
self.assertIsNotNone(msg)
|
||||
self.assertEqual(msg["type"], "ready")
|
||||
self.assertEqual(msg["vm"], "chv-t1")
|
||||
|
||||
def test_wait_ready_timeout_returns_none(self):
|
||||
vm_dir = os.path.join(self.dir, "chv-t2")
|
||||
os.makedirs(vm_dir)
|
||||
self.assertIsNone(
|
||||
chv_vsock.wait_ready(self.dir, "chv-t2", timeout=0.3))
|
||||
|
||||
|
||||
class GuestNotifyScriptTest(unittest.TestCase):
|
||||
def test_script_exists_and_stdlib_only(self):
|
||||
self.assertTrue(GUEST_SCRIPT.is_file())
|
||||
self.assertTrue(_stdlib_only(GUEST_SCRIPT))
|
||||
self.assertTrue(
|
||||
_stdlib_only(ROOT / "cloud" / "chv" / "vsock.py"))
|
||||
|
||||
def test_vm_name_sources(self):
|
||||
mod = _load_guest_module()
|
||||
self.assertEqual(mod._vm_name(["prog", "chv-t1"]), "chv-t1")
|
||||
old = os.environ.pop("CHV_VM_NAME", None)
|
||||
try:
|
||||
os.environ["CHV_VM_NAME"] = "chv-env"
|
||||
self.assertEqual(mod._vm_name(["prog"]), "chv-env")
|
||||
finally:
|
||||
if old is None:
|
||||
os.environ.pop("CHV_VM_NAME", None)
|
||||
else:
|
||||
os.environ["CHV_VM_NAME"] = old
|
||||
name = mod._vm_name(["prog"])
|
||||
self.assertIsInstance(name, str)
|
||||
self.assertTrue(name)
|
||||
|
||||
def test_send_writes_json_line(self):
|
||||
mod = _load_guest_module()
|
||||
client, server = socket.socketpair()
|
||||
try:
|
||||
mod._send(client, "heartbeat", "chv-t1")
|
||||
server.settimeout(2)
|
||||
data = server.recv(4096)
|
||||
msg = json.loads(data.decode("utf-8"))
|
||||
self.assertEqual(msg["type"], "heartbeat")
|
||||
self.assertEqual(msg["vm"], "chv-t1")
|
||||
self.assertIsInstance(msg["ts"], float)
|
||||
finally:
|
||||
client.close()
|
||||
server.close()
|
||||
|
||||
def test_main_exits_silently_when_host_unreachable(self):
|
||||
# 本机无 vsock 监听：main 应在重试窗口内静默返回 0，
|
||||
# 绝不抛异常、不影响调用方。
|
||||
mod = _load_guest_module()
|
||||
old = mod.CONNECT_RETRY_SECONDS
|
||||
mod.CONNECT_RETRY_SECONDS = 0
|
||||
try:
|
||||
self.assertEqual(mod.main(["prog", "chv-t1"]), 0)
|
||||
finally:
|
||||
mod.CONNECT_RETRY_SECONDS = old
|
||||
|
||||
def test_ports_agree(self):
|
||||
mod = _load_guest_module()
|
||||
self.assertEqual(mod.NOTIFY_PORT, chv_vsock.VSOCK_NOTIFY_PORT)
|
||||
self.assertEqual(mod.HOST_CID, chv_vsock.HOST_CID)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,730 @@
|
||||
"""CHV 常驻池（feat-chv-muse-p1）测试。
|
||||
|
||||
覆盖 cloud/chv/resident_pool.py、executor 的常驻接线
|
||||
（_run_with_vm 常驻/冷启动两路、finally 语义）与
|
||||
worker_adapter.reconcile_resident_pool facade。全部用假
|
||||
执行器/假 VMM，绝不起真实 VM、不连真实网络。
|
||||
|
||||
断言点（按任务书）：
|
||||
- acquire/resume 失败回落冷启动，失败条目丢弃；
|
||||
- release 只 pause 不销毁，pause 失败条目留 running；
|
||||
- registry 损坏 quarantine、单坏条目不连坐；
|
||||
- 重启后条目一律 orphan 不复用，只能 reconcile 清理；
|
||||
- _run_with_vm 常驻命中跑完 pause、冷启动跑完 shutdown，
|
||||
客内抛错时两路 finally 都执行；
|
||||
- 带密钥 env / 未启用 / 建池失败时回落冷启动，任务不丢；
|
||||
- 每用户隔离：release 只动本用户条目；
|
||||
- 空闲回收只动超阈值 paused 条目，阈值 0 即关闭；
|
||||
- WarmPool 实验模块不动（wired_to_dispatch 仍 False）。
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import tempfile
|
||||
import unittest
|
||||
from types import SimpleNamespace
|
||||
from unittest import mock
|
||||
|
||||
from cloud.chv import worker_adapter
|
||||
from cloud.chv.errors import ChvFailure
|
||||
from cloud.chv.executor import CloudHypervisorExecutor
|
||||
from cloud.chv.experimental_pool import WarmPool
|
||||
from cloud.chv.resident_pool import (
|
||||
ResidentPool,
|
||||
reconcile_resident_orphans_for_executor,
|
||||
resident_task_for_user,
|
||||
resident_user_key,
|
||||
)
|
||||
|
||||
|
||||
class _FakeExecutor:
|
||||
"""ResidentPool 用的鸭子型执行器：记录调用，可注入失败。
|
||||
|
||||
wait 失败时模拟真实语义：先自行 shutdown 再抛错
|
||||
（wait_guest_ready 的失败路径就是 stop_vm 后 raise）。
|
||||
"""
|
||||
|
||||
def __init__(self, state_dir, env=None):
|
||||
self.state_dir = state_dir
|
||||
self._env = env or {}
|
||||
self.resumes = []
|
||||
self.pauses = []
|
||||
self.wakes = []
|
||||
self.waits = []
|
||||
self.shutdowns = []
|
||||
self.hibernates = []
|
||||
self.emits = []
|
||||
self.resume_fail = False
|
||||
self.wait_fail = False
|
||||
self.wake_fail = False
|
||||
self.pause_fail = False
|
||||
self.hibernate_fail = False
|
||||
self.shutdown_fail = False
|
||||
|
||||
def vm_name(self, task_or_name):
|
||||
raw = task_or_name.get("id") if isinstance(
|
||||
task_or_name, dict) else str(task_or_name)
|
||||
safe = re.sub(r"[^A-Za-z0-9_.-]", "-", raw)
|
||||
if safe.startswith("chv-"):
|
||||
return safe
|
||||
return "chv-" + safe
|
||||
|
||||
def resume_vm(self, task_or_name):
|
||||
self.resumes.append(self.vm_name(task_or_name))
|
||||
if self.resume_fail:
|
||||
raise RuntimeError("resume boom")
|
||||
return True
|
||||
|
||||
def pause_vm(self, task_or_name):
|
||||
self.pauses.append(self.vm_name(task_or_name))
|
||||
if self.pause_fail:
|
||||
raise RuntimeError("pause boom")
|
||||
return True
|
||||
|
||||
def wake_vm(self, task_or_name):
|
||||
self.wakes.append(self.vm_name(task_or_name))
|
||||
if self.wake_fail:
|
||||
raise RuntimeError("wake boom")
|
||||
return {"vm": self.vm_name(task_or_name), "mode": "cold"}
|
||||
|
||||
def wait_guest_ready(self, task):
|
||||
self.waits.append(self.vm_name(task))
|
||||
if self.wait_fail:
|
||||
self.shutdown_vm(task)
|
||||
raise ChvFailure("guest_not_ready", "ssh 未就绪")
|
||||
return True
|
||||
|
||||
def shutdown_vm(self, task_or_name):
|
||||
self.shutdowns.append(self.vm_name(task_or_name))
|
||||
if self.shutdown_fail:
|
||||
raise RuntimeError("shutdown boom")
|
||||
return True
|
||||
|
||||
def hibernate_vm(self, task_or_name, snapshot_url):
|
||||
self.hibernates.append(
|
||||
(self.vm_name(task_or_name), snapshot_url))
|
||||
if self.hibernate_fail:
|
||||
raise RuntimeError("hibernate boom")
|
||||
return {"vm": self.vm_name(task_or_name),
|
||||
"state": "hibernated", "snapshot": snapshot_url}
|
||||
|
||||
def _emit(self, event, task, detail="", kind=None):
|
||||
self.emits.append(event)
|
||||
|
||||
|
||||
def _write(path, content):
|
||||
with open(path, "w") as fh:
|
||||
fh.write(content)
|
||||
return path
|
||||
|
||||
|
||||
def _registry_path(state_dir):
|
||||
return os.path.join(state_dir, "resident-pool.json")
|
||||
|
||||
|
||||
class ResidentKeyTest(unittest.TestCase):
|
||||
def test_user_key_sanitizes_and_defaults(self):
|
||||
self.assertEqual(resident_user_key(None), "default")
|
||||
self.assertEqual(resident_user_key(""), "default")
|
||||
self.assertEqual(resident_user_key("alice"), "alice")
|
||||
self.assertEqual(resident_user_key("a/b c"), "a-b-c")
|
||||
self.assertEqual(resident_task_for_user("alice"),
|
||||
{"id": "resident-alice"})
|
||||
self.assertEqual(resident_task_for_user(None),
|
||||
{"id": "resident-default"})
|
||||
|
||||
|
||||
class ResidentPoolAcquireTest(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.tmp = tempfile.TemporaryDirectory()
|
||||
self.fake = _FakeExecutor(self.tmp.name)
|
||||
|
||||
def tearDown(self):
|
||||
self.tmp.cleanup()
|
||||
|
||||
def _provisioned_pool(self, user="alice"):
|
||||
pool = ResidentPool(self.fake)
|
||||
got = pool.provision(user)
|
||||
self.assertEqual(got["mode"], "resident")
|
||||
return pool
|
||||
|
||||
def test_provision_then_acquire_then_release(self):
|
||||
pool = self._provisioned_pool()
|
||||
self.assertEqual(self.fake.wakes, ["chv-resident-alice"])
|
||||
self.assertEqual(self.fake.pauses, ["chv-resident-alice"])
|
||||
self.assertEqual(pool.paused_count(), 1)
|
||||
got = pool.acquire("alice")
|
||||
self.assertEqual(got["mode"], "resident")
|
||||
self.assertEqual(got["vm"], "chv-resident-alice")
|
||||
self.assertEqual(got["task"], {"id": "resident-alice"})
|
||||
self.assertEqual(self.fake.resumes, ["chv-resident-alice"])
|
||||
self.assertEqual(self.fake.waits, ["chv-resident-alice"])
|
||||
self.assertEqual(pool.paused_count(), 0)
|
||||
# running 中的条目不接受第二个 acquire（回落冷启动）。
|
||||
self.assertEqual(pool.acquire("alice")["mode"], "cold")
|
||||
self.assertEqual(len(self.fake.resumes), 1)
|
||||
# release 只 pause，不销毁。
|
||||
self.assertTrue(pool.release("alice"))
|
||||
self.assertEqual(
|
||||
self.fake.pauses,
|
||||
["chv-resident-alice", "chv-resident-alice"])
|
||||
self.assertEqual(self.fake.shutdowns, [])
|
||||
self.assertEqual(pool.paused_count(), 1)
|
||||
# 再取仍是同一台 VM。
|
||||
got = pool.acquire("alice")
|
||||
self.assertEqual(got["vm"], "chv-resident-alice")
|
||||
|
||||
def test_resume_failure_falls_back_to_cold_and_drops_entry(self):
|
||||
pool = self._provisioned_pool()
|
||||
self.fake.resume_fail = True
|
||||
got = pool.acquire("alice")
|
||||
self.assertEqual(got, {"mode": "cold", "vm": None,
|
||||
"task": None})
|
||||
self.assertEqual(pool.paused_count(), 0)
|
||||
with open(_registry_path(self.tmp.name)) as fh:
|
||||
self.assertEqual(json.load(fh), {})
|
||||
|
||||
def test_wait_failure_falls_back_to_cold_and_drops_entry(self):
|
||||
pool = self._provisioned_pool()
|
||||
self.fake.wait_fail = True
|
||||
got = pool.acquire("alice")
|
||||
self.assertEqual(got["mode"], "cold")
|
||||
self.assertEqual(pool.paused_count(), 0)
|
||||
# wait 的失败路径已自行 stop_vm（模拟真实语义）。
|
||||
self.assertEqual(self.fake.shutdowns,
|
||||
["chv-resident-alice"])
|
||||
with open(_registry_path(self.tmp.name)) as fh:
|
||||
self.assertEqual(json.load(fh), {})
|
||||
|
||||
def test_release_without_running_entry_is_false(self):
|
||||
pool = self._provisioned_pool()
|
||||
self.assertFalse(pool.release("alice"))
|
||||
self.assertFalse(pool.release("nobody"))
|
||||
self.assertEqual(self.fake.pauses, ["chv-resident-alice"])
|
||||
|
||||
def test_release_pause_failure_keeps_entry_running(self):
|
||||
pool = self._provisioned_pool()
|
||||
pool.acquire("alice")
|
||||
self.fake.pause_fail = True
|
||||
self.assertFalse(pool.release("alice"))
|
||||
# 不删条目、不销毁：下次 acquire 仍是 running→cold，
|
||||
# VM 本身没被 shutdown，留待 worker 关闭时清理。
|
||||
self.assertEqual(self.fake.shutdowns, [])
|
||||
self.assertEqual(pool.acquire("alice")["mode"], "cold")
|
||||
|
||||
def test_provision_is_idempotent_and_per_user_isolated(self):
|
||||
pool = ResidentPool(self.fake)
|
||||
self.assertEqual(pool.provision("alice")["mode"], "resident")
|
||||
self.assertEqual(pool.provision("bob")["mode"], "resident")
|
||||
self.assertEqual(pool.provision("alice")["mode"], "resident")
|
||||
self.assertEqual(self.fake.wakes.count("chv-resident-alice"), 1)
|
||||
self.assertEqual(self.fake.wakes.count("chv-resident-bob"), 1)
|
||||
self.assertEqual(pool.paused_count(), 2)
|
||||
# release 只动本用户条目。
|
||||
self.assertFalse(pool.release("bob")) # bob 未 running
|
||||
pool.acquire("alice")
|
||||
self.assertTrue(pool.release("alice"))
|
||||
self.assertEqual(pool.paused_count(), 2)
|
||||
got = pool.acquire("bob")
|
||||
self.assertEqual(got["vm"], "chv-resident-bob")
|
||||
self.assertEqual(pool.paused_count(), 1)
|
||||
self.assertEqual(self.fake.resumes, ["chv-resident-alice",
|
||||
"chv-resident-bob"])
|
||||
|
||||
def test_provision_wake_failure_falls_back_and_cleans_up(self):
|
||||
pool = ResidentPool(self.fake)
|
||||
self.fake.wake_fail = True
|
||||
got = pool.provision("alice")
|
||||
self.assertEqual(got, {"mode": "cold", "vm": None,
|
||||
"task": None})
|
||||
# 建池失败尽力 shutdown，不留半醒 VM 与条目。
|
||||
self.assertEqual(self.fake.shutdowns,
|
||||
["chv-resident-alice"])
|
||||
self.assertEqual(pool.paused_count(), 0)
|
||||
with open(_registry_path(self.tmp.name)) as fh:
|
||||
self.assertEqual(json.load(fh), {})
|
||||
|
||||
|
||||
class ResidentPoolRegistryTest(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.tmp = tempfile.TemporaryDirectory()
|
||||
self.fake = _FakeExecutor(self.tmp.name)
|
||||
|
||||
def tearDown(self):
|
||||
self.tmp.cleanup()
|
||||
|
||||
def test_corrupt_registry_is_quarantined(self):
|
||||
path = _write(_registry_path(self.tmp.name), "{not json")
|
||||
pool = ResidentPool(self.fake)
|
||||
self.assertTrue(pool.registry_corrupt)
|
||||
self.assertEqual(pool.paused_count(), 0)
|
||||
self.assertEqual(pool.orphan_count(), 0)
|
||||
self.assertTrue(os.path.exists(path + ".corrupt"))
|
||||
self.assertFalse(os.path.exists(path))
|
||||
|
||||
def test_top_level_non_dict_is_corrupt(self):
|
||||
path = _write(_registry_path(self.tmp.name), "[1, 2]")
|
||||
pool = ResidentPool(self.fake)
|
||||
self.assertTrue(pool.registry_corrupt)
|
||||
self.assertTrue(os.path.exists(path + ".corrupt"))
|
||||
|
||||
def test_bad_entries_dropped_without_tainting_good_ones(self):
|
||||
_write(_registry_path(self.tmp.name), json.dumps({
|
||||
"good": {"state": "paused", "vm": "chv-resident-good",
|
||||
"user_id": "good"},
|
||||
"no-vm": {"state": "paused"},
|
||||
"not-a-dict": "x",
|
||||
"wrong-state": {"state": "flying",
|
||||
"vm": "chv-resident-x"},
|
||||
}))
|
||||
pool = ResidentPool(self.fake)
|
||||
self.assertEqual(pool.registry_corrupt_entries, 3)
|
||||
self.assertEqual(pool.orphan_count(), 1)
|
||||
self.assertEqual(pool.paused_count(), 0)
|
||||
|
||||
def test_restart_never_reuses_entries(self):
|
||||
pool1 = ResidentPool(self.fake)
|
||||
pool1.provision("alice")
|
||||
pool2 = ResidentPool(self.fake)
|
||||
self.assertEqual(pool2.orphan_count(), 1)
|
||||
self.assertEqual(pool2.paused_count(), 0)
|
||||
# 新池绝不把上一世条目发给任务。
|
||||
self.assertEqual(pool2.acquire("alice")["mode"], "cold")
|
||||
self.assertEqual(self.fake.resumes, [])
|
||||
cleaned = pool2.reconcile_orphans()
|
||||
self.assertEqual(cleaned, ["chv-resident-alice"])
|
||||
self.assertEqual(self.fake.shutdowns,
|
||||
["chv-resident-alice"])
|
||||
with open(_registry_path(self.tmp.name)) as fh:
|
||||
self.assertEqual(json.load(fh), {})
|
||||
|
||||
def test_idle_minutes_env_parsing(self):
|
||||
self.assertEqual(
|
||||
ResidentPool(self.fake).idle_minutes, 0)
|
||||
fake = _FakeExecutor(
|
||||
self.tmp.name, env={"CHV_RESIDENT_IDLE_MINUTES": "45"})
|
||||
self.assertEqual(ResidentPool(fake).idle_minutes, 45)
|
||||
fake = _FakeExecutor(
|
||||
self.tmp.name, env={"CHV_RESIDENT_IDLE_MINUTES": "junk"})
|
||||
self.assertEqual(ResidentPool(fake).idle_minutes, 0)
|
||||
fake = _FakeExecutor(
|
||||
self.tmp.name, env={"CHV_RESIDENT_IDLE_MINUTES": "-5"})
|
||||
self.assertEqual(ResidentPool(fake).idle_minutes, 0)
|
||||
pool = ResidentPool(self.fake, idle_minutes=10)
|
||||
self.assertEqual(pool.idle_minutes, 10)
|
||||
|
||||
|
||||
class ResidentPoolIdleTest(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.tmp = tempfile.TemporaryDirectory()
|
||||
self.fake = _FakeExecutor(self.tmp.name)
|
||||
|
||||
def tearDown(self):
|
||||
self.tmp.cleanup()
|
||||
|
||||
def test_idle_disabled_by_default(self):
|
||||
pool = ResidentPool(self.fake)
|
||||
pool.provision("alice")
|
||||
import time
|
||||
self.assertEqual(
|
||||
pool.reclaim_idle(now=time.time() + 10 ** 9), [])
|
||||
self.assertEqual(self.fake.hibernates, [])
|
||||
|
||||
def test_overdue_paused_entry_is_hibernated(self):
|
||||
import time
|
||||
pool = ResidentPool(self.fake, idle_minutes=30)
|
||||
pool.provision("alice")
|
||||
pool.provision("bob")
|
||||
now = time.time() + 31 * 60
|
||||
reclaimed = pool.reclaim_idle(now=now)
|
||||
self.assertEqual(reclaimed, ["chv-resident-alice",
|
||||
"chv-resident-bob"])
|
||||
self.assertEqual(len(self.fake.hibernates), 2)
|
||||
for vm, url in self.fake.hibernates:
|
||||
self.assertTrue(url.startswith("file://"))
|
||||
self.assertIn(self.tmp.name, url)
|
||||
self.assertTrue(url.endswith(
|
||||
os.path.join(vm, "resident.snapshot")))
|
||||
self.assertEqual(pool.paused_count(), 0)
|
||||
with open(_registry_path(self.tmp.name)) as fh:
|
||||
self.assertEqual(json.load(fh), {})
|
||||
|
||||
def test_not_yet_idle_entry_is_kept(self):
|
||||
import time
|
||||
pool = ResidentPool(self.fake, idle_minutes=30)
|
||||
pool.provision("alice")
|
||||
self.assertEqual(
|
||||
pool.reclaim_idle(now=time.time() + 10 * 60), [])
|
||||
self.assertEqual(pool.paused_count(), 1)
|
||||
|
||||
def test_hibernate_failure_keeps_entry_for_retry(self):
|
||||
import time
|
||||
pool = ResidentPool(self.fake, idle_minutes=30)
|
||||
pool.provision("alice")
|
||||
self.fake.hibernate_fail = True
|
||||
self.assertEqual(
|
||||
pool.reclaim_idle(now=time.time() + 31 * 60), [])
|
||||
self.assertEqual(pool.paused_count(), 1)
|
||||
|
||||
|
||||
class ResidentFacadeTest(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.tmp = tempfile.TemporaryDirectory()
|
||||
self.fake = _FakeExecutor(self.tmp.name)
|
||||
|
||||
def tearDown(self):
|
||||
self.tmp.cleanup()
|
||||
|
||||
def test_module_facade_cleans_and_empties_registry(self):
|
||||
_write(_registry_path(self.tmp.name), json.dumps({
|
||||
"a": {"state": "paused", "vm": "chv-resident-a"},
|
||||
"b": {"state": "running", "vm": "chv-resident-b"},
|
||||
"c": {"state": "provisioning", "vm": "chv-resident-c"},
|
||||
"d": {"state": "warm", "vm": "chv-resident-d"},
|
||||
"e": {"state": "paused"},
|
||||
"f": "not-a-dict",
|
||||
}))
|
||||
cleaned = reconcile_resident_orphans_for_executor(self.fake)
|
||||
self.assertEqual(cleaned, ["chv-resident-a", "chv-resident-b",
|
||||
"chv-resident-c"])
|
||||
self.assertEqual(self.fake.shutdowns, cleaned)
|
||||
with open(_registry_path(self.tmp.name)) as fh:
|
||||
self.assertEqual(json.load(fh), {})
|
||||
self.assertIn("resident_pool_reconcile", self.fake.emits)
|
||||
|
||||
def test_module_facade_without_registry_is_empty(self):
|
||||
self.assertEqual(
|
||||
reconcile_resident_orphans_for_executor(self.fake), [])
|
||||
self.assertEqual(self.fake.shutdowns, [])
|
||||
|
||||
def test_module_facade_without_state_dir_is_empty(self):
|
||||
fake = _FakeExecutor(None)
|
||||
fake.state_dir = None
|
||||
self.assertEqual(
|
||||
reconcile_resident_orphans_for_executor(fake), [])
|
||||
|
||||
class WorkerAdapterFacadeTest(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.tmp = tempfile.TemporaryDirectory()
|
||||
self.fake = _FakeExecutor(self.tmp.name)
|
||||
self.audits = []
|
||||
|
||||
def tearDown(self):
|
||||
self.tmp.cleanup()
|
||||
|
||||
def _audit(self, project_id, event, detail):
|
||||
self.audits.append((project_id, event, detail))
|
||||
|
||||
def test_facade_cleans_resident_registry(self):
|
||||
# worker_adapter 的 isinstance 门要求真执行器：用假
|
||||
# 的 shutdown 替身包一层真执行器实例（只借类型）。
|
||||
ex = _RealExecCase.make_executor(self.tmp.name, self.tmp.name)
|
||||
ex.shutdown_vm = self.fake.shutdown_vm
|
||||
ex._emit = self.fake._emit
|
||||
_write(_registry_path(self.tmp.name), json.dumps({
|
||||
"a": {"state": "paused", "vm": "chv-resident-a"},
|
||||
"b": {"state": "running", "vm": "chv-resident-b"},
|
||||
}))
|
||||
cleaned = worker_adapter.reconcile_resident_pool(
|
||||
ex, self._audit, "startup")
|
||||
self.assertEqual(cleaned, ["chv-resident-a", "chv-resident-b"])
|
||||
self.assertEqual(self.fake.shutdowns, cleaned)
|
||||
self.assertTrue(any(
|
||||
event == "chv_resident_pool_reconcile"
|
||||
for _p, event, _d in self.audits))
|
||||
with open(_registry_path(self.tmp.name)) as fh:
|
||||
self.assertEqual(json.load(fh), {})
|
||||
|
||||
def test_facade_never_constructs_resident_pool(self):
|
||||
ex = _RealExecCase.make_executor(self.tmp.name, self.tmp.name)
|
||||
ex.shutdown_vm = self.fake.shutdown_vm
|
||||
ex._emit = self.fake._emit
|
||||
_write(_registry_path(self.tmp.name), json.dumps({
|
||||
"a": {"state": "paused", "vm": "chv-resident-a"},
|
||||
}))
|
||||
with mock.patch(
|
||||
"cloud.chv.resident_pool.ResidentPool",
|
||||
side_effect=AssertionError("must not construct")):
|
||||
cleaned = worker_adapter.reconcile_resident_pool(
|
||||
ex, self._audit, "shutdown")
|
||||
self.assertEqual(cleaned, ["chv-resident-a"])
|
||||
|
||||
def test_facade_ignores_non_chv_executor(self):
|
||||
cleaned = worker_adapter.reconcile_resident_pool(
|
||||
object(), self._audit, "startup")
|
||||
self.assertEqual(cleaned, [])
|
||||
self.assertEqual(self.audits, [])
|
||||
|
||||
def test_facade_does_not_touch_warm_pool_reconcile(self):
|
||||
# reconcile_warm_pool 仍走 warm-pool.json，与常驻无关。
|
||||
ex = _RealExecCase.make_executor(self.tmp.name, self.tmp.name)
|
||||
ex.shutdown_vm = self.fake.shutdown_vm
|
||||
ex._emit = self.fake._emit
|
||||
warm_path = os.path.join(self.tmp.name, "warm-pool.json")
|
||||
_write(warm_path, json.dumps({
|
||||
"w": {"state": "warm", "vm": "chv-warm-1",
|
||||
"task_id": "w"},
|
||||
}))
|
||||
cleaned = worker_adapter.reconcile_warm_pool(
|
||||
ex, self._audit, "startup")
|
||||
self.assertEqual(cleaned, ["chv-warm-1"])
|
||||
self.assertFalse(os.path.exists(_registry_path(self.tmp.name)))
|
||||
|
||||
|
||||
def _write_file(path, content=b"x"):
|
||||
with open(path, "wb") as fh:
|
||||
fh.write(content)
|
||||
os.chmod(path, 0o755)
|
||||
return path
|
||||
|
||||
|
||||
class _RealExecCase(unittest.TestCase):
|
||||
"""真执行器 + 全方法替身：测 _run_with_vm 的接线本身，
|
||||
不测 VMM。"""
|
||||
|
||||
@staticmethod
|
||||
def make_executor(state_dir, base_dir, env_extra=None):
|
||||
env = {"CHV_EGRESS_PROXY": "http://10.0.0.1:8888",
|
||||
"CHV_HOST_KEY_MODE": "keyscan",
|
||||
"CHV_SSH_FORWARD": "1"}
|
||||
if env_extra:
|
||||
env.update(env_extra)
|
||||
cpuinfo = os.path.join(base_dir, "cpuinfo")
|
||||
with open(cpuinfo, "w") as fh:
|
||||
fh.write("flags : fpu vmx sse\n")
|
||||
ex = CloudHypervisorExecutor(
|
||||
binary=_write_file(os.path.join(base_dir,
|
||||
"cloud-hypervisor")),
|
||||
ch_remote=_write_file(os.path.join(base_dir, "ch-remote")),
|
||||
kernel=_write_file(os.path.join(base_dir, "vmlinux")),
|
||||
rootfs=_write_file(os.path.join(base_dir, "disk.raw"),
|
||||
b"R" * 100),
|
||||
ssh_key=_write_file(os.path.join(base_dir, "id_chv")),
|
||||
kvm_path=_write_file(os.path.join(base_dir, "kvm")),
|
||||
cpuinfo_path=cpuinfo,
|
||||
state_dir=state_dir,
|
||||
sleeper=lambda s: None, env=env)
|
||||
ex.audit_callback = lambda e, t, d, k=None: None
|
||||
return ex
|
||||
|
||||
def setUp(self):
|
||||
self.tmp = tempfile.TemporaryDirectory()
|
||||
self.state_dir = os.path.join(self.tmp.name, "state")
|
||||
self.calls = []
|
||||
|
||||
def tearDown(self):
|
||||
self.tmp.cleanup()
|
||||
|
||||
def _stubbed_exec(self, env_extra=None, wake_fail_resident=False):
|
||||
ex = self.make_executor(self.state_dir, self.tmp.name, env_extra)
|
||||
calls = self.calls
|
||||
|
||||
def wake(task, extra_env=None):
|
||||
calls.append(("wake", ex.vm_name(task)))
|
||||
if wake_fail_resident and ex.vm_name(task).startswith(
|
||||
"chv-resident-"):
|
||||
raise RuntimeError("provision boom")
|
||||
return {"vm": ex.vm_name(task), "mode": "cold"}
|
||||
|
||||
def pause(task_or_name):
|
||||
calls.append(("pause", ex.vm_name(task_or_name)))
|
||||
return True
|
||||
|
||||
def resume(task_or_name):
|
||||
calls.append(("resume", ex.vm_name(task_or_name)))
|
||||
return True
|
||||
|
||||
def wait(task):
|
||||
calls.append(("wait", ex.vm_name(task)))
|
||||
return True
|
||||
|
||||
def shutdown(task_or_name):
|
||||
calls.append(("shutdown", ex.vm_name(task_or_name)))
|
||||
return True
|
||||
|
||||
def run_guest(task, inner_args, extra_env=None, **kwargs):
|
||||
calls.append(("guest", task["id"], list(inner_args)))
|
||||
return {"task_id": task["id"], "status": "completed",
|
||||
"stdout": "ok", "vm": ex.vm_name(task)}
|
||||
|
||||
def run_guest_goal(task, inner_args, extra_env=None, **kwargs):
|
||||
calls.append(("guest-goal", task["id"], list(inner_args)))
|
||||
return {"task_id": task["id"], "status": "completed",
|
||||
"goal_met": True, "vm": ex.vm_name(task)}
|
||||
|
||||
ex.wake_vm = wake
|
||||
ex.pause_vm = pause
|
||||
ex.resume_vm = resume
|
||||
ex.wait_guest_ready = wait
|
||||
ex.shutdown_vm = shutdown
|
||||
ex._run_guest = run_guest
|
||||
ex._run_guest_goal = run_guest_goal
|
||||
return ex
|
||||
|
||||
|
||||
class RunWithVmResidentTest(_RealExecCase):
|
||||
def test_first_task_provisions_and_second_reuses(self):
|
||||
ex = self._stubbed_exec()
|
||||
task = {"id": "task-1", "prompt": "hi"}
|
||||
result = ex._run_with_vm(task, ["dispatch", "--task", "hi"])
|
||||
self.assertEqual(result["status"], "completed")
|
||||
# 常驻伪任务跑命令，但 task_id 改回真实任务。
|
||||
self.assertEqual(result["task_id"], "task-1")
|
||||
self.assertEqual(result["vm"], "chv-resident-default")
|
||||
kinds = [c[0] for c in self.calls]
|
||||
self.assertEqual(kinds, ["wake", "pause", "resume", "wait",
|
||||
"guest", "pause"])
|
||||
self.assertEqual(self.calls[0][1], "chv-resident-default")
|
||||
self.assertEqual(self.calls[4][1], "resident-default")
|
||||
# 第二个任务复用同一台常驻 VM，不再 wake。
|
||||
self.calls.clear()
|
||||
task2 = {"id": "task-2", "prompt": "hi2"}
|
||||
result2 = ex._run_with_vm(task2, ["dispatch", "--task", "hi2"])
|
||||
self.assertEqual(result2["task_id"], "task-2")
|
||||
self.assertEqual(result2["vm"], "chv-resident-default")
|
||||
self.assertEqual([c[0] for c in self.calls],
|
||||
["resume", "wait", "guest", "pause"])
|
||||
# 落盘条目是 paused，可供下次 acquire。
|
||||
with open(_registry_path(self.state_dir)) as fh:
|
||||
data = json.load(fh)
|
||||
self.assertEqual(data["default"]["state"], "paused")
|
||||
self.assertEqual(data["default"]["vm"], "chv-resident-default")
|
||||
|
||||
def test_goal_path_runs_on_resident_and_rewrites_task_id(self):
|
||||
ex = self._stubbed_exec()
|
||||
task = {"id": "goal-1", "prompt": "g"}
|
||||
result = ex._run_with_vm(task, ["goal", "--task", "g"],
|
||||
goal=True)
|
||||
self.assertEqual(result["status"], "completed")
|
||||
self.assertEqual(result["task_id"], "goal-1")
|
||||
self.assertTrue(result["goal_met"])
|
||||
self.assertIn(("guest-goal", "resident-default",
|
||||
["goal", "--task", "g"]), self.calls)
|
||||
|
||||
def test_dispatch_task_flows_through_resident(self):
|
||||
ex = self._stubbed_exec()
|
||||
result = ex.dispatch_task({"id": "d-1", "prompt": "hello"})
|
||||
self.assertEqual(result["status"], "completed")
|
||||
self.assertEqual(result["task_id"], "d-1")
|
||||
self.assertEqual(result["vm"], "chv-resident-default")
|
||||
|
||||
def test_per_user_vms_are_isolated(self):
|
||||
ex = self._stubbed_exec()
|
||||
ex._run_with_vm({"id": "t-a", "user_id": "alice",
|
||||
"prompt": "x"}, ["dispatch"])
|
||||
ex._run_with_vm({"id": "t-b", "user_id": "bob",
|
||||
"prompt": "x"}, ["dispatch"])
|
||||
wakes = [c[1] for c in self.calls if c[0] == "wake"]
|
||||
self.assertEqual(wakes, ["chv-resident-alice",
|
||||
"chv-resident-bob"])
|
||||
with open(_registry_path(self.state_dir)) as fh:
|
||||
data = json.load(fh)
|
||||
self.assertEqual(set(data), {"alice", "bob"})
|
||||
|
||||
def test_disabled_flag_goes_cold(self):
|
||||
ex = self._stubbed_exec(
|
||||
env_extra={"CHV_RESIDENT_ENABLED": "0"})
|
||||
result = ex._run_with_vm({"id": "task-1", "prompt": "hi"},
|
||||
["dispatch"])
|
||||
self.assertEqual(result["status"], "completed")
|
||||
self.assertEqual(result["task_id"], "task-1")
|
||||
kinds = [c[0] for c in self.calls]
|
||||
self.assertEqual(kinds, ["wake", "guest", "shutdown"])
|
||||
self.assertEqual(self.calls[0][1], "chv-task-1")
|
||||
self.assertIsNone(getattr(ex, "_resident_pool", None))
|
||||
self.assertFalse(os.path.exists(
|
||||
_registry_path(self.state_dir)))
|
||||
|
||||
def test_secret_env_goes_cold(self):
|
||||
ex = self._stubbed_exec()
|
||||
result = ex._run_with_vm(
|
||||
{"id": "task-1", "prompt": "hi"}, ["dispatch"],
|
||||
extra_env={"SUB2API_API_KEY": "sekret"})
|
||||
self.assertEqual(result["status"], "completed")
|
||||
kinds = [c[0] for c in self.calls]
|
||||
self.assertEqual(kinds, ["wake", "guest", "shutdown"])
|
||||
self.assertFalse(os.path.exists(
|
||||
_registry_path(self.state_dir)))
|
||||
|
||||
def test_provision_failure_falls_back_to_cold(self):
|
||||
ex = self._stubbed_exec(wake_fail_resident=True)
|
||||
result = ex._run_with_vm({"id": "task-1", "prompt": "hi"},
|
||||
["dispatch"])
|
||||
self.assertEqual(result["status"], "completed")
|
||||
self.assertEqual(result["task_id"], "task-1")
|
||||
kinds = [c[0] for c in self.calls]
|
||||
# 建池 wake 失败（尽力 shutdown）→ 冷启动照常跑任务。
|
||||
self.assertEqual(kinds, ["wake", "shutdown", "wake",
|
||||
"guest", "shutdown"])
|
||||
self.assertEqual(self.calls[1][1], "chv-resident-default")
|
||||
self.assertEqual(self.calls[2][1], "chv-task-1")
|
||||
|
||||
def test_guest_error_still_pauses_resident_in_finally(self):
|
||||
ex = self._stubbed_exec()
|
||||
|
||||
def boom(task, inner_args, extra_env=None, **kwargs):
|
||||
raise RuntimeError("guest boom")
|
||||
|
||||
ex._run_guest = boom
|
||||
with self.assertRaises(RuntimeError):
|
||||
ex._run_with_vm({"id": "task-1", "prompt": "hi"},
|
||||
["dispatch"])
|
||||
kinds = [c[0] for c in self.calls]
|
||||
self.assertEqual(kinds, ["wake", "pause", "resume", "wait",
|
||||
"pause"])
|
||||
self.assertEqual(self.calls[-1][1], "chv-resident-default")
|
||||
|
||||
def test_guest_error_still_shuts_down_cold_in_finally(self):
|
||||
ex = self._stubbed_exec(
|
||||
env_extra={"CHV_RESIDENT_ENABLED": "0"})
|
||||
|
||||
def boom(task, inner_args, extra_env=None, **kwargs):
|
||||
raise RuntimeError("guest boom")
|
||||
|
||||
ex._run_guest = boom
|
||||
with self.assertRaises(RuntimeError):
|
||||
ex._run_with_vm({"id": "task-1", "prompt": "hi"},
|
||||
["dispatch"])
|
||||
self.assertEqual([c[0] for c in self.calls],
|
||||
["wake", "shutdown"])
|
||||
|
||||
def test_wake_failure_propagates_without_shutdown(self):
|
||||
ex = self._stubbed_exec(
|
||||
env_extra={"CHV_RESIDENT_ENABLED": "0"})
|
||||
|
||||
def wake_fail(task, extra_env=None):
|
||||
raise ChvFailure("vm_create_failed", "no vm")
|
||||
|
||||
ex.wake_vm = wake_fail
|
||||
with self.assertRaises(ChvFailure):
|
||||
ex._run_with_vm({"id": "task-1", "prompt": "hi"},
|
||||
["dispatch"])
|
||||
self.assertEqual(self.calls, [])
|
||||
|
||||
|
||||
class WarmPoolUntouchedTest(unittest.TestCase):
|
||||
def test_wired_to_dispatch_still_false(self):
|
||||
self.assertFalse(WarmPool.wired_to_dispatch)
|
||||
|
||||
def test_executor_source_never_imports_experimental(self):
|
||||
import ast
|
||||
from pathlib import Path
|
||||
tree = ast.parse((Path(__file__).resolve().parents[2]
|
||||
/ "cloud" / "chv" / "executor.py").read_text())
|
||||
for node in ast.walk(tree):
|
||||
if isinstance(node, ast.Import):
|
||||
for alias in node.names:
|
||||
self.assertNotIn(alias.name, (
|
||||
"cloud.chv.pool",
|
||||
"cloud.chv.experimental_pool",
|
||||
"cloud.chv.orphans"))
|
||||
if isinstance(node, ast.ImportFrom):
|
||||
self.assertNotIn(node.module, (
|
||||
"cloud.chv.pool",
|
||||
"cloud.chv.experimental_pool",
|
||||
"cloud.chv.orphans"))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -227,7 +227,15 @@ class PreflightTest(ChvCase):
|
||||
self.assertIn("内核", text)
|
||||
|
||||
def test_dispatch_missing_dependency_does_not_call_runner(self):
|
||||
ex = self.make_exec(binary="/tmp/no-chv")
|
||||
# P1 常驻默认开：这里显式关常驻，测的仍是冷启动
|
||||
# preflight 路径（缺依赖 → 零 runner 调用）。不开
|
||||
# 关的话，_resident_acquire 会先试建池，wake 失败
|
||||
# 后的清理 shutdown 会多一次 ch-remote 调用——那是
|
||||
# 建池失败清理的既定行为（见 P1 的 provision 失败
|
||||
# 回落测试），不是本测试要守的“缺依赖零调用”。
|
||||
ex = self.make_exec(
|
||||
binary="/tmp/no-chv",
|
||||
env_extra={"CHV_RESIDENT_ENABLED": "0"})
|
||||
with self.assertRaises(ChvFailure):
|
||||
ex.dispatch_task(self._task())
|
||||
self.assertEqual(self.calls, [])
|
||||
@@ -538,7 +546,12 @@ class WarmPoolTest(ChvCase):
|
||||
# -- dispatch / goal（wake → 跑 → finally 关） ---------------------------
|
||||
class DispatchTest(ChvCase):
|
||||
def test_dispatch_wakes_runs_shuts_down(self):
|
||||
ex = self.make_exec()
|
||||
# 本测试钉的是冷启动 dispatch 序列（wake chv-abc123
|
||||
# → 跑 → finally shutdown）；P1 起常驻默认开，须显
|
||||
# 式关常驻才走这条路。常驻 dispatch 由
|
||||
# test_chv_muse_p1_resident 覆盖。
|
||||
ex = self.make_exec(
|
||||
env_extra={"CHV_RESIDENT_ENABLED": "0"})
|
||||
out = ex.dispatch_task(self._task())
|
||||
self.assertEqual(out["executor"], "cloud-hypervisor")
|
||||
self.assertEqual(out["status"], "completed")
|
||||
|
||||
Reference in new issue
Block a user