"""CHV 第三棒（feat-chv-v3）回归测试：WSL 真机 V-1～V-4/F1/F2 实测缺陷的代码层回归。 - V-3① snapshot 传裸 URL（source_url= 是 restore 的键）； - V-3② 快照产物是目录（v54），校验改为目录非空； - V-3③ restore 起 VMM 前清旧 VMM + 旧 socket； - V-3 附带：hibernate 留盘（快照 config.json 引用原盘）， GC 不删有登记快照的 VM 的盘，restore 前预检快照引用的盘； - F1：re-wake 删旧盘（H5 删私钥致新 key 对 + 旧盘 cloud-init 不重应用 = host_key_mismatch）； - F2：跨实例 stop 按 socket 进程扫描清未登记 VMM； - V-4：remote_command source 失败 fail-fast（exit 126）， guest-env 归 SSH 用户所有；V-9：owner 恒 root:root（write_files 先于建用户）+ 客内 runcmd chown，wake 在密钥投递后等 guest-env 对 SSH 用户可读再报 ready（runcmd chown 比 SSH 就绪晚约 30 秒），超时关 VM 再抛 guest_env_not_ready； - V-1 附带：build.sh 预建 /app/work（1777 粘滞位 + 断言）； - V-1 附带：host.py default_codex_bin() 走 PATH 查找。 - V-7 复修（feat-chv-v3-repair）：D1 snapshot 先建目录（旧 fixture 预建目录 + mock ch-remote 不校验，458 测试全过也 盖不住缺失目录的真实路径）、D2 hibernate 留 seed.iso + 残留清理保护有登记快照的 VM 的 seed、F1 任务 SSH 连接建立 阶段瞬时 reset 重试恰好 1 次。 全部用假 runner/launcher/api_client + 真实 /proc 扫描（F2 的被杀进程是真实 spawn 的 sleep 进程，argv[0] 伪装成 cloud-hypervisor），绝不启动真实 VM。 """ import importlib.util import json import os import subprocess import sys import tempfile import unittest from pathlib import Path from unittest import mock try: from cloud.chv.executor import CloudHypervisorExecutor from cloud.chv.hostkeys import write_seed from cloud.chv.ssh_transport import remote_command from cloud.sandbox import ChvFailure except ImportError: sys.path.insert(0, str(Path(__file__).resolve().parents[2])) from cloud.chv.executor import CloudHypervisorExecutor from cloud.chv.hostkeys import write_seed from cloud.chv.ssh_transport import remote_command from cloud.sandbox import ChvFailure ROOT = Path(__file__).resolve().parents[2] class FakeResult: def __init__(self, returncode=0, stdout="", stderr=""): self.returncode = returncode self.stdout = stdout self.stderr = stderr class FakeProc: def __init__(self, alive=True): self._alive = alive self.terminated = False self.killed = False def poll(self): return None if self._alive else 1 def terminate(self): self.terminated = True self._alive = False def wait(self, timeout=None): return 0 def kill(self): self.killed = True self._alive = False def _write(path, content=b"x"): with open(path, "wb") as fh: fh.write(content) os.chmod(path, 0o755) return path def _load_host_py(): spec = importlib.util.spec_from_file_location( "chv_host_py", str(ROOT / "host.py")) mod = importlib.util.module_from_spec(spec) spec.loader.exec_module(mod) return mod class V3Case(unittest.TestCase): def setUp(self): self.tmp = tempfile.TemporaryDirectory() self.dir = self.tmp.name self.binary = _write(os.path.join(self.dir, "cloud-hypervisor")) self.ch_remote = _write(os.path.join(self.dir, "ch-remote")) self.kernel = _write(os.path.join(self.dir, "vmlinux")) self.rootfs = _write(os.path.join(self.dir, "disk.raw"), b"R" * 100) self.ssh_key = _write(os.path.join(self.dir, "id_chv")) self.kvm = _write(os.path.join(self.dir, "kvm")) self.cpuinfo = os.path.join(self.dir, "cpuinfo") with open(self.cpuinfo, "w") as fh: fh.write("flags : fpu vmx sse\n") self.state_dir = os.path.join(self.dir, "state") self.calls = [] self.api_calls = [] self.launches = [] self.audit = [] self.proc = None self.spawned = [] def tearDown(self): for proc in self.spawned: try: if proc.poll() is None: proc.terminate() proc.wait(timeout=5) except Exception: pass self.tmp.cleanup() def make_exec(self, runner=None, api_client=None, launcher=None, env_extra=None, **overrides): env = {"CHV_EGRESS_PROXY": "http://10.0.0.1:8888", "CHV_HOST_KEY_MODE": "keyscan", "CHV_SSH_FORWARD": "1"} if env_extra: env.update(env_extra) kwargs = dict( binary=self.binary, ch_remote=self.ch_remote, kernel=self.kernel, rootfs=self.rootfs, ssh_key=self.ssh_key, kvm_path=self.kvm, cpuinfo_path=self.cpuinfo, state_dir=self.state_dir, runner=runner or self._default_runner, launcher=launcher or self._default_launcher, api_client=api_client or self._default_api, sleeper=lambda s: None, env=env) kwargs.update(overrides) ex = CloudHypervisorExecutor(**kwargs) ex.audit_callback = lambda e, t, d, k=None: self.audit.append( (e, t.get("id") if isinstance(t, dict) else t, d, k)) return ex def _default_runner(self, cmd, timeout=None): self.calls.append(list(cmd)) if cmd and cmd[0] == "ssh": if cmd[-1] == "true": return FakeResult(0, "", "") return FakeResult(0, "", "") if cmd and cmd[0] == "ssh-keygen": priv = cmd[cmd.index("-f") + 1] with open(priv, "w") as fh: fh.write("-----BEGIN OPENSSH PRIVATE KEY-----\nFAKE\n") os.chmod(priv, 0o600) with open(priv + ".pub", "w") as fh: fh.write("ssh-ed25519 AAAAfake1 vm-key\n") return FakeResult(0, "", "") if cmd and cmd[0] == "genisoimage": out = cmd[cmd.index("-output") + 1] with open(out, "wb") as fh: fh.write(b"CIDATA" * 16) return FakeResult(0, "", "") return FakeResult(0, "", "") def _default_api(self, socket_path, method, path, payload=None): self.api_calls.append((method, path, payload)) return (200, {}) def _default_launcher(self, cmd): self.launches.append(list(cmd)) self.proc = FakeProc(alive=True) return self.proc def _task(self, task_id="abc123"): return {"id": task_id, "prompt": "做一个加法函数", "project_id": "proj1"} def _snap_dir(self, name="snap1", files=None): """建一个 v54 形态的快照目录，返回 (url, path)。""" path = os.path.join(self.state_dir, name) os.makedirs(path, exist_ok=True) for fname, content in (files or { "config.json": b"{}", "state.json": b"{}", "memory-ranges": b"m"}).items(): with open(os.path.join(path, fname), "wb") as fh: fh.write(content) return "file://" + path, path def _spawn_fake_vmm(self, sock_path, argv0="cloud-hypervisor"): """真实 spawn 一个长眠进程，argv[0] 伪装成 VMM， 命令行带 --api-socket （F2/V-3③ 的扫描目标）。""" proc = subprocess.Popen( [argv0, "-c", "import time; time.sleep(30)", "--api-socket", sock_path], executable=sys.executable, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) self.spawned.append(proc) return proc # -- V-3①/② 快照：裸 URL + 目录校验 --------------------------------------- class SnapshotV3Test(V3Case): def test_snapshot_passes_bare_url(self): url, _ = self._snap_dir() ex = self.make_exec() out = ex.snapshot_vm(self._task(), url) self.assertEqual(out, url) snap_calls = [c for c in self.calls if c[0] == self.ch_remote and "snapshot" in c] self.assertEqual(len(snap_calls), 1) self.assertEqual(snap_calls[0][-1], url) self.assertFalse(any(a.startswith("source_url") for a in snap_calls[0])) def test_restore_still_passes_source_url(self): url, _ = self._snap_dir() ex = self.make_exec() out = ex.restore_vm(self._task(), url) self.assertEqual(out["mode"], "restored") restore_calls = [c for c in self.calls if c[0] == self.ch_remote and "restore" in c] self.assertEqual(len(restore_calls), 1) self.assertEqual(restore_calls[0][-1], "source_url=%s" % url) def test_snapshot_dir_records_dir_total_size(self): url, path = self._snap_dir(files={ "config.json": b"C" * 30, "state.json": b"S" * 30}) ex = self.make_exec() ex.snapshot_vm(self._task(), url) with open(os.path.join(self.state_dir, "chv-abc123", "snapshots.json")) as fh: entries = json.load(fh) self.assertEqual(len(entries), 1) self.assertEqual(entries[0]["size"], 60) self.assertEqual(entries[0]["path"], path) # -- V-3③ restore 准备：清旧 VMM + 旧 socket ------------------------------ class RestorePrepareV3Test(V3Case): def test_restore_kills_registered_old_vmm_and_socket(self): url, _ = self._snap_dir() ex = self.make_exec() ex.wake_vm(self._task()) old_proc = self.proc self.assertTrue(old_proc.poll() is None) sock = os.path.join(self.state_dir, "chv-abc123.sock") with open(sock, "w") as fh: fh.write("") out = ex.restore_vm(self._task(), url) self.assertEqual(out["mode"], "restored") self.assertTrue(old_proc.terminated) # 新 VMM 已起（第 2 次 launch），旧 socket 文件在 # launch 前已被清掉（fake launcher 不重建 socket 文件）。 self.assertEqual(len(self.launches), 2) self.assertFalse(os.path.exists(sock)) def test_restore_kills_unregistered_vmm_by_socket(self): url, _ = self._snap_dir() ex = self.make_exec() sock = os.path.join(self.state_dir, "chv-abc123.sock") os.makedirs(self.state_dir, exist_ok=True) with open(sock, "w") as fh: fh.write("") victim = self._spawn_fake_vmm(sock) self.assertIsNone(victim.poll()) out = ex.restore_vm(self._task(), url) self.assertEqual(out["mode"], "restored") self.assertIsNotNone(victim.wait(timeout=5)) # -- V-3 附带：hibernate 留盘 + GC 保护 + restore 盘预检 ------------------- class HibernateDiskV3Test(V3Case): def test_hibernate_keeps_task_disk(self): ex = self.make_exec() ex.wake_vm(self._task()) disk = os.path.join(self.state_dir, "chv-abc123", "disk.raw") self.assertTrue(os.path.exists(disk)) url, _ = self._snap_dir() out = ex.hibernate_vm(self._task(), url) self.assertEqual(out["state"], "hibernated") self.assertTrue(os.path.exists(disk), "hibernate 的 stop 不得删盘：快照 config.json " "引用该盘，删了 restore 即失败") def test_gc_spares_disk_while_snapshot_recorded(self): ex = self.make_exec() ex.wake_vm(self._task()) url, _ = self._snap_dir() ex.hibernate_vm(self._task(), url) disk = os.path.join(self.state_dir, "chv-abc123", "disk.raw") result = ex.gc_stale_disks() self.assertEqual(result["deleted"], []) self.assertTrue(os.path.exists(disk)) # 快照登记清掉后，盘成孤儿，GC 才可删。 os.remove(os.path.join(self.state_dir, "chv-abc123", "snapshots.json")) result = ex.gc_stale_disks() self.assertEqual(result["deleted"], ["chv-abc123"]) self.assertFalse(os.path.exists(disk)) def test_restore_missing_snapshot_disk_fail_fast(self): ex = self.make_exec() missing_disk = os.path.join(self.state_dir, "gone", "disk.raw") url, _ = self._snap_dir(files={ "config.json": json.dumps( {"disks": [{"path": missing_disk}]}).encode(), "state.json": b"{}"}) with self.assertRaises(ChvFailure) as ctx: ex.restore_vm(self._task(), url) self.assertEqual(ctx.exception.kind, "snapshot_disk_missing") self.assertEqual(self.launches, []) def test_restore_snapshot_disk_present_passes_precheck(self): ex = self.make_exec() ex.wake_vm(self._task()) disk = os.path.join(self.state_dir, "chv-abc123", "disk.raw") url, _ = self._snap_dir(files={ "config.json": json.dumps( {"disks": [{"path": disk}]}).encode(), "state.json": b"{}"}) out = ex.restore_vm(self._task(), url) self.assertEqual(out["mode"], "restored") # -- F1：re-wake 删旧盘 ----------------------------------------------------- class RewakeDiskV3Test(V3Case): def _corrupt_disk(self, task_id="abc123"): disk = os.path.join(self.state_dir, "chv-" + task_id, "disk.raw") with open(disk, "wb") as fh: fh.write(b"STALE" * 20) return disk def test_rewake_live_vm_replaces_stale_disk(self): ex = self.make_exec() ex.wake_vm(self._task()) disk = self._corrupt_disk() out = ex.wake_vm(self._task()) self.assertEqual(out["state"], "running") with open(disk, "rb") as fh: self.assertEqual(fh.read(), b"R" * 100, "re-wake 须重拷新盘：旧盘复用 + 新 key 对 " "= cloud-init 不重应用 = host_key_mismatch") def test_rewake_after_crash_replaces_stale_disk(self): ex = self.make_exec() ex.wake_vm(self._task()) disk = self._corrupt_disk() ex2 = self.make_exec() out = ex2.wake_vm(self._task()) self.assertEqual(out["state"], "running") with open(disk, "rb") as fh: self.assertEqual(fh.read(), b"R" * 100) def test_provisioned_rewake_regenerates_keypair_on_fresh_disk(self): ex = self.make_exec( env_extra={"CHV_HOST_KEY_MODE": "provisioned"}) out = ex.wake_vm(self._task()) self.assertEqual(out["state"], "running") vm_dir = os.path.join(self.state_dir, "chv-abc123") # H5：首 wake 后宿主私钥已删，re-wake 必然重新生成 key 对。 self.assertFalse(os.path.exists( os.path.join(vm_dir, "ssh_host_ed25519_key"))) self.assertTrue(os.path.exists( os.path.join(vm_dir, "known_hosts"))) disk = self._corrupt_disk() geniso_before = sum(1 for c in self.calls if c and c[0] == "genisoimage") out2 = ex.wake_vm(self._task()) self.assertEqual(out2["state"], "running") with open(disk, "rb") as fh: self.assertEqual(fh.read(), b"R" * 100) geniso_after = sum(1 for c in self.calls if c and c[0] == "genisoimage") self.assertEqual(geniso_after, geniso_before + 1, "re-wake 须重建 seed 盘（新 key 对进新 seed）") # -- F2：跨实例 stop 按 socket 进程扫描 ------------------------------------- class CrossInstanceStopV3Test(V3Case): def test_stop_kills_unregistered_vmm_by_socket(self): ex = self.make_exec() sock = os.path.join(self.state_dir, "chv-abc123.sock") os.makedirs(self.state_dir, exist_ok=True) with open(sock, "w") as fh: fh.write("") victim = self._spawn_fake_vmm(sock) self.assertIsNone(victim.poll()) self.assertTrue(ex.stop_vm(self._task())) self.assertIsNotNone(victim.wait(timeout=5)) self.assertFalse(os.path.exists(sock)) def test_stop_does_not_kill_others_vmm_or_chremote(self): ex = self.make_exec() sock = os.path.join(self.state_dir, "chv-abc123.sock") other_sock = os.path.join(self.state_dir, "chv-other.sock") os.makedirs(self.state_dir, exist_ok=True) for path in (sock, other_sock): with open(path, "w") as fh: fh.write("") other_vmm = self._spawn_fake_vmm(other_sock) # 同带本 socket 的 ch-remote 客户端：真实形态是 # `ch-remote --api-socket= shutdown`，不是 VMM， # 绝不杀。 chremote = subprocess.Popen( ["ch-remote", "-c", "import time; time.sleep(30)", "--api-socket=" + sock], executable=sys.executable, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) self.spawned.append(chremote) self.assertIsNone(other_vmm.poll()) self.assertIsNone(chremote.poll()) ex.stop_vm(self._task()) self.assertIsNone(other_vmm.poll()) self.assertIsNone(chremote.poll()) # -- V-4：guest-env source 失败 fail-fast + 归 SSH 用户所有 ------------------ class GuestEnvV4Test(V3Case): def test_remote_command_source_failure_exits(self): tokens = remote_command(["python3", "/app/host.py", "status"], guest_env_file="/etc/chv/guest-env") joined = " ".join(tokens) self.assertIn(". /etc/chv/guest-env || exit 126;", joined) self.assertNotIn("/etc/chv/guest-env';", joined) def test_remote_command_without_guest_env_unchanged(self): tokens = remote_command(["true"]) self.assertEqual(tokens, ["true"]) def test_write_seed_guest_env_owner_always_root(self): # V-9 真机根因：write_files 在 users-groups 之前跑， # owner 写 chv:chv 时 getpwnam 失败、整个模块失败， # 文件根本不产生；故 owner 恒为 root:root，改归 SSH # 用户由客内 final 阶段 99-chv-sync.cfg runcmd chown。 import yaml for ssh_user in (None, "root", "chv"): seed_dir = os.path.join(self.dir, "seed-%s" % ssh_user) write_seed(seed_dir, "vm1", "PRIV", "ssh-ed25519 AAAAx", secret_env={"SUB2API_API_KEY": "s3cr3t"}, ssh_user=ssh_user) with open(os.path.join(seed_dir, "user-data")) as fh: parsed = yaml.safe_load(fh.read()) guest = [e for e in parsed["write_files"] if e["path"] == "/etc/chv/guest-env"] self.assertEqual(len(guest), 1) self.assertEqual(guest[0]["owner"], "root:root") self.assertEqual(guest[0]["permissions"], "0600") def test_provisioned_wake_writes_guest_env_for_chv_user(self): import yaml ex = self.make_exec( env_extra={"CHV_HOST_KEY_MODE": "provisioned"}, ssh_user="chv") out = ex.wake_vm(self._task(), extra_env={"SUB2API_API_KEY": "s3cr3t"}) self.assertEqual(out["state"], "running") user_data = os.path.join( self.state_dir, "chv-abc123", "seed", "user-data") with open(user_data) as fh: parsed = yaml.safe_load(fh.read()) guest = [e for e in parsed["write_files"] if e["path"] == "/etc/chv/guest-env"] self.assertEqual(len(guest), 1) # V-9：owner 恒为 root:root（write_files 先于建用户）， # 改归 chv 由客内 99-chv-sync.cfg runcmd chown。 self.assertEqual(guest[0]["owner"], "root:root") def _readable_runner(self, answers): """Fake runner：`test -r /etc/chv/guest-env` 按 answers 依次回可读与否（用完后重复最后一个），其余命令走默认 runner。answers 是 bool 列表。""" state = {"n": 0} def run(cmd, timeout=None): if ("test" in cmd and cmd[-1] == "/etc/chv/guest-env"): idx = min(state["n"], len(answers) - 1) state["n"] += 1 return FakeResult(0 if answers[idx] else 1, "", "") return self._default_runner(cmd, timeout) run.readable_calls = lambda: state["n"] return run def test_wake_waits_for_guest_env_readable(self): # V-9：runcmd chown 比 SSH 就绪晚约 30 秒，wake 须等 # guest-env 对 SSH 用户可读再报 ready，否则首个派发 # source 它即 Permission denied。 runner = self._readable_runner([False, False, True]) ex = self.make_exec( runner=runner, env_extra={"CHV_HOST_KEY_MODE": "provisioned"}, ssh_user="chv") out = ex.wake_vm(self._task(), extra_env={"SUB2API_API_KEY": "s3cr3t"}) self.assertEqual(out["state"], "running") self.assertEqual(runner.readable_calls(), 3) def test_wake_fails_when_guest_env_never_readable(self): # V-9：guest-env 一直对 SSH 用户不可读时 wake 超时即 # 关 VM 再抛 guest_env_not_ready，绝不带着读不到的密钥 # 跑任务。 runner = self._readable_runner([False]) ex = self.make_exec( runner=runner, env_extra={"CHV_HOST_KEY_MODE": "provisioned"}, ssh_user="chv") with self.assertRaises(ChvFailure) as ctx: ex.wake_vm(self._task(), extra_env={"SUB2API_API_KEY": "s3cr3t"}) self.assertEqual(ctx.exception.kind, "guest_env_not_ready") self.assertGreater(runner.readable_calls(), 1) # -- V-1 附带：build.sh /app/work + host.py default_codex_bin -------------- class GuestBuildAndHostBinV3Test(unittest.TestCase): def test_build_sh_precreates_app_work_writable(self): text = (ROOT / "chv-guest" / "build.sh").read_text() self.assertIn("mkdir -p /app/work", text) self.assertIn("chmod 1777 /app/work", text) self.assertIn('stat -c %a /app/work', text) def test_build_sh_runcmd_chowns_guest_env_before_sync_boot(self): # V-9：guest-env 由 write_files 以 root:root 写（建用户 # 之前），须在 final 阶段 chv-sync boot 之前改归 chv， # 否则 SSH 派发 source 它会 Permission denied；无文件 # 时 test -f 护栏使该行空操作，runcmd 失败不影响启动。 text = (ROOT / "chv-guest" / "build.sh").read_text() chown = ("test -f /etc/chv/guest-env && " "chown chv:chv /etc/chv/guest-env || true") self.assertIn(chown, text) # "chv-sync boot" 在文件前部注释里先出现过一次，故从 # chown 位置往后找同一 runcmd 行里的 boot。 chown_at = text.index(chown) boot_at = text.index("/usr/local/bin/chv-sync boot", chown_at) self.assertLess(chown_at, boot_at) def test_build_sh_dry_run_prints_app_work_step(self): tmp = tempfile.TemporaryDirectory() self.addCleanup(tmp.cleanup) bindir = os.path.join(tmp.name, "bin") os.makedirs(os.path.join(bindir, "codex-resources")) for name in ("codex", "codex-code-mode-host"): p = os.path.join(bindir, name) with open(p, "w") as fh: fh.write("#!/bin/sh\necho \"codex-cli 0.164.0-alpha.9\"\n") os.chmod(p, 0o755) repo = os.path.join(tmp.name, "repo") os.makedirs(repo) with open(os.path.join(repo, "host.py"), "w") as fh: fh.write("# host\n") subprocess.run(["git", "init", "-q", repo], check=True, capture_output=True) subprocess.run(["git", "-C", repo, "config", "user.email", "t@t"], check=True, capture_output=True) subprocess.run(["git", "-C", repo, "config", "user.name", "t"], check=True, capture_output=True) subprocess.run(["git", "-C", repo, "add", "-A"], check=True, capture_output=True) subprocess.run(["git", "-C", repo, "commit", "-qm", "init"], check=True, capture_output=True) commit = subprocess.run( ["git", "-C", repo, "rev-parse", "--short", "HEAD"], check=True, capture_output=True, text=True).stdout.strip() env = dict(os.environ) env.update({"CODEX_BIN_DIR": bindir, "UBUNTU_VERSION": "24.04", "CODEX_VERSION": "0.164.0-alpha.9", "REPO_COMMIT": commit, "UBUNTU_IMAGE_SHA256": "0" * 64, "REPO_DIR": repo}) r = subprocess.run( ["bash", str(ROOT / "chv-guest" / "build.sh"), "--dry-run", os.path.join(tmp.name, "out")], check=False, capture_output=True, text=True, env=env) self.assertEqual(r.returncode, 0, r.stderr) self.assertIn("/app/work", r.stdout) self.assertIn("1777", r.stdout) def test_default_codex_bin_uses_path_lookup(self): mod = _load_host_py() tmp = tempfile.TemporaryDirectory() self.addCleanup(tmp.cleanup) fake = os.path.join(tmp.name, "codex") with open(fake, "w") as fh: fh.write("#!/bin/sh\n") os.chmod(fake, 0o755) env = dict(os.environ) env.pop("CODEX_BIN", None) env["PATH"] = tmp.name + os.pathsep + env.get("PATH", "") with mock.patch.dict(os.environ, env, clear=True): self.assertEqual(mod.default_codex_bin(), fake) def test_default_codex_bin_falls_back_to_bare_name(self): mod = _load_host_py() tmp = tempfile.TemporaryDirectory() self.addCleanup(tmp.cleanup) env = {"PATH": tmp.name} with mock.patch.dict(os.environ, env, clear=True): self.assertEqual(mod.default_codex_bin(), "codex") def test_default_codex_bin_env_still_wins(self): mod = _load_host_py() with mock.patch.dict(os.environ, {"CODEX_BIN": "/opt/custom/codex"}): self.assertEqual(mod.default_codex_bin(), "/opt/custom/codex") # -- V-7/D1：snapshot 先建目标目录 --------------------------------------- class SnapshotMkdirV7Test(V3Case): def _snapshot_creating_runner(self): """模拟真实 ch-remote：往目标目录写快照文件。目录不 存在时 open 即 FileNotFoundError——若执行器没先 mkdir，测试直接失败，这正是旧 fixture 盖不住的点 （旧 fixture 预建目录，mock 又不写文件）。""" def runner(cmd, timeout=None): if cmd and cmd[0] == self.ch_remote and "snapshot" in cmd: self.calls.append(list(cmd)) target = cmd[-1][len("file://"):] for fname, content in ( ("config.json", b"{}"), ("state.json", b"{}"), ("memory-ranges", b"m")): with open(os.path.join(target, fname), "wb") as fh: fh.write(content) return FakeResult(0, "", "") return self._default_runner(cmd, timeout) return runner def test_snapshot_creates_missing_snapshot_dir(self): ex = self.make_exec(runner=self._snapshot_creating_runner()) url = "file://" + os.path.join(self.state_dir, "fresh-snap") self.assertFalse(os.path.exists(url[len("file://"):])) out = ex.snapshot_vm(self._task(), url) self.assertEqual(out, url) snap_path = url[len("file://"):] self.assertTrue(os.path.isdir(snap_path)) self.assertIn("config.json", os.listdir(snap_path)) with open(os.path.join(self.state_dir, "chv-abc123", "snapshots.json")) as fh: entries = json.load(fh) self.assertEqual(entries[0]["path"], snap_path) def test_hibernate_creates_missing_snapshot_dir(self): ex = self.make_exec(runner=self._snapshot_creating_runner()) ex.wake_vm(self._task()) url = "file://" + os.path.join(self.state_dir, "fresh-snap") out = ex.hibernate_vm(self._task(), url) self.assertEqual(out["state"], "hibernated") self.assertTrue(os.path.isdir(url[len("file://"):])) def test_snapshot_target_existing_file_rejected(self): ex = self.make_exec() target = os.path.join(self.state_dir, "not-a-dir") os.makedirs(self.state_dir, exist_ok=True) with open(target, "wb") as fh: fh.write(b"x") with self.assertRaises(ChvFailure) as ctx: ex.snapshot_vm(self._task(), "file://" + target) self.assertEqual(ctx.exception.kind, "snapshot_not_persisted") self.assertEqual(self.calls, [], "目标不是目录时 ch-remote 根本不应被调") # -- V-7/D2：hibernate 留 seed.iso ---------------------------------------- class HibernateSeedIsoV7Test(V3Case): def _prov_exec(self): # keyscan 模式不制 seed.iso；provisioned 模式 wake 即 # 建 seed.iso（真机休眠的形态），用它走全链路。 return self.make_exec( env_extra={"CHV_HOST_KEY_MODE": "provisioned"}) def _seed_iso(self): return os.path.join(self.state_dir, "chv-abc123", "seed.iso") def _disk(self): return os.path.join(self.state_dir, "chv-abc123", "disk.raw") def _snap_with_disks(self): """快照 config.json 同时引用 disk.raw 与 seed.iso （真机快照的 disks 形态）。""" return self._snap_dir(files={ "config.json": json.dumps( {"disks": [{"path": self._disk()}, {"path": self._seed_iso()}]}).encode(), "state.json": b"{}"}) def test_hibernate_keeps_seed_iso(self): ex = self._prov_exec() ex.wake_vm(self._task()) self.assertTrue(os.path.exists(self._seed_iso())) url, _ = self._snap_dir() out = ex.hibernate_vm(self._task(), url) self.assertEqual(out["state"], "hibernated") self.assertTrue(os.path.exists(self._disk()), "hibernate 仍须留 disk.raw") self.assertTrue(os.path.exists(self._seed_iso()), "hibernate 须留 seed.iso：快照 config.json " "的 disks 引用它，删了 restore 即失败") # 只留 iso：seed 目录（制盘输入）与宿主私钥仍删， # config.json 不引用它们，私钥明文不留宿主。 self.assertFalse(os.path.exists( os.path.join(self.state_dir, "chv-abc123", "seed"))) self.assertFalse(os.path.exists( os.path.join(self.state_dir, "chv-abc123", "ssh_host_ed25519_key"))) def test_stop_without_keep_disk_removes_seed_iso(self): ex = self._prov_exec() ex.wake_vm(self._task()) self.assertTrue(ex.stop_vm(self._task())) self.assertFalse(os.path.exists(self._seed_iso()), "正常 stop 仍须删 seed.iso：它含宿主私钥" "明文，不得随 VM 残留") self.assertFalse(os.path.exists(self._disk())) def test_cleanup_stale_secrets_spares_seed_iso_while_snapshot_recorded(self): ex = self._prov_exec() ex.wake_vm(self._task()) url, _ = self._snap_dir() ex.hibernate_vm(self._task(), url) cleaned = ex.cleanup_stale_secrets() self.assertNotIn("chv-abc123", cleaned) self.assertTrue(os.path.exists(self._seed_iso()), "有登记快照的 VM 的 seed.iso 不是孤儿，" "残留清理不得清掉（与 gc_stale_disks 的" "盘保护同款逻辑）") # 快照登记清掉后，seed.iso 成孤儿，才可清。 os.remove(os.path.join(self.state_dir, "chv-abc123", "snapshots.json")) cleaned = ex.cleanup_stale_secrets() self.assertIn("chv-abc123", cleaned) self.assertFalse(os.path.exists(self._seed_iso())) def test_restore_after_hibernate_with_seed_iso_passes_precheck(self): ex = self._prov_exec() ex.wake_vm(self._task()) url, _ = self._snap_with_disks() ex.hibernate_vm(self._task(), url) out = ex.restore_vm(self._task(), url) self.assertEqual(out["mode"], "restored") # -- V-7/F1：任务 SSH 连接建立阶段瞬时 reset 重试 1 次 -------------------- class SshTransientRetryV7Test(V3Case): def _dispatch_runner(self, first, second=None): """首条 dispatch SSH 返回 first，之后返回 second （默认成功）。只记 dispatch SSH 的调用次数。""" state = {"n": 0} def runner(cmd, timeout=None): if cmd and cmd[0] == "ssh" and "dispatch" in cmd: state["n"] += 1 if state["n"] == 1: return first return second or FakeResult( 0, json.dumps({"status": "completed", "text": "ok"}), "") return self._default_runner(cmd, timeout) runner.state = state return runner def test_dispatch_retries_once_on_kex_reset_then_succeeds(self): runner = self._dispatch_runner(FakeResult( 255, "", "kex_exchange_identification: Connection reset by peer")) ex = self.make_exec(runner=runner) out = ex.dispatch_task(self._task()) self.assertEqual(out["status"], "completed") self.assertEqual(runner.state["n"], 2, "kex 阶段瞬时 reset 只重试恰好 1 次") self.assertTrue(any(e == "vm_ssh_transient_retry" for e, _t, _d, _k in self.audit)) def test_dispatch_does_not_retry_on_host_key_mismatch(self): runner = self._dispatch_runner(FakeResult( 255, "", "Host key verification failed.")) ex = self.make_exec(runner=runner) out = ex.dispatch_task(self._task()) self.assertEqual(out["status"], "failed") self.assertEqual(runner.state["n"], 1, "host key 拒绝（同为 255）绝不重试") self.assertFalse(any(e == "vm_ssh_transient_retry" for e, _t, _d, _k in self.audit)) def test_dispatch_does_not_retry_on_remote_failure(self): runner = self._dispatch_runner(FakeResult( 1, json.dumps({"status": "failed", "text": "boom"}), "")) ex = self.make_exec(runner=runner) out = ex.dispatch_task(self._task()) self.assertEqual(out["status"], "failed") self.assertEqual(runner.state["n"], 1, "远端命令失败不重试") def test_dispatch_retries_at_most_once(self): runner = self._dispatch_runner( FakeResult(255, "", "Connection reset by peer"), second=FakeResult(255, "", "Connection reset by peer")) ex = self.make_exec(runner=runner) out = ex.dispatch_task(self._task()) self.assertEqual(out["status"], "failed") self.assertEqual(runner.state["n"], 2, "第二次失败即停，不再追") def test_transient_retry_helper_retries_once(self): """helper 本身（_run_guest 与 _run_guest_goal 共用）： 首条 255 + reset 标记即重试 1 次，第二次成功即返回。""" calls = [] def runner(cmd, timeout=None): calls.append(list(cmd)) if len(calls) == 1: return FakeResult(255, "", "Connection reset by peer") return FakeResult(0, "ok", "") ex = self.make_exec(runner=runner) result = ex._call_ssh_with_transient_retry( self._task(), ["ssh", "true"]) self.assertEqual(result.returncode, 0) self.assertEqual(len(calls), 2) if __name__ == "__main__": unittest.main()