"""Cloud Hypervisor 优化轨（feat-chv-optimize，堆叠于 #216 之上）测试。 只用标准库与 mock runner/launcher/api_client，绝不启动真实 VM（与 test_cloud_hypervisor.py 同款边界）。覆盖： - 拆模块：cloud/chv/* 新路径与 cloud/sandbox 旧路径是同一对象 - 去重复：Local 与 CHV 的 goal 解析在矩阵输入下结论全等； 三执行器 host 顶层参数顺序全等；runner 超时 helper - 安全：每 VM known_hosts 钉死、ssh-keyscan 首次捕获（TOFU）、 非 root 用户与端口显式配置与收紧校验 - execute 语义：只读已运行 VM，未运行报 vm_not_running - 预热池：registry 持久化、重启载入只当孤儿、reconcile 清理、 损坏 quarantine、一 次性不复用 - 长任务：VM 中途死亡判 interrupted 而非 failed/抛异常 """ import json import os import sys import tempfile import unittest from pathlib import Path try: from cloud.chv import ChvFailure from cloud.chv import CloudHypervisorExecutor from cloud.chv import reconcile_orphans_for_executor from cloud.chv.experimental_pool import WarmPool from cloud.chv.api_client import _unix_socket_api as _chv_api from cloud.chv.config import _parse_memory_bytes as _chv_mem from cloud.sandbox import ChvFailure as _sb_failure from cloud.sandbox import CloudHypervisorExecutor as _sb_exec from cloud.sandbox import DockerExecutor from cloud.sandbox import reconcile_orphans_for_executor as _sb_reconcile from cloud.sandbox import LocalExecutor from cloud.sandbox import _call_runner_with_timeout from cloud.sandbox import _parse_memory_bytes as _sb_mem from cloud.sandbox import _unix_socket_api as _sb_api except ImportError: # discover -s cloud/tests 时把项目根目录补进 sys.path sys.path.insert(0, str(Path(__file__).resolve().parents[2])) from cloud.chv import ChvFailure from cloud.chv import CloudHypervisorExecutor from cloud.chv import reconcile_orphans_for_executor from cloud.chv.experimental_pool import WarmPool from cloud.chv.api_client import _unix_socket_api as _chv_api from cloud.chv.config import _parse_memory_bytes as _chv_mem from cloud.sandbox import ChvFailure as _sb_failure from cloud.sandbox import CloudHypervisorExecutor as _sb_exec from cloud.sandbox import DockerExecutor from cloud.sandbox import reconcile_orphans_for_executor as _sb_reconcile from cloud.sandbox import LocalExecutor from cloud.sandbox import _call_runner_with_timeout from cloud.sandbox import _parse_memory_bytes as _sb_mem from cloud.sandbox import _unix_socket_api as _sb_api class FakeResult: def __init__(self, returncode=0, stdout="", stderr=""): self.returncode = returncode self.stdout = stdout self.stderr = stderr class FakeProc: def __init__(self, alive=True): self._alive = alive self.terminated = False def poll(self): return None if self._alive else 1 def terminate(self): self.terminated = True self._alive = False def wait(self, timeout=None): return 0 def kill(self): self._alive = False def _write(path, content=b"x"): with open(path, "wb") as fh: fh.write(content) os.chmod(path, 0o755) return path class ChvOptCase(unittest.TestCase): def setUp(self): self.tmp = tempfile.TemporaryDirectory() self.dir = self.tmp.name self.binary = _write(os.path.join(self.dir, "cloud-hypervisor")) self.ch_remote = _write(os.path.join(self.dir, "ch-remote")) self.kernel = _write(os.path.join(self.dir, "vmlinux")) self.rootfs = _write(os.path.join(self.dir, "disk.raw"), b"R" * 100) self.ssh_key = _write(os.path.join(self.dir, "id_chv")) self.kvm = _write(os.path.join(self.dir, "kvm")) self.cpuinfo = os.path.join(self.dir, "cpuinfo") with open(self.cpuinfo, "w") as fh: fh.write("flags : fpu vmx sse\n") self.state_dir = os.path.join(self.dir, "state") self.calls = [] self.api_calls = [] self.launches = [] def tearDown(self): self.tmp.cleanup() def make_exec(self, runner=None, api_client=None, launcher=None, env_extra=None, **overrides): env = {"CHV_EGRESS_PROXY": "http://10.0.0.1:8888", # 实验室宣称：mock runner 假装回环端口已转发到客机 #（feat-chv-fix 起，执行器不再无条件假设回环能通）。 # 实验室显式 opt-in keyscan：生产默认已是 # provisioned；本套件旧 fake 不实现 ssh-keygen/ # genisoimage，测的不是 host key 模式，故显式 # 声明实验室路径，绝不把它写成生产默认。 "CHV_HOST_KEY_MODE": "keyscan", "CHV_SSH_FORWARD": "1"} if env_extra: env.update(env_extra) if runner is None: runner = self._default_runner if api_client is None: api_client = self._default_api if launcher is None: launcher = self._default_launcher kwargs = dict( binary=self.binary, ch_remote=self.ch_remote, kernel=self.kernel, rootfs=self.rootfs, ssh_key=self.ssh_key, kvm_path=self.kvm, cpuinfo_path=self.cpuinfo, state_dir=self.state_dir, runner=runner, launcher=launcher, api_client=api_client, sleeper=lambda s: None, env=env) kwargs.update(overrides) return CloudHypervisorExecutor(**kwargs) def _default_runner(self, cmd, timeout=None): self.calls.append(list(cmd)) if cmd[0] == "ssh": if cmd[-1] == "true": return FakeResult(0, "", "") if "dispatch" in cmd: return FakeResult(0, json.dumps( {"status": "completed", "text": "ok-dispatch"}), "") if "goal" in cmd: return FakeResult(0, "goal met after rounds\n", "") if "status" in cmd: return FakeResult(0, json.dumps( {"status": "completed", "text": "ok-status"}), "") return FakeResult(0, "", "") return FakeResult(0, "", "") def _default_api(self, socket_path, method, path, payload=None): self.api_calls.append((method, path, payload)) return (200, {}) def _default_launcher(self, cmd): self.launches.append(list(cmd)) return FakeProc(alive=True) def _task(self, task_id="abc123"): return {"id": task_id, "prompt": "做一个加法函数", "project_id": "proj1"} # -- 拆模块 --------------------------------------------------------------- class SplitTest(unittest.TestCase): def test_new_and_old_import_paths_are_same_objects(self): self.assertIs(CloudHypervisorExecutor, _sb_exec) self.assertIs(ChvFailure, _sb_failure) self.assertIs(_chv_mem, _sb_mem) self.assertIs(_chv_api, _sb_api) self.assertIs(reconcile_orphans_for_executor, _sb_reconcile) def test_classes_live_in_chv_modules(self): self.assertEqual(CloudHypervisorExecutor.__module__, "cloud.chv.executor") self.assertEqual(WarmPool.__module__, "cloud.chv.experimental_pool") self.assertEqual(ChvFailure.__module__, "cloud.chv.errors") def test_old_warm_pool_import_paths_are_gone(self): # 本轨破环性清理：旧路径一律 ImportError，实验路径可用。 with self.assertRaises(ImportError): from cloud.chv import WarmPool # noqa: F401 with self.assertRaises(ImportError): from cloud.sandbox import WarmPool # noqa: F401 with self.assertRaises(ImportError): import cloud.chv.pool # noqa: F401 # -- 去重复：goal 解析与 host 命令 ------------------------------------------ class SharedGoalTest(unittest.TestCase): CASES = [ # (returncode, stdout, stderr) (0, "goal met after rounds\n", ""), (0, "round 1 verify_out=bad\ngoal unmet after rounds\n", ""), (1, "goal interrupted after rounds: boom\n", ""), (1, "KeyError: 'x'\n", ""), (1, "some odd failure\n", ""), (0, "nothing conclusive\n", ""), (1, "goal-resume 拒绝：断点丢失\n", ""), (1, "任务已存在\n", ""), (1, "goal met after rounds\n", ""), (1, "goal unmet after rounds\n", ""), (1, "", "HTTP 503 Service Unavailable\n"), ] def test_local_and_chv_goal_conclusions_match(self): local = LocalExecutor("/tmp/host.py") chv = ChvOptCase() chv.setUp() try: ex = chv.make_exec() for rc, out, err in self.CASES: res = FakeResult(rc, out, err) a = local._goal_result_from_runner("t1", res) b = ex._goal_result("t1", res) self.assertEqual( (a["status"], a["goal_met"]), (b["status"], b["goal_met"]), "rc=%s stdout=%r" % (rc, out)) self.assertEqual(a["executor"], "local") self.assertEqual(b["executor"], "cloud-hypervisor") finally: chv.tearDown() def test_already_exists_message_points_to_continue(self): chv = ChvOptCase() chv.setUp() try: ex = chv.make_exec() out = ex._goal_result( "t1", FakeResult(1, "任务已存在\n", "")) self.assertEqual(out["status"], "failed") self.assertIn("继续", out["goal_error"]) finally: chv.tearDown() def test_host_command_options_identical_across_executors(self): chv = ChvOptCase() chv.setUp() try: ex = chv.make_exec() local = LocalExecutor("/tmp/host.py") docker = DockerExecutor() for kwargs in ( {"model": "m1", "effort": "low", "approval": "ask"}, {"model": "m1", "effort": "low", "approval": "full", "sandbox": "read-only", "model_provider": "sub2api"}, ): lcmd = local._base_command(**kwargs) dcmd = docker._inner_host_command("status", **kwargs) ccmd = ex._guest_host_command("status", **kwargs) self.assertEqual(lcmd[2:], dcmd[2:-1]) self.assertEqual(lcmd[2:], ccmd[2:-1]) finally: chv.tearDown() def test_runner_timeout_helper(self): seen = [] def only_cmd(cmd): seen.append(("cmd-only", list(cmd))) return FakeResult(0, "", "") out = _call_runner_with_timeout(only_cmd, ["x"], timeout=5) self.assertEqual(out.returncode, 0) self.assertEqual(seen, [("cmd-only", ["x"])]) def with_timeout(cmd, timeout=None): seen.append(("with-timeout", timeout)) return FakeResult(0, "", "") _call_runner_with_timeout(with_timeout, ["x"], timeout=7) self.assertEqual(seen[-1], ("with-timeout", 7)) _call_runner_with_timeout(with_timeout, ["x"], timeout=None) self.assertEqual(seen[-1], ("with-timeout", None)) # -- 安全：known_hosts 钉死 ------------------------------------------------- class SshPinningTest(ChvOptCase): def test_known_hosts_path_is_per_vm(self): ex = self.make_exec() a = ex.known_hosts_path({"id": "t-a"}) b = ex.known_hosts_path({"id": "t-b"}) self.assertNotEqual(a, b) self.assertEqual(a, os.path.join(self.state_dir, "chv-t-a", "known_hosts")) def test_capture_host_key_writes_pinned_file_0600(self): def keyscan(cmd, timeout=None): self.calls.append(list(cmd)) return FakeResult( 0, "[127.0.0.1]:2222 ssh-ed25519 AAAAB3test\n" "evil.example.com ssh-rsa AAAAEvil\n" "# comment\n", "") ex = self.make_exec(runner=keyscan) self.assertTrue(ex.capture_host_key(self._task())) path = ex.known_hosts_path(self._task()) with open(path) as fh: content = fh.read() self.assertIn("[127.0.0.1]:2222 ssh-ed25519 AAAAB3test", content) self.assertNotIn("evil.example.com", content) self.assertEqual(os.stat(path).st_mode & 0o777, 0o600) def test_capture_host_key_empty_is_false_not_error(self): ex = self.make_exec( runner=lambda cmd, timeout=None: FakeResult(0, "", "")) self.assertFalse(ex.capture_host_key(self._task())) self.assertFalse(os.path.exists( ex.known_hosts_path(self._task()))) def test_wait_guest_ready_captures_before_ssh(self): def runner(cmd, timeout=None): self.calls.append(list(cmd)) if cmd[0] == "ssh-keyscan": return FakeResult( 0, "[127.0.0.1]:2222 ssh-ed25519 AAAAB3test\n", "") if cmd[0] == "ssh" and cmd[-1] == "true": return FakeResult(0, "", "") return FakeResult(0, "", "") ex = self.make_exec(runner=runner) self.assertTrue(ex.wait_guest_ready(self._task())) self.assertTrue(os.path.exists( ex.known_hosts_path(self._task()))) keyscan_idx = next(i for i, c in enumerate(self.calls) if c[0] == "ssh-keyscan") ssh_idx = next(i for i, c in enumerate(self.calls) if c[0] == "ssh" and c[-1] == "true") self.assertLess(keyscan_idx, ssh_idx) def test_ssh_user_port_explicit_config(self): ex = self.make_exec(ssh_user="codex", ssh_port=2201) cmd = ex.build_ssh_command(self._task(), ["true"]) joined = " ".join(cmd) self.assertIn("codex@127.0.0.1", joined) self.assertNotIn("root@127.0.0.1", joined) self.assertIn("2201", joined) def test_ssh_user_port_tightened_validation(self): with self.assertRaises(ValueError): self.make_exec(ssh_user="") with self.assertRaises(ValueError): self.make_exec(ssh_port=0) with self.assertRaises(ValueError): self.make_exec(ssh_port=70000) with self.assertRaises(ValueError): self.make_exec(ssh_port="abc") with self.assertRaises(ValueError): self.make_exec(env_extra={"CHV_SSH_PORT": "0"}) # -- execute 语义 ------------------------------------------------------------- class ExecuteGateTest(ChvOptCase): def test_execute_without_running_vm_is_classified(self): ex = self.make_exec() with self.assertRaises(ChvFailure) as ctx: ex.execute(self._task()) self.assertEqual(ctx.exception.kind, "vm_not_running") self.assertEqual(self.launches, []) self.assertFalse(any(c[0] == "ssh" for c in self.calls)) def test_execute_with_dead_proc_and_no_socket_is_classified(self): ex = self.make_exec() ex._vms["chv-abc123"] = FakeProc(alive=False) with self.assertRaises(ChvFailure) as ctx: ex.execute(self._task()) self.assertEqual(ctx.exception.kind, "vm_not_running") def test_execute_with_socket_but_no_registry_proceeds(self): ex = self.make_exec() os.makedirs(self.state_dir, exist_ok=True) with open(os.path.join(self.state_dir, "chv-abc123.sock"), "w") as fh: fh.write("") out = ex.execute(self._task()) self.assertIn("ok-status", out["stdout"]) self.assertEqual(self.launches, []) # -- 预热池持久化与孤儿清理 ---------------------------------------------------- class StubExecutor: """WarmPool 测试替身：只实现池子调用的鸭子接口。""" def __init__(self, state_dir): self.state_dir = state_dir self.warm_pool_size = 2 self.snapshot_url = "file:///snap/base" self.wake_calls = [] self.pause_calls = [] self.shutdown_calls = [] self.audit = [] def vm_name(self, task): raw = task.get("id") if isinstance(task, dict) else str(task) return raw if raw.startswith("chv-") else "chv-" + raw def wake_vm(self, task): self.wake_calls.append(task["id"]) return {"vm": self.vm_name(task), "state": "running"} def pause_vm(self, task): self.pause_calls.append(task["id"]) return True def shutdown_vm(self, task_or_name): name = task_or_name if isinstance(task_or_name, str) \ else self.vm_name(task_or_name) self.shutdown_calls.append(name) return True def _emit(self, event, task, detail="", kind=None): self.audit.append(event) class PoolRegistryTest(unittest.TestCase): def setUp(self): self.tmp = tempfile.TemporaryDirectory() self.state_dir = os.path.join(self.tmp.name, "state") os.makedirs(self.state_dir, exist_ok=True) self.registry = os.path.join(self.state_dir, "warm-pool.json") def tearDown(self): self.tmp.cleanup() def test_persist_load_orphan_reconcile(self): stub = StubExecutor(self.state_dir) pool = WarmPool(stub, size=2, snapshot_url="file:///snap/base") added = pool.prewarm() self.assertEqual(len(added), 2) self.assertTrue(os.path.exists(self.registry)) with open(self.registry) as fh: data = json.load(fh) self.assertEqual(len(data), 2) # 新池实例载入：上一世条目只当孤儿，绝不发给任务： stub2 = StubExecutor(self.state_dir) pool2 = WarmPool(stub2, size=2, snapshot_url="file:///snap/base") self.assertEqual(pool2.warm_count(), 0) self.assertEqual(pool2.orphan_count(), 2) self.assertEqual(pool2.acquire({"id": "t1"})["mode"], "cold") cleaned = pool2.reconcile_orphans() self.assertEqual(len(cleaned), 2) self.assertEqual(sorted(stub2.shutdown_calls), sorted(cleaned)) self.assertEqual(pool2.orphan_count(), 0) with open(self.registry) as fh: self.assertEqual(json.load(fh), {}) # 第三实例：无残留： pool3 = WarmPool(StubExecutor(self.state_dir), size=2, snapshot_url="file:///snap/base") self.assertEqual(pool3.orphan_count(), 0) def test_corrupt_registry_quarantines_and_starts_empty(self): with open(self.registry, "w") as fh: fh.write("{not json") pool = WarmPool(StubExecutor(self.state_dir), size=2, snapshot_url="file:///snap/base") self.assertTrue(pool.registry_corrupt) self.assertEqual(pool.warm_count(), 0) self.assertEqual(pool.orphan_count(), 0) self.assertTrue(os.path.exists(self.registry + ".corrupt")) self.assertFalse(os.path.exists(self.registry)) # 损坏后仍可正常预热（registry 重建）： self.assertEqual(len(pool.prewarm()), 2) self.assertTrue(os.path.exists(self.registry)) def test_assigned_entry_on_load_is_never_served(self): with open(self.registry, "w") as fh: json.dump({"warm-1": {"state": "assigned", "vm": "chv-warm-1", "task_id": "old-task", "snapshot_url": "file:///snap/base"}}, fh) pool = WarmPool(StubExecutor(self.state_dir), size=2, snapshot_url="file:///snap/base") self.assertEqual(pool.warm_count(), 0) self.assertEqual(pool.acquire({"id": "t1"})["mode"], "cold") self.assertEqual(pool.reconcile_orphans(), ["chv-warm-1"]) def test_release_never_rewarms_and_registry_clears(self): stub = StubExecutor(self.state_dir) pool = WarmPool(stub, size=1, snapshot_url="file:///snap/base") added = pool.prewarm() got = pool.acquire({"id": "t1"}) self.assertEqual(got["mode"], "warm") self.assertTrue(pool.release({"id": "t1"}, pool_vm=added[0])) self.assertEqual(pool.warm_count(), 0) with open(self.registry) as fh: self.assertEqual(json.load(fh), {}) # -- 长任务中途死亡 -------------------------------------------------------------- class MidDeathTest(ChvOptCase): def _dispatch_with(self, runner): ex = self.make_exec(runner=runner) return ex.dispatch_task(self._task()) def test_failed_result_with_dead_vm_is_interrupted(self): proc_holder = [] def launcher(cmd): proc = FakeProc(alive=True) proc_holder.append(proc) self.launches.append(list(cmd)) return proc def runner(cmd, timeout=None): self.calls.append(list(cmd)) if cmd[0] == "ssh" and "dispatch" in cmd: proc_holder[0]._alive = False return FakeResult(1, "", "Connection closed by remote host") if cmd[0] == "ssh": return FakeResult(0, "", "") return FakeResult(0, "", "") ex = self.make_exec(runner=runner, launcher=launcher) out = ex.dispatch_task(self._task()) self.assertEqual(out["status"], "interrupted") self.assertTrue(out["vm_dead"]) def test_failed_result_with_live_vm_stays_failed(self): def runner(cmd, timeout=None): self.calls.append(list(cmd)) if cmd[0] == "ssh" and "dispatch" in cmd: return FakeResult(1, "", "host dispatch failed") if cmd[0] == "ssh": return FakeResult(0, "", "") return FakeResult(0, "", "") ex = self.make_exec(runner=runner) out = ex.dispatch_task(self._task()) self.assertEqual(out["status"], "failed") self.assertNotIn("vm_dead", out) def test_exception_with_dead_vm_is_interrupted_not_raised(self): proc_holder = [] def launcher(cmd): proc = FakeProc(alive=True) proc_holder.append(proc) self.launches.append(list(cmd)) return proc def runner(cmd, timeout=None): self.calls.append(list(cmd)) if cmd[0] == "ssh" and "dispatch" in cmd: proc_holder[0]._alive = False raise RuntimeError("ssh transport broken") if cmd[0] == "ssh": return FakeResult(0, "", "") return FakeResult(0, "", "") ex = self.make_exec(runner=runner, launcher=launcher) out = ex.dispatch_task(self._task()) self.assertEqual(out["status"], "interrupted") self.assertTrue(out["vm_dead"]) def test_exception_with_live_vm_propagates(self): def runner(cmd, timeout=None): self.calls.append(list(cmd)) if cmd[0] == "ssh" and "dispatch" in cmd: raise RuntimeError("ssh transport broken") if cmd[0] == "ssh": return FakeResult(0, "", "") return FakeResult(0, "", "") ex = self.make_exec(runner=runner) with self.assertRaises(RuntimeError): ex.dispatch_task(self._task()) def test_goal_exception_with_dead_vm_is_interrupted(self): proc_holder = [] def launcher(cmd): proc = FakeProc(alive=True) proc_holder.append(proc) self.launches.append(list(cmd)) return proc def runner(cmd, timeout=None): self.calls.append(list(cmd)) if cmd[0] == "ssh" and "goal" in cmd: proc_holder[0]._alive = False raise RuntimeError("ssh transport broken") if cmd[0] == "ssh": return FakeResult(0, "", "") return FakeResult(0, "", "") ex = self.make_exec(runner=runner, launcher=launcher) out = ex.goal_task({"id": "g1", "prompt": "做目标", "goal_mode": True}) self.assertEqual(out["status"], "interrupted") self.assertIsNone(out["goal_met"]) self.assertTrue(out["vm_dead"]) if __name__ == "__main__": unittest.main()