"""CHV 补缺口轨测试（堆叠于 #221，base=feat-chv-optimize）。 只用标准库与 mock runner/launcher/api_client，绝不启动真实 VM：当前环境没有可跑真 VM 的宿主（生产主机是 CVM、无 /dev/kvm、无嵌套虚拟化；本 VM 容器内同样无 /dev/kvm）。 真实 VM 启动、快照恢复、压测一律不在本文件声称已验证。 """ import json import os import subprocess import sys import tempfile import unittest from pathlib import Path from types import SimpleNamespace try: from cloud.chv.hostkeys import ( build_seed_user_data, generate_host_keypair, host_key_paths, known_hosts_line, write_seed, ) from cloud.chv.pool import WarmPool from cloud.sandbox import ( ChvFailure, CloudHypervisorExecutor, _parse_memory_bytes, ) from cloud.worker import Worker, _call_goal_resume_breakpoint except ImportError: sys.path.insert(0, str(Path(__file__).resolve().parents[2])) from cloud.chv.hostkeys import ( build_seed_user_data, generate_host_keypair, host_key_paths, known_hosts_line, write_seed, ) from cloud.chv.pool import WarmPool from cloud.sandbox import ( ChvFailure, CloudHypervisorExecutor, _parse_memory_bytes, ) from cloud.worker import Worker, _call_goal_resume_breakpoint class FakeResult: def __init__(self, returncode=0, stdout="", stderr=""): self.returncode = returncode self.stdout = stdout self.stderr = stderr class FakeProc: def __init__(self, alive=True): self._alive = alive def poll(self): return None if self._alive else 1 def terminate(self): self._alive = False def wait(self, timeout=None): return 0 def kill(self): self._alive = False def _write(path, content=b"x"): with open(path, "wb") as fh: fh.write(content) os.chmod(path, 0o755) return path class ChvGapCase(unittest.TestCase): def setUp(self): self.tmp = tempfile.TemporaryDirectory() self.dir = self.tmp.name self.binary = _write(os.path.join(self.dir, "cloud-hypervisor")) self.ch_remote = _write(os.path.join(self.dir, "ch-remote")) self.kernel = _write(os.path.join(self.dir, "vmlinux")) self.rootfs = _write(os.path.join(self.dir, "disk.raw"), b"R" * 100) self.ssh_key = _write(os.path.join(self.dir, "id_chv")) self.kvm = _write(os.path.join(self.dir, "kvm")) self.cpuinfo = os.path.join(self.dir, "cpuinfo") with open(self.cpuinfo, "w") as fh: fh.write("flags : fpu vmx sse\n") self.state_dir = os.path.join(self.dir, "state") self.calls = [] self.audit = [] def tearDown(self): self.tmp.cleanup() def _default_runner(self, cmd, timeout=None): self.calls.append(list(cmd)) if cmd and cmd[0] == "ssh": if cmd[-1] == "true": return FakeResult(0, "", "") if "goal-resume" in cmd: return FakeResult(0, "goal met after rounds\n", "") if "goal" in cmd: return FakeResult(0, "goal met after rounds\n", "") return FakeResult(0, "", "") return FakeResult(0, "", "") def _default_api(self, socket_path, method, path, payload=None): return (200, {}) def _default_launcher(self, cmd): return FakeProc(alive=True) def make_exec(self, runner=None, env_extra=None, **overrides): env = {"CHV_EGRESS_PROXY": "http://10.0.0.1:8888"} if env_extra: env.update(env_extra) kwargs = dict( binary=self.binary, ch_remote=self.ch_remote, kernel=self.kernel, rootfs=self.rootfs, ssh_key=self.ssh_key, kvm_path=self.kvm, cpuinfo_path=self.cpuinfo, state_dir=self.state_dir, runner=runner or self._default_runner, launcher=self._default_launcher, api_client=self._default_api, sleeper=lambda s: None, env=env) kwargs.update(overrides) ex = CloudHypervisorExecutor(**kwargs) ex.audit_callback = lambda e, t, d, k=None: self.audit.append( (e, t.get("id") if isinstance(t, dict) else t, d, k)) return ex # -- (1) worker 断点接线 --------------------------------------------- class BreakpointWiringTest(ChvGapCase): def test_call_helper_passes_model_provider_when_accepted(self): seen = {} class FakeChv: def goal_resume_breakpoint_task(self, task, model_provider=None): seen["provider"] = model_provider return {"status": "completed", "task_id": task["id"]} out = _call_goal_resume_breakpoint( FakeChv(), {"id": "bp1"}, model_provider="prov-x") self.assertEqual(out["status"], "completed") self.assertEqual(seen["provider"], "prov-x") def test_call_helper_single_arg_executor_still_works(self): class FakeOld: def goal_resume_breakpoint_task(self, task): return {"status": "completed", "task_id": task["id"]} out = _call_goal_resume_breakpoint( FakeOld(), {"id": "bp2"}, model_provider="prov-x") self.assertEqual(out["status"], "completed") def test_chv_breakpoint_accepts_model_provider(self): ex = self.make_exec() out = ex.goal_resume_breakpoint_task( {"id": "g1", "prompt": "续跑"}, model_provider="prov-x") self.assertEqual(out["status"], "completed") self.assertTrue(out["goal_met"]) # 客内命令带 --model-provider（网关 key 本身不进命令值）。 ssh_calls = [c for c in self.calls if c and c[0] == "ssh"] self.assertTrue(any("--model-provider" in c for c in ssh_calls)) def test_worker_branch_uses_hasattr_not_only_local(self): # worker.py 通用分支的文字证据：Local 优先、其次 hasattr。 worker_path = Path(__file__).resolve().parents[1] / "worker.py" text = worker_path.read_text() self.assertIn("_call_goal_resume_breakpoint", text) self.assertIn("hasattr(", text) self.assertIn("goal_resume_breakpoint_task", text) # -- (2) reconcile_orphans 接线 --------------------------------------- class ReconcileWiringTest(ChvGapCase): def test_worker_startup_and_close_call_reconcile(self): from cloud.server import init_db db_path = os.path.join(self.dir, "control.db") init_db(db_path) ex = self.make_exec() # 手工造一个 orphan：先建池写 registry，再新开池载入。 pool1 = WarmPool(ex) pool1._entries["chv-old"] = { "state": "warm", "vm": "chv-old", "task": {"id": "old"}, "snapshot_url": None} pool1._save_registry() self.assertTrue( os.path.exists(os.path.join(self.state_dir, "warm-pool.json"))) # Worker 构造即 startup reconcile：用假 shutdown 记录。 shutdowns = [] orig_shutdown = ex.shutdown_vm ex.shutdown_vm = lambda name: shutdowns.append(name) or True worker = Worker(db_path, "/fake/host.py", executor=ex, env={"XCM_API_KEY": "test-placeholder"}) self.assertIn("chv-old", shutdowns) self.assertEqual(worker._warm_pool.orphan_count(), 0) worker.close() # close 时再次 reconcile 不炸（池已空，返回 [] 即可）。 self.assertEqual(worker._reconcile_warm_pool("shutdown"), []) def test_reconcile_failure_does_not_break_close(self): from cloud.server import init_db db_path = os.path.join(self.dir, "control.db") init_db(db_path) ex = self.make_exec() worker = Worker(db_path, "/fake/host.py", executor=ex, env={"XCM_API_KEY": "test-placeholder"}) # 池的 reconcile 抛错时 close 仍不炸。 worker._warm_pool.reconcile_orphans = lambda: (_ for _ in ()).throw( RuntimeError("boom")) worker.close() # 不抛即通过 # -- (3) SSH host key 预置 --------------------------------------------- class HostKeyProvisionTest(ChvGapCase): def _fake_keygen_runner(self, cmd, timeout=None): self.calls.append(list(cmd)) if cmd and cmd[0] == "ssh-keygen": priv = cmd[cmd.index("-f") + 1] with open(priv, "w") as fh: fh.write("-----BEGIN OPENSSH PRIVATE KEY-----\nFAKE\n") os.chmod(priv, 0o600) with open(priv + ".pub", "w") as fh: fh.write("ssh-ed25519 AAAAfake1 vm-key\n") return FakeResult(0, "", "") return self._default_runner(cmd, timeout) def test_per_vm_keys_differ_and_known_hosts_pinned(self): ex = self.make_exec(runner=self._fake_keygen_runner) r1 = ex.provision_host_key({"id": "vm-a"}) r2 = ex.provision_host_key({"id": "vm-b"}) # 不同 VM 的私钥文件不同路径、known_hosts 不同文件。 self.assertNotEqual(r1["paths"]["private"], r2["paths"]["private"]) self.assertNotEqual(r1["known_hosts"], r2["known_hosts"]) for r in (r1, r2): with open(r["known_hosts"]) as fh: line = fh.read().strip() self.assertTrue(line.startswith("[127.0.0.1]:")) self.assertIn("ssh-ed25519 AAAAfake1", line) self.assertNotIn("PRIVATE KEY", line) # 同一 VM 再次预置复用已有对（不重生成、不改 known_hosts）。 with open(r1["known_hosts"]) as fh: before = fh.read() ex.provision_host_key({"id": "vm-a"}) with open(r1["known_hosts"]) as fh: self.assertEqual(fh.read(), before) def test_seed_contains_private_key_and_is_deleted_on_stop(self): ex = self.make_exec(runner=self._fake_keygen_runner) r = ex.provision_host_key({"id": "vm-c"}) seed_user = os.path.join(r["seed_dir"], "user-data") self.assertTrue(os.path.exists(seed_user)) with open(seed_user) as fh: seed_text = fh.read() self.assertIn("ssh_host_ed25519_key", seed_text) self.assertIn("ssh_deletekeys: false", seed_text) # stop 时 seed 目录与私钥删掉，known_hosts 公钥保留审计。 ex.stop_vm({"id": "vm-c"}) self.assertFalse(os.path.exists(r["seed_dir"])) self.assertFalse(os.path.exists(r["paths"]["private"])) self.assertTrue(os.path.exists(r["known_hosts"])) def test_known_hosts_line_rejects_bad_key(self): with self.assertRaises(ValueError): known_hosts_line(2222, "not-a-key") self.assertEqual( known_hosts_line(2222, "ssh-ed25519 AAAAx comment"), "[127.0.0.1]:2222 ssh-ed25519 AAAAx") def test_seed_user_data_shape(self): text = build_seed_user_data("PRIV\n", "ssh-ed25519 AAAAx\n") self.assertIn("/etc/ssh/ssh_host_ed25519_key", text) self.assertIn("ssh_deletekeys: false", text) with self.assertRaises(ValueError): build_seed_user_data("", "ssh-ed25519 AAAAx") def test_keyscan_is_weak_fallback_only(self): # provisioned 模式下 wake 前必须预置；keyscan 不应被 # 当成生产路径：配置值本身可解析，且文档/代码里标弱。 ex = self.make_exec(env_extra={"CHV_HOST_KEY_MODE": "provisioned"}) self.assertEqual(ex.host_key_mode, "provisioned") with self.assertRaises(ValueError): self.make_exec(env_extra={"CHV_HOST_KEY_MODE": "bogus"}) # -- (4) egress tap 校验 ------------------------------------------------- class EgressValidationTest(ChvGapCase): def test_direct_egress_without_flag_is_fail_closed(self): ex = self.make_exec() ex.egress_proxy = None ex.allow_direct_egress = False with self.assertRaises(ChvFailure) as ctx: ex.preflight() self.assertEqual(ctx.exception.kind, "egress_not_configured") def test_direct_egress_with_flag_passes_and_is_audited(self): ex = self.make_exec(env_extra={"CHV_ALLOW_DIRECT_EGRESS": "1"}) # preflight 本身不拦显式直连（守卫在 wake 的审计里）。 self.assertTrue(ex.preflight()) out = ex.goal_task({"id": "g1", "prompt": "做目标", "goal_mode": True}) self.assertEqual(out["status"], "completed") self.assertTrue(any(e == "egress_direct_allowed" for e, _, _, _ in self.audit)) def test_bad_net_json_is_preflight_error(self): ex = self.make_exec(env_extra={"CHV_NET_JSON": "{not json"}) with self.assertRaises(ChvFailure) as ctx: ex.preflight() self.assertEqual(ctx.exception.kind, "net_config_invalid") # -- (5) 构建脚本 -------------------------------------------------------- class BuildScriptTest(unittest.TestCase): def setUp(self): self.tmp = tempfile.TemporaryDirectory() self.dir = self.tmp.name self.bin_dir = os.path.join(self.dir, "bin") os.makedirs(os.path.join(self.bin_dir, "codex-resources")) for name in ("codex", "codex-code-mode-host"): p = os.path.join(self.bin_dir, name) with open(p, "w") as fh: fh.write("#!/bin/sh\n") os.chmod(p, 0o755) self.repo = os.path.join(self.dir, "repo") os.makedirs(os.path.join(self.repo, "cloud")) with open(os.path.join(self.repo, "host.py"), "w") as fh: fh.write("# host\n") self.script = str(Path(__file__).resolve().parents[2] / "chv-guest" / "build.sh") def tearDown(self): self.tmp.cleanup() def _run(self, args, env_extra=None): env = dict(os.environ) env.update({"CODEX_BIN_DIR": self.bin_dir, "UBUNTU_VERSION": "24.04", "CODEX_VERSION": "0.164.0-alpha.9", "REPO_COMMIT": "test123", "REPO_DIR": self.repo}) if env_extra: env.update(env_extra) return subprocess.run(["bash", self.script] + args, capture_output=True, text=True, env=env) def test_dry_run_prints_real_steps_and_builds_nothing(self): out_dir = os.path.join(self.dir, "out") r = self._run(["--dry-run", out_dir]) self.assertEqual(r.returncode, 0, r.stderr) self.assertIn("qemu-img convert", r.stdout) self.assertIn("virt-customize", r.stdout) self.assertIn("virt-get-kernel", r.stdout) self.assertIn("BUILDINFO", r.stdout) self.assertIn("chv-guest-24.04-0.164.0-alpha.9-test123", r.stdout) self.assertFalse(os.path.exists(os.path.join(out_dir, "disk.raw"))) def test_full_mode_without_kvm_fails_fast(self): # 本 VM 容器内无 /dev/kvm，全量模式必须明确失败， # 绝不声称已建成镜像。 r = self._run([os.path.join(self.dir, "out2")]) self.assertNotEqual(r.returncode, 0) self.assertIn("/dev/kvm", r.stderr + r.stdout) def test_missing_binary_fails_fast(self): os.remove(os.path.join(self.bin_dir, "codex")) r = self._run(["--dry-run", os.path.join(self.dir, "out3")]) self.assertNotEqual(r.returncode, 0) # -- (6) 中途死亡 → interrupted → 断点续跑闭环 --------------------------- class MidDeathLoopTest(ChvGapCase): def test_vm_death_is_interrupted_then_breakpoint_resumes(self): state = {"dead": False} def runner(cmd, timeout=None): self.calls.append(list(cmd)) if cmd and cmd[0] == "ssh": if cmd[-1] == "true": return FakeResult(0, "", "") if "goal-resume" in cmd and state["dead"]: # 第二次断点调用：VM 已换新，健康返回。 return FakeResult(0, "goal met after rounds\n", "") if "goal" in cmd: # 第一次 goal：模拟传输中断，并把 VMM 进程标死。 state["dead"] = True # 找到登记的 VMM 进程并标死（wake 已登记）。 for proc in list(ex._vms.values()): proc._alive = False raise RuntimeError("connection lost mid-task") return FakeResult(0, "", "") ex = self.make_exec(runner=runner) # 第一次 goal：中途死应判 interrupted，不是 failed 死信。 out = ex.goal_task({"id": "g9", "prompt": "做目标", "goal_mode": True}) self.assertEqual(out["status"], "interrupted") # 断点续跑：同一执行器的断点方法能走通并完成。 state["dead"] = True # 标记已进入断点阶段（runner 按此分支） out2 = ex.goal_resume_breakpoint_task( {"id": "g9", "prompt": "续跑"}) self.assertEqual(out2["status"], "completed") if __name__ == "__main__": unittest.main()