"""CHV 补缺口轨测试（堆叠于 #221，base=feat-chv-optimize）。 只用标准库与 mock runner/launcher/api_client，绝不启动真实 VM：当前环境没有可跑真 VM 的宿主（生产主机是 CVM、无 /dev/kvm、无嵌套虚拟化；本 VM 容器内同样无 /dev/kvm）。 真实 VM 启动、快照恢复、压测一律不在本文件声称已验证。 """ import json import os import subprocess import sys import tempfile import unittest from pathlib import Path from types import SimpleNamespace try: from cloud.chv.hostkeys import ( build_seed_user_data, generate_host_keypair, host_key_paths, known_hosts_line, write_seed, ) from cloud.chv.pool import WarmPool from cloud.sandbox import ( ChvFailure, CloudHypervisorExecutor, _parse_memory_bytes, ) from cloud.worker import Worker, _call_goal_resume_breakpoint except ImportError: sys.path.insert(0, str(Path(__file__).resolve().parents[2])) from cloud.chv.hostkeys import ( build_seed_user_data, generate_host_keypair, host_key_paths, known_hosts_line, write_seed, ) from cloud.chv.pool import WarmPool from cloud.sandbox import ( ChvFailure, CloudHypervisorExecutor, _parse_memory_bytes, ) from cloud.worker import Worker, _call_goal_resume_breakpoint class FakeResult: def __init__(self, returncode=0, stdout="", stderr=""): self.returncode = returncode self.stdout = stdout self.stderr = stderr class FakeProc: def __init__(self, alive=True): self._alive = alive def poll(self): return None if self._alive else 1 def terminate(self): self._alive = False def wait(self, timeout=None): return 0 def kill(self): self._alive = False def _write(path, content=b"x"): with open(path, "wb") as fh: fh.write(content) os.chmod(path, 0o755) return path class ChvGapCase(unittest.TestCase): def setUp(self): self.tmp = tempfile.TemporaryDirectory() self.dir = self.tmp.name self.binary = _write(os.path.join(self.dir, "cloud-hypervisor")) self.ch_remote = _write(os.path.join(self.dir, "ch-remote")) self.kernel = _write(os.path.join(self.dir, "vmlinux")) self.rootfs = _write(os.path.join(self.dir, "disk.raw"), b"R" * 100) self.ssh_key = _write(os.path.join(self.dir, "id_chv")) self.kvm = _write(os.path.join(self.dir, "kvm")) self.cpuinfo = os.path.join(self.dir, "cpuinfo") with open(self.cpuinfo, "w") as fh: fh.write("flags : fpu vmx sse\n") self.state_dir = os.path.join(self.dir, "state") self.calls = [] self.audit = [] def tearDown(self): self.tmp.cleanup() def _default_runner(self, cmd, timeout=None): self.calls.append(list(cmd)) if cmd and cmd[0] == "ssh": if cmd[-1] == "true": return FakeResult(0, "", "") if "goal-resume" in cmd: return FakeResult(0, "goal met after rounds\n", "") if "goal" in cmd: return FakeResult(0, "goal met after rounds\n", "") return FakeResult(0, "", "") return FakeResult(0, "", "") def _default_api(self, socket_path, method, path, payload=None): return (200, {}) def _default_launcher(self, cmd): return FakeProc(alive=True) def make_exec(self, runner=None, env_extra=None, **overrides): env = {"CHV_EGRESS_PROXY": "http://10.0.0.1:8888", # 实验室宣称：mock runner 假装回环端口已转发到客机 #（feat-chv-fix 起，执行器不再无条件假设回环能通）。 "CHV_SSH_FORWARD": "1"} if env_extra: env.update(env_extra) kwargs = dict( binary=self.binary, ch_remote=self.ch_remote, kernel=self.kernel, rootfs=self.rootfs, ssh_key=self.ssh_key, kvm_path=self.kvm, cpuinfo_path=self.cpuinfo, state_dir=self.state_dir, runner=runner or self._default_runner, launcher=self._default_launcher, api_client=self._default_api, sleeper=lambda s: None, env=env) kwargs.update(overrides) ex = CloudHypervisorExecutor(**kwargs) ex.audit_callback = lambda e, t, d, k=None: self.audit.append( (e, t.get("id") if isinstance(t, dict) else t, d, k)) return ex # -- (1) worker 断点接线 --------------------------------------------- class BreakpointWiringTest(ChvGapCase): def test_call_helper_passes_model_provider_when_accepted(self): seen = {} class FakeChv: def goal_resume_breakpoint_task(self, task, model_provider=None): seen["provider"] = model_provider return {"status": "completed", "task_id": task["id"]} out = _call_goal_resume_breakpoint( FakeChv(), {"id": "bp1"}, model_provider="prov-x") self.assertEqual(out["status"], "completed") self.assertEqual(seen["provider"], "prov-x") def test_call_helper_single_arg_executor_still_works(self): class FakeOld: def goal_resume_breakpoint_task(self, task): return {"status": "completed", "task_id": task["id"]} out = _call_goal_resume_breakpoint( FakeOld(), {"id": "bp2"}, model_provider="prov-x") self.assertEqual(out["status"], "completed") def test_chv_breakpoint_accepts_model_provider(self): ex = self.make_exec() out = ex.goal_resume_breakpoint_task( {"id": "g1", "prompt": "续跑"}, model_provider="prov-x") self.assertEqual(out["status"], "completed") self.assertTrue(out["goal_met"]) # 客内命令带 --model-provider（网关 key 本身不进命令值）。 ssh_calls = [c for c in self.calls if c and c[0] == "ssh"] self.assertTrue(any("--model-provider" in c for c in ssh_calls)) def test_worker_breakpoint_branch_calls_generic_executor(self): # feat-chv-fix（P2-3）：不再用源码 grep 充数，改为行为 # 断言——真实 Worker._run_attempt 走通用断点分支，调到 # 非 Local 执行器的 goal_resume_breakpoint_task，并把 # 结果原样返回（参数、守卫顺序、返回值都覆盖）。 from cloud.server import init_db db_path = os.path.join(self.dir, "control.db") init_db(db_path) seen = {} class FakeGeneric: def goal_resume_breakpoint_task(self, task, model_provider=None, extra_env=None): seen["provider"] = model_provider seen["task_id"] = task["id"] return {"executor": "generic", "task_id": task["id"], "status": "completed", "stdout": "", "stderr": "", "returncode": 0, "host_status": None, "goal_met": True, "goal_error": None} worker = Worker(db_path, "/fake/host.py", executor=FakeGeneric(), env={"XCM_API_KEY": "test-placeholder"}) task = {"id": "bp-behav", "project_id": "p1", "prompt": "续跑", "goal_mode": 1, "_goal_resume_breakpoint": True} result, error = worker._run_attempt(task) self.assertIsNone(error) self.assertEqual(result["status"], "completed") self.assertTrue(result["goal_met"]) self.assertEqual(seen["task_id"], "bp-behav") self.assertNotIn("executor 不支持", str(result)) # -- (2) reconcile_orphans 接线 --------------------------------------- class ReconcileWiringTest(ChvGapCase): def test_worker_startup_and_close_call_reconcile(self): from cloud.server import init_db db_path = os.path.join(self.dir, "control.db") init_db(db_path) ex = self.make_exec() # 手工造一个 orphan：先建池写 registry，再新开池载入。 pool1 = WarmPool(ex) pool1._entries["chv-old"] = { "state": "warm", "vm": "chv-old", "task": {"id": "old"}, "snapshot_url": None} pool1._save_registry() self.assertTrue( os.path.exists(os.path.join(self.state_dir, "warm-pool.json"))) # Worker 构造即 startup reconcile：用假 shutdown 记录。 shutdowns = [] orig_shutdown = ex.shutdown_vm ex.shutdown_vm = lambda name: shutdowns.append(name) or True worker = Worker(db_path, "/fake/host.py", executor=ex, env={"XCM_API_KEY": "test-placeholder"}) self.assertIn("chv-old", shutdowns) self.assertEqual(worker._warm_pool.orphan_count(), 0) worker.close() # close 时再次 reconcile 不炸（池已空，返回 [] 即可）。 self.assertEqual(worker._reconcile_warm_pool("shutdown"), []) def test_reconcile_failure_does_not_break_close(self): from cloud.server import init_db db_path = os.path.join(self.dir, "control.db") init_db(db_path) ex = self.make_exec() worker = Worker(db_path, "/fake/host.py", executor=ex, env={"XCM_API_KEY": "test-placeholder"}) # 池的 reconcile 抛错时 close 仍不炸。 worker._warm_pool.reconcile_orphans = lambda: (_ for _ in ()).throw( RuntimeError("boom")) worker.close() # 不抛即通过 # -- (3) SSH host key 预置 --------------------------------------------- class HostKeyProvisionTest(ChvGapCase): def _fake_keygen_runner(self, cmd, timeout=None): self.calls.append(list(cmd)) if cmd and cmd[0] == "ssh-keygen": priv = cmd[cmd.index("-f") + 1] with open(priv, "w") as fh: fh.write("-----BEGIN OPENSSH PRIVATE KEY-----\nFAKE\n") os.chmod(priv, 0o600) with open(priv + ".pub", "w") as fh: fh.write("ssh-ed25519 AAAAfake1 vm-key\n") return FakeResult(0, "", "") return self._default_runner(cmd, timeout) def test_per_vm_keys_differ_and_known_hosts_pinned(self): ex = self.make_exec(runner=self._fake_keygen_runner) r1 = ex.provision_host_key({"id": "vm-a"}) r2 = ex.provision_host_key({"id": "vm-b"}) # 不同 VM 的私钥文件不同路径、known_hosts 不同文件。 self.assertNotEqual(r1["paths"]["private"], r2["paths"]["private"]) self.assertNotEqual(r1["known_hosts"], r2["known_hosts"]) for r in (r1, r2): with open(r["known_hosts"]) as fh: line = fh.read().strip() self.assertTrue(line.startswith("[127.0.0.1]:")) self.assertIn("ssh-ed25519 AAAAfake1", line) self.assertNotIn("PRIVATE KEY", line) # 同一 VM 再次预置复用已有对（不重生成、不改 known_hosts）。 with open(r1["known_hosts"]) as fh: before = fh.read() ex.provision_host_key({"id": "vm-a"}) with open(r1["known_hosts"]) as fh: self.assertEqual(fh.read(), before) def test_seed_contains_private_key_and_is_deleted_on_stop(self): ex = self.make_exec(runner=self._fake_keygen_runner) r = ex.provision_host_key({"id": "vm-c"}) seed_user = os.path.join(r["seed_dir"], "user-data") self.assertTrue(os.path.exists(seed_user)) with open(seed_user) as fh: seed_text = fh.read() self.assertIn("ssh_host_ed25519_key", seed_text) self.assertIn("ssh_deletekeys: false", seed_text) # stop 时 seed 目录与私钥删掉，known_hosts 公钥保留审计。 ex.stop_vm({"id": "vm-c"}) self.assertFalse(os.path.exists(r["seed_dir"])) self.assertFalse(os.path.exists(r["paths"]["private"])) self.assertTrue(os.path.exists(r["known_hosts"])) def test_known_hosts_line_rejects_bad_key(self): with self.assertRaises(ValueError): known_hosts_line(2222, "not-a-key") self.assertEqual( known_hosts_line(2222, "ssh-ed25519 AAAAx comment"), "[127.0.0.1]:2222 ssh-ed25519 AAAAx") def test_seed_user_data_shape(self): text = build_seed_user_data("PRIV\n", "ssh-ed25519 AAAAx\n") self.assertIn("/etc/ssh/ssh_host_ed25519_key", text) self.assertIn("ssh_deletekeys: false", text) with self.assertRaises(ValueError): build_seed_user_data("", "ssh-ed25519 AAAAx") def test_keyscan_is_weak_fallback_only(self): # provisioned 模式下 wake 前必须预置；keyscan 不应被 # 当成生产路径：配置值本身可解析，且文档/代码里标弱。 ex = self.make_exec(env_extra={"CHV_HOST_KEY_MODE": "provisioned"}) self.assertEqual(ex.host_key_mode, "provisioned") with self.assertRaises(ValueError): self.make_exec(env_extra={"CHV_HOST_KEY_MODE": "bogus"}) # -- (4) egress tap 校验 ------------------------------------------------- class EgressValidationTest(ChvGapCase): def test_direct_egress_without_flag_is_fail_closed(self): ex = self.make_exec() ex.egress_proxy = None ex.allow_direct_egress = False with self.assertRaises(ChvFailure) as ctx: ex.preflight() self.assertEqual(ctx.exception.kind, "egress_not_configured") def test_direct_egress_with_flag_passes_and_is_audited(self): ex = self.make_exec(env_extra={"CHV_ALLOW_DIRECT_EGRESS": "1"}) # preflight 本身不拦显式直连（守卫在 wake 的审计里）。 self.assertTrue(ex.preflight()) out = ex.goal_task({"id": "g1", "prompt": "做目标", "goal_mode": True}) self.assertEqual(out["status"], "completed") self.assertTrue(any(e == "egress_direct_allowed" for e, _, _, _ in self.audit)) def test_bad_net_json_is_preflight_error(self): ex = self.make_exec(env_extra={"CHV_NET_JSON": "{not json"}) with self.assertRaises(ChvFailure) as ctx: ex.preflight() self.assertEqual(ctx.exception.kind, "net_config_invalid") # -- (5) 构建脚本 -------------------------------------------------------- class BuildScriptTest(unittest.TestCase): def setUp(self): self.tmp = tempfile.TemporaryDirectory() self.dir = self.tmp.name self.bin_dir = os.path.join(self.dir, "bin") os.makedirs(os.path.join(self.bin_dir, "codex-resources")) for name in ("codex", "codex-code-mode-host"): p = os.path.join(self.bin_dir, name) with open(p, "w") as fh: fh.write("#!/bin/sh\n") os.chmod(p, 0o755) self.repo = os.path.join(self.dir, "repo") os.makedirs(os.path.join(self.repo, "cloud")) with open(os.path.join(self.repo, "host.py"), "w") as fh: fh.write("# host\n") self.script = str(Path(__file__).resolve().parents[2] / "chv-guest" / "build.sh") def tearDown(self): self.tmp.cleanup() def _run(self, args, env_extra=None): env = dict(os.environ) env.update({"CODEX_BIN_DIR": self.bin_dir, "UBUNTU_VERSION": "24.04", "CODEX_VERSION": "0.164.0-alpha.9", "REPO_COMMIT": "test123", "REPO_DIR": self.repo}) if env_extra: env.update(env_extra) return subprocess.run(["bash", self.script] + args, capture_output=True, text=True, env=env) def test_dry_run_prints_real_steps_and_builds_nothing(self): out_dir = os.path.join(self.dir, "out") r = self._run(["--dry-run", out_dir]) self.assertEqual(r.returncode, 0, r.stderr) self.assertIn("qemu-img convert", r.stdout) self.assertIn("virt-customize", r.stdout) self.assertIn("virt-get-kernel", r.stdout) self.assertIn("BUILDINFO", r.stdout) self.assertIn("chv-guest-24.04-0.164.0-alpha.9-test123", r.stdout) self.assertFalse(os.path.exists(os.path.join(out_dir, "disk.raw"))) def test_full_mode_without_kvm_fails_fast(self): # 本 VM 容器内无 /dev/kvm，全量模式必须明确失败， # 绝不声称已建成镜像。 r = self._run([os.path.join(self.dir, "out2")]) self.assertNotEqual(r.returncode, 0) self.assertIn("/dev/kvm", r.stderr + r.stdout) def test_missing_binary_fails_fast(self): os.remove(os.path.join(self.bin_dir, "codex")) r = self._run(["--dry-run", os.path.join(self.dir, "out3")]) self.assertNotEqual(r.returncode, 0) # -- (6) 中途死亡 → interrupted → 断点续跑闭环 --------------------------- class MidDeathLoopTest(ChvGapCase): def test_vm_death_is_interrupted_then_breakpoint_resumes(self): state = {"dead": False} def runner(cmd, timeout=None): self.calls.append(list(cmd)) if cmd and cmd[0] == "ssh": if cmd[-1] == "true": return FakeResult(0, "", "") if "goal-resume" in cmd and state["dead"]: # 第二次断点调用：VM 已换新，健康返回。 return FakeResult(0, "goal met after rounds\n", "") if "goal" in cmd: # 第一次 goal：模拟传输中断，并把 VMM 进程标死。 state["dead"] = True # 找到登记的 VMM 进程并标死（wake 已登记）。 for proc in list(ex._vms.values()): proc._alive = False raise RuntimeError("connection lost mid-task") return FakeResult(0, "", "") ex = self.make_exec(runner=runner) # 第一次 goal：中途死应判 interrupted，不是 failed 死信。 out = ex.goal_task({"id": "g9", "prompt": "做目标", "goal_mode": True}) self.assertEqual(out["status"], "interrupted") # 断点续跑：同一执行器的断点方法能走通并完成。 state["dead"] = True # 标记已进入断点阶段（runner 按此分支） out2 = ex.goal_resume_breakpoint_task( {"id": "g9", "prompt": "续跑"}) self.assertEqual(out2["status"], "completed") if __name__ == "__main__": unittest.main()