host
22b83f118a
Stack: #216 -> #221 -> this. Worker breakpoint branch now calls any executor providing goal_resume_breakpoint_task (Local keeps its original path, gateway tasks still loud-fail); WarmPool reconcile_orphans is called on CHV worker startup and close with audit-on-failure; per-VM SSH host keys are provisioned before VMM start via cloud-init NoCloud seed (keyscan kept only as weak fallback, CHV_HOST_KEY_MODE=provisioned|keyscan); CHV_NET_JSON is validated in preflight and explicit direct egress is audited; chv-guest/build.sh is now real steps with --dry-run and fails fast without /dev/kvm. Tests: new test_chv_gaps.py (18), cloud full 1514 OK (skipped=4). No real VM verification: no KVM host.
418 lines
17 KiB
Python
418 lines
17 KiB
Python
"""CHV 补缺口轨测试（堆叠于 #221，base=feat-chv-optimize）。
|
||
|
||
只用标准库与 mock runner/launcher/api_client，绝不启动真实
|
||
VM：当前环境没有可跑真 VM 的宿主（生产主机是 CVM、无
|
||
/dev/kvm、无嵌套虚拟化；本 VM 容器内同样无 /dev/kvm）。
|
||
真实 VM 启动、快照恢复、压测一律不在本文件声称已验证。
|
||
"""
|
||
|
||
import json
|
||
import os
|
||
import subprocess
|
||
import sys
|
||
import tempfile
|
||
import unittest
|
||
from pathlib import Path
|
||
from types import SimpleNamespace
|
||
|
||
try:
|
||
from cloud.chv.hostkeys import (
|
||
build_seed_user_data,
|
||
generate_host_keypair,
|
||
host_key_paths,
|
||
known_hosts_line,
|
||
write_seed,
|
||
)
|
||
from cloud.chv.pool import WarmPool
|
||
from cloud.sandbox import (
|
||
ChvFailure,
|
||
CloudHypervisorExecutor,
|
||
_parse_memory_bytes,
|
||
)
|
||
from cloud.worker import Worker, _call_goal_resume_breakpoint
|
||
except ImportError:
|
||
sys.path.insert(0, str(Path(__file__).resolve().parents[2]))
|
||
from cloud.chv.hostkeys import (
|
||
build_seed_user_data,
|
||
generate_host_keypair,
|
||
host_key_paths,
|
||
known_hosts_line,
|
||
write_seed,
|
||
)
|
||
from cloud.chv.pool import WarmPool
|
||
from cloud.sandbox import (
|
||
ChvFailure,
|
||
CloudHypervisorExecutor,
|
||
_parse_memory_bytes,
|
||
)
|
||
from cloud.worker import Worker, _call_goal_resume_breakpoint
|
||
|
||
|
||
class FakeResult:
|
||
def __init__(self, returncode=0, stdout="", stderr=""):
|
||
self.returncode = returncode
|
||
self.stdout = stdout
|
||
self.stderr = stderr
|
||
|
||
|
||
class FakeProc:
|
||
def __init__(self, alive=True):
|
||
self._alive = alive
|
||
|
||
def poll(self):
|
||
return None if self._alive else 1
|
||
|
||
def terminate(self):
|
||
self._alive = False
|
||
|
||
def wait(self, timeout=None):
|
||
return 0
|
||
|
||
def kill(self):
|
||
self._alive = False
|
||
|
||
|
||
def _write(path, content=b"x"):
|
||
with open(path, "wb") as fh:
|
||
fh.write(content)
|
||
os.chmod(path, 0o755)
|
||
return path
|
||
|
||
|
||
class ChvGapCase(unittest.TestCase):
|
||
def setUp(self):
|
||
self.tmp = tempfile.TemporaryDirectory()
|
||
self.dir = self.tmp.name
|
||
self.binary = _write(os.path.join(self.dir, "cloud-hypervisor"))
|
||
self.ch_remote = _write(os.path.join(self.dir, "ch-remote"))
|
||
self.kernel = _write(os.path.join(self.dir, "vmlinux"))
|
||
self.rootfs = _write(os.path.join(self.dir, "disk.raw"),
|
||
b"R" * 100)
|
||
self.ssh_key = _write(os.path.join(self.dir, "id_chv"))
|
||
self.kvm = _write(os.path.join(self.dir, "kvm"))
|
||
self.cpuinfo = os.path.join(self.dir, "cpuinfo")
|
||
with open(self.cpuinfo, "w") as fh:
|
||
fh.write("flags : fpu vmx sse\n")
|
||
self.state_dir = os.path.join(self.dir, "state")
|
||
self.calls = []
|
||
self.audit = []
|
||
|
||
def tearDown(self):
|
||
self.tmp.cleanup()
|
||
|
||
def _default_runner(self, cmd, timeout=None):
|
||
self.calls.append(list(cmd))
|
||
if cmd and cmd[0] == "ssh":
|
||
if cmd[-1] == "true":
|
||
return FakeResult(0, "", "")
|
||
if "goal-resume" in cmd:
|
||
return FakeResult(0, "goal met after rounds\n", "")
|
||
if "goal" in cmd:
|
||
return FakeResult(0, "goal met after rounds\n", "")
|
||
return FakeResult(0, "", "")
|
||
return FakeResult(0, "", "")
|
||
|
||
def _default_api(self, socket_path, method, path, payload=None):
|
||
return (200, {})
|
||
|
||
def _default_launcher(self, cmd):
|
||
return FakeProc(alive=True)
|
||
|
||
def make_exec(self, runner=None, env_extra=None, **overrides):
|
||
env = {"CHV_EGRESS_PROXY": "http://10.0.0.1:8888"}
|
||
if env_extra:
|
||
env.update(env_extra)
|
||
kwargs = dict(
|
||
binary=self.binary, ch_remote=self.ch_remote,
|
||
kernel=self.kernel, rootfs=self.rootfs,
|
||
ssh_key=self.ssh_key, kvm_path=self.kvm,
|
||
cpuinfo_path=self.cpuinfo, state_dir=self.state_dir,
|
||
runner=runner or self._default_runner,
|
||
launcher=self._default_launcher,
|
||
api_client=self._default_api,
|
||
sleeper=lambda s: None, env=env)
|
||
kwargs.update(overrides)
|
||
ex = CloudHypervisorExecutor(**kwargs)
|
||
ex.audit_callback = lambda e, t, d, k=None: self.audit.append(
|
||
(e, t.get("id") if isinstance(t, dict) else t, d, k))
|
||
return ex
|
||
|
||
|
||
# -- (1) worker 断点接线 ---------------------------------------------
|
||
class BreakpointWiringTest(ChvGapCase):
|
||
def test_call_helper_passes_model_provider_when_accepted(self):
|
||
seen = {}
|
||
|
||
class FakeChv:
|
||
def goal_resume_breakpoint_task(self, task, model_provider=None):
|
||
seen["provider"] = model_provider
|
||
return {"status": "completed", "task_id": task["id"]}
|
||
|
||
out = _call_goal_resume_breakpoint(
|
||
FakeChv(), {"id": "bp1"}, model_provider="prov-x")
|
||
self.assertEqual(out["status"], "completed")
|
||
self.assertEqual(seen["provider"], "prov-x")
|
||
|
||
def test_call_helper_single_arg_executor_still_works(self):
|
||
class FakeOld:
|
||
def goal_resume_breakpoint_task(self, task):
|
||
return {"status": "completed", "task_id": task["id"]}
|
||
|
||
out = _call_goal_resume_breakpoint(
|
||
FakeOld(), {"id": "bp2"}, model_provider="prov-x")
|
||
self.assertEqual(out["status"], "completed")
|
||
|
||
def test_chv_breakpoint_accepts_model_provider(self):
|
||
ex = self.make_exec()
|
||
out = ex.goal_resume_breakpoint_task(
|
||
{"id": "g1", "prompt": "续跑"}, model_provider="prov-x")
|
||
self.assertEqual(out["status"], "completed")
|
||
self.assertTrue(out["goal_met"])
|
||
# 客内命令带 --model-provider（网关 key 本身不进命令值）。
|
||
ssh_calls = [c for c in self.calls if c and c[0] == "ssh"]
|
||
self.assertTrue(any("--model-provider" in c for c in ssh_calls))
|
||
|
||
def test_worker_branch_uses_hasattr_not_only_local(self):
|
||
# worker.py 通用分支的文字证据：Local 优先、其次 hasattr。
|
||
worker_path = Path(__file__).resolve().parents[1] / "worker.py"
|
||
text = worker_path.read_text()
|
||
self.assertIn("_call_goal_resume_breakpoint", text)
|
||
self.assertIn("hasattr(", text)
|
||
self.assertIn("goal_resume_breakpoint_task", text)
|
||
|
||
|
||
# -- (2) reconcile_orphans 接线 ---------------------------------------
|
||
class ReconcileWiringTest(ChvGapCase):
|
||
def test_worker_startup_and_close_call_reconcile(self):
|
||
from cloud.server import init_db
|
||
db_path = os.path.join(self.dir, "control.db")
|
||
init_db(db_path)
|
||
ex = self.make_exec()
|
||
# 手工造一个 orphan：先建池写 registry，再新开池载入。
|
||
pool1 = WarmPool(ex)
|
||
pool1._entries["chv-old"] = {
|
||
"state": "warm", "vm": "chv-old",
|
||
"task": {"id": "old"}, "snapshot_url": None}
|
||
pool1._save_registry()
|
||
self.assertTrue(
|
||
os.path.exists(os.path.join(self.state_dir, "warm-pool.json")))
|
||
# Worker 构造即 startup reconcile：用假 shutdown 记录。
|
||
shutdowns = []
|
||
orig_shutdown = ex.shutdown_vm
|
||
ex.shutdown_vm = lambda name: shutdowns.append(name) or True
|
||
worker = Worker(db_path, "/fake/host.py", executor=ex,
|
||
env={"XCM_API_KEY": "test-placeholder"})
|
||
self.assertIn("chv-old", shutdowns)
|
||
self.assertEqual(worker._warm_pool.orphan_count(), 0)
|
||
worker.close()
|
||
# close 时再次 reconcile 不炸（池已空，返回 [] 即可）。
|
||
self.assertEqual(worker._reconcile_warm_pool("shutdown"), [])
|
||
|
||
def test_reconcile_failure_does_not_break_close(self):
|
||
from cloud.server import init_db
|
||
db_path = os.path.join(self.dir, "control.db")
|
||
init_db(db_path)
|
||
ex = self.make_exec()
|
||
worker = Worker(db_path, "/fake/host.py", executor=ex,
|
||
env={"XCM_API_KEY": "test-placeholder"})
|
||
# 池的 reconcile 抛错时 close 仍不炸。
|
||
worker._warm_pool.reconcile_orphans = lambda: (_ for _ in ()).throw(
|
||
RuntimeError("boom"))
|
||
worker.close() # 不抛即通过
|
||
|
||
|
||
# -- (3) SSH host key 预置 ---------------------------------------------
|
||
class HostKeyProvisionTest(ChvGapCase):
|
||
def _fake_keygen_runner(self, cmd, timeout=None):
|
||
self.calls.append(list(cmd))
|
||
if cmd and cmd[0] == "ssh-keygen":
|
||
priv = cmd[cmd.index("-f") + 1]
|
||
with open(priv, "w") as fh:
|
||
fh.write("-----BEGIN OPENSSH PRIVATE KEY-----\nFAKE\n")
|
||
os.chmod(priv, 0o600)
|
||
with open(priv + ".pub", "w") as fh:
|
||
fh.write("ssh-ed25519 AAAAfake1 vm-key\n")
|
||
return FakeResult(0, "", "")
|
||
return self._default_runner(cmd, timeout)
|
||
|
||
def test_per_vm_keys_differ_and_known_hosts_pinned(self):
|
||
ex = self.make_exec(runner=self._fake_keygen_runner)
|
||
r1 = ex.provision_host_key({"id": "vm-a"})
|
||
r2 = ex.provision_host_key({"id": "vm-b"})
|
||
# 不同 VM 的私钥文件不同路径、known_hosts 不同文件。
|
||
self.assertNotEqual(r1["paths"]["private"],
|
||
r2["paths"]["private"])
|
||
self.assertNotEqual(r1["known_hosts"], r2["known_hosts"])
|
||
for r in (r1, r2):
|
||
with open(r["known_hosts"]) as fh:
|
||
line = fh.read().strip()
|
||
self.assertTrue(line.startswith("[127.0.0.1]:"))
|
||
self.assertIn("ssh-ed25519 AAAAfake1", line)
|
||
self.assertNotIn("PRIVATE KEY", line)
|
||
# 同一 VM 再次预置复用已有对（不重生成、不改 known_hosts）。
|
||
with open(r1["known_hosts"]) as fh:
|
||
before = fh.read()
|
||
ex.provision_host_key({"id": "vm-a"})
|
||
with open(r1["known_hosts"]) as fh:
|
||
self.assertEqual(fh.read(), before)
|
||
|
||
def test_seed_contains_private_key_and_is_deleted_on_stop(self):
|
||
ex = self.make_exec(runner=self._fake_keygen_runner)
|
||
r = ex.provision_host_key({"id": "vm-c"})
|
||
seed_user = os.path.join(r["seed_dir"], "user-data")
|
||
self.assertTrue(os.path.exists(seed_user))
|
||
with open(seed_user) as fh:
|
||
seed_text = fh.read()
|
||
self.assertIn("ssh_host_ed25519_key", seed_text)
|
||
self.assertIn("ssh_deletekeys: false", seed_text)
|
||
# stop 时 seed 目录与私钥删掉，known_hosts 公钥保留审计。
|
||
ex.stop_vm({"id": "vm-c"})
|
||
self.assertFalse(os.path.exists(r["seed_dir"]))
|
||
self.assertFalse(os.path.exists(r["paths"]["private"]))
|
||
self.assertTrue(os.path.exists(r["known_hosts"]))
|
||
|
||
def test_known_hosts_line_rejects_bad_key(self):
|
||
with self.assertRaises(ValueError):
|
||
known_hosts_line(2222, "not-a-key")
|
||
self.assertEqual(
|
||
known_hosts_line(2222, "ssh-ed25519 AAAAx comment"),
|
||
"[127.0.0.1]:2222 ssh-ed25519 AAAAx")
|
||
|
||
def test_seed_user_data_shape(self):
|
||
text = build_seed_user_data("PRIV\n", "ssh-ed25519 AAAAx\n")
|
||
self.assertIn("/etc/ssh/ssh_host_ed25519_key", text)
|
||
self.assertIn("ssh_deletekeys: false", text)
|
||
with self.assertRaises(ValueError):
|
||
build_seed_user_data("", "ssh-ed25519 AAAAx")
|
||
|
||
def test_keyscan_is_weak_fallback_only(self):
|
||
# provisioned 模式下 wake 前必须预置；keyscan 不应被
|
||
# 当成生产路径：配置值本身可解析，且文档/代码里标弱。
|
||
ex = self.make_exec(env_extra={"CHV_HOST_KEY_MODE": "provisioned"})
|
||
self.assertEqual(ex.host_key_mode, "provisioned")
|
||
with self.assertRaises(ValueError):
|
||
self.make_exec(env_extra={"CHV_HOST_KEY_MODE": "bogus"})
|
||
|
||
|
||
# -- (4) egress tap 校验 -------------------------------------------------
|
||
class EgressValidationTest(ChvGapCase):
|
||
def test_direct_egress_without_flag_is_fail_closed(self):
|
||
ex = self.make_exec()
|
||
ex.egress_proxy = None
|
||
ex.allow_direct_egress = False
|
||
with self.assertRaises(ChvFailure) as ctx:
|
||
ex.preflight()
|
||
self.assertEqual(ctx.exception.kind, "egress_not_configured")
|
||
|
||
def test_direct_egress_with_flag_passes_and_is_audited(self):
|
||
ex = self.make_exec(env_extra={"CHV_ALLOW_DIRECT_EGRESS": "1"})
|
||
# preflight 本身不拦显式直连（守卫在 wake 的审计里）。
|
||
self.assertTrue(ex.preflight())
|
||
out = ex.goal_task({"id": "g1", "prompt": "做目标",
|
||
"goal_mode": True})
|
||
self.assertEqual(out["status"], "completed")
|
||
self.assertTrue(any(e == "egress_direct_allowed"
|
||
for e, _, _, _ in self.audit))
|
||
|
||
def test_bad_net_json_is_preflight_error(self):
|
||
ex = self.make_exec(env_extra={"CHV_NET_JSON": "{not json"})
|
||
with self.assertRaises(ChvFailure) as ctx:
|
||
ex.preflight()
|
||
self.assertEqual(ctx.exception.kind, "net_config_invalid")
|
||
|
||
|
||
# -- (5) 构建脚本 --------------------------------------------------------
|
||
class BuildScriptTest(unittest.TestCase):
|
||
def setUp(self):
|
||
self.tmp = tempfile.TemporaryDirectory()
|
||
self.dir = self.tmp.name
|
||
self.bin_dir = os.path.join(self.dir, "bin")
|
||
os.makedirs(os.path.join(self.bin_dir, "codex-resources"))
|
||
for name in ("codex", "codex-code-mode-host"):
|
||
p = os.path.join(self.bin_dir, name)
|
||
with open(p, "w") as fh:
|
||
fh.write("#!/bin/sh\n")
|
||
os.chmod(p, 0o755)
|
||
self.repo = os.path.join(self.dir, "repo")
|
||
os.makedirs(os.path.join(self.repo, "cloud"))
|
||
with open(os.path.join(self.repo, "host.py"), "w") as fh:
|
||
fh.write("# host\n")
|
||
self.script = str(Path(__file__).resolve().parents[2]
|
||
/ "chv-guest" / "build.sh")
|
||
|
||
def tearDown(self):
|
||
self.tmp.cleanup()
|
||
|
||
def _run(self, args, env_extra=None):
|
||
env = dict(os.environ)
|
||
env.update({"CODEX_BIN_DIR": self.bin_dir,
|
||
"UBUNTU_VERSION": "24.04",
|
||
"CODEX_VERSION": "0.164.0-alpha.9",
|
||
"REPO_COMMIT": "test123",
|
||
"REPO_DIR": self.repo})
|
||
if env_extra:
|
||
env.update(env_extra)
|
||
return subprocess.run(["bash", self.script] + args,
|
||
capture_output=True, text=True, env=env)
|
||
|
||
def test_dry_run_prints_real_steps_and_builds_nothing(self):
|
||
out_dir = os.path.join(self.dir, "out")
|
||
r = self._run(["--dry-run", out_dir])
|
||
self.assertEqual(r.returncode, 0, r.stderr)
|
||
self.assertIn("qemu-img convert", r.stdout)
|
||
self.assertIn("virt-customize", r.stdout)
|
||
self.assertIn("virt-get-kernel", r.stdout)
|
||
self.assertIn("BUILDINFO", r.stdout)
|
||
self.assertIn("chv-guest-24.04-0.164.0-alpha.9-test123", r.stdout)
|
||
self.assertFalse(os.path.exists(os.path.join(out_dir, "disk.raw")))
|
||
|
||
def test_full_mode_without_kvm_fails_fast(self):
|
||
# 本 VM 容器内无 /dev/kvm，全量模式必须明确失败，
|
||
# 绝不声称已建成镜像。
|
||
r = self._run([os.path.join(self.dir, "out2")])
|
||
self.assertNotEqual(r.returncode, 0)
|
||
self.assertIn("/dev/kvm", r.stderr + r.stdout)
|
||
|
||
def test_missing_binary_fails_fast(self):
|
||
os.remove(os.path.join(self.bin_dir, "codex"))
|
||
r = self._run(["--dry-run", os.path.join(self.dir, "out3")])
|
||
self.assertNotEqual(r.returncode, 0)
|
||
|
||
|
||
# -- (6) 中途死亡 → interrupted → 断点续跑闭环 ---------------------------
|
||
class MidDeathLoopTest(ChvGapCase):
|
||
def test_vm_death_is_interrupted_then_breakpoint_resumes(self):
|
||
state = {"dead": False}
|
||
|
||
def runner(cmd, timeout=None):
|
||
self.calls.append(list(cmd))
|
||
if cmd and cmd[0] == "ssh":
|
||
if cmd[-1] == "true":
|
||
return FakeResult(0, "", "")
|
||
if "goal-resume" in cmd and state["dead"]:
|
||
# 第二次断点调用：VM 已换新，健康返回。
|
||
return FakeResult(0, "goal met after rounds\n", "")
|
||
if "goal" in cmd:
|
||
# 第一次 goal：模拟传输中断，并把 VMM 进程标死。
|
||
state["dead"] = True
|
||
# 找到登记的 VMM 进程并标死（wake 已登记）。
|
||
for proc in list(ex._vms.values()):
|
||
proc._alive = False
|
||
raise RuntimeError("connection lost mid-task")
|
||
return FakeResult(0, "", "")
|
||
|
||
ex = self.make_exec(runner=runner)
|
||
# 第一次 goal：中途死应判 interrupted，不是 failed 死信。
|
||
out = ex.goal_task({"id": "g9", "prompt": "做目标",
|
||
"goal_mode": True})
|
||
self.assertEqual(out["status"], "interrupted")
|
||
# 断点续跑：同一执行器的断点方法能走通并完成。
|
||
state["dead"] = True # 标记已进入断点阶段（runner 按此分支）
|
||
out2 = ex.goal_resume_breakpoint_task(
|
||
{"id": "g9", "prompt": "续跑"})
|
||
self.assertEqual(out2["status"], "completed")
|
||
|
||
|
||
if __name__ == "__main__":
|
||
unittest.main()
|