host
445376743d
P2-1：preflight reuse/CPU/内存校验块小提取为 _preflight_config_checks（纯移动，深耦合处如实停）。 P2-2：build_vm_command 降级为私有 _build_vm_command； Docker goal 内联解析顶部加正典分叉说明 + 同步检查清单。 P2-3：gaps grep 文本断言改成真实 Worker._run_attempt 行为断言。P2-4：README/build.sh 口径对齐（章节顺序、 BUILDINFO 名字列表、Node.js 不装写实）。P2-5：stop_vm 删除失败记审计。P2-6：CPU/内存校验统一进 preflight。 测试：原 84 + 新增 33 = 117 项，mock，无真 VM。
448 lines
18 KiB
Python
448 lines
18 KiB
Python
"""CHV 补缺口轨测试（堆叠于 #221，base=feat-chv-optimize）。
|
||
|
||
只用标准库与 mock runner/launcher/api_client，绝不启动真实
|
||
VM：当前环境没有可跑真 VM 的宿主（生产主机是 CVM、无
|
||
/dev/kvm、无嵌套虚拟化；本 VM 容器内同样无 /dev/kvm）。
|
||
真实 VM 启动、快照恢复、压测一律不在本文件声称已验证。
|
||
"""
|
||
|
||
import json
|
||
import os
|
||
import subprocess
|
||
import sys
|
||
import tempfile
|
||
import unittest
|
||
from pathlib import Path
|
||
from types import SimpleNamespace
|
||
|
||
try:
|
||
from cloud.chv.hostkeys import (
|
||
build_seed_user_data,
|
||
generate_host_keypair,
|
||
host_key_paths,
|
||
known_hosts_line,
|
||
write_seed,
|
||
)
|
||
from cloud.chv.pool import WarmPool
|
||
from cloud.sandbox import (
|
||
ChvFailure,
|
||
CloudHypervisorExecutor,
|
||
_parse_memory_bytes,
|
||
)
|
||
from cloud.worker import Worker, _call_goal_resume_breakpoint
|
||
except ImportError:
|
||
sys.path.insert(0, str(Path(__file__).resolve().parents[2]))
|
||
from cloud.chv.hostkeys import (
|
||
build_seed_user_data,
|
||
generate_host_keypair,
|
||
host_key_paths,
|
||
known_hosts_line,
|
||
write_seed,
|
||
)
|
||
from cloud.chv.pool import WarmPool
|
||
from cloud.sandbox import (
|
||
ChvFailure,
|
||
CloudHypervisorExecutor,
|
||
_parse_memory_bytes,
|
||
)
|
||
from cloud.worker import Worker, _call_goal_resume_breakpoint
|
||
|
||
|
||
class FakeResult:
|
||
def __init__(self, returncode=0, stdout="", stderr=""):
|
||
self.returncode = returncode
|
||
self.stdout = stdout
|
||
self.stderr = stderr
|
||
|
||
|
||
class FakeProc:
|
||
def __init__(self, alive=True):
|
||
self._alive = alive
|
||
|
||
def poll(self):
|
||
return None if self._alive else 1
|
||
|
||
def terminate(self):
|
||
self._alive = False
|
||
|
||
def wait(self, timeout=None):
|
||
return 0
|
||
|
||
def kill(self):
|
||
self._alive = False
|
||
|
||
|
||
def _write(path, content=b"x"):
|
||
with open(path, "wb") as fh:
|
||
fh.write(content)
|
||
os.chmod(path, 0o755)
|
||
return path
|
||
|
||
|
||
class ChvGapCase(unittest.TestCase):
|
||
def setUp(self):
|
||
self.tmp = tempfile.TemporaryDirectory()
|
||
self.dir = self.tmp.name
|
||
self.binary = _write(os.path.join(self.dir, "cloud-hypervisor"))
|
||
self.ch_remote = _write(os.path.join(self.dir, "ch-remote"))
|
||
self.kernel = _write(os.path.join(self.dir, "vmlinux"))
|
||
self.rootfs = _write(os.path.join(self.dir, "disk.raw"),
|
||
b"R" * 100)
|
||
self.ssh_key = _write(os.path.join(self.dir, "id_chv"))
|
||
self.kvm = _write(os.path.join(self.dir, "kvm"))
|
||
self.cpuinfo = os.path.join(self.dir, "cpuinfo")
|
||
with open(self.cpuinfo, "w") as fh:
|
||
fh.write("flags : fpu vmx sse\n")
|
||
self.state_dir = os.path.join(self.dir, "state")
|
||
self.calls = []
|
||
self.audit = []
|
||
|
||
def tearDown(self):
|
||
self.tmp.cleanup()
|
||
|
||
def _default_runner(self, cmd, timeout=None):
|
||
self.calls.append(list(cmd))
|
||
if cmd and cmd[0] == "ssh":
|
||
if cmd[-1] == "true":
|
||
return FakeResult(0, "", "")
|
||
if "goal-resume" in cmd:
|
||
return FakeResult(0, "goal met after rounds\n", "")
|
||
if "goal" in cmd:
|
||
return FakeResult(0, "goal met after rounds\n", "")
|
||
return FakeResult(0, "", "")
|
||
return FakeResult(0, "", "")
|
||
|
||
def _default_api(self, socket_path, method, path, payload=None):
|
||
return (200, {})
|
||
|
||
def _default_launcher(self, cmd):
|
||
return FakeProc(alive=True)
|
||
|
||
def make_exec(self, runner=None, env_extra=None, **overrides):
|
||
env = {"CHV_EGRESS_PROXY": "http://10.0.0.1:8888",
|
||
# 实验室宣称：mock runner 假装回环端口已转发到客机
|
||
#（feat-chv-fix 起，执行器不再无条件假设回环能通）。
|
||
"CHV_SSH_FORWARD": "1"}
|
||
if env_extra:
|
||
env.update(env_extra)
|
||
kwargs = dict(
|
||
binary=self.binary, ch_remote=self.ch_remote,
|
||
kernel=self.kernel, rootfs=self.rootfs,
|
||
ssh_key=self.ssh_key, kvm_path=self.kvm,
|
||
cpuinfo_path=self.cpuinfo, state_dir=self.state_dir,
|
||
runner=runner or self._default_runner,
|
||
launcher=self._default_launcher,
|
||
api_client=self._default_api,
|
||
sleeper=lambda s: None, env=env)
|
||
kwargs.update(overrides)
|
||
ex = CloudHypervisorExecutor(**kwargs)
|
||
ex.audit_callback = lambda e, t, d, k=None: self.audit.append(
|
||
(e, t.get("id") if isinstance(t, dict) else t, d, k))
|
||
return ex
|
||
|
||
|
||
# -- (1) worker 断点接线 ---------------------------------------------
|
||
class BreakpointWiringTest(ChvGapCase):
|
||
def test_call_helper_passes_model_provider_when_accepted(self):
|
||
seen = {}
|
||
|
||
class FakeChv:
|
||
def goal_resume_breakpoint_task(self, task, model_provider=None):
|
||
seen["provider"] = model_provider
|
||
return {"status": "completed", "task_id": task["id"]}
|
||
|
||
out = _call_goal_resume_breakpoint(
|
||
FakeChv(), {"id": "bp1"}, model_provider="prov-x")
|
||
self.assertEqual(out["status"], "completed")
|
||
self.assertEqual(seen["provider"], "prov-x")
|
||
|
||
def test_call_helper_single_arg_executor_still_works(self):
|
||
class FakeOld:
|
||
def goal_resume_breakpoint_task(self, task):
|
||
return {"status": "completed", "task_id": task["id"]}
|
||
|
||
out = _call_goal_resume_breakpoint(
|
||
FakeOld(), {"id": "bp2"}, model_provider="prov-x")
|
||
self.assertEqual(out["status"], "completed")
|
||
|
||
def test_chv_breakpoint_accepts_model_provider(self):
|
||
ex = self.make_exec()
|
||
out = ex.goal_resume_breakpoint_task(
|
||
{"id": "g1", "prompt": "续跑"}, model_provider="prov-x")
|
||
self.assertEqual(out["status"], "completed")
|
||
self.assertTrue(out["goal_met"])
|
||
# 客内命令带 --model-provider（网关 key 本身不进命令值）。
|
||
ssh_calls = [c for c in self.calls if c and c[0] == "ssh"]
|
||
self.assertTrue(any("--model-provider" in c for c in ssh_calls))
|
||
|
||
def test_worker_breakpoint_branch_calls_generic_executor(self):
|
||
# feat-chv-fix（P2-3）：不再用源码 grep 充数，改为行为
|
||
# 断言——真实 Worker._run_attempt 走通用断点分支，调到
|
||
# 非 Local 执行器的 goal_resume_breakpoint_task，并把
|
||
# 结果原样返回（参数、守卫顺序、返回值都覆盖）。
|
||
from cloud.server import init_db
|
||
db_path = os.path.join(self.dir, "control.db")
|
||
init_db(db_path)
|
||
seen = {}
|
||
|
||
class FakeGeneric:
|
||
def goal_resume_breakpoint_task(self, task,
|
||
model_provider=None,
|
||
extra_env=None):
|
||
seen["provider"] = model_provider
|
||
seen["task_id"] = task["id"]
|
||
return {"executor": "generic", "task_id": task["id"],
|
||
"status": "completed", "stdout": "",
|
||
"stderr": "", "returncode": 0,
|
||
"host_status": None, "goal_met": True,
|
||
"goal_error": None}
|
||
|
||
worker = Worker(db_path, "/fake/host.py",
|
||
executor=FakeGeneric(),
|
||
env={"XCM_API_KEY": "test-placeholder"})
|
||
task = {"id": "bp-behav", "project_id": "p1",
|
||
"prompt": "续跑", "goal_mode": 1,
|
||
"_goal_resume_breakpoint": True}
|
||
result, error = worker._run_attempt(task)
|
||
self.assertIsNone(error)
|
||
self.assertEqual(result["status"], "completed")
|
||
self.assertTrue(result["goal_met"])
|
||
self.assertEqual(seen["task_id"], "bp-behav")
|
||
self.assertNotIn("executor 不支持", str(result))
|
||
|
||
|
||
# -- (2) reconcile_orphans 接线 ---------------------------------------
|
||
class ReconcileWiringTest(ChvGapCase):
|
||
def test_worker_startup_and_close_call_reconcile(self):
|
||
from cloud.server import init_db
|
||
db_path = os.path.join(self.dir, "control.db")
|
||
init_db(db_path)
|
||
ex = self.make_exec()
|
||
# 手工造一个 orphan：先建池写 registry，再新开池载入。
|
||
pool1 = WarmPool(ex)
|
||
pool1._entries["chv-old"] = {
|
||
"state": "warm", "vm": "chv-old",
|
||
"task": {"id": "old"}, "snapshot_url": None}
|
||
pool1._save_registry()
|
||
self.assertTrue(
|
||
os.path.exists(os.path.join(self.state_dir, "warm-pool.json")))
|
||
# Worker 构造即 startup reconcile：用假 shutdown 记录。
|
||
shutdowns = []
|
||
orig_shutdown = ex.shutdown_vm
|
||
ex.shutdown_vm = lambda name: shutdowns.append(name) or True
|
||
worker = Worker(db_path, "/fake/host.py", executor=ex,
|
||
env={"XCM_API_KEY": "test-placeholder"})
|
||
self.assertIn("chv-old", shutdowns)
|
||
self.assertEqual(worker._warm_pool.orphan_count(), 0)
|
||
worker.close()
|
||
# close 时再次 reconcile 不炸（池已空，返回 [] 即可）。
|
||
self.assertEqual(worker._reconcile_warm_pool("shutdown"), [])
|
||
|
||
def test_reconcile_failure_does_not_break_close(self):
|
||
from cloud.server import init_db
|
||
db_path = os.path.join(self.dir, "control.db")
|
||
init_db(db_path)
|
||
ex = self.make_exec()
|
||
worker = Worker(db_path, "/fake/host.py", executor=ex,
|
||
env={"XCM_API_KEY": "test-placeholder"})
|
||
# 池的 reconcile 抛错时 close 仍不炸。
|
||
worker._warm_pool.reconcile_orphans = lambda: (_ for _ in ()).throw(
|
||
RuntimeError("boom"))
|
||
worker.close() # 不抛即通过
|
||
|
||
|
||
# -- (3) SSH host key 预置 ---------------------------------------------
|
||
class HostKeyProvisionTest(ChvGapCase):
|
||
def _fake_keygen_runner(self, cmd, timeout=None):
|
||
self.calls.append(list(cmd))
|
||
if cmd and cmd[0] == "ssh-keygen":
|
||
priv = cmd[cmd.index("-f") + 1]
|
||
with open(priv, "w") as fh:
|
||
fh.write("-----BEGIN OPENSSH PRIVATE KEY-----\nFAKE\n")
|
||
os.chmod(priv, 0o600)
|
||
with open(priv + ".pub", "w") as fh:
|
||
fh.write("ssh-ed25519 AAAAfake1 vm-key\n")
|
||
return FakeResult(0, "", "")
|
||
return self._default_runner(cmd, timeout)
|
||
|
||
def test_per_vm_keys_differ_and_known_hosts_pinned(self):
|
||
ex = self.make_exec(runner=self._fake_keygen_runner)
|
||
r1 = ex.provision_host_key({"id": "vm-a"})
|
||
r2 = ex.provision_host_key({"id": "vm-b"})
|
||
# 不同 VM 的私钥文件不同路径、known_hosts 不同文件。
|
||
self.assertNotEqual(r1["paths"]["private"],
|
||
r2["paths"]["private"])
|
||
self.assertNotEqual(r1["known_hosts"], r2["known_hosts"])
|
||
for r in (r1, r2):
|
||
with open(r["known_hosts"]) as fh:
|
||
line = fh.read().strip()
|
||
self.assertTrue(line.startswith("[127.0.0.1]:"))
|
||
self.assertIn("ssh-ed25519 AAAAfake1", line)
|
||
self.assertNotIn("PRIVATE KEY", line)
|
||
# 同一 VM 再次预置复用已有对（不重生成、不改 known_hosts）。
|
||
with open(r1["known_hosts"]) as fh:
|
||
before = fh.read()
|
||
ex.provision_host_key({"id": "vm-a"})
|
||
with open(r1["known_hosts"]) as fh:
|
||
self.assertEqual(fh.read(), before)
|
||
|
||
def test_seed_contains_private_key_and_is_deleted_on_stop(self):
|
||
ex = self.make_exec(runner=self._fake_keygen_runner)
|
||
r = ex.provision_host_key({"id": "vm-c"})
|
||
seed_user = os.path.join(r["seed_dir"], "user-data")
|
||
self.assertTrue(os.path.exists(seed_user))
|
||
with open(seed_user) as fh:
|
||
seed_text = fh.read()
|
||
self.assertIn("ssh_host_ed25519_key", seed_text)
|
||
self.assertIn("ssh_deletekeys: false", seed_text)
|
||
# stop 时 seed 目录与私钥删掉，known_hosts 公钥保留审计。
|
||
ex.stop_vm({"id": "vm-c"})
|
||
self.assertFalse(os.path.exists(r["seed_dir"]))
|
||
self.assertFalse(os.path.exists(r["paths"]["private"]))
|
||
self.assertTrue(os.path.exists(r["known_hosts"]))
|
||
|
||
def test_known_hosts_line_rejects_bad_key(self):
|
||
with self.assertRaises(ValueError):
|
||
known_hosts_line(2222, "not-a-key")
|
||
self.assertEqual(
|
||
known_hosts_line(2222, "ssh-ed25519 AAAAx comment"),
|
||
"[127.0.0.1]:2222 ssh-ed25519 AAAAx")
|
||
|
||
def test_seed_user_data_shape(self):
|
||
text = build_seed_user_data("PRIV\n", "ssh-ed25519 AAAAx\n")
|
||
self.assertIn("/etc/ssh/ssh_host_ed25519_key", text)
|
||
self.assertIn("ssh_deletekeys: false", text)
|
||
with self.assertRaises(ValueError):
|
||
build_seed_user_data("", "ssh-ed25519 AAAAx")
|
||
|
||
def test_keyscan_is_weak_fallback_only(self):
|
||
# provisioned 模式下 wake 前必须预置；keyscan 不应被
|
||
# 当成生产路径：配置值本身可解析，且文档/代码里标弱。
|
||
ex = self.make_exec(env_extra={"CHV_HOST_KEY_MODE": "provisioned"})
|
||
self.assertEqual(ex.host_key_mode, "provisioned")
|
||
with self.assertRaises(ValueError):
|
||
self.make_exec(env_extra={"CHV_HOST_KEY_MODE": "bogus"})
|
||
|
||
|
||
# -- (4) egress tap 校验 -------------------------------------------------
|
||
class EgressValidationTest(ChvGapCase):
|
||
def test_direct_egress_without_flag_is_fail_closed(self):
|
||
ex = self.make_exec()
|
||
ex.egress_proxy = None
|
||
ex.allow_direct_egress = False
|
||
with self.assertRaises(ChvFailure) as ctx:
|
||
ex.preflight()
|
||
self.assertEqual(ctx.exception.kind, "egress_not_configured")
|
||
|
||
def test_direct_egress_with_flag_passes_and_is_audited(self):
|
||
ex = self.make_exec(env_extra={"CHV_ALLOW_DIRECT_EGRESS": "1"})
|
||
# preflight 本身不拦显式直连（守卫在 wake 的审计里）。
|
||
self.assertTrue(ex.preflight())
|
||
out = ex.goal_task({"id": "g1", "prompt": "做目标",
|
||
"goal_mode": True})
|
||
self.assertEqual(out["status"], "completed")
|
||
self.assertTrue(any(e == "egress_direct_allowed"
|
||
for e, _, _, _ in self.audit))
|
||
|
||
def test_bad_net_json_is_preflight_error(self):
|
||
ex = self.make_exec(env_extra={"CHV_NET_JSON": "{not json"})
|
||
with self.assertRaises(ChvFailure) as ctx:
|
||
ex.preflight()
|
||
self.assertEqual(ctx.exception.kind, "net_config_invalid")
|
||
|
||
|
||
# -- (5) 构建脚本 --------------------------------------------------------
|
||
class BuildScriptTest(unittest.TestCase):
|
||
def setUp(self):
|
||
self.tmp = tempfile.TemporaryDirectory()
|
||
self.dir = self.tmp.name
|
||
self.bin_dir = os.path.join(self.dir, "bin")
|
||
os.makedirs(os.path.join(self.bin_dir, "codex-resources"))
|
||
for name in ("codex", "codex-code-mode-host"):
|
||
p = os.path.join(self.bin_dir, name)
|
||
with open(p, "w") as fh:
|
||
fh.write("#!/bin/sh\n")
|
||
os.chmod(p, 0o755)
|
||
self.repo = os.path.join(self.dir, "repo")
|
||
os.makedirs(os.path.join(self.repo, "cloud"))
|
||
with open(os.path.join(self.repo, "host.py"), "w") as fh:
|
||
fh.write("# host\n")
|
||
self.script = str(Path(__file__).resolve().parents[2]
|
||
/ "chv-guest" / "build.sh")
|
||
|
||
def tearDown(self):
|
||
self.tmp.cleanup()
|
||
|
||
def _run(self, args, env_extra=None):
|
||
env = dict(os.environ)
|
||
env.update({"CODEX_BIN_DIR": self.bin_dir,
|
||
"UBUNTU_VERSION": "24.04",
|
||
"CODEX_VERSION": "0.164.0-alpha.9",
|
||
"REPO_COMMIT": "test123",
|
||
"REPO_DIR": self.repo})
|
||
if env_extra:
|
||
env.update(env_extra)
|
||
return subprocess.run(["bash", self.script] + args,
|
||
capture_output=True, text=True, env=env)
|
||
|
||
def test_dry_run_prints_real_steps_and_builds_nothing(self):
|
||
out_dir = os.path.join(self.dir, "out")
|
||
r = self._run(["--dry-run", out_dir])
|
||
self.assertEqual(r.returncode, 0, r.stderr)
|
||
self.assertIn("qemu-img convert", r.stdout)
|
||
self.assertIn("virt-customize", r.stdout)
|
||
self.assertIn("virt-get-kernel", r.stdout)
|
||
self.assertIn("BUILDINFO", r.stdout)
|
||
self.assertIn("chv-guest-24.04-0.164.0-alpha.9-test123", r.stdout)
|
||
self.assertFalse(os.path.exists(os.path.join(out_dir, "disk.raw")))
|
||
|
||
def test_full_mode_without_kvm_fails_fast(self):
|
||
# 本 VM 容器内无 /dev/kvm，全量模式必须明确失败，
|
||
# 绝不声称已建成镜像。
|
||
r = self._run([os.path.join(self.dir, "out2")])
|
||
self.assertNotEqual(r.returncode, 0)
|
||
self.assertIn("/dev/kvm", r.stderr + r.stdout)
|
||
|
||
def test_missing_binary_fails_fast(self):
|
||
os.remove(os.path.join(self.bin_dir, "codex"))
|
||
r = self._run(["--dry-run", os.path.join(self.dir, "out3")])
|
||
self.assertNotEqual(r.returncode, 0)
|
||
|
||
|
||
# -- (6) 中途死亡 → interrupted → 断点续跑闭环 ---------------------------
|
||
class MidDeathLoopTest(ChvGapCase):
|
||
def test_vm_death_is_interrupted_then_breakpoint_resumes(self):
|
||
state = {"dead": False}
|
||
|
||
def runner(cmd, timeout=None):
|
||
self.calls.append(list(cmd))
|
||
if cmd and cmd[0] == "ssh":
|
||
if cmd[-1] == "true":
|
||
return FakeResult(0, "", "")
|
||
if "goal-resume" in cmd and state["dead"]:
|
||
# 第二次断点调用：VM 已换新，健康返回。
|
||
return FakeResult(0, "goal met after rounds\n", "")
|
||
if "goal" in cmd:
|
||
# 第一次 goal：模拟传输中断，并把 VMM 进程标死。
|
||
state["dead"] = True
|
||
# 找到登记的 VMM 进程并标死（wake 已登记）。
|
||
for proc in list(ex._vms.values()):
|
||
proc._alive = False
|
||
raise RuntimeError("connection lost mid-task")
|
||
return FakeResult(0, "", "")
|
||
|
||
ex = self.make_exec(runner=runner)
|
||
# 第一次 goal：中途死应判 interrupted，不是 failed 死信。
|
||
out = ex.goal_task({"id": "g9", "prompt": "做目标",
|
||
"goal_mode": True})
|
||
self.assertEqual(out["status"], "interrupted")
|
||
# 断点续跑：同一执行器的断点方法能走通并完成。
|
||
state["dead"] = True # 标记已进入断点阶段（runner 按此分支）
|
||
out2 = ex.goal_resume_breakpoint_task(
|
||
{"id": "g9", "prompt": "续跑"})
|
||
self.assertEqual(out2["status"], "completed")
|
||
|
||
|
||
if __name__ == "__main__":
|
||
unittest.main()
|