Files
codex-app-server/cloud/tests/test_chv_fix.py
T
host 445376743d feat: CHV 审核修复 P2(范围恢复为全做)
P2-1:preflight reuse/CPU/内存校验块小提取为
_preflight_config_checks(纯移动,深耦合处如实停)。
P2-2:build_vm_command 降级为私有 _build_vm_command;
Docker goal 内联解析顶部加正典分叉说明 + 同步检查清单。
P2-3:gaps grep 文本断言改成真实 Worker._run_attempt
行为断言。P2-4:README/build.sh 口径对齐(章节顺序、
BUILDINFO 名字列表、Node.js 不装写实)。P2-5:stop_vm
删除失败记审计。P2-6:CPU/内存校验统一进 preflight。
测试:原 84 + 新增 33 = 117 项,mock,无真 VM。
2026-10-31 02:01:03 +00:00

595 lines
23 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
"""CHV 审核修复轨测试(堆叠于 #230,base=feat-chv-gaps)。
只用标准库与 mock runner/launcher/api_client,绝不启动真实
VM:当前环境没有可跑真 VM 的宿主。真实 VM 启动、seed 盘
在真机上的 cloud-init 应用、SSH 可达实测一律不在本文件
声称已验证;本文件只证明代码层面的接线(VmConfig 含 seed
盘、restore 不回落、reuse 拦截、SSH 目标解析、配额/快照/
API 完整性等)成立。
"""
import json
import os
import socket
import sys
import tempfile
import threading
import unittest
from unittest import mock
from pathlib import Path
from types import SimpleNamespace
try:
from cloud.chv.api_client import _unix_socket_api
from cloud.chv.disk import check_disk_quota, provision_task_disk
from cloud.chv.hostkeys import build_seed_disk, seed_disk_path
from cloud.chv.pool import WarmPool
from cloud.sandbox import (
ChvFailure,
CloudHypervisorExecutor,
)
from cloud.worker import Worker
except ImportError:
sys.path.insert(0, str(Path(__file__).resolve().parents[2]))
from cloud.chv.api_client import _unix_socket_api
from cloud.chv.disk import check_disk_quota, provision_task_disk
from cloud.chv.hostkeys import build_seed_disk, seed_disk_path
from cloud.chv.pool import WarmPool
from cloud.sandbox import (
ChvFailure,
CloudHypervisorExecutor,
)
from cloud.worker import Worker
class FakeResult:
def __init__(self, returncode=0, stdout="", stderr=""):
self.returncode = returncode
self.stdout = stdout
self.stderr = stderr
class FakeProc:
def __init__(self, alive=True):
self._alive = alive
def poll(self):
return None if self._alive else 1
def terminate(self):
self._alive = False
def wait(self, timeout=None):
return 0
def kill(self):
self._alive = False
def _write(path, content=b"x"):
with open(path, "wb") as fh:
fh.write(content)
os.chmod(path, 0o755)
return path
class ChvFixCase(unittest.TestCase):
def setUp(self):
self.tmp = tempfile.TemporaryDirectory()
self.dir = self.tmp.name
self.binary = _write(os.path.join(self.dir, "cloud-hypervisor"))
self.ch_remote = _write(os.path.join(self.dir, "ch-remote"))
self.kernel = _write(os.path.join(self.dir, "vmlinux"))
self.rootfs = _write(os.path.join(self.dir, "disk.raw"),
b"R" * 100)
self.ssh_key = _write(os.path.join(self.dir, "id_chv"))
self.kvm = _write(os.path.join(self.dir, "kvm"))
self.cpuinfo = os.path.join(self.dir, "cpuinfo")
with open(self.cpuinfo, "w") as fh:
fh.write("flags : fpu vmx sse\n")
self.state_dir = os.path.join(self.dir, "state")
self.calls = []
self.api_calls = []
self.api_payloads = {}
self.audit = []
def tearDown(self):
self.tmp.cleanup()
def _default_runner(self, cmd, timeout=None):
self.calls.append(list(cmd))
if cmd and cmd[0] == "ssh":
if cmd[-1] == "true":
return FakeResult(0, "", "")
if "goal" in cmd:
return FakeResult(0, "goal met after rounds\n", "")
return FakeResult(0, "", "")
if cmd and cmd[0] == "ssh-keygen":
priv = cmd[cmd.index("-f") + 1]
with open(priv, "w") as fh:
fh.write("-----BEGIN OPENSSH PRIVATE KEY-----\nFAKE\n")
os.chmod(priv, 0o600)
with open(priv + ".pub", "w") as fh:
fh.write("ssh-ed25519 AAAAfake1 vm-key\n")
return FakeResult(0, "", "")
if cmd and cmd[0] == "genisoimage":
out = cmd[cmd.index("-output") + 1]
with open(out, "wb") as fh:
fh.write(b"CIDATA" * 16)
return FakeResult(0, "", "")
return FakeResult(0, "", "")
def _default_api(self, socket_path, method, path, payload=None):
self.api_calls.append((method, path))
if path == "/vm.create":
self.api_payloads["vm.create"] = payload
return (200, {})
def make_exec(self, runner=None, env_extra=None, **overrides):
env = {"CHV_EGRESS_PROXY": "http://10.0.0.1:8888",
"CHV_SSH_FORWARD": "1"}
if env_extra:
env.update(env_extra)
kwargs = dict(
binary=self.binary, ch_remote=self.ch_remote,
kernel=self.kernel, rootfs=self.rootfs,
ssh_key=self.ssh_key, kvm_path=self.kvm,
cpuinfo_path=self.cpuinfo, state_dir=self.state_dir,
runner=runner or self._default_runner,
launcher=lambda cmd: FakeProc(alive=True),
api_client=self._default_api,
sleeper=lambda s: None, env=env)
kwargs.update(overrides)
ex = CloudHypervisorExecutor(**kwargs)
ex.audit_callback = lambda e, t, d, k=None: self.audit.append(
(e, t.get("id") if isinstance(t, dict) else t, d, k))
return ex
# -- P0-1 seed 磁盘接入 -------------------------------------------------
class SeedDiskTest(ChvFixCase):
def test_vm_config_carries_seed_disk_readonly(self):
ex = self.make_exec()
cfg = ex.build_vm_config("/tmp/x/disk.raw",
seed_disk="/tmp/x/seed.iso")
self.assertEqual(len(cfg["disks"]), 2)
self.assertEqual(cfg["disks"][1]["path"], "/tmp/x/seed.iso")
self.assertTrue(cfg["disks"][1]["readonly"])
cfg2 = ex.build_vm_config("/tmp/x/disk.raw")
self.assertEqual(len(cfg2["disks"]), 1)
def test_provisioned_wake_mounts_seed_disk(self):
ex = self.make_exec(
env_extra={"CHV_HOST_KEY_MODE": "provisioned"})
out = ex.wake_vm({"id": "seed1"})
self.assertEqual(out["state"], "running")
payload = self.api_payloads["vm.create"]
self.assertEqual(len(payload["disks"]), 2)
seed_disk = payload["disks"][1]["path"]
self.assertTrue(seed_disk.endswith("seed.iso"))
self.assertTrue(payload["disks"][1]["readonly"])
self.assertTrue(os.path.exists(seed_disk))
self.assertIn("genisoimage", [c[0] for c in self.calls])
def test_seed_disk_build_failure_is_fail_fast(self):
def bad_runner(cmd, timeout=None):
self.calls.append(list(cmd))
if cmd and cmd[0] == "genisoimage":
return FakeResult(1, "", "no genisoimage")
return self._default_runner(cmd, timeout)
ex = self.make_exec(
runner=bad_runner,
env_extra={"CHV_HOST_KEY_MODE": "provisioned"})
with self.assertRaises(RuntimeError):
ex.wake_vm({"id": "seedfail"})
# 制盘失败在起 VMM 之前,不应走到 vm.create。
self.assertNotIn("/vm.create",
[p for _, p in self.api_calls])
def test_build_seed_disk_rejects_missing_product(self):
with self.assertRaises(RuntimeError):
build_seed_disk(self.dir, os.path.join(self.dir, "no.iso"),
runner=lambda cmd: FakeResult(0, "", ""))
# -- P0-2 restore 不回落 ------------------------------------------------
class RestoreNoFallbackTest(ChvFixCase):
def test_wait_guest_ready_provisioned_missing_known_hosts(self):
ex = self.make_exec(
env_extra={"CHV_HOST_KEY_MODE": "provisioned"})
with self.assertRaises(ChvFailure) as ctx:
ex.wait_guest_ready({"id": "nopin"})
self.assertEqual(ctx.exception.kind, "host_key_not_pinned")
self.assertFalse(any(c[0] == "ssh-keyscan" for c in self.calls))
def test_restore_provisioned_missing_known_hosts_fails_before_launch(self):
ex = self.make_exec(
env_extra={"CHV_HOST_KEY_MODE": "provisioned"})
launches = []
ex.launcher = lambda cmd: launches.append(cmd) or FakeProc(True)
with self.assertRaises(ChvFailure) as ctx:
ex.restore_vm({"id": "r1"}, "file:///snap/x")
self.assertEqual(ctx.exception.kind, "host_key_not_pinned")
self.assertEqual(launches, [])
# -- P0-3 reuse 拦截 ----------------------------------------------------
class ReuseBlockTest(ChvFixCase):
def test_reuse_blocked_without_experiment_flag(self):
ex = self.make_exec(env_extra={"CHV_DISK_MODE": "reuse"})
with self.assertRaises(ChvFailure) as ctx:
ex.preflight()
self.assertEqual(ctx.exception.kind,
"disk_mode_reuse_blocked")
def test_reuse_passes_with_experiment_flag_and_is_audited(self):
ex = self.make_exec(env_extra={
"CHV_DISK_MODE": "reuse",
"CHV_REUSE_SINGLE_TENANT_EXPERIMENT": "1"})
self.assertTrue(ex.preflight())
out = ex.wake_vm({"id": "reuse1"})
self.assertEqual(out["state"], "running")
self.assertTrue(any(e == "disk_reuse_experiment_allowed"
for e, _, _, _ in self.audit))
# -- P0-4 SSH 目标解析 --------------------------------------------------
class SshTargetTest(ChvFixCase):
def test_explicit_host_wins(self):
ex = self.make_exec(ssh_host="10.9.8.7")
self.assertEqual(ex.resolve_ssh_host(), "10.9.8.7")
def test_net_guest_ip_used_when_no_explicit_host(self):
ex = self.make_exec(env_extra={
"CHV_NET_JSON": '[{"tap": "", "ip": "192.168.100.5",'
'"mask": "255.255.255.0"}]',
"CHV_SSH_FORWARD": "0"})
# env_extra 覆盖 forward=0 后仍应走 net ip,不报未声明。
self.assertEqual(ex.resolve_ssh_host(), "192.168.100.5")
cmd = ex.build_ssh_command({"id": "t"}, ["true"])
self.assertIn("root@192.168.100.5", " ".join(cmd))
def test_undeclared_target_is_fail_fast(self):
ex = self.make_exec(env_extra={"CHV_SSH_FORWARD": "0"})
with self.assertRaises(ChvFailure) as ctx:
ex.resolve_ssh_host()
self.assertEqual(ctx.exception.kind, "ssh_target_undeclared")
def test_forward_attestation_allows_loopback(self):
ex = self.make_exec()
self.assertEqual(ex.resolve_ssh_host(), "127.0.0.1")
# -- P1-1 网关 extra_env ------------------------------------------------
class GatewayExtraEnvTest(ChvFixCase):
def test_chv_ssh_command_carries_extra_env(self):
ex = self.make_exec()
cmd = ex.build_ssh_command({"id": "t"}, ["true"],
extra_env={"SUB2API_API_KEY": "k1"})
joined = " ".join(cmd)
self.assertIn("SUB2API_API_KEY=k1", joined)
self.assertIn("HTTPS_PROXY=http://10.0.0.1:8888", joined)
def test_worker_gateway_guard_allows_chv(self):
from cloud.server import init_db
db_path = os.path.join(self.dir, "control.db")
init_db(db_path)
ex = self.make_exec()
worker = Worker(db_path, "/fake/host.py", executor=ex,
env={"XCM_API_KEY": "test-placeholder"})
task = {"id": "gw1", "project_id": "p1", "prompt": "做目标",
"goal_mode": 1,
"_gateway_provider": "sub2api",
"_gateway_env": {"SUB2API_API_KEY": "gwkey",
"SUB2API_BASE_URL": "http://gw/"}}
result, error = worker._run_attempt(task)
self.assertIsNone(error)
self.assertEqual(result["status"], "completed")
ssh_calls = [c for c in self.calls if c and c[0] == "ssh"]
self.assertTrue(any("SUB2API_API_KEY=gwkey" in " ".join(c)
for c in ssh_calls))
# -- P1-2 取消/清理/看门狗 ----------------------------------------------
class CancelCleanupTest(ChvFixCase):
def test_worker_cancel_calls_cancel_vm(self):
from cloud.server import init_db
db_path = os.path.join(self.dir, "control.db")
init_db(db_path)
ex = self.make_exec()
cancelled = []
ex.cancel_vm = lambda name: cancelled.append(name) or True
worker = Worker(db_path, "/fake/host.py", executor=ex,
env={"XCM_API_KEY": "test-placeholder"})
worker._running_set("t-cancel")
import sqlite3, time
conn = sqlite3.connect(db_path)
conn.execute(
"INSERT INTO tasks(id, project_id, prompt, status,"
" idempotency_key, depends_on, priority, created_by,"
" created_at, updated_at, last_heartbeat)"
" VALUES(?, ?, ?, ?, NULL, ?, ?, 'tester', ?, ?, NULL)",
("t-cancel", "p1", "做事", "running", "[]", 0,
time.time(), time.time()))
conn.commit()
conn.close()
self.assertTrue(worker.cancel("t-cancel"))
self.assertEqual(cancelled, ["t-cancel"])
def test_worker_cleanup_calls_cleanup_stale_vms(self):
from cloud.server import init_db
db_path = os.path.join(self.dir, "control.db")
init_db(db_path)
ex = self.make_exec()
ex.cleanup_stale_vms = lambda: ["chv-stale"]
worker = Worker(db_path, "/fake/host.py", executor=ex,
env={"XCM_API_KEY": "test-placeholder"})
self.assertIn("chv-stale", worker.cleanup_stale_containers())
def test_chv_cancel_watchdog_kills_vm_on_cancel(self):
import threading
from cloud.server import init_db
db_path = os.path.join(self.dir, "control.db")
init_db(db_path)
ex = self.make_exec()
killed = []
ex.cancel_vm = lambda name: killed.append(name) or True
worker = Worker(db_path, "/fake/host.py", executor=ex,
env={"XCM_API_KEY": "test-placeholder"})
worker._cancel_requested.add("t-wd")
stop = threading.Event()
worker._chv_cancel_watchdog("t-wd", "chv-t-wd", stop)
self.assertEqual(killed, ["chv-t-wd"])
# -- P1-3 WarmPool acquire resume ---------------------------------------
class PoolAcquireResumeTest(unittest.TestCase):
def test_acquire_resumes_paused_vm(self):
class Stub:
def __init__(self):
self.resumed = []
self.state_dir = None
self.warm_pool_size = 1
self.snapshot_url = "file:///snap/base"
def vm_name(self, task):
raw = task.get("id") if isinstance(task, dict) \
else str(task)
return raw if raw.startswith("chv-") \
else "chv-" + raw
def wake_vm(self, task):
return {"vm": self.vm_name(task)}
def pause_vm(self, task):
return True
def resume_vm(self, task):
self.resumed.append(task["id"])
return True
def shutdown_vm(self, task_or_name):
return True
def _emit(self, event, task, detail="", kind=None):
pass
stub = Stub()
pool = WarmPool(stub, size=1, snapshot_url="file:///snap/base",
registry_path=None)
pool.prewarm()
got = pool.acquire({"id": "t1"})
self.assertEqual(got["mode"], "warm")
self.assertEqual(stub.resumed, ["warm-1"])
def test_acquire_discards_vm_whose_resume_fails(self):
class Stub:
def __init__(self):
self.state_dir = None
self.warm_pool_size = 1
self.snapshot_url = "file:///snap/base"
def vm_name(self, task):
raw = task.get("id") if isinstance(task, dict) \
else str(task)
return raw if raw.startswith("chv-") \
else "chv-" + raw
def wake_vm(self, task):
return {"vm": self.vm_name(task)}
def pause_vm(self, task):
return True
def resume_vm(self, task):
raise RuntimeError("resume 失败")
def shutdown_vm(self, task_or_name):
return True
def _emit(self, event, task, detail="", kind=None):
pass
pool = WarmPool(Stub(), size=1, snapshot_url="file:///snap/base",
registry_path=None)
pool.prewarm()
self.assertEqual(pool.acquire({"id": "t1"})["mode"], "cold")
# -- P1-4 容量持久化计数 -------------------------------------------------
class CapacityPersistentTest(ChvFixCase):
def test_socket_counts_after_restart(self):
ex = self.make_exec(env_extra={"CHV_MAX_VMS": "1"})
os.makedirs(self.state_dir, exist_ok=True)
with open(os.path.join(self.state_dir, "chv-other.sock"),
"w") as fh:
fh.write("")
with self.assertRaises(ChvFailure) as ctx:
ex.wake_vm({"id": "new1"})
self.assertEqual(ctx.exception.kind, "host_capacity_exceeded")
def test_own_socket_does_not_block_rewake(self):
ex = self.make_exec(env_extra={"CHV_MAX_VMS": "1"})
os.makedirs(self.state_dir, exist_ok=True)
with open(os.path.join(self.state_dir, "chv-t1.sock"),
"w") as fh:
fh.write("")
# 同任务重复 wake:自己的残留 socket 不计入容量。
out = ex.wake_vm({"id": "t1"})
self.assertEqual(out["state"], "running")
# -- P1-5 磁盘配额 -------------------------------------------------------
class DiskQuotaTest(ChvFixCase):
def test_unreadable_image_is_not_silently_skipped(self):
with self.assertRaises(ChvFailure) as ctx:
check_disk_quota("/nonexistent/disk.raw", 1.0)
self.assertEqual(ctx.exception.kind, "disk_quota_unverified")
def test_total_usage_over_quota(self):
os.makedirs(self.state_dir, exist_ok=True)
with open(os.path.join(self.state_dir, "old.raw"),
"wb") as fh:
fh.write(b"X" * 100)
# 配额 150 字节:已用 100 + 本次 100 > 150。
with self.assertRaises(ChvFailure) as ctx:
check_disk_quota(self.rootfs, 150 / (1024 ** 3),
state_dir=self.state_dir)
self.assertEqual(ctx.exception.kind, "disk_quota_exceeded")
def test_existing_disk_size_mismatch_is_recopied(self):
task_disk = provision_task_disk(self.state_dir, "chv-a",
self.rootfs, "copy")
with open(task_disk, "wb") as fh:
fh.write(b"Z" * 10)
task_disk2 = provision_task_disk(self.state_dir, "chv-a",
self.rootfs, "copy")
self.assertEqual(task_disk, task_disk2)
with open(task_disk2, "rb") as fh:
self.assertEqual(fh.read(), b"R" * 100)
# -- P1-6 API 截断 -------------------------------------------------------
class ApiTruncationTest(unittest.TestCase):
def _serve(self, raw):
tmp = tempfile.TemporaryDirectory()
sock_path = os.path.join(tmp.name, "vmm.sock")
srv = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
srv.bind(sock_path)
srv.listen(1)
def _run():
conn, _ = srv.accept()
try:
conn.recv(65536)
conn.sendall(raw)
finally:
conn.close()
srv.close()
threading.Thread(target=_run, daemon=True).start()
return tmp, sock_path
def test_truncated_body_raises(self):
body = b'{"ok": true}'
raw = (b"HTTP/1.1 200 OK\r\nContent-Length: 100\r\n\r\n"
+ body)
tmp, sock_path = self._serve(raw)
with self.assertRaises(RuntimeError):
_unix_socket_api(sock_path, "GET", "/vmm.ping")
tmp.cleanup()
def test_chunked_rejected(self):
raw = (b"HTTP/1.1 200 OK\r\nTransfer-Encoding: chunked\r\n\r\n"
b"5\r\nhello\r\n0\r\n\r\n")
tmp, sock_path = self._serve(raw)
with self.assertRaises(RuntimeError):
_unix_socket_api(sock_path, "GET", "/vmm.ping")
tmp.cleanup()
def test_full_body_parses(self):
body = b'{"ok": true}'
raw = (b"HTTP/1.1 200 OK\r\nContent-Length: %d\r\n\r\n"
% len(body)) + body
tmp, sock_path = self._serve(raw)
status, parsed = _unix_socket_api(sock_path, "GET", "/vmm.ping")
self.assertEqual(status, 200)
self.assertTrue(parsed["ok"])
tmp.cleanup()
# -- P1-7 快照落盘验证 ---------------------------------------------------
class SnapshotPersistTest(ChvFixCase):
def test_snapshot_missing_file_does_not_shutdown(self):
ex = self.make_exec()
with self.assertRaises(ChvFailure) as ctx:
ex.hibernate_vm({"id": "h1"}, "file:///nonexistent/snap")
self.assertEqual(ctx.exception.kind, "snapshot_not_persisted")
kinds = [c[-1] for c in self.calls if c[0] == self.ch_remote]
self.assertIn("pause", kinds)
self.assertNotIn("shutdown", kinds)
# -- P1-8 execute 死端口归类 ---------------------------------------------
class ExecuteDeadPortTest(ChvFixCase):
def test_dead_socket_ssh_failure_is_vm_not_running(self):
def raising_runner(cmd, timeout=None):
self.calls.append(list(cmd))
if cmd and cmd[0] == "ssh":
raise RuntimeError("connection refused")
return FakeResult(0, "", "")
ex = self.make_exec(runner=raising_runner)
os.makedirs(self.state_dir, exist_ok=True)
with open(os.path.join(self.state_dir, "chv-dead.sock"),
"w") as fh:
fh.write("")
ex._vms["chv-dead"] = FakeProc(alive=False)
with self.assertRaises(ChvFailure) as ctx:
ex.execute({"id": "dead"})
self.assertEqual(ctx.exception.kind, "vm_not_running")
# -- P2 配置校验时机 / 文档口径 ------------------------------------------
class ConfigTimingAndDocsTest(ChvFixCase):
def test_preflight_rejects_bad_cpu_and_memory(self):
ex = self.make_exec(cpus="abc")
with self.assertRaises(ChvFailure) as ctx:
ex.preflight()
self.assertEqual(ctx.exception.kind, "config_invalid")
ex2 = self.make_exec(memory="not-a-size")
with self.assertRaises(ChvFailure) as ctx2:
ex2.preflight()
self.assertEqual(ctx2.exception.kind, "config_invalid")
def test_stop_cleanup_failure_is_audited(self):
ex = self.make_exec()
ex.stop_vm({"id": "ck"})
os.makedirs(os.path.join(self.state_dir, "chv-ck", "seed"),
exist_ok=True)
with mock.patch("cloud.chv.executor.shutil.rmtree",
side_effect=OSError("rm failed")):
ex.stop_vm({"id": "ck"})
self.assertTrue(any(e == "vm_key_cleanup_failed"
for e, _, _, _ in self.audit))
def test_readme_sections_in_order_and_buildinfo_honest(self):
readme = Path(__file__).resolve().parents[2] \
/ "chv-guest" / "README.md"
text = readme.read_text()
self.assertLess(text.index("## 5. Host key"),
text.index("## 6. 与执行器的对接"))
self.assertIn("contents: [清单逐项名]", text)
self.assertIn("Node.js 不装", text)
def test_manual_vm_command_is_private(self):
ex = self.make_exec()
self.assertFalse(hasattr(ex, "build_vm_command"))
cmd = ex._build_vm_command({"id": "t"})
self.assertIn("--api-socket", cmd)
if __name__ == "__main__":
unittest.main()