Files
codex-app-server/cloud/tests/test_chv_fix.py
T
host 9771ed9e8c feat: CHV 审核修复(P0+P1,堆叠于 #230)
P0:seed 磁盘 CIDATA ISO 挂进 VmConfig 第二块只读盘;
restore provisioned 下复用钉死 known_hosts、缺失 fail-fast
绝不 keyscan;preflight 拦截 CHV_DISK_MODE=reuse(显式
实验二次确认 + 审计放行);SSH 目标选客机 IP 方案(显式
host > net 静态 ip > 显式转发宣称回环,三者皆无 fail-fast)。
P1:网关 extra_env 经 SSH env 前缀注入、worker 守卫收窄到
Local/CHV;取消/清理/看门狗接 CHV 通用路径;WarmPool
acquire 内 resume;容量门持久化计数;磁盘配额总量+余量
与已存在磁盘校验重拷;API 按 Content-Length 验完整性;
快照 file:// 验落盘再 shutdown;execute 死端口归类
vm_not_running。
P2 全部排除未做(用户范围收窄令,留待以后一轨)。
测试:原 84 + 新增 29 = 113 项,mock,无真 VM。
2026-10-31 01:54:13 +00:00

555 lines
21 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
"""CHV 审核修复轨测试(堆叠于 #230,base=feat-chv-gaps)。
只用标准库与 mock runner/launcher/api_client,绝不启动真实
VM:当前环境没有可跑真 VM 的宿主。真实 VM 启动、seed 盘
在真机上的 cloud-init 应用、SSH 可达实测一律不在本文件
声称已验证;本文件只证明代码层面的接线(VmConfig 含 seed
盘、restore 不回落、reuse 拦截、SSH 目标解析、配额/快照/
API 完整性等)成立。
"""
import json
import os
import socket
import sys
import tempfile
import threading
import unittest
from pathlib import Path
from types import SimpleNamespace
try:
from cloud.chv.api_client import _unix_socket_api
from cloud.chv.disk import check_disk_quota, provision_task_disk
from cloud.chv.hostkeys import build_seed_disk, seed_disk_path
from cloud.chv.pool import WarmPool
from cloud.sandbox import (
ChvFailure,
CloudHypervisorExecutor,
)
from cloud.worker import Worker
except ImportError:
sys.path.insert(0, str(Path(__file__).resolve().parents[2]))
from cloud.chv.api_client import _unix_socket_api
from cloud.chv.disk import check_disk_quota, provision_task_disk
from cloud.chv.hostkeys import build_seed_disk, seed_disk_path
from cloud.chv.pool import WarmPool
from cloud.sandbox import (
ChvFailure,
CloudHypervisorExecutor,
)
from cloud.worker import Worker
class FakeResult:
def __init__(self, returncode=0, stdout="", stderr=""):
self.returncode = returncode
self.stdout = stdout
self.stderr = stderr
class FakeProc:
def __init__(self, alive=True):
self._alive = alive
def poll(self):
return None if self._alive else 1
def terminate(self):
self._alive = False
def wait(self, timeout=None):
return 0
def kill(self):
self._alive = False
def _write(path, content=b"x"):
with open(path, "wb") as fh:
fh.write(content)
os.chmod(path, 0o755)
return path
class ChvFixCase(unittest.TestCase):
def setUp(self):
self.tmp = tempfile.TemporaryDirectory()
self.dir = self.tmp.name
self.binary = _write(os.path.join(self.dir, "cloud-hypervisor"))
self.ch_remote = _write(os.path.join(self.dir, "ch-remote"))
self.kernel = _write(os.path.join(self.dir, "vmlinux"))
self.rootfs = _write(os.path.join(self.dir, "disk.raw"),
b"R" * 100)
self.ssh_key = _write(os.path.join(self.dir, "id_chv"))
self.kvm = _write(os.path.join(self.dir, "kvm"))
self.cpuinfo = os.path.join(self.dir, "cpuinfo")
with open(self.cpuinfo, "w") as fh:
fh.write("flags : fpu vmx sse\n")
self.state_dir = os.path.join(self.dir, "state")
self.calls = []
self.api_calls = []
self.api_payloads = {}
self.audit = []
def tearDown(self):
self.tmp.cleanup()
def _default_runner(self, cmd, timeout=None):
self.calls.append(list(cmd))
if cmd and cmd[0] == "ssh":
if cmd[-1] == "true":
return FakeResult(0, "", "")
if "goal" in cmd:
return FakeResult(0, "goal met after rounds\n", "")
return FakeResult(0, "", "")
if cmd and cmd[0] == "ssh-keygen":
priv = cmd[cmd.index("-f") + 1]
with open(priv, "w") as fh:
fh.write("-----BEGIN OPENSSH PRIVATE KEY-----\nFAKE\n")
os.chmod(priv, 0o600)
with open(priv + ".pub", "w") as fh:
fh.write("ssh-ed25519 AAAAfake1 vm-key\n")
return FakeResult(0, "", "")
if cmd and cmd[0] == "genisoimage":
out = cmd[cmd.index("-output") + 1]
with open(out, "wb") as fh:
fh.write(b"CIDATA" * 16)
return FakeResult(0, "", "")
return FakeResult(0, "", "")
def _default_api(self, socket_path, method, path, payload=None):
self.api_calls.append((method, path))
if path == "/vm.create":
self.api_payloads["vm.create"] = payload
return (200, {})
def make_exec(self, runner=None, env_extra=None, **overrides):
env = {"CHV_EGRESS_PROXY": "http://10.0.0.1:8888",
"CHV_SSH_FORWARD": "1"}
if env_extra:
env.update(env_extra)
kwargs = dict(
binary=self.binary, ch_remote=self.ch_remote,
kernel=self.kernel, rootfs=self.rootfs,
ssh_key=self.ssh_key, kvm_path=self.kvm,
cpuinfo_path=self.cpuinfo, state_dir=self.state_dir,
runner=runner or self._default_runner,
launcher=lambda cmd: FakeProc(alive=True),
api_client=self._default_api,
sleeper=lambda s: None, env=env)
kwargs.update(overrides)
ex = CloudHypervisorExecutor(**kwargs)
ex.audit_callback = lambda e, t, d, k=None: self.audit.append(
(e, t.get("id") if isinstance(t, dict) else t, d, k))
return ex
# -- P0-1 seed 磁盘接入 -------------------------------------------------
class SeedDiskTest(ChvFixCase):
def test_vm_config_carries_seed_disk_readonly(self):
ex = self.make_exec()
cfg = ex.build_vm_config("/tmp/x/disk.raw",
seed_disk="/tmp/x/seed.iso")
self.assertEqual(len(cfg["disks"]), 2)
self.assertEqual(cfg["disks"][1]["path"], "/tmp/x/seed.iso")
self.assertTrue(cfg["disks"][1]["readonly"])
cfg2 = ex.build_vm_config("/tmp/x/disk.raw")
self.assertEqual(len(cfg2["disks"]), 1)
def test_provisioned_wake_mounts_seed_disk(self):
ex = self.make_exec(
env_extra={"CHV_HOST_KEY_MODE": "provisioned"})
out = ex.wake_vm({"id": "seed1"})
self.assertEqual(out["state"], "running")
payload = self.api_payloads["vm.create"]
self.assertEqual(len(payload["disks"]), 2)
seed_disk = payload["disks"][1]["path"]
self.assertTrue(seed_disk.endswith("seed.iso"))
self.assertTrue(payload["disks"][1]["readonly"])
self.assertTrue(os.path.exists(seed_disk))
self.assertIn("genisoimage", [c[0] for c in self.calls])
def test_seed_disk_build_failure_is_fail_fast(self):
def bad_runner(cmd, timeout=None):
self.calls.append(list(cmd))
if cmd and cmd[0] == "genisoimage":
return FakeResult(1, "", "no genisoimage")
return self._default_runner(cmd, timeout)
ex = self.make_exec(
runner=bad_runner,
env_extra={"CHV_HOST_KEY_MODE": "provisioned"})
with self.assertRaises(RuntimeError):
ex.wake_vm({"id": "seedfail"})
# 制盘失败在起 VMM 之前,不应走到 vm.create。
self.assertNotIn("/vm.create",
[p for _, p in self.api_calls])
def test_build_seed_disk_rejects_missing_product(self):
with self.assertRaises(RuntimeError):
build_seed_disk(self.dir, os.path.join(self.dir, "no.iso"),
runner=lambda cmd: FakeResult(0, "", ""))
# -- P0-2 restore 不回落 ------------------------------------------------
class RestoreNoFallbackTest(ChvFixCase):
def test_wait_guest_ready_provisioned_missing_known_hosts(self):
ex = self.make_exec(
env_extra={"CHV_HOST_KEY_MODE": "provisioned"})
with self.assertRaises(ChvFailure) as ctx:
ex.wait_guest_ready({"id": "nopin"})
self.assertEqual(ctx.exception.kind, "host_key_not_pinned")
self.assertFalse(any(c[0] == "ssh-keyscan" for c in self.calls))
def test_restore_provisioned_missing_known_hosts_fails_before_launch(self):
ex = self.make_exec(
env_extra={"CHV_HOST_KEY_MODE": "provisioned"})
launches = []
ex.launcher = lambda cmd: launches.append(cmd) or FakeProc(True)
with self.assertRaises(ChvFailure) as ctx:
ex.restore_vm({"id": "r1"}, "file:///snap/x")
self.assertEqual(ctx.exception.kind, "host_key_not_pinned")
self.assertEqual(launches, [])
# -- P0-3 reuse 拦截 ----------------------------------------------------
class ReuseBlockTest(ChvFixCase):
def test_reuse_blocked_without_experiment_flag(self):
ex = self.make_exec(env_extra={"CHV_DISK_MODE": "reuse"})
with self.assertRaises(ChvFailure) as ctx:
ex.preflight()
self.assertEqual(ctx.exception.kind,
"disk_mode_reuse_blocked")
def test_reuse_passes_with_experiment_flag_and_is_audited(self):
ex = self.make_exec(env_extra={
"CHV_DISK_MODE": "reuse",
"CHV_REUSE_SINGLE_TENANT_EXPERIMENT": "1"})
self.assertTrue(ex.preflight())
out = ex.wake_vm({"id": "reuse1"})
self.assertEqual(out["state"], "running")
self.assertTrue(any(e == "disk_reuse_experiment_allowed"
for e, _, _, _ in self.audit))
# -- P0-4 SSH 目标解析 --------------------------------------------------
class SshTargetTest(ChvFixCase):
def test_explicit_host_wins(self):
ex = self.make_exec(ssh_host="10.9.8.7")
self.assertEqual(ex.resolve_ssh_host(), "10.9.8.7")
def test_net_guest_ip_used_when_no_explicit_host(self):
ex = self.make_exec(env_extra={
"CHV_NET_JSON": '[{"tap": "", "ip": "192.168.100.5",'
'"mask": "255.255.255.0"}]',
"CHV_SSH_FORWARD": "0"})
# env_extra 覆盖 forward=0 后仍应走 net ip,不报未声明。
self.assertEqual(ex.resolve_ssh_host(), "192.168.100.5")
cmd = ex.build_ssh_command({"id": "t"}, ["true"])
self.assertIn("root@192.168.100.5", " ".join(cmd))
def test_undeclared_target_is_fail_fast(self):
ex = self.make_exec(env_extra={"CHV_SSH_FORWARD": "0"})
with self.assertRaises(ChvFailure) as ctx:
ex.resolve_ssh_host()
self.assertEqual(ctx.exception.kind, "ssh_target_undeclared")
def test_forward_attestation_allows_loopback(self):
ex = self.make_exec()
self.assertEqual(ex.resolve_ssh_host(), "127.0.0.1")
# -- P1-1 网关 extra_env ------------------------------------------------
class GatewayExtraEnvTest(ChvFixCase):
def test_chv_ssh_command_carries_extra_env(self):
ex = self.make_exec()
cmd = ex.build_ssh_command({"id": "t"}, ["true"],
extra_env={"SUB2API_API_KEY": "k1"})
joined = " ".join(cmd)
self.assertIn("SUB2API_API_KEY=k1", joined)
self.assertIn("HTTPS_PROXY=http://10.0.0.1:8888", joined)
def test_worker_gateway_guard_allows_chv(self):
from cloud.server import init_db
db_path = os.path.join(self.dir, "control.db")
init_db(db_path)
ex = self.make_exec()
worker = Worker(db_path, "/fake/host.py", executor=ex,
env={"XCM_API_KEY": "test-placeholder"})
task = {"id": "gw1", "project_id": "p1", "prompt": "做目标",
"goal_mode": 1,
"_gateway_provider": "sub2api",
"_gateway_env": {"SUB2API_API_KEY": "gwkey",
"SUB2API_BASE_URL": "http://gw/"}}
result, error = worker._run_attempt(task)
self.assertIsNone(error)
self.assertEqual(result["status"], "completed")
ssh_calls = [c for c in self.calls if c and c[0] == "ssh"]
self.assertTrue(any("SUB2API_API_KEY=gwkey" in " ".join(c)
for c in ssh_calls))
# -- P1-2 取消/清理/看门狗 ----------------------------------------------
class CancelCleanupTest(ChvFixCase):
def test_worker_cancel_calls_cancel_vm(self):
from cloud.server import init_db
db_path = os.path.join(self.dir, "control.db")
init_db(db_path)
ex = self.make_exec()
cancelled = []
ex.cancel_vm = lambda name: cancelled.append(name) or True
worker = Worker(db_path, "/fake/host.py", executor=ex,
env={"XCM_API_KEY": "test-placeholder"})
worker._running_set("t-cancel")
import sqlite3, time
conn = sqlite3.connect(db_path)
conn.execute(
"INSERT INTO tasks(id, project_id, prompt, status,"
" idempotency_key, depends_on, priority, created_by,"
" created_at, updated_at, last_heartbeat)"
" VALUES(?, ?, ?, ?, NULL, ?, ?, 'tester', ?, ?, NULL)",
("t-cancel", "p1", "做事", "running", "[]", 0,
time.time(), time.time()))
conn.commit()
conn.close()
self.assertTrue(worker.cancel("t-cancel"))
self.assertEqual(cancelled, ["t-cancel"])
def test_worker_cleanup_calls_cleanup_stale_vms(self):
from cloud.server import init_db
db_path = os.path.join(self.dir, "control.db")
init_db(db_path)
ex = self.make_exec()
ex.cleanup_stale_vms = lambda: ["chv-stale"]
worker = Worker(db_path, "/fake/host.py", executor=ex,
env={"XCM_API_KEY": "test-placeholder"})
self.assertIn("chv-stale", worker.cleanup_stale_containers())
def test_chv_cancel_watchdog_kills_vm_on_cancel(self):
import threading
from cloud.server import init_db
db_path = os.path.join(self.dir, "control.db")
init_db(db_path)
ex = self.make_exec()
killed = []
ex.cancel_vm = lambda name: killed.append(name) or True
worker = Worker(db_path, "/fake/host.py", executor=ex,
env={"XCM_API_KEY": "test-placeholder"})
worker._cancel_requested.add("t-wd")
stop = threading.Event()
worker._chv_cancel_watchdog("t-wd", "chv-t-wd", stop)
self.assertEqual(killed, ["chv-t-wd"])
# -- P1-3 WarmPool acquire resume ---------------------------------------
class PoolAcquireResumeTest(unittest.TestCase):
def test_acquire_resumes_paused_vm(self):
class Stub:
def __init__(self):
self.resumed = []
self.state_dir = None
self.warm_pool_size = 1
self.snapshot_url = "file:///snap/base"
def vm_name(self, task):
raw = task.get("id") if isinstance(task, dict) \
else str(task)
return raw if raw.startswith("chv-") \
else "chv-" + raw
def wake_vm(self, task):
return {"vm": self.vm_name(task)}
def pause_vm(self, task):
return True
def resume_vm(self, task):
self.resumed.append(task["id"])
return True
def shutdown_vm(self, task_or_name):
return True
def _emit(self, event, task, detail="", kind=None):
pass
stub = Stub()
pool = WarmPool(stub, size=1, snapshot_url="file:///snap/base",
registry_path=None)
pool.prewarm()
got = pool.acquire({"id": "t1"})
self.assertEqual(got["mode"], "warm")
self.assertEqual(stub.resumed, ["warm-1"])
def test_acquire_discards_vm_whose_resume_fails(self):
class Stub:
def __init__(self):
self.state_dir = None
self.warm_pool_size = 1
self.snapshot_url = "file:///snap/base"
def vm_name(self, task):
raw = task.get("id") if isinstance(task, dict) \
else str(task)
return raw if raw.startswith("chv-") \
else "chv-" + raw
def wake_vm(self, task):
return {"vm": self.vm_name(task)}
def pause_vm(self, task):
return True
def resume_vm(self, task):
raise RuntimeError("resume 失败")
def shutdown_vm(self, task_or_name):
return True
def _emit(self, event, task, detail="", kind=None):
pass
pool = WarmPool(Stub(), size=1, snapshot_url="file:///snap/base",
registry_path=None)
pool.prewarm()
self.assertEqual(pool.acquire({"id": "t1"})["mode"], "cold")
# -- P1-4 容量持久化计数 -------------------------------------------------
class CapacityPersistentTest(ChvFixCase):
def test_socket_counts_after_restart(self):
ex = self.make_exec(env_extra={"CHV_MAX_VMS": "1"})
os.makedirs(self.state_dir, exist_ok=True)
with open(os.path.join(self.state_dir, "chv-other.sock"),
"w") as fh:
fh.write("")
with self.assertRaises(ChvFailure) as ctx:
ex.wake_vm({"id": "new1"})
self.assertEqual(ctx.exception.kind, "host_capacity_exceeded")
def test_own_socket_does_not_block_rewake(self):
ex = self.make_exec(env_extra={"CHV_MAX_VMS": "1"})
os.makedirs(self.state_dir, exist_ok=True)
with open(os.path.join(self.state_dir, "chv-t1.sock"),
"w") as fh:
fh.write("")
# 同任务重复 wake:自己的残留 socket 不计入容量。
out = ex.wake_vm({"id": "t1"})
self.assertEqual(out["state"], "running")
# -- P1-5 磁盘配额 -------------------------------------------------------
class DiskQuotaTest(ChvFixCase):
def test_unreadable_image_is_not_silently_skipped(self):
with self.assertRaises(ChvFailure) as ctx:
check_disk_quota("/nonexistent/disk.raw", 1.0)
self.assertEqual(ctx.exception.kind, "disk_quota_unverified")
def test_total_usage_over_quota(self):
os.makedirs(self.state_dir, exist_ok=True)
with open(os.path.join(self.state_dir, "old.raw"),
"wb") as fh:
fh.write(b"X" * 100)
# 配额 150 字节:已用 100 + 本次 100 > 150。
with self.assertRaises(ChvFailure) as ctx:
check_disk_quota(self.rootfs, 150 / (1024 ** 3),
state_dir=self.state_dir)
self.assertEqual(ctx.exception.kind, "disk_quota_exceeded")
def test_existing_disk_size_mismatch_is_recopied(self):
task_disk = provision_task_disk(self.state_dir, "chv-a",
self.rootfs, "copy")
with open(task_disk, "wb") as fh:
fh.write(b"Z" * 10)
task_disk2 = provision_task_disk(self.state_dir, "chv-a",
self.rootfs, "copy")
self.assertEqual(task_disk, task_disk2)
with open(task_disk2, "rb") as fh:
self.assertEqual(fh.read(), b"R" * 100)
# -- P1-6 API 截断 -------------------------------------------------------
class ApiTruncationTest(unittest.TestCase):
def _serve(self, raw):
tmp = tempfile.TemporaryDirectory()
sock_path = os.path.join(tmp.name, "vmm.sock")
srv = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
srv.bind(sock_path)
srv.listen(1)
def _run():
conn, _ = srv.accept()
try:
conn.recv(65536)
conn.sendall(raw)
finally:
conn.close()
srv.close()
threading.Thread(target=_run, daemon=True).start()
return tmp, sock_path
def test_truncated_body_raises(self):
body = b'{"ok": true}'
raw = (b"HTTP/1.1 200 OK\r\nContent-Length: 100\r\n\r\n"
+ body)
tmp, sock_path = self._serve(raw)
with self.assertRaises(RuntimeError):
_unix_socket_api(sock_path, "GET", "/vmm.ping")
tmp.cleanup()
def test_chunked_rejected(self):
raw = (b"HTTP/1.1 200 OK\r\nTransfer-Encoding: chunked\r\n\r\n"
b"5\r\nhello\r\n0\r\n\r\n")
tmp, sock_path = self._serve(raw)
with self.assertRaises(RuntimeError):
_unix_socket_api(sock_path, "GET", "/vmm.ping")
tmp.cleanup()
def test_full_body_parses(self):
body = b'{"ok": true}'
raw = (b"HTTP/1.1 200 OK\r\nContent-Length: %d\r\n\r\n"
% len(body)) + body
tmp, sock_path = self._serve(raw)
status, parsed = _unix_socket_api(sock_path, "GET", "/vmm.ping")
self.assertEqual(status, 200)
self.assertTrue(parsed["ok"])
tmp.cleanup()
# -- P1-7 快照落盘验证 ---------------------------------------------------
class SnapshotPersistTest(ChvFixCase):
def test_snapshot_missing_file_does_not_shutdown(self):
ex = self.make_exec()
with self.assertRaises(ChvFailure) as ctx:
ex.hibernate_vm({"id": "h1"}, "file:///nonexistent/snap")
self.assertEqual(ctx.exception.kind, "snapshot_not_persisted")
kinds = [c[-1] for c in self.calls if c[0] == self.ch_remote]
self.assertIn("pause", kinds)
self.assertNotIn("shutdown", kinds)
# -- P1-8 execute 死端口归类 ---------------------------------------------
class ExecuteDeadPortTest(ChvFixCase):
def test_dead_socket_ssh_failure_is_vm_not_running(self):
def raising_runner(cmd, timeout=None):
self.calls.append(list(cmd))
if cmd and cmd[0] == "ssh":
raise RuntimeError("connection refused")
return FakeResult(0, "", "")
ex = self.make_exec(runner=raising_runner)
os.makedirs(self.state_dir, exist_ok=True)
with open(os.path.join(self.state_dir, "chv-dead.sock"),
"w") as fh:
fh.write("")
ex._vms["chv-dead"] = FakeProc(alive=False)
with self.assertRaises(ChvFailure) as ctx:
ex.execute({"id": "dead"})
self.assertEqual(ctx.exception.kind, "vm_not_running")
if __name__ == "__main__":
unittest.main()