host
1836c5af32
D1：snapshot_vm 在配额预检后、ch-remote 前建好 file:// 目标 目录（新 helper _ensure_snapshot_dir；fresh URL 直接 snapshot 曾 rc=1 Destination is not a directory）。建目录只在 snapshot_vm 一处，hibernate 经此调用；配额拒绝仍先于任何文件改动。 D2：keep_disk=True 时 stop 保留 seed.iso（_remove_vm_seed_files 抽出；快照 config.json 的 disks 同时引用 disk.raw 与 seed.iso）， 且 cleanup_stale_secrets 不清有登记快照的 VM 的 seed（与 gc_stale_disks 的盘保护同款逻辑）；正常 stop 仍删两者。 F1：首条任务 SSH 撞 kex 阶段瞬时 reset（rc=255 + kex/reset 标记） 时重试恰好 1 次（_call_ssh_with_transient_retry，_run_guest 与 _run_guest_goal 共用；host key 拒绝/认证失败/远端失败不重试； 审计 vm_ssh_transient_retry）。§5“执行器内部不重试”的唯一例外， 结论写入 docs/chv-production-readiness.md §3/§5/§7。 测试：test_chv_v3_fixes 新增 12 项（D1×3、D2×4、F1×5）；旧代码 上 12 项中 9 项失败（余 3 项为守卫既有行为）。全量 test_chv* 470 测试通过。
767 lines
32 KiB
Python
767 lines
32 KiB
Python
"""CHV 第三棒（feat-chv-v3）回归测试：WSL 真机 V-1～V-4/F1/F2
|
||
实测缺陷的代码层回归。
|
||
|
||
- V-3① snapshot 传裸 URL（source_url= 是 restore 的键）；
|
||
- V-3② 快照产物是目录（v54），校验改为目录非空；
|
||
- V-3③ restore 起 VMM 前清旧 VMM + 旧 socket；
|
||
- V-3 附带：hibernate 留盘（快照 config.json 引用原盘），
|
||
GC 不删有登记快照的 VM 的盘，restore 前预检快照引用的盘；
|
||
- F1：re-wake 删旧盘（H5 删私钥致新 key 对 + 旧盘 cloud-init
|
||
不重应用 = host_key_mismatch）；
|
||
- F2：跨实例 stop 按 socket 进程扫描清未登记 VMM；
|
||
- V-4：remote_command source 失败 fail-fast（exit 126），
|
||
guest-env 归 SSH 用户所有；
|
||
- V-1 附带：build.sh 预建 /app/work（1777 粘滞位 + 断言）；
|
||
- V-1 附带：host.py default_codex_bin() 走 PATH 查找。
|
||
- V-7 复修（feat-chv-v3-repair）：D1 snapshot 先建目录（旧
|
||
fixture 预建目录 + mock ch-remote 不校验，458 测试全过也
|
||
盖不住缺失目录的真实路径）、D2 hibernate 留 seed.iso +
|
||
残留清理保护有登记快照的 VM 的 seed、F1 任务 SSH 连接建立
|
||
阶段瞬时 reset 重试恰好 1 次。
|
||
|
||
全部用假 runner/launcher/api_client + 真实 /proc 扫描（F2
|
||
的被杀进程是真实 spawn 的 sleep 进程，argv[0] 伪装成
|
||
cloud-hypervisor），绝不启动真实 VM。
|
||
"""
|
||
|
||
import importlib.util
|
||
import json
|
||
import os
|
||
import subprocess
|
||
import sys
|
||
import tempfile
|
||
import unittest
|
||
from pathlib import Path
|
||
from unittest import mock
|
||
|
||
try:
|
||
from cloud.chv.executor import CloudHypervisorExecutor
|
||
from cloud.chv.hostkeys import write_seed
|
||
from cloud.chv.ssh_transport import remote_command
|
||
from cloud.sandbox import ChvFailure
|
||
except ImportError:
|
||
sys.path.insert(0, str(Path(__file__).resolve().parents[2]))
|
||
from cloud.chv.executor import CloudHypervisorExecutor
|
||
from cloud.chv.hostkeys import write_seed
|
||
from cloud.chv.ssh_transport import remote_command
|
||
from cloud.sandbox import ChvFailure
|
||
|
||
ROOT = Path(__file__).resolve().parents[2]
|
||
|
||
|
||
class FakeResult:
|
||
def __init__(self, returncode=0, stdout="", stderr=""):
|
||
self.returncode = returncode
|
||
self.stdout = stdout
|
||
self.stderr = stderr
|
||
|
||
|
||
class FakeProc:
|
||
def __init__(self, alive=True):
|
||
self._alive = alive
|
||
self.terminated = False
|
||
self.killed = False
|
||
|
||
def poll(self):
|
||
return None if self._alive else 1
|
||
|
||
def terminate(self):
|
||
self.terminated = True
|
||
self._alive = False
|
||
|
||
def wait(self, timeout=None):
|
||
return 0
|
||
|
||
def kill(self):
|
||
self.killed = True
|
||
self._alive = False
|
||
|
||
|
||
def _write(path, content=b"x"):
|
||
with open(path, "wb") as fh:
|
||
fh.write(content)
|
||
os.chmod(path, 0o755)
|
||
return path
|
||
|
||
|
||
def _load_host_py():
|
||
spec = importlib.util.spec_from_file_location(
|
||
"chv_host_py", str(ROOT / "host.py"))
|
||
mod = importlib.util.module_from_spec(spec)
|
||
spec.loader.exec_module(mod)
|
||
return mod
|
||
|
||
|
||
class V3Case(unittest.TestCase):
|
||
def setUp(self):
|
||
self.tmp = tempfile.TemporaryDirectory()
|
||
self.dir = self.tmp.name
|
||
self.binary = _write(os.path.join(self.dir, "cloud-hypervisor"))
|
||
self.ch_remote = _write(os.path.join(self.dir, "ch-remote"))
|
||
self.kernel = _write(os.path.join(self.dir, "vmlinux"))
|
||
self.rootfs = _write(os.path.join(self.dir, "disk.raw"),
|
||
b"R" * 100)
|
||
self.ssh_key = _write(os.path.join(self.dir, "id_chv"))
|
||
self.kvm = _write(os.path.join(self.dir, "kvm"))
|
||
self.cpuinfo = os.path.join(self.dir, "cpuinfo")
|
||
with open(self.cpuinfo, "w") as fh:
|
||
fh.write("flags : fpu vmx sse\n")
|
||
self.state_dir = os.path.join(self.dir, "state")
|
||
self.calls = []
|
||
self.api_calls = []
|
||
self.launches = []
|
||
self.audit = []
|
||
self.proc = None
|
||
self.spawned = []
|
||
|
||
def tearDown(self):
|
||
for proc in self.spawned:
|
||
try:
|
||
if proc.poll() is None:
|
||
proc.terminate()
|
||
proc.wait(timeout=5)
|
||
except Exception:
|
||
pass
|
||
self.tmp.cleanup()
|
||
|
||
def make_exec(self, runner=None, api_client=None, launcher=None,
|
||
env_extra=None, **overrides):
|
||
env = {"CHV_EGRESS_PROXY": "http://10.0.0.1:8888",
|
||
"CHV_HOST_KEY_MODE": "keyscan",
|
||
"CHV_SSH_FORWARD": "1"}
|
||
if env_extra:
|
||
env.update(env_extra)
|
||
kwargs = dict(
|
||
binary=self.binary, ch_remote=self.ch_remote,
|
||
kernel=self.kernel, rootfs=self.rootfs,
|
||
ssh_key=self.ssh_key, kvm_path=self.kvm,
|
||
cpuinfo_path=self.cpuinfo, state_dir=self.state_dir,
|
||
runner=runner or self._default_runner,
|
||
launcher=launcher or self._default_launcher,
|
||
api_client=api_client or self._default_api,
|
||
sleeper=lambda s: None, env=env)
|
||
kwargs.update(overrides)
|
||
ex = CloudHypervisorExecutor(**kwargs)
|
||
ex.audit_callback = lambda e, t, d, k=None: self.audit.append(
|
||
(e, t.get("id") if isinstance(t, dict) else t, d, k))
|
||
return ex
|
||
|
||
def _default_runner(self, cmd, timeout=None):
|
||
self.calls.append(list(cmd))
|
||
if cmd and cmd[0] == "ssh":
|
||
if cmd[-1] == "true":
|
||
return FakeResult(0, "", "")
|
||
return FakeResult(0, "", "")
|
||
if cmd and cmd[0] == "ssh-keygen":
|
||
priv = cmd[cmd.index("-f") + 1]
|
||
with open(priv, "w") as fh:
|
||
fh.write("-----BEGIN OPENSSH PRIVATE KEY-----\nFAKE\n")
|
||
os.chmod(priv, 0o600)
|
||
with open(priv + ".pub", "w") as fh:
|
||
fh.write("ssh-ed25519 AAAAfake1 vm-key\n")
|
||
return FakeResult(0, "", "")
|
||
if cmd and cmd[0] == "genisoimage":
|
||
out = cmd[cmd.index("-output") + 1]
|
||
with open(out, "wb") as fh:
|
||
fh.write(b"CIDATA" * 16)
|
||
return FakeResult(0, "", "")
|
||
return FakeResult(0, "", "")
|
||
|
||
def _default_api(self, socket_path, method, path, payload=None):
|
||
self.api_calls.append((method, path, payload))
|
||
return (200, {})
|
||
|
||
def _default_launcher(self, cmd):
|
||
self.launches.append(list(cmd))
|
||
self.proc = FakeProc(alive=True)
|
||
return self.proc
|
||
|
||
def _task(self, task_id="abc123"):
|
||
return {"id": task_id, "prompt": "做一个加法函数",
|
||
"project_id": "proj1"}
|
||
|
||
def _snap_dir(self, name="snap1", files=None):
|
||
"""建一个 v54 形态的快照目录，返回 (url, path)。"""
|
||
path = os.path.join(self.state_dir, name)
|
||
os.makedirs(path, exist_ok=True)
|
||
for fname, content in (files or {
|
||
"config.json": b"{}",
|
||
"state.json": b"{}",
|
||
"memory-ranges": b"m"}).items():
|
||
with open(os.path.join(path, fname), "wb") as fh:
|
||
fh.write(content)
|
||
return "file://" + path, path
|
||
|
||
def _spawn_fake_vmm(self, sock_path, argv0="cloud-hypervisor"):
|
||
"""真实 spawn 一个长眠进程，argv[0] 伪装成 VMM，
|
||
命令行带 --api-socket <sock>（F2/V-3③ 的扫描目标）。"""
|
||
proc = subprocess.Popen(
|
||
[argv0, "-c", "import time; time.sleep(30)",
|
||
"--api-socket", sock_path],
|
||
executable=sys.executable,
|
||
stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
|
||
self.spawned.append(proc)
|
||
return proc
|
||
|
||
|
||
# -- V-3①/② 快照：裸 URL + 目录校验 ---------------------------------------
|
||
class SnapshotV3Test(V3Case):
|
||
def test_snapshot_passes_bare_url(self):
|
||
url, _ = self._snap_dir()
|
||
ex = self.make_exec()
|
||
out = ex.snapshot_vm(self._task(), url)
|
||
self.assertEqual(out, url)
|
||
snap_calls = [c for c in self.calls
|
||
if c[0] == self.ch_remote and "snapshot" in c]
|
||
self.assertEqual(len(snap_calls), 1)
|
||
self.assertEqual(snap_calls[0][-1], url)
|
||
self.assertFalse(any(a.startswith("source_url")
|
||
for a in snap_calls[0]))
|
||
|
||
def test_restore_still_passes_source_url(self):
|
||
url, _ = self._snap_dir()
|
||
ex = self.make_exec()
|
||
out = ex.restore_vm(self._task(), url)
|
||
self.assertEqual(out["mode"], "restored")
|
||
restore_calls = [c for c in self.calls
|
||
if c[0] == self.ch_remote and "restore" in c]
|
||
self.assertEqual(len(restore_calls), 1)
|
||
self.assertEqual(restore_calls[0][-1], "source_url=%s" % url)
|
||
|
||
def test_snapshot_dir_records_dir_total_size(self):
|
||
url, path = self._snap_dir(files={
|
||
"config.json": b"C" * 30, "state.json": b"S" * 30})
|
||
ex = self.make_exec()
|
||
ex.snapshot_vm(self._task(), url)
|
||
with open(os.path.join(self.state_dir, "chv-abc123",
|
||
"snapshots.json")) as fh:
|
||
entries = json.load(fh)
|
||
self.assertEqual(len(entries), 1)
|
||
self.assertEqual(entries[0]["size"], 60)
|
||
self.assertEqual(entries[0]["path"], path)
|
||
|
||
|
||
# -- V-3③ restore 准备：清旧 VMM + 旧 socket ------------------------------
|
||
class RestorePrepareV3Test(V3Case):
|
||
def test_restore_kills_registered_old_vmm_and_socket(self):
|
||
url, _ = self._snap_dir()
|
||
ex = self.make_exec()
|
||
ex.wake_vm(self._task())
|
||
old_proc = self.proc
|
||
self.assertTrue(old_proc.poll() is None)
|
||
sock = os.path.join(self.state_dir, "chv-abc123.sock")
|
||
with open(sock, "w") as fh:
|
||
fh.write("")
|
||
out = ex.restore_vm(self._task(), url)
|
||
self.assertEqual(out["mode"], "restored")
|
||
self.assertTrue(old_proc.terminated)
|
||
# 新 VMM 已起（第 2 次 launch），旧 socket 文件在
|
||
# launch 前已被清掉（fake launcher 不重建 socket 文件）。
|
||
self.assertEqual(len(self.launches), 2)
|
||
self.assertFalse(os.path.exists(sock))
|
||
|
||
def test_restore_kills_unregistered_vmm_by_socket(self):
|
||
url, _ = self._snap_dir()
|
||
ex = self.make_exec()
|
||
sock = os.path.join(self.state_dir, "chv-abc123.sock")
|
||
os.makedirs(self.state_dir, exist_ok=True)
|
||
with open(sock, "w") as fh:
|
||
fh.write("")
|
||
victim = self._spawn_fake_vmm(sock)
|
||
self.assertIsNone(victim.poll())
|
||
out = ex.restore_vm(self._task(), url)
|
||
self.assertEqual(out["mode"], "restored")
|
||
self.assertIsNotNone(victim.wait(timeout=5))
|
||
|
||
|
||
# -- V-3 附带：hibernate 留盘 + GC 保护 + restore 盘预检 -------------------
|
||
class HibernateDiskV3Test(V3Case):
|
||
def test_hibernate_keeps_task_disk(self):
|
||
ex = self.make_exec()
|
||
ex.wake_vm(self._task())
|
||
disk = os.path.join(self.state_dir, "chv-abc123", "disk.raw")
|
||
self.assertTrue(os.path.exists(disk))
|
||
url, _ = self._snap_dir()
|
||
out = ex.hibernate_vm(self._task(), url)
|
||
self.assertEqual(out["state"], "hibernated")
|
||
self.assertTrue(os.path.exists(disk),
|
||
"hibernate 的 stop 不得删盘：快照 config.json "
|
||
"引用该盘，删了 restore 即失败")
|
||
|
||
def test_gc_spares_disk_while_snapshot_recorded(self):
|
||
ex = self.make_exec()
|
||
ex.wake_vm(self._task())
|
||
url, _ = self._snap_dir()
|
||
ex.hibernate_vm(self._task(), url)
|
||
disk = os.path.join(self.state_dir, "chv-abc123", "disk.raw")
|
||
result = ex.gc_stale_disks()
|
||
self.assertEqual(result["deleted"], [])
|
||
self.assertTrue(os.path.exists(disk))
|
||
# 快照登记清掉后，盘成孤儿，GC 才可删。
|
||
os.remove(os.path.join(self.state_dir, "chv-abc123",
|
||
"snapshots.json"))
|
||
result = ex.gc_stale_disks()
|
||
self.assertEqual(result["deleted"], ["chv-abc123"])
|
||
self.assertFalse(os.path.exists(disk))
|
||
|
||
def test_restore_missing_snapshot_disk_fail_fast(self):
|
||
ex = self.make_exec()
|
||
missing_disk = os.path.join(self.state_dir, "gone", "disk.raw")
|
||
url, _ = self._snap_dir(files={
|
||
"config.json": json.dumps(
|
||
{"disks": [{"path": missing_disk}]}).encode(),
|
||
"state.json": b"{}"})
|
||
with self.assertRaises(ChvFailure) as ctx:
|
||
ex.restore_vm(self._task(), url)
|
||
self.assertEqual(ctx.exception.kind, "snapshot_disk_missing")
|
||
self.assertEqual(self.launches, [])
|
||
|
||
def test_restore_snapshot_disk_present_passes_precheck(self):
|
||
ex = self.make_exec()
|
||
ex.wake_vm(self._task())
|
||
disk = os.path.join(self.state_dir, "chv-abc123", "disk.raw")
|
||
url, _ = self._snap_dir(files={
|
||
"config.json": json.dumps(
|
||
{"disks": [{"path": disk}]}).encode(),
|
||
"state.json": b"{}"})
|
||
out = ex.restore_vm(self._task(), url)
|
||
self.assertEqual(out["mode"], "restored")
|
||
|
||
|
||
# -- F1：re-wake 删旧盘 -----------------------------------------------------
|
||
class RewakeDiskV3Test(V3Case):
|
||
def _corrupt_disk(self, task_id="abc123"):
|
||
disk = os.path.join(self.state_dir, "chv-" + task_id,
|
||
"disk.raw")
|
||
with open(disk, "wb") as fh:
|
||
fh.write(b"STALE" * 20)
|
||
return disk
|
||
|
||
def test_rewake_live_vm_replaces_stale_disk(self):
|
||
ex = self.make_exec()
|
||
ex.wake_vm(self._task())
|
||
disk = self._corrupt_disk()
|
||
out = ex.wake_vm(self._task())
|
||
self.assertEqual(out["state"], "running")
|
||
with open(disk, "rb") as fh:
|
||
self.assertEqual(fh.read(), b"R" * 100,
|
||
"re-wake 须重拷新盘：旧盘复用 + 新 key 对 "
|
||
"= cloud-init 不重应用 = host_key_mismatch")
|
||
|
||
def test_rewake_after_crash_replaces_stale_disk(self):
|
||
ex = self.make_exec()
|
||
ex.wake_vm(self._task())
|
||
disk = self._corrupt_disk()
|
||
ex2 = self.make_exec()
|
||
out = ex2.wake_vm(self._task())
|
||
self.assertEqual(out["state"], "running")
|
||
with open(disk, "rb") as fh:
|
||
self.assertEqual(fh.read(), b"R" * 100)
|
||
|
||
def test_provisioned_rewake_regenerates_keypair_on_fresh_disk(self):
|
||
ex = self.make_exec(
|
||
env_extra={"CHV_HOST_KEY_MODE": "provisioned"})
|
||
out = ex.wake_vm(self._task())
|
||
self.assertEqual(out["state"], "running")
|
||
vm_dir = os.path.join(self.state_dir, "chv-abc123")
|
||
# H5：首 wake 后宿主私钥已删，re-wake 必然重新生成 key 对。
|
||
self.assertFalse(os.path.exists(
|
||
os.path.join(vm_dir, "ssh_host_ed25519_key")))
|
||
self.assertTrue(os.path.exists(
|
||
os.path.join(vm_dir, "known_hosts")))
|
||
disk = self._corrupt_disk()
|
||
geniso_before = sum(1 for c in self.calls
|
||
if c and c[0] == "genisoimage")
|
||
out2 = ex.wake_vm(self._task())
|
||
self.assertEqual(out2["state"], "running")
|
||
with open(disk, "rb") as fh:
|
||
self.assertEqual(fh.read(), b"R" * 100)
|
||
geniso_after = sum(1 for c in self.calls
|
||
if c and c[0] == "genisoimage")
|
||
self.assertEqual(geniso_after, geniso_before + 1,
|
||
"re-wake 须重建 seed 盘（新 key 对进新 seed）")
|
||
|
||
|
||
# -- F2：跨实例 stop 按 socket 进程扫描 -------------------------------------
|
||
class CrossInstanceStopV3Test(V3Case):
|
||
def test_stop_kills_unregistered_vmm_by_socket(self):
|
||
ex = self.make_exec()
|
||
sock = os.path.join(self.state_dir, "chv-abc123.sock")
|
||
os.makedirs(self.state_dir, exist_ok=True)
|
||
with open(sock, "w") as fh:
|
||
fh.write("")
|
||
victim = self._spawn_fake_vmm(sock)
|
||
self.assertIsNone(victim.poll())
|
||
self.assertTrue(ex.stop_vm(self._task()))
|
||
self.assertIsNotNone(victim.wait(timeout=5))
|
||
self.assertFalse(os.path.exists(sock))
|
||
|
||
def test_stop_does_not_kill_others_vmm_or_chremote(self):
|
||
ex = self.make_exec()
|
||
sock = os.path.join(self.state_dir, "chv-abc123.sock")
|
||
other_sock = os.path.join(self.state_dir, "chv-other.sock")
|
||
os.makedirs(self.state_dir, exist_ok=True)
|
||
for path in (sock, other_sock):
|
||
with open(path, "w") as fh:
|
||
fh.write("")
|
||
other_vmm = self._spawn_fake_vmm(other_sock)
|
||
# 同带本 socket 的 ch-remote 客户端：真实形态是
|
||
# `ch-remote --api-socket=<sock> shutdown`，不是 VMM，
|
||
# 绝不杀。
|
||
chremote = subprocess.Popen(
|
||
["ch-remote", "-c", "import time; time.sleep(30)",
|
||
"--api-socket=" + sock],
|
||
executable=sys.executable,
|
||
stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
|
||
self.spawned.append(chremote)
|
||
self.assertIsNone(other_vmm.poll())
|
||
self.assertIsNone(chremote.poll())
|
||
ex.stop_vm(self._task())
|
||
self.assertIsNone(other_vmm.poll())
|
||
self.assertIsNone(chremote.poll())
|
||
|
||
|
||
# -- V-4：guest-env source 失败 fail-fast + 归 SSH 用户所有 ------------------
|
||
class GuestEnvV4Test(V3Case):
|
||
def test_remote_command_source_failure_exits(self):
|
||
tokens = remote_command(["python3", "/app/host.py", "status"],
|
||
guest_env_file="/etc/chv/guest-env")
|
||
joined = " ".join(tokens)
|
||
self.assertIn(". /etc/chv/guest-env || exit 126;", joined)
|
||
self.assertNotIn("/etc/chv/guest-env';", joined)
|
||
|
||
def test_remote_command_without_guest_env_unchanged(self):
|
||
tokens = remote_command(["true"])
|
||
self.assertEqual(tokens, ["true"])
|
||
|
||
def test_write_seed_owner_follows_ssh_user(self):
|
||
import yaml
|
||
for ssh_user, want_owner in (
|
||
(None, "root:root"), ("root", "root:root"),
|
||
("chv", "chv:chv")):
|
||
seed_dir = os.path.join(self.dir, "seed-%s" % want_owner)
|
||
write_seed(seed_dir, "vm1", "PRIV",
|
||
"ssh-ed25519 AAAAx",
|
||
secret_env={"SUB2API_API_KEY": "s3cr3t"},
|
||
ssh_user=ssh_user)
|
||
with open(os.path.join(seed_dir, "user-data")) as fh:
|
||
parsed = yaml.safe_load(fh.read())
|
||
guest = [e for e in parsed["write_files"]
|
||
if e["path"] == "/etc/chv/guest-env"]
|
||
self.assertEqual(len(guest), 1)
|
||
self.assertEqual(guest[0]["owner"], want_owner)
|
||
self.assertEqual(guest[0]["permissions"], "0600")
|
||
|
||
def test_provisioned_wake_writes_guest_env_for_chv_user(self):
|
||
import yaml
|
||
ex = self.make_exec(
|
||
env_extra={"CHV_HOST_KEY_MODE": "provisioned"},
|
||
ssh_user="chv")
|
||
out = ex.wake_vm(self._task(),
|
||
extra_env={"SUB2API_API_KEY": "s3cr3t"})
|
||
self.assertEqual(out["state"], "running")
|
||
user_data = os.path.join(
|
||
self.state_dir, "chv-abc123", "seed", "user-data")
|
||
with open(user_data) as fh:
|
||
parsed = yaml.safe_load(fh.read())
|
||
guest = [e for e in parsed["write_files"]
|
||
if e["path"] == "/etc/chv/guest-env"]
|
||
self.assertEqual(len(guest), 1)
|
||
self.assertEqual(guest[0]["owner"], "chv:chv")
|
||
|
||
|
||
# -- V-1 附带：build.sh /app/work + host.py default_codex_bin --------------
|
||
class GuestBuildAndHostBinV3Test(unittest.TestCase):
|
||
def test_build_sh_precreates_app_work_writable(self):
|
||
text = (ROOT / "chv-guest" / "build.sh").read_text()
|
||
self.assertIn("mkdir -p /app/work", text)
|
||
self.assertIn("chmod 1777 /app/work", text)
|
||
self.assertIn('stat -c %a /app/work', text)
|
||
|
||
def test_build_sh_dry_run_prints_app_work_step(self):
|
||
tmp = tempfile.TemporaryDirectory()
|
||
self.addCleanup(tmp.cleanup)
|
||
bindir = os.path.join(tmp.name, "bin")
|
||
os.makedirs(os.path.join(bindir, "codex-resources"))
|
||
for name in ("codex", "codex-code-mode-host"):
|
||
p = os.path.join(bindir, name)
|
||
with open(p, "w") as fh:
|
||
fh.write("#!/bin/sh\necho \"codex-cli 0.164.0-alpha.9\"\n")
|
||
os.chmod(p, 0o755)
|
||
repo = os.path.join(tmp.name, "repo")
|
||
os.makedirs(repo)
|
||
with open(os.path.join(repo, "host.py"), "w") as fh:
|
||
fh.write("# host\n")
|
||
subprocess.run(["git", "init", "-q", repo],
|
||
check=True, capture_output=True)
|
||
subprocess.run(["git", "-C", repo, "config", "user.email", "t@t"],
|
||
check=True, capture_output=True)
|
||
subprocess.run(["git", "-C", repo, "config", "user.name", "t"],
|
||
check=True, capture_output=True)
|
||
subprocess.run(["git", "-C", repo, "add", "-A"],
|
||
check=True, capture_output=True)
|
||
subprocess.run(["git", "-C", repo, "commit", "-qm", "init"],
|
||
check=True, capture_output=True)
|
||
commit = subprocess.run(
|
||
["git", "-C", repo, "rev-parse", "--short", "HEAD"],
|
||
check=True, capture_output=True, text=True).stdout.strip()
|
||
env = dict(os.environ)
|
||
env.update({"CODEX_BIN_DIR": bindir,
|
||
"UBUNTU_VERSION": "24.04",
|
||
"CODEX_VERSION": "0.164.0-alpha.9",
|
||
"REPO_COMMIT": commit,
|
||
"UBUNTU_IMAGE_SHA256": "0" * 64,
|
||
"REPO_DIR": repo})
|
||
r = subprocess.run(
|
||
["bash", str(ROOT / "chv-guest" / "build.sh"),
|
||
"--dry-run", os.path.join(tmp.name, "out")],
|
||
check=False, capture_output=True, text=True, env=env)
|
||
self.assertEqual(r.returncode, 0, r.stderr)
|
||
self.assertIn("/app/work", r.stdout)
|
||
self.assertIn("1777", r.stdout)
|
||
|
||
def test_default_codex_bin_uses_path_lookup(self):
|
||
mod = _load_host_py()
|
||
tmp = tempfile.TemporaryDirectory()
|
||
self.addCleanup(tmp.cleanup)
|
||
fake = os.path.join(tmp.name, "codex")
|
||
with open(fake, "w") as fh:
|
||
fh.write("#!/bin/sh\n")
|
||
os.chmod(fake, 0o755)
|
||
env = dict(os.environ)
|
||
env.pop("CODEX_BIN", None)
|
||
env["PATH"] = tmp.name + os.pathsep + env.get("PATH", "")
|
||
with mock.patch.dict(os.environ, env, clear=True):
|
||
self.assertEqual(mod.default_codex_bin(), fake)
|
||
|
||
def test_default_codex_bin_falls_back_to_bare_name(self):
|
||
mod = _load_host_py()
|
||
tmp = tempfile.TemporaryDirectory()
|
||
self.addCleanup(tmp.cleanup)
|
||
env = {"PATH": tmp.name}
|
||
with mock.patch.dict(os.environ, env, clear=True):
|
||
self.assertEqual(mod.default_codex_bin(), "codex")
|
||
|
||
def test_default_codex_bin_env_still_wins(self):
|
||
mod = _load_host_py()
|
||
with mock.patch.dict(os.environ,
|
||
{"CODEX_BIN": "/opt/custom/codex"}):
|
||
self.assertEqual(mod.default_codex_bin(),
|
||
"/opt/custom/codex")
|
||
|
||
|
||
# -- V-7/D1：snapshot 先建目标目录 ---------------------------------------
|
||
class SnapshotMkdirV7Test(V3Case):
|
||
def _snapshot_creating_runner(self):
|
||
"""模拟真实 ch-remote：往目标目录写快照文件。目录不
|
||
存在时 open 即 FileNotFoundError——若执行器没先
|
||
mkdir，测试直接失败，这正是旧 fixture 盖不住的点
|
||
（旧 fixture 预建目录，mock 又不写文件）。"""
|
||
def runner(cmd, timeout=None):
|
||
if cmd and cmd[0] == self.ch_remote and "snapshot" in cmd:
|
||
self.calls.append(list(cmd))
|
||
target = cmd[-1][len("file://"):]
|
||
for fname, content in (
|
||
("config.json", b"{}"),
|
||
("state.json", b"{}"),
|
||
("memory-ranges", b"m")):
|
||
with open(os.path.join(target, fname), "wb") as fh:
|
||
fh.write(content)
|
||
return FakeResult(0, "", "")
|
||
return self._default_runner(cmd, timeout)
|
||
return runner
|
||
|
||
def test_snapshot_creates_missing_snapshot_dir(self):
|
||
ex = self.make_exec(runner=self._snapshot_creating_runner())
|
||
url = "file://" + os.path.join(self.state_dir, "fresh-snap")
|
||
self.assertFalse(os.path.exists(url[len("file://"):]))
|
||
out = ex.snapshot_vm(self._task(), url)
|
||
self.assertEqual(out, url)
|
||
snap_path = url[len("file://"):]
|
||
self.assertTrue(os.path.isdir(snap_path))
|
||
self.assertIn("config.json", os.listdir(snap_path))
|
||
with open(os.path.join(self.state_dir, "chv-abc123",
|
||
"snapshots.json")) as fh:
|
||
entries = json.load(fh)
|
||
self.assertEqual(entries[0]["path"], snap_path)
|
||
|
||
def test_hibernate_creates_missing_snapshot_dir(self):
|
||
ex = self.make_exec(runner=self._snapshot_creating_runner())
|
||
ex.wake_vm(self._task())
|
||
url = "file://" + os.path.join(self.state_dir, "fresh-snap")
|
||
out = ex.hibernate_vm(self._task(), url)
|
||
self.assertEqual(out["state"], "hibernated")
|
||
self.assertTrue(os.path.isdir(url[len("file://"):]))
|
||
|
||
def test_snapshot_target_existing_file_rejected(self):
|
||
ex = self.make_exec()
|
||
target = os.path.join(self.state_dir, "not-a-dir")
|
||
os.makedirs(self.state_dir, exist_ok=True)
|
||
with open(target, "wb") as fh:
|
||
fh.write(b"x")
|
||
with self.assertRaises(ChvFailure) as ctx:
|
||
ex.snapshot_vm(self._task(), "file://" + target)
|
||
self.assertEqual(ctx.exception.kind, "snapshot_not_persisted")
|
||
self.assertEqual(self.calls, [],
|
||
"目标不是目录时 ch-remote 根本不应被调")
|
||
|
||
|
||
# -- V-7/D2：hibernate 留 seed.iso ----------------------------------------
|
||
class HibernateSeedIsoV7Test(V3Case):
|
||
def _prov_exec(self):
|
||
# keyscan 模式不制 seed.iso；provisioned 模式 wake 即
|
||
# 建 seed.iso（真机休眠的形态），用它走全链路。
|
||
return self.make_exec(
|
||
env_extra={"CHV_HOST_KEY_MODE": "provisioned"})
|
||
|
||
def _seed_iso(self):
|
||
return os.path.join(self.state_dir, "chv-abc123", "seed.iso")
|
||
|
||
def _disk(self):
|
||
return os.path.join(self.state_dir, "chv-abc123", "disk.raw")
|
||
|
||
def _snap_with_disks(self):
|
||
"""快照 config.json 同时引用 disk.raw 与 seed.iso
|
||
（真机快照的 disks 形态）。"""
|
||
return self._snap_dir(files={
|
||
"config.json": json.dumps(
|
||
{"disks": [{"path": self._disk()},
|
||
{"path": self._seed_iso()}]}).encode(),
|
||
"state.json": b"{}"})
|
||
|
||
def test_hibernate_keeps_seed_iso(self):
|
||
ex = self._prov_exec()
|
||
ex.wake_vm(self._task())
|
||
self.assertTrue(os.path.exists(self._seed_iso()))
|
||
url, _ = self._snap_dir()
|
||
out = ex.hibernate_vm(self._task(), url)
|
||
self.assertEqual(out["state"], "hibernated")
|
||
self.assertTrue(os.path.exists(self._disk()),
|
||
"hibernate 仍须留 disk.raw")
|
||
self.assertTrue(os.path.exists(self._seed_iso()),
|
||
"hibernate 须留 seed.iso：快照 config.json "
|
||
"的 disks 引用它，删了 restore 即失败")
|
||
# 只留 iso：seed 目录（制盘输入）与宿主私钥仍删，
|
||
# config.json 不引用它们，私钥明文不留宿主。
|
||
self.assertFalse(os.path.exists(
|
||
os.path.join(self.state_dir, "chv-abc123", "seed")))
|
||
self.assertFalse(os.path.exists(
|
||
os.path.join(self.state_dir, "chv-abc123",
|
||
"ssh_host_ed25519_key")))
|
||
|
||
def test_stop_without_keep_disk_removes_seed_iso(self):
|
||
ex = self._prov_exec()
|
||
ex.wake_vm(self._task())
|
||
self.assertTrue(ex.stop_vm(self._task()))
|
||
self.assertFalse(os.path.exists(self._seed_iso()),
|
||
"正常 stop 仍须删 seed.iso：它含宿主私钥"
|
||
"明文，不得随 VM 残留")
|
||
self.assertFalse(os.path.exists(self._disk()))
|
||
|
||
def test_cleanup_stale_secrets_spares_seed_iso_while_snapshot_recorded(self):
|
||
ex = self._prov_exec()
|
||
ex.wake_vm(self._task())
|
||
url, _ = self._snap_dir()
|
||
ex.hibernate_vm(self._task(), url)
|
||
cleaned = ex.cleanup_stale_secrets()
|
||
self.assertNotIn("chv-abc123", cleaned)
|
||
self.assertTrue(os.path.exists(self._seed_iso()),
|
||
"有登记快照的 VM 的 seed.iso 不是孤儿，"
|
||
"残留清理不得清掉（与 gc_stale_disks 的"
|
||
"盘保护同款逻辑）")
|
||
# 快照登记清掉后，seed.iso 成孤儿，才可清。
|
||
os.remove(os.path.join(self.state_dir, "chv-abc123",
|
||
"snapshots.json"))
|
||
cleaned = ex.cleanup_stale_secrets()
|
||
self.assertIn("chv-abc123", cleaned)
|
||
self.assertFalse(os.path.exists(self._seed_iso()))
|
||
|
||
def test_restore_after_hibernate_with_seed_iso_passes_precheck(self):
|
||
ex = self._prov_exec()
|
||
ex.wake_vm(self._task())
|
||
url, _ = self._snap_with_disks()
|
||
ex.hibernate_vm(self._task(), url)
|
||
out = ex.restore_vm(self._task(), url)
|
||
self.assertEqual(out["mode"], "restored")
|
||
|
||
|
||
# -- V-7/F1：任务 SSH 连接建立阶段瞬时 reset 重试 1 次 --------------------
|
||
class SshTransientRetryV7Test(V3Case):
|
||
def _dispatch_runner(self, first, second=None):
|
||
"""首条 dispatch SSH 返回 first，之后返回 second
|
||
（默认成功）。只记 dispatch SSH 的调用次数。"""
|
||
state = {"n": 0}
|
||
|
||
def runner(cmd, timeout=None):
|
||
if cmd and cmd[0] == "ssh" and "dispatch" in cmd:
|
||
state["n"] += 1
|
||
if state["n"] == 1:
|
||
return first
|
||
return second or FakeResult(
|
||
0, json.dumps({"status": "completed",
|
||
"text": "ok"}), "")
|
||
return self._default_runner(cmd, timeout)
|
||
runner.state = state
|
||
return runner
|
||
|
||
def test_dispatch_retries_once_on_kex_reset_then_succeeds(self):
|
||
runner = self._dispatch_runner(FakeResult(
|
||
255, "",
|
||
"kex_exchange_identification: Connection reset by peer"))
|
||
ex = self.make_exec(runner=runner)
|
||
out = ex.dispatch_task(self._task())
|
||
self.assertEqual(out["status"], "completed")
|
||
self.assertEqual(runner.state["n"], 2,
|
||
"kex 阶段瞬时 reset 只重试恰好 1 次")
|
||
self.assertTrue(any(e == "vm_ssh_transient_retry"
|
||
for e, _t, _d, _k in self.audit))
|
||
|
||
def test_dispatch_does_not_retry_on_host_key_mismatch(self):
|
||
runner = self._dispatch_runner(FakeResult(
|
||
255, "", "Host key verification failed."))
|
||
ex = self.make_exec(runner=runner)
|
||
out = ex.dispatch_task(self._task())
|
||
self.assertEqual(out["status"], "failed")
|
||
self.assertEqual(runner.state["n"], 1,
|
||
"host key 拒绝（同为 255）绝不重试")
|
||
self.assertFalse(any(e == "vm_ssh_transient_retry"
|
||
for e, _t, _d, _k in self.audit))
|
||
|
||
def test_dispatch_does_not_retry_on_remote_failure(self):
|
||
runner = self._dispatch_runner(FakeResult(
|
||
1, json.dumps({"status": "failed", "text": "boom"}), ""))
|
||
ex = self.make_exec(runner=runner)
|
||
out = ex.dispatch_task(self._task())
|
||
self.assertEqual(out["status"], "failed")
|
||
self.assertEqual(runner.state["n"], 1, "远端命令失败不重试")
|
||
|
||
def test_dispatch_retries_at_most_once(self):
|
||
runner = self._dispatch_runner(
|
||
FakeResult(255, "", "Connection reset by peer"),
|
||
second=FakeResult(255, "", "Connection reset by peer"))
|
||
ex = self.make_exec(runner=runner)
|
||
out = ex.dispatch_task(self._task())
|
||
self.assertEqual(out["status"], "failed")
|
||
self.assertEqual(runner.state["n"], 2, "第二次失败即停，不再追")
|
||
|
||
def test_transient_retry_helper_retries_once(self):
|
||
"""helper 本身（_run_guest 与 _run_guest_goal 共用）：
|
||
首条 255 + reset 标记即重试 1 次，第二次成功即返回。"""
|
||
calls = []
|
||
|
||
def runner(cmd, timeout=None):
|
||
calls.append(list(cmd))
|
||
if len(calls) == 1:
|
||
return FakeResult(255, "", "Connection reset by peer")
|
||
return FakeResult(0, "ok", "")
|
||
|
||
ex = self.make_exec(runner=runner)
|
||
result = ex._call_ssh_with_transient_retry(
|
||
self._task(), ["ssh", "true"])
|
||
self.assertEqual(result.returncode, 0)
|
||
self.assertEqual(len(calls), 2)
|
||
|
||
|
||
if __name__ == "__main__":
|
||
unittest.main()
|