731 lines
28 KiB
Python
731 lines
28 KiB
Python
"""CHV 生产级审核 P1 修复轨（feat-chv-p1fix，堆叠于 #270 之上）测试。
|
||
|
||
逐项对应审核报告 §5 的 P1-01～P1-10（P1-10 是文档项，
|
||
无代码测试；文档数字见 docs/chv-production-readiness.md
|
||
§15 与 docs/chv-architecture.md 头范围）：
|
||
- P1-01：生产 reconcile 失败项不进 cleaned、不清库、
|
||
落盘保留失败项原始条目、可重试；实验池同款语义。
|
||
- P1-02：registry 损坏与非法条目发审计并计数，损坏不
|
||
静默返回空。
|
||
- P1-03：残留私钥删失败发审计、写脏标记，调用方可区分
|
||
成功与失败（failed VM 不进 cleaned）。
|
||
- P1-04：build_ssh_command 删除回环回落，构造失败即抛；
|
||
显式回环（CHV_SSH_HOST / CHV_SSH_FORWARD=1）仍可用。
|
||
- P1-05：state_dir flock（locking.StateLock）——线程内
|
||
互斥、跨进程互斥、wake 全程串行化、purge 快照遍历。
|
||
- P1-06：单条快照超总量帽时拒绝记录，且不删旧快照、
|
||
不落盘新条目。
|
||
- P1-07：CHV_CPU / CHV_MEM / CHV_TASK_TIMEOUT 的 >0
|
||
校验（config 构造层 + preflight 事后改写兜底）。
|
||
- P1-08：快照保留期剪枝删失败发审计；snapshot_vm 失败
|
||
路径发 vm_snapshot_failed 审计。
|
||
- P1-09：4 处三元 else 分支——str 形态 pause/resume/
|
||
snapshot + 非 dict spill task。
|
||
|
||
全部用标准库与 mock runner/launcher/api_client，绝不启动
|
||
真实 VM（本机无 /dev/kvm，真机验证仍是未验证项）。
|
||
"""
|
||
|
||
import json
|
||
import os
|
||
import subprocess
|
||
import sys
|
||
import tempfile
|
||
import threading
|
||
import time
|
||
import unittest
|
||
from pathlib import Path
|
||
from unittest import mock
|
||
|
||
try:
|
||
from cloud.chv import (
|
||
ChvFailure,
|
||
CloudHypervisorExecutor,
|
||
reconcile_orphans_for_executor,
|
||
)
|
||
from cloud.chv.config import build_chv_config
|
||
from cloud.chv.disk import (
|
||
prune_snapshots,
|
||
record_snapshot,
|
||
)
|
||
from cloud.chv.locking import StateLock, lock_path_for_state_dir
|
||
except ImportError: # discover -s cloud/tests 时把项目根目录补进 sys.path
|
||
sys.path.insert(0, str(Path(__file__).resolve().parents[2]))
|
||
from cloud.chv import (
|
||
ChvFailure,
|
||
CloudHypervisorExecutor,
|
||
reconcile_orphans_for_executor,
|
||
)
|
||
from cloud.chv.config import build_chv_config
|
||
from cloud.chv.disk import (
|
||
prune_snapshots,
|
||
record_snapshot,
|
||
)
|
||
from cloud.chv.locking import StateLock, lock_path_for_state_dir
|
||
|
||
|
||
class FakeResult:
|
||
def __init__(self, returncode=0, stdout="", stderr=""):
|
||
self.returncode = returncode
|
||
self.stdout = stdout
|
||
self.stderr = stderr
|
||
|
||
|
||
class FakeProc:
|
||
def __init__(self, alive=True):
|
||
self._alive = alive
|
||
|
||
def poll(self):
|
||
return None if self._alive else 1
|
||
|
||
def terminate(self):
|
||
self._alive = False
|
||
|
||
def wait(self, timeout=None):
|
||
return 0
|
||
|
||
def kill(self):
|
||
self._alive = False
|
||
|
||
|
||
def _write(path, content=b"x"):
|
||
with open(path, "wb") as fh:
|
||
fh.write(content)
|
||
os.chmod(path, 0o755)
|
||
return path
|
||
|
||
|
||
class P1FixCase(unittest.TestCase):
|
||
def setUp(self):
|
||
self.tmp = tempfile.TemporaryDirectory()
|
||
self.dir = self.tmp.name
|
||
self.binary = _write(os.path.join(self.dir, "cloud-hypervisor"))
|
||
self.ch_remote = _write(os.path.join(self.dir, "ch-remote"))
|
||
self.kernel = _write(os.path.join(self.dir, "vmlinux"))
|
||
self.rootfs = _write(os.path.join(self.dir, "disk.raw"),
|
||
b"R" * 100)
|
||
self.ssh_key = _write(os.path.join(self.dir, "id_chv"))
|
||
self.kvm = _write(os.path.join(self.dir, "kvm"))
|
||
self.cpuinfo = os.path.join(self.dir, "cpuinfo")
|
||
with open(self.cpuinfo, "w") as fh:
|
||
fh.write("flags : fpu vmx sse\n")
|
||
self.state_dir = os.path.join(self.dir, "state")
|
||
self.calls = []
|
||
self.audit = []
|
||
|
||
def tearDown(self):
|
||
self.tmp.cleanup()
|
||
|
||
def make_exec(self, runner=None, api_client=None, launcher=None,
|
||
env_extra=None, **overrides):
|
||
env = {"CHV_EGRESS_PROXY": "http://10.0.0.1:8888",
|
||
"CHV_SSH_FORWARD": "1",
|
||
"CHV_HOST_KEY_MODE": "keyscan"}
|
||
if env_extra:
|
||
env.update(env_extra)
|
||
if runner is None:
|
||
runner = self._default_runner
|
||
if api_client is None:
|
||
api_client = self._default_api
|
||
if launcher is None:
|
||
launcher = self._default_launcher
|
||
kwargs = dict(
|
||
binary=self.binary, ch_remote=self.ch_remote,
|
||
kernel=self.kernel, rootfs=self.rootfs,
|
||
ssh_key=self.ssh_key, kvm_path=self.kvm,
|
||
cpuinfo_path=self.cpuinfo, state_dir=self.state_dir,
|
||
runner=runner, launcher=launcher, api_client=api_client,
|
||
sleeper=lambda s: None, env=env)
|
||
kwargs.update(overrides)
|
||
ex = CloudHypervisorExecutor(**kwargs)
|
||
ex.audit_callback = lambda *a: self.audit.append(a)
|
||
return ex
|
||
|
||
def _default_runner(self, cmd, timeout=None):
|
||
self.calls.append(list(cmd))
|
||
return FakeResult(0, "", "")
|
||
|
||
def _default_api(self, socket_path, method, path, payload=None):
|
||
return (200, {})
|
||
|
||
def _default_launcher(self, cmd):
|
||
return FakeProc(alive=True)
|
||
|
||
def _task(self, task_id="abc123"):
|
||
return {"id": task_id, "prompt": "x", "project_id": "proj1"}
|
||
|
||
def _audit_events(self):
|
||
return [a[0] for a in self.audit]
|
||
|
||
|
||
# -- P1-01 / P1-02：生产 reconcile --------------------------------------
|
||
|
||
class FakeProdExecutor:
|
||
def __init__(self, state_dir, fail_vms=()):
|
||
self.state_dir = state_dir
|
||
self.fail_vms = set(fail_vms)
|
||
self.shutdowns = []
|
||
self.emits = []
|
||
|
||
def shutdown_vm(self, vm):
|
||
self.shutdowns.append(vm)
|
||
if vm in self.fail_vms:
|
||
raise RuntimeError("shutdown down")
|
||
|
||
def _emit(self, event, task, detail):
|
||
self.emits.append(event)
|
||
|
||
|
||
def _write_registry(state_dir, data):
|
||
path = os.path.join(state_dir, "warm-pool.json")
|
||
os.makedirs(state_dir, exist_ok=True)
|
||
if isinstance(data, str):
|
||
with open(path, "w") as fh:
|
||
fh.write(data)
|
||
else:
|
||
with open(path, "w") as fh:
|
||
json.dump(data, fh)
|
||
return path
|
||
|
||
|
||
class ReconcileP1FixTest(unittest.TestCase):
|
||
def test_failure_not_cleaned_registry_retained_and_retryable(self):
|
||
with tempfile.TemporaryDirectory() as d:
|
||
path = _write_registry(d, {
|
||
"a": {"state": "warm", "vm": "chv-good"},
|
||
"b": {"state": "warm", "vm": "chv-bad"},
|
||
})
|
||
ex = FakeProdExecutor(d, fail_vms={"chv-bad"})
|
||
self.assertEqual(reconcile_orphans_for_executor(ex),
|
||
["chv-good"])
|
||
self.assertIn("warm_pool_reconcile_failed", ex.emits)
|
||
with open(path) as fh:
|
||
remaining = json.load(fh)
|
||
self.assertEqual(
|
||
remaining,
|
||
{"b": {"state": "warm", "vm": "chv-bad"}})
|
||
# 修复后重试：失败项可被清理，库最终为空。
|
||
ex.fail_vms = set()
|
||
self.assertEqual(reconcile_orphans_for_executor(ex),
|
||
["chv-bad"])
|
||
with open(path) as fh:
|
||
self.assertEqual(json.load(fh), {})
|
||
|
||
def test_corrupt_registry_audited_not_silent(self):
|
||
with tempfile.TemporaryDirectory() as d:
|
||
path = _write_registry(d, "{not json")
|
||
ex = FakeProdExecutor(d)
|
||
self.assertEqual(reconcile_orphans_for_executor(ex), [])
|
||
self.assertIn("warm_pool_registry_corrupt", ex.emits)
|
||
self.assertTrue(os.path.exists(path + ".corrupt"))
|
||
|
||
def test_invalid_entries_audited_and_counted(self):
|
||
with tempfile.TemporaryDirectory() as d:
|
||
path = _write_registry(d, {
|
||
"a": {"state": "warm", "vm": "chv-a"},
|
||
"c": {"state": "warm"},
|
||
"e": {"state": "running", "vm": "chv-e"},
|
||
})
|
||
ex = FakeProdExecutor(d)
|
||
self.assertEqual(reconcile_orphans_for_executor(ex),
|
||
["chv-a"])
|
||
self.assertIn("warm_pool_registry_invalid_entries",
|
||
ex.emits)
|
||
with open(path) as fh:
|
||
self.assertEqual(json.load(fh), {})
|
||
|
||
|
||
# -- P1-03：残留私钥删失败 ----------------------------------------------
|
||
|
||
class StaleSecretsP1FixTest(P1FixCase):
|
||
def test_delete_failure_not_cleaned_audited_and_dirty_marked(self):
|
||
from cloud.chv.hostkeys import dirty_marker_path
|
||
ex = self.make_exec()
|
||
vm_dir = os.path.join(self.state_dir, "chv-bad")
|
||
os.makedirs(vm_dir, exist_ok=True)
|
||
priv = os.path.join(vm_dir, "ssh_host_ed25519_key")
|
||
with open(priv, "w") as fh:
|
||
fh.write("PRIVATE")
|
||
real_remove = os.remove
|
||
|
||
def remove(path):
|
||
if path == priv:
|
||
raise OSError("删不掉")
|
||
return real_remove(path)
|
||
|
||
with mock.patch.object(os, "remove", side_effect=remove):
|
||
cleaned = ex.cleanup_stale_secrets()
|
||
self.assertNotIn("chv-bad", cleaned)
|
||
self.assertIn("host_secret_residue_delete_failed",
|
||
self._audit_events())
|
||
self.assertTrue(os.path.exists(
|
||
dirty_marker_path(self.state_dir, "chv-bad")))
|
||
self.assertTrue(os.path.exists(priv))
|
||
|
||
def test_delete_success_cleaned(self):
|
||
ex = self.make_exec()
|
||
vm_dir = os.path.join(self.state_dir, "chv-ok")
|
||
os.makedirs(vm_dir, exist_ok=True)
|
||
with open(os.path.join(vm_dir, "ssh_host_ed25519_key"),
|
||
"w") as fh:
|
||
fh.write("PRIVATE")
|
||
cleaned = ex.cleanup_stale_secrets()
|
||
self.assertIn("chv-ok", cleaned)
|
||
self.assertIn("host_secret_residue_cleaned",
|
||
self._audit_events())
|
||
|
||
|
||
# -- P1-04：SSH 回落删除 -------------------------------------------------
|
||
|
||
class SshFallbackP1FixTest(P1FixCase):
|
||
def test_undeclared_build_raises_not_loopback(self):
|
||
ex = self.make_exec(env_extra={"CHV_SSH_FORWARD": "0"})
|
||
with self.assertRaises(ChvFailure) as ctx:
|
||
ex.build_ssh_command(self._task(), ["true"])
|
||
self.assertEqual(ctx.exception.kind, "ssh_target_undeclared")
|
||
|
||
def test_explicit_loopback_still_allowed(self):
|
||
ex = self.make_exec(env_extra={"CHV_SSH_HOST": "127.0.0.1"})
|
||
cmd = ex.build_ssh_command(self._task(), ["true"])
|
||
self.assertIn("root@127.0.0.1", " ".join(cmd))
|
||
ex2 = self.make_exec(env_extra={"CHV_SSH_FORWARD": "1"})
|
||
cmd2 = ex2.build_ssh_command(self._task(), ["true"])
|
||
self.assertIn("root@127.0.0.1", " ".join(cmd2))
|
||
|
||
|
||
# -- P1-05：flock ---------------------------------------------------------
|
||
|
||
class StateLockP1FixTest(unittest.TestCase):
|
||
def test_thread_mutual_exclusion(self):
|
||
with tempfile.TemporaryDirectory() as d:
|
||
state_dir = os.path.join(d, "state")
|
||
lock_a = StateLock(state_dir)
|
||
lock_b = StateLock(state_dir)
|
||
acquired = []
|
||
lock_a.acquire()
|
||
try:
|
||
def _take():
|
||
# RLock 须由加锁线程释放，故获取与释放同线程。
|
||
lock_b.acquire()
|
||
acquired.append(True)
|
||
lock_b.release()
|
||
|
||
t = threading.Thread(target=_take)
|
||
t.start()
|
||
time.sleep(0.3)
|
||
self.assertEqual(acquired, [])
|
||
self.assertTrue(t.is_alive())
|
||
finally:
|
||
lock_a.release()
|
||
t.join(timeout=5)
|
||
self.assertEqual(acquired, [True])
|
||
self.assertTrue(os.path.exists(
|
||
lock_path_for_state_dir(state_dir)))
|
||
|
||
def test_cross_process_mutual_exclusion(self):
|
||
with tempfile.TemporaryDirectory() as d:
|
||
state_dir = os.path.join(d, "state")
|
||
os.makedirs(state_dir, exist_ok=True)
|
||
root = str(Path(__file__).resolve().parents[2])
|
||
ready = os.path.join(state_dir, "child-locked")
|
||
script = (
|
||
"import sys, time; sys.path.insert(0, %r); "
|
||
"from cloud.chv.locking import StateLock; "
|
||
"lock = StateLock(%r); lock.acquire(); "
|
||
"open(%r, 'w').write('locked'); "
|
||
"time.sleep(10)" % (root, state_dir, ready))
|
||
proc = subprocess.Popen(
|
||
[sys.executable, "-c", script],
|
||
stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
|
||
try:
|
||
deadline = time.time() + 10
|
||
while not os.path.exists(ready) and time.time() < deadline:
|
||
time.sleep(0.05)
|
||
self.assertTrue(os.path.exists(ready))
|
||
self.assertIsNone(proc.poll())
|
||
lock = StateLock(state_dir)
|
||
acquired = []
|
||
|
||
def _take():
|
||
lock.acquire()
|
||
acquired.append(True)
|
||
lock.release()
|
||
|
||
t = threading.Thread(target=_take)
|
||
t.start()
|
||
time.sleep(0.5)
|
||
# 子进程持锁期间，本进程的加锁必须阻塞。
|
||
self.assertEqual(acquired, [])
|
||
proc.terminate()
|
||
proc.wait(timeout=5)
|
||
t.join(timeout=5)
|
||
self.assertEqual(acquired, [True])
|
||
finally:
|
||
if proc.poll() is None:
|
||
proc.kill()
|
||
proc.wait(timeout=5)
|
||
|
||
def test_purge_snapshot_traversal_under_concurrent_mutation(self):
|
||
# 旧无锁 pop + 持锁遍历并发可致字典迭代异常；新实现
|
||
# 快照键列表后逐项持锁删，并发变更加锁路径下不得抛错。
|
||
with tempfile.TemporaryDirectory():
|
||
case = P1FixCase()
|
||
case.setUp()
|
||
try:
|
||
ex = case.make_exec()
|
||
ex.state_dir = case.state_dir
|
||
stop = threading.Event()
|
||
|
||
def mutate():
|
||
i = 0
|
||
while not stop.is_set():
|
||
name = "chv-m%d" % (i % 5)
|
||
with ex._vms_lock:
|
||
ex._vms[name] = FakeProc(alive=False)
|
||
i += 1
|
||
|
||
t = threading.Thread(target=mutate)
|
||
t.start()
|
||
try:
|
||
for _ in range(50):
|
||
ex._purge_dead_registry()
|
||
finally:
|
||
stop.set()
|
||
t.join(timeout=5)
|
||
finally:
|
||
case.tearDown()
|
||
|
||
|
||
class WakeSerializationP1FixTest(P1FixCase):
|
||
def test_concurrent_wakes_serialized_by_state_lock(self):
|
||
# 两路并发 wake：每路 api 往返睡 0.2 秒；串行则总耗时
|
||
# 至少 0.35 秒，并行则约 0.2 秒。阈值留足余量防抖动。
|
||
def sleeping_api(socket_path, method, path, payload=None):
|
||
if path == "/vmm.ping":
|
||
time.sleep(0.2)
|
||
return (200, {})
|
||
|
||
ex1 = self.make_exec(api_client=sleeping_api)
|
||
ex2 = self.make_exec(api_client=sleeping_api)
|
||
results = []
|
||
start = time.time()
|
||
|
||
def wake(ex, task_id):
|
||
try:
|
||
results.append(ex.wake_vm({"id": task_id})["state"])
|
||
except Exception as exc: # noqa: BLE001 - 失败即测试失败
|
||
results.append("error:%s" % exc)
|
||
|
||
t1 = threading.Thread(target=wake, args=(ex1, "s1"))
|
||
t2 = threading.Thread(target=wake, args=(ex2, "s2"))
|
||
t1.start()
|
||
t2.start()
|
||
t1.join(timeout=30)
|
||
t2.join(timeout=30)
|
||
elapsed = time.time() - start
|
||
self.assertEqual(sorted(results), ["running", "running"])
|
||
self.assertGreaterEqual(elapsed, 0.35)
|
||
self.assertTrue(os.path.exists(
|
||
lock_path_for_state_dir(self.state_dir)))
|
||
|
||
|
||
# -- P1-06：单条快照超帽 ---------------------------------------------------
|
||
|
||
class SingleSnapshotCapP1FixTest(unittest.TestCase):
|
||
def test_single_oversize_snapshot_rejected_without_side_effects(self):
|
||
with tempfile.TemporaryDirectory() as tmp:
|
||
old = os.path.join(tmp, "vm1", "old.snap")
|
||
os.makedirs(os.path.join(tmp, "vm1"), exist_ok=True)
|
||
with open(old, "wb") as fh:
|
||
fh.write(b"O" * 100)
|
||
from cloud.chv.disk import _save_snapshots
|
||
_save_snapshots(tmp, "vm1", [{
|
||
"url": "file://" + old, "path": old, "size": 100,
|
||
"created_at": 0}])
|
||
new = os.path.join(tmp, "vm1", "new.snap")
|
||
with open(new, "wb") as fh:
|
||
fh.write(b"N" * 1000)
|
||
# 总量帽 500 字节：新单条 1000 字节本身已超帽。
|
||
with self.assertRaises(ChvFailure) as ctx:
|
||
record_snapshot(tmp, "vm1", "file://" + new,
|
||
500 / (1024 ** 3))
|
||
self.assertEqual(ctx.exception.kind,
|
||
"snapshot_quota_exceeded")
|
||
# 拒绝无副作用：旧快照文件仍在、登记仍只有旧条目。
|
||
self.assertTrue(os.path.exists(old))
|
||
from cloud.chv.disk import _load_snapshots
|
||
entries = _load_snapshots(tmp, "vm1")
|
||
self.assertEqual(len(entries), 1)
|
||
self.assertEqual(entries[0]["path"], old)
|
||
|
||
|
||
# -- P1-07：>0 校验 ---------------------------------------------------------
|
||
|
||
class PositiveConfigP1FixTest(P1FixCase):
|
||
def test_config_layer_rejects_non_positive(self):
|
||
base_env = {}
|
||
for env_extra, field in (
|
||
({"CHV_CPU": "0"}, "cpu"),
|
||
({"CHV_CPU": "-2"}, "cpu"),
|
||
({"CHV_MEM": "0"}, "mem"),
|
||
({"CHV_MEM": "0G"}, "mem"),
|
||
({"CHV_TASK_TIMEOUT": "0"}, "timeout"),
|
||
({"CHV_TASK_TIMEOUT": "-5"}, "timeout")):
|
||
env = dict(base_env)
|
||
env.update(env_extra)
|
||
with self.assertRaises(ValueError, msg=field):
|
||
build_chv_config({}, env)
|
||
# 合法值与 None（不限）不受影响。
|
||
cfg = build_chv_config({}, {"CHV_TASK_TIMEOUT": "30"})
|
||
self.assertEqual(cfg.task_timeout, 30.0)
|
||
cfg = build_chv_config({}, {})
|
||
self.assertIsNone(cfg.task_timeout)
|
||
|
||
def test_preflight_rejects_tampered_attributes(self):
|
||
for attr, value in (("cpus", "0"), ("cpus", "-1"),
|
||
("memory", "0"), ("task_timeout", 0),
|
||
("task_timeout", -3)):
|
||
ex = self.make_exec()
|
||
setattr(ex, attr, value)
|
||
with self.assertRaises(ChvFailure) as ctx:
|
||
ex.preflight()
|
||
self.assertEqual(ctx.exception.kind, "config_invalid")
|
||
|
||
|
||
# -- P1-08：剪枝审计与快照失败审计 -----------------------------------------
|
||
|
||
class SnapshotAuditP1FixTest(P1FixCase):
|
||
def test_retention_prune_delete_failure_audited(self):
|
||
ex = self.make_exec()
|
||
vm_dir = os.path.join(self.state_dir, "chv-abc123")
|
||
os.makedirs(vm_dir, exist_ok=True)
|
||
old = os.path.join(self.state_dir, "old.snap")
|
||
with open(old, "wb") as fh:
|
||
fh.write(b"O" * 10)
|
||
with open(os.path.join(vm_dir, "snapshots.json"), "w") as fh:
|
||
json.dump([{"url": "file://" + old, "path": old,
|
||
"size": 10, "created_at": 0}], fh)
|
||
new = os.path.join(self.state_dir, "new.snap")
|
||
with open(new, "wb") as fh:
|
||
fh.write(b"N" * 10)
|
||
real_remove = os.remove
|
||
|
||
def remove(path):
|
||
if path == old:
|
||
raise OSError("删不掉")
|
||
return real_remove(path)
|
||
|
||
with mock.patch.object(os, "remove", side_effect=remove):
|
||
out = ex.snapshot_vm(self._task(), "file://" + new)
|
||
self.assertEqual(out, "file://" + new)
|
||
self.assertIn("vm_snapshot_prune_failed",
|
||
self._audit_events())
|
||
self.assertTrue(os.path.exists(old))
|
||
|
||
def test_snapshot_failure_audited(self):
|
||
def failing_runner(cmd, timeout=None):
|
||
if cmd and cmd[0] == "ch-remote":
|
||
raise RuntimeError("ch-remote 挂了")
|
||
return FakeResult(0, "", "")
|
||
|
||
ex = self.make_exec(runner=failing_runner)
|
||
# 目标须在 state_dir 内才过白名单（ch-remote 失败在后）。
|
||
target = "file://" + os.path.join(self.state_dir, "x.snap")
|
||
with self.assertRaises(RuntimeError):
|
||
ex.snapshot_vm(self._task(), target)
|
||
self.assertIn("vm_snapshot_failed", self._audit_events())
|
||
|
||
|
||
# -- P1-09：4 处三元 else 分支 ----------------------------------------------
|
||
|
||
class BranchCoverP1FixTest(P1FixCase):
|
||
def test_pause_resume_snapshot_str_forms(self):
|
||
ex = self.make_exec()
|
||
os.makedirs(self.state_dir, exist_ok=True)
|
||
self.assertTrue(ex.pause_vm("chv-abc"))
|
||
self.assertTrue(ex.resume_vm("chv-abc"))
|
||
snap = os.path.join(self.state_dir, "snap.raw")
|
||
with open(snap, "wb") as fh:
|
||
fh.write(b"S" * 10)
|
||
out = ex.snapshot_vm("chv-abc", "file://" + snap)
|
||
self.assertEqual(out, "file://" + snap)
|
||
self.assertIn("vm_pause", self._audit_events())
|
||
self.assertIn("vm_resume", self._audit_events())
|
||
self.assertIn("vm_snapshot", self._audit_events())
|
||
|
||
def test_spill_audit_non_dict_task(self):
|
||
ex = self.make_exec()
|
||
os.makedirs(self.state_dir, exist_ok=True)
|
||
ex._spill_audit("ev", "plain-task-id", "detail", None)
|
||
path = os.path.join(self.state_dir, "chv-audit-spill.jsonl")
|
||
with open(path) as fh:
|
||
row = json.loads(fh.readline())
|
||
self.assertEqual(row["task_id"], "plain-task-id")
|
||
self.assertEqual(row["event"], "ev")
|
||
|
||
|
||
# -- 覆盖补齐：新分支的行覆盖（诚实口径，不凑数改代码） --------
|
||
|
||
class CoverageCompletionP1FixTest(P1FixCase):
|
||
def test_prune_events_no_file_and_no_delete(self):
|
||
from cloud.chv.disk import _save_snapshots
|
||
with tempfile.TemporaryDirectory() as tmp:
|
||
_save_snapshots(tmp, "vm1", [
|
||
{"url": "file:///gone.snap", "path": "/gone.snap",
|
||
"size": 5, "created_at": 0},
|
||
{"url": "https://example.invalid/x", "path": None,
|
||
"size": 5, "created_at": 0}])
|
||
events = []
|
||
kept, pruned = prune_snapshots(
|
||
tmp, "vm1", retention_days=30, now=10 ** 9,
|
||
events=events)
|
||
self.assertEqual(kept, [])
|
||
self.assertEqual(len(pruned), 2)
|
||
kinds = [e[0] for e in events]
|
||
self.assertEqual(kinds, ["pruned_no_file", "pruned_no_file"])
|
||
events2 = []
|
||
_save_snapshots(tmp, "vm1", [
|
||
{"url": "file:///gone2.snap", "path": "/gone2.snap",
|
||
"size": 5, "created_at": 0}])
|
||
kept2, _ = prune_snapshots(
|
||
tmp, "vm1", retention_days=30, now=10 ** 9,
|
||
delete_files=False, events=events2)
|
||
self.assertEqual(kept2, [])
|
||
self.assertEqual([e[0] for e in events2],
|
||
["pruned_no_file"])
|
||
|
||
def test_record_without_cap_and_tighten_delete_failure_event(self):
|
||
with tempfile.TemporaryDirectory() as tmp:
|
||
entries, removed = record_snapshot(
|
||
tmp, "vm1", "https://example.invalid/new", None)
|
||
self.assertEqual(len(entries), 1)
|
||
self.assertEqual(removed, [])
|
||
old = os.path.join(tmp, "vm1", "old.snap")
|
||
os.makedirs(os.path.join(tmp, "vm1"), exist_ok=True)
|
||
with open(old, "wb") as fh:
|
||
fh.write(b"O" * 100)
|
||
from cloud.chv.disk import _save_snapshots
|
||
_save_snapshots(tmp, "vm1", [
|
||
{"url": "file://" + old, "path": old, "size": 100,
|
||
"created_at": 0}])
|
||
real_remove = os.remove
|
||
|
||
def remove(path):
|
||
if path == old:
|
||
raise OSError("删不掉")
|
||
return real_remove(path)
|
||
|
||
events = []
|
||
with mock.patch.object(os, "remove", side_effect=remove):
|
||
entries2, removed2 = record_snapshot(
|
||
tmp, "vm1", "https://example.invalid/new2",
|
||
0.0, events=events)
|
||
self.assertEqual(len(removed2), 1)
|
||
self.assertEqual([e[0] for e in events], ["delete_failed"])
|
||
self.assertTrue(os.path.exists(old))
|
||
|
||
def test_tighten_no_file_entry_event(self):
|
||
from cloud.chv.disk import _save_snapshots
|
||
with tempfile.TemporaryDirectory() as tmp:
|
||
_save_snapshots(tmp, "vm1", [
|
||
{"url": "https://example.invalid/old", "path": None,
|
||
"size": 100, "created_at": 0}])
|
||
events = []
|
||
entries, removed = record_snapshot(
|
||
tmp, "vm1", "https://example.invalid/new",
|
||
0.0, events=events)
|
||
self.assertEqual(len(removed), 1)
|
||
self.assertEqual([e[0] for e in events],
|
||
["pruned_no_file"])
|
||
|
||
def test_state_lock_ctx_refreshes_on_state_dir_change(self):
|
||
ex = self.make_exec()
|
||
old_lock = ex._state_lock
|
||
new_dir = os.path.join(self.dir, "state2")
|
||
ex.state_dir = new_dir
|
||
self.assertIsNot(ex._state_lock_ctx(), old_lock)
|
||
self.assertEqual(ex._state_lock.state_dir, new_dir)
|
||
|
||
def test_gc_and_secrets_listdir_failure_returns_empty(self):
|
||
ex = self.make_exec()
|
||
ex.state_dir = os.path.join(self.dir, "missing-dir")
|
||
self.assertEqual(ex.gc_stale_disks(),
|
||
{"deleted": [], "failed": []})
|
||
self.assertEqual(ex.cleanup_stale_secrets(), [])
|
||
|
||
def test_cleanup_listdir_failure_returns_empty(self):
|
||
ex = self.make_exec()
|
||
os.makedirs(self.state_dir, exist_ok=True)
|
||
real_listdir = os.listdir
|
||
calls = {"n": 0}
|
||
|
||
def listdir(path):
|
||
calls["n"] += 1
|
||
if calls["n"] >= 3:
|
||
raise OSError("listdir 挂了")
|
||
return real_listdir(path)
|
||
|
||
with mock.patch.object(os, "listdir", side_effect=listdir):
|
||
self.assertEqual(ex.cleanup_stale_vms(), [])
|
||
|
||
def test_orphan_entries_compat_wrapper_and_no_emit_executor(self):
|
||
from cloud.chv.orphans import _orphan_entries
|
||
data = {"a": {"state": "warm", "vm": "chv-a",
|
||
"task_id": "a", "snapshot_url": None},
|
||
"b": {"state": "running", "vm": "chv-b"}}
|
||
self.assertEqual(_orphan_entries(data),
|
||
[{"vm": "chv-a", "task_id": "a",
|
||
"snapshot_url": None}])
|
||
|
||
class NoEmit:
|
||
def __init__(self, state_dir):
|
||
self.state_dir = state_dir
|
||
self.shutdowns = []
|
||
|
||
def shutdown_vm(self, vm):
|
||
self.shutdowns.append(vm)
|
||
|
||
with tempfile.TemporaryDirectory() as d:
|
||
_write_registry(d, {
|
||
"a": {"state": "warm", "vm": "chv-a"}})
|
||
ex = NoEmit(d)
|
||
self.assertEqual(reconcile_orphans_for_executor(ex),
|
||
["chv-a"])
|
||
|
||
def test_state_lock_flock_failure_closes_and_releases(self):
|
||
import fcntl
|
||
with tempfile.TemporaryDirectory() as d:
|
||
state_dir = os.path.join(d, "state")
|
||
lock = StateLock(state_dir)
|
||
with mock.patch.object(fcntl, "flock",
|
||
side_effect=OSError("flock 挂了")):
|
||
with self.assertRaises(OSError):
|
||
lock.acquire()
|
||
self.assertIsNone(lock._fd)
|
||
# 失败后锁仍可用（RLock 已释放、fd 已关闭）。
|
||
lock.acquire()
|
||
lock.release()
|
||
|
||
def test_pool_reconcile_failed_emit_failure_swallowed(self):
|
||
from cloud.chv.experimental_pool import WarmPool
|
||
stub = _PoolStubForP1Fix()
|
||
pool = WarmPool(stub, registry_path=None)
|
||
pool._entries["old-1"] = {"state": "orphan", "vm": "vm-old",
|
||
"task": {"id": "old-1"}}
|
||
self.assertEqual(pool.reconcile_orphans(), [])
|
||
self.assertIn("old-1", pool._entries)
|
||
|
||
|
||
class _PoolStubForP1Fix:
|
||
"""实验池失败审计吞错测试用假执行器。"""
|
||
|
||
def shutdown_vm(self, task_or_name):
|
||
raise RuntimeError("shutdown 失败")
|
||
|
||
def _emit(self, event, task, detail="", kind=None):
|
||
raise RuntimeError("audit 失败")
|
||
|
||
|
||
if __name__ == "__main__":
|
||
unittest.main()
|