fix(chv): 生产级审核 P1-01～P1-10 修复（堆叠于 #270 之上）
This commit is contained in:
muse-agent committed
1 parent
8c8d62f70d
commit
971e054938
12 files changed
+1545
-120
No files matched your search
@@ -156,6 +156,8 @@ class BreakingCleanupTest(unittest.TestCase):
|
||||
|
||||
class OrphansTest(unittest.TestCase):
|
||||
def test_valid_warm_and_assigned_cleaned_invalid_dropped(self):
|
||||
# feat-chv-p1fix（P1-02）：非法条目不再静默丢弃，
|
||||
# 先发 invalid 审计并计数，再发 cleaned 审计。
|
||||
with tempfile.TemporaryDirectory() as d:
|
||||
path = _write_registry(d, {
|
||||
"a": {"state": "warm", "vm": "chv-a",
|
||||
@@ -171,19 +173,26 @@ class OrphansTest(unittest.TestCase):
|
||||
reconcile_orphans_for_executor(ex),
|
||||
["chv-a", "chv-b"])
|
||||
self.assertEqual(ex.shutdowns, ["chv-a", "chv-b"])
|
||||
self.assertEqual(ex.emits, ["warm_pool_reconcile"])
|
||||
self.assertEqual(ex.emits, [
|
||||
"warm_pool_registry_invalid_entries",
|
||||
"warm_pool_reconcile"])
|
||||
with open(path) as fh:
|
||||
self.assertEqual(json.load(fh), {})
|
||||
|
||||
def test_invalid_only_returns_empty_and_leaves_file(self):
|
||||
def test_invalid_only_returns_empty_and_drops_audited_entries(self):
|
||||
# feat-chv-p1fix（P1-02）：仅非法条目时返回 [] 且
|
||||
# 不 shutdown；非法条目已审计计数，落盘时丢弃，
|
||||
# 不再每次重启重复审计同一批非法条目。
|
||||
with tempfile.TemporaryDirectory() as d:
|
||||
path = _write_registry(d, {
|
||||
"c": {"state": "warm", "task_id": "c"}})
|
||||
ex = FakeProdExecutor(d)
|
||||
self.assertEqual(reconcile_orphans_for_executor(ex), [])
|
||||
self.assertEqual(ex.shutdowns, [])
|
||||
self.assertEqual(
|
||||
ex.emits, ["warm_pool_registry_invalid_entries"])
|
||||
with open(path) as fh:
|
||||
self.assertIn("task_id", fh.read())
|
||||
self.assertEqual(json.load(fh), {})
|
||||
|
||||
def test_corrupt_json_quarantined(self):
|
||||
with tempfile.TemporaryDirectory() as d:
|
||||
@@ -212,9 +221,12 @@ class OrphansTest(unittest.TestCase):
|
||||
self.assertEqual(
|
||||
reconcile_orphans_for_executor(NoDir()), [])
|
||||
|
||||
def test_shutdown_and_emit_failures_do_not_break_cleanup(self):
|
||||
def test_shutdown_failure_not_cleaned_and_retained(self):
|
||||
# feat-chv-p1fix（P1-01）：shutdown 失败的条目不进
|
||||
# cleaned、不触发清库，落盘保留原始条目可重试；
|
||||
# _emit 失败仍 best-effort，不炸清理流程。
|
||||
with tempfile.TemporaryDirectory() as d:
|
||||
_write_registry(d, {
|
||||
path = _write_registry(d, {
|
||||
"a": {"state": "warm", "vm": "chv-a"}})
|
||||
ex = FakeProdExecutor(d)
|
||||
|
||||
@@ -225,7 +237,11 @@ class OrphansTest(unittest.TestCase):
|
||||
ex._emit = lambda *a, **k: (_ for _ in ()).throw(
|
||||
RuntimeError("audit down"))
|
||||
self.assertEqual(
|
||||
reconcile_orphans_for_executor(ex), ["chv-a"])
|
||||
reconcile_orphans_for_executor(ex), [])
|
||||
with open(path) as fh:
|
||||
self.assertEqual(
|
||||
json.load(fh),
|
||||
{"a": {"state": "warm", "vm": "chv-a"}})
|
||||
|
||||
def test_experimental_pool_registry_roundtrip_unchanged(self):
|
||||
# 实验池自身的 registry 语义（载入即 orphan、reconcile
|
||||
|
||||
@@ -889,25 +889,31 @@ class DefensiveBranchTest(CloseupCase):
|
||||
ex.runner = keyscan_runner
|
||||
self.assertFalse(ex.capture_host_key(self._task()))
|
||||
|
||||
def test_build_ssh_command_fallback_and_secret_not_delivered(self):
|
||||
def test_build_ssh_command_undeclared_raises_and_secret_not_delivered(self):
|
||||
# feat-chv-p1fix（P1-04）：删除回环回落——目标未声明
|
||||
# 时构造即抛 ssh_target_undeclared，不再悄悄连回环。
|
||||
ex = self.make_exec(env_extra={"CHV_SSH_FORWARD": "0"})
|
||||
cmd = ex.build_ssh_command(self._task(), ["true"])
|
||||
self.assertIn("127.0.0.1", " ".join(cmd))
|
||||
with self.assertRaises(ChvFailure) as ctx:
|
||||
ex.build_ssh_command(self._task(), ["true"])
|
||||
self.assertEqual(ctx.exception.kind, "ssh_target_undeclared")
|
||||
ex = self.make_exec()
|
||||
with self.assertRaises(ChvFailure) as ctx:
|
||||
ex.build_ssh_command(self._task(), ["true"],
|
||||
extra_env={"SUB2API_API_KEY": "v"})
|
||||
self.assertEqual(ctx.exception.kind, "secret_env_not_delivered")
|
||||
|
||||
def test_write_provisioned_known_hosts_fallback(self):
|
||||
def test_write_provisioned_known_hosts_undeclared_raises(self):
|
||||
# feat-chv-p1fix（P1-04 同款）：预置 known_hosts 解析
|
||||
# 失败即抛，不再回落回环写钉死。
|
||||
ex = self.make_exec(env_extra={"CHV_SSH_FORWARD": "0",
|
||||
"CHV_HOST_KEY_MODE": "provisioned"})
|
||||
vm_dir = os.path.join(self.state_dir, "chv-abc123")
|
||||
os.makedirs(vm_dir, exist_ok=True)
|
||||
paths = {"known_hosts": os.path.join(vm_dir, "known_hosts")}
|
||||
ex._write_provisioned_known_hosts(paths, "ssh-ed25519 AAAA")
|
||||
with open(paths["known_hosts"]) as fh:
|
||||
self.assertIn("127.0.0.1", fh.read())
|
||||
with self.assertRaises(ChvFailure) as ctx:
|
||||
ex._write_provisioned_known_hosts(paths, "ssh-ed25519 AAAA")
|
||||
self.assertEqual(ctx.exception.kind, "ssh_target_undeclared")
|
||||
self.assertFalse(os.path.exists(paths["known_hosts"]))
|
||||
|
||||
def test_provision_host_key_dirty_refused(self):
|
||||
ex = self.make_exec()
|
||||
@@ -1083,22 +1089,26 @@ class DefensiveBranchTest(CloseupCase):
|
||||
self.assertFalse(ex._vm_is_running("chv-bad"))
|
||||
|
||||
def test_clean_one_vm_secrets_remove_failures(self):
|
||||
# feat-chv-p1fix（P1-03）：返回 3 元组，删失败以
|
||||
# failed=True 区分，不再静默 pass。
|
||||
ex = self.make_exec()
|
||||
vm_dir = os.path.join(self.state_dir, "chv-x")
|
||||
os.makedirs(os.path.join(vm_dir, "ssh_host_ed25519_key"),
|
||||
exist_ok=True)
|
||||
removed_priv, removed_seed = ex._clean_one_vm_secrets(
|
||||
removed_priv, removed_seed, failed = ex._clean_one_vm_secrets(
|
||||
"chv-x", vm_dir)
|
||||
self.assertFalse(removed_priv)
|
||||
self.assertTrue(failed)
|
||||
vm2 = os.path.join(self.state_dir, "chv-y")
|
||||
os.makedirs(os.path.join(vm2, "seed"), exist_ok=True)
|
||||
import shutil
|
||||
real_rmtree = shutil.rmtree
|
||||
with mock.patch("shutil.rmtree",
|
||||
side_effect=OSError("rmtree boom")):
|
||||
removed_priv, removed_seed = ex._clean_one_vm_secrets(
|
||||
removed_priv, removed_seed, failed = ex._clean_one_vm_secrets(
|
||||
"chv-y", vm2)
|
||||
self.assertFalse(removed_seed)
|
||||
self.assertTrue(failed)
|
||||
self.assertTrue(real_rmtree is not None)
|
||||
|
||||
def test_cleanup_stale_vms_subcall_failures_and_poll_branches(self):
|
||||
|
||||
@@ -559,14 +559,17 @@ class ExperimentalPoolCover100Test(unittest.TestCase):
|
||||
self.assertIn("warm-1", pool._entries)
|
||||
self.assertNotIn("old-1", pool._entries)
|
||||
|
||||
def test_reconcile_shutdown_failure_still_cleans(self):
|
||||
def test_reconcile_shutdown_failure_not_cleaned(self):
|
||||
# feat-chv-p1fix（P1-01 同款语义，实验池）：shutdown
|
||||
# 失败的条目不进 cleaned、不从内存条目里 pop，
|
||||
# 下次 reconcile 可重试。
|
||||
stub = _PoolStub()
|
||||
stub.shutdown_raises = True
|
||||
pool = WarmPool(stub, registry_path=None)
|
||||
pool._entries["old-1"] = {"state": "orphan", "vm": "vm-old",
|
||||
"task": {"id": "old-1"}}
|
||||
self.assertEqual(pool.reconcile_orphans(), ["vm-old"])
|
||||
self.assertEqual(pool._entries, {})
|
||||
self.assertEqual(pool.reconcile_orphans(), [])
|
||||
self.assertIn("old-1", pool._entries)
|
||||
|
||||
def test_reconcile_emit_failure_is_best_effort(self):
|
||||
stub = _PoolStub()
|
||||
|
||||
@@ -0,0 +1,730 @@
|
||||
"""CHV 生产级审核 P1 修复轨（feat-chv-p1fix，堆叠于 #270 之上）测试。
|
||||
|
||||
逐项对应审核报告 §5 的 P1-01～P1-10（P1-10 是文档项，
|
||||
无代码测试；文档数字见 docs/chv-production-readiness.md
|
||||
§15 与 docs/chv-architecture.md 头范围）：
|
||||
- P1-01：生产 reconcile 失败项不进 cleaned、不清库、
|
||||
落盘保留失败项原始条目、可重试；实验池同款语义。
|
||||
- P1-02：registry 损坏与非法条目发审计并计数，损坏不
|
||||
静默返回空。
|
||||
- P1-03：残留私钥删失败发审计、写脏标记，调用方可区分
|
||||
成功与失败（failed VM 不进 cleaned）。
|
||||
- P1-04：build_ssh_command 删除回环回落，构造失败即抛；
|
||||
显式回环（CHV_SSH_HOST / CHV_SSH_FORWARD=1）仍可用。
|
||||
- P1-05：state_dir flock（locking.StateLock）——线程内
|
||||
互斥、跨进程互斥、wake 全程串行化、purge 快照遍历。
|
||||
- P1-06：单条快照超总量帽时拒绝记录，且不删旧快照、
|
||||
不落盘新条目。
|
||||
- P1-07：CHV_CPU / CHV_MEM / CHV_TASK_TIMEOUT 的 >0
|
||||
校验（config 构造层 + preflight 事后改写兜底）。
|
||||
- P1-08：快照保留期剪枝删失败发审计；snapshot_vm 失败
|
||||
路径发 vm_snapshot_failed 审计。
|
||||
- P1-09：4 处三元 else 分支——str 形态 pause/resume/
|
||||
snapshot + 非 dict spill task。
|
||||
|
||||
全部用标准库与 mock runner/launcher/api_client，绝不启动
|
||||
真实 VM（本机无 /dev/kvm，真机验证仍是未验证项）。
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import threading
|
||||
import time
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
from unittest import mock
|
||||
|
||||
try:
|
||||
from cloud.chv import (
|
||||
ChvFailure,
|
||||
CloudHypervisorExecutor,
|
||||
reconcile_orphans_for_executor,
|
||||
)
|
||||
from cloud.chv.config import build_chv_config
|
||||
from cloud.chv.disk import (
|
||||
prune_snapshots,
|
||||
record_snapshot,
|
||||
)
|
||||
from cloud.chv.locking import StateLock, lock_path_for_state_dir
|
||||
except ImportError: # discover -s cloud/tests 时把项目根目录补进 sys.path
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[2]))
|
||||
from cloud.chv import (
|
||||
ChvFailure,
|
||||
CloudHypervisorExecutor,
|
||||
reconcile_orphans_for_executor,
|
||||
)
|
||||
from cloud.chv.config import build_chv_config
|
||||
from cloud.chv.disk import (
|
||||
prune_snapshots,
|
||||
record_snapshot,
|
||||
)
|
||||
from cloud.chv.locking import StateLock, lock_path_for_state_dir
|
||||
|
||||
|
||||
class FakeResult:
|
||||
def __init__(self, returncode=0, stdout="", stderr=""):
|
||||
self.returncode = returncode
|
||||
self.stdout = stdout
|
||||
self.stderr = stderr
|
||||
|
||||
|
||||
class FakeProc:
|
||||
def __init__(self, alive=True):
|
||||
self._alive = alive
|
||||
|
||||
def poll(self):
|
||||
return None if self._alive else 1
|
||||
|
||||
def terminate(self):
|
||||
self._alive = False
|
||||
|
||||
def wait(self, timeout=None):
|
||||
return 0
|
||||
|
||||
def kill(self):
|
||||
self._alive = False
|
||||
|
||||
|
||||
def _write(path, content=b"x"):
|
||||
with open(path, "wb") as fh:
|
||||
fh.write(content)
|
||||
os.chmod(path, 0o755)
|
||||
return path
|
||||
|
||||
|
||||
class P1FixCase(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.tmp = tempfile.TemporaryDirectory()
|
||||
self.dir = self.tmp.name
|
||||
self.binary = _write(os.path.join(self.dir, "cloud-hypervisor"))
|
||||
self.ch_remote = _write(os.path.join(self.dir, "ch-remote"))
|
||||
self.kernel = _write(os.path.join(self.dir, "vmlinux"))
|
||||
self.rootfs = _write(os.path.join(self.dir, "disk.raw"),
|
||||
b"R" * 100)
|
||||
self.ssh_key = _write(os.path.join(self.dir, "id_chv"))
|
||||
self.kvm = _write(os.path.join(self.dir, "kvm"))
|
||||
self.cpuinfo = os.path.join(self.dir, "cpuinfo")
|
||||
with open(self.cpuinfo, "w") as fh:
|
||||
fh.write("flags : fpu vmx sse\n")
|
||||
self.state_dir = os.path.join(self.dir, "state")
|
||||
self.calls = []
|
||||
self.audit = []
|
||||
|
||||
def tearDown(self):
|
||||
self.tmp.cleanup()
|
||||
|
||||
def make_exec(self, runner=None, api_client=None, launcher=None,
|
||||
env_extra=None, **overrides):
|
||||
env = {"CHV_EGRESS_PROXY": "http://10.0.0.1:8888",
|
||||
"CHV_SSH_FORWARD": "1",
|
||||
"CHV_HOST_KEY_MODE": "keyscan"}
|
||||
if env_extra:
|
||||
env.update(env_extra)
|
||||
if runner is None:
|
||||
runner = self._default_runner
|
||||
if api_client is None:
|
||||
api_client = self._default_api
|
||||
if launcher is None:
|
||||
launcher = self._default_launcher
|
||||
kwargs = dict(
|
||||
binary=self.binary, ch_remote=self.ch_remote,
|
||||
kernel=self.kernel, rootfs=self.rootfs,
|
||||
ssh_key=self.ssh_key, kvm_path=self.kvm,
|
||||
cpuinfo_path=self.cpuinfo, state_dir=self.state_dir,
|
||||
runner=runner, launcher=launcher, api_client=api_client,
|
||||
sleeper=lambda s: None, env=env)
|
||||
kwargs.update(overrides)
|
||||
ex = CloudHypervisorExecutor(**kwargs)
|
||||
ex.audit_callback = lambda *a: self.audit.append(a)
|
||||
return ex
|
||||
|
||||
def _default_runner(self, cmd, timeout=None):
|
||||
self.calls.append(list(cmd))
|
||||
return FakeResult(0, "", "")
|
||||
|
||||
def _default_api(self, socket_path, method, path, payload=None):
|
||||
return (200, {})
|
||||
|
||||
def _default_launcher(self, cmd):
|
||||
return FakeProc(alive=True)
|
||||
|
||||
def _task(self, task_id="abc123"):
|
||||
return {"id": task_id, "prompt": "x", "project_id": "proj1"}
|
||||
|
||||
def _audit_events(self):
|
||||
return [a[0] for a in self.audit]
|
||||
|
||||
|
||||
# -- P1-01 / P1-02：生产 reconcile --------------------------------------
|
||||
|
||||
class FakeProdExecutor:
|
||||
def __init__(self, state_dir, fail_vms=()):
|
||||
self.state_dir = state_dir
|
||||
self.fail_vms = set(fail_vms)
|
||||
self.shutdowns = []
|
||||
self.emits = []
|
||||
|
||||
def shutdown_vm(self, vm):
|
||||
self.shutdowns.append(vm)
|
||||
if vm in self.fail_vms:
|
||||
raise RuntimeError("shutdown down")
|
||||
|
||||
def _emit(self, event, task, detail):
|
||||
self.emits.append(event)
|
||||
|
||||
|
||||
def _write_registry(state_dir, data):
|
||||
path = os.path.join(state_dir, "warm-pool.json")
|
||||
os.makedirs(state_dir, exist_ok=True)
|
||||
if isinstance(data, str):
|
||||
with open(path, "w") as fh:
|
||||
fh.write(data)
|
||||
else:
|
||||
with open(path, "w") as fh:
|
||||
json.dump(data, fh)
|
||||
return path
|
||||
|
||||
|
||||
class ReconcileP1FixTest(unittest.TestCase):
|
||||
def test_failure_not_cleaned_registry_retained_and_retryable(self):
|
||||
with tempfile.TemporaryDirectory() as d:
|
||||
path = _write_registry(d, {
|
||||
"a": {"state": "warm", "vm": "chv-good"},
|
||||
"b": {"state": "warm", "vm": "chv-bad"},
|
||||
})
|
||||
ex = FakeProdExecutor(d, fail_vms={"chv-bad"})
|
||||
self.assertEqual(reconcile_orphans_for_executor(ex),
|
||||
["chv-good"])
|
||||
self.assertIn("warm_pool_reconcile_failed", ex.emits)
|
||||
with open(path) as fh:
|
||||
remaining = json.load(fh)
|
||||
self.assertEqual(
|
||||
remaining,
|
||||
{"b": {"state": "warm", "vm": "chv-bad"}})
|
||||
# 修复后重试：失败项可被清理，库最终为空。
|
||||
ex.fail_vms = set()
|
||||
self.assertEqual(reconcile_orphans_for_executor(ex),
|
||||
["chv-bad"])
|
||||
with open(path) as fh:
|
||||
self.assertEqual(json.load(fh), {})
|
||||
|
||||
def test_corrupt_registry_audited_not_silent(self):
|
||||
with tempfile.TemporaryDirectory() as d:
|
||||
path = _write_registry(d, "{not json")
|
||||
ex = FakeProdExecutor(d)
|
||||
self.assertEqual(reconcile_orphans_for_executor(ex), [])
|
||||
self.assertIn("warm_pool_registry_corrupt", ex.emits)
|
||||
self.assertTrue(os.path.exists(path + ".corrupt"))
|
||||
|
||||
def test_invalid_entries_audited_and_counted(self):
|
||||
with tempfile.TemporaryDirectory() as d:
|
||||
path = _write_registry(d, {
|
||||
"a": {"state": "warm", "vm": "chv-a"},
|
||||
"c": {"state": "warm"},
|
||||
"e": {"state": "running", "vm": "chv-e"},
|
||||
})
|
||||
ex = FakeProdExecutor(d)
|
||||
self.assertEqual(reconcile_orphans_for_executor(ex),
|
||||
["chv-a"])
|
||||
self.assertIn("warm_pool_registry_invalid_entries",
|
||||
ex.emits)
|
||||
with open(path) as fh:
|
||||
self.assertEqual(json.load(fh), {})
|
||||
|
||||
|
||||
# -- P1-03：残留私钥删失败 ----------------------------------------------
|
||||
|
||||
class StaleSecretsP1FixTest(P1FixCase):
|
||||
def test_delete_failure_not_cleaned_audited_and_dirty_marked(self):
|
||||
from cloud.chv.hostkeys import dirty_marker_path
|
||||
ex = self.make_exec()
|
||||
vm_dir = os.path.join(self.state_dir, "chv-bad")
|
||||
os.makedirs(vm_dir, exist_ok=True)
|
||||
priv = os.path.join(vm_dir, "ssh_host_ed25519_key")
|
||||
with open(priv, "w") as fh:
|
||||
fh.write("PRIVATE")
|
||||
real_remove = os.remove
|
||||
|
||||
def remove(path):
|
||||
if path == priv:
|
||||
raise OSError("删不掉")
|
||||
return real_remove(path)
|
||||
|
||||
with mock.patch.object(os, "remove", side_effect=remove):
|
||||
cleaned = ex.cleanup_stale_secrets()
|
||||
self.assertNotIn("chv-bad", cleaned)
|
||||
self.assertIn("host_secret_residue_delete_failed",
|
||||
self._audit_events())
|
||||
self.assertTrue(os.path.exists(
|
||||
dirty_marker_path(self.state_dir, "chv-bad")))
|
||||
self.assertTrue(os.path.exists(priv))
|
||||
|
||||
def test_delete_success_cleaned(self):
|
||||
ex = self.make_exec()
|
||||
vm_dir = os.path.join(self.state_dir, "chv-ok")
|
||||
os.makedirs(vm_dir, exist_ok=True)
|
||||
with open(os.path.join(vm_dir, "ssh_host_ed25519_key"),
|
||||
"w") as fh:
|
||||
fh.write("PRIVATE")
|
||||
cleaned = ex.cleanup_stale_secrets()
|
||||
self.assertIn("chv-ok", cleaned)
|
||||
self.assertIn("host_secret_residue_cleaned",
|
||||
self._audit_events())
|
||||
|
||||
|
||||
# -- P1-04：SSH 回落删除 -------------------------------------------------
|
||||
|
||||
class SshFallbackP1FixTest(P1FixCase):
|
||||
def test_undeclared_build_raises_not_loopback(self):
|
||||
ex = self.make_exec(env_extra={"CHV_SSH_FORWARD": "0"})
|
||||
with self.assertRaises(ChvFailure) as ctx:
|
||||
ex.build_ssh_command(self._task(), ["true"])
|
||||
self.assertEqual(ctx.exception.kind, "ssh_target_undeclared")
|
||||
|
||||
def test_explicit_loopback_still_allowed(self):
|
||||
ex = self.make_exec(env_extra={"CHV_SSH_HOST": "127.0.0.1"})
|
||||
cmd = ex.build_ssh_command(self._task(), ["true"])
|
||||
self.assertIn("root@127.0.0.1", " ".join(cmd))
|
||||
ex2 = self.make_exec(env_extra={"CHV_SSH_FORWARD": "1"})
|
||||
cmd2 = ex2.build_ssh_command(self._task(), ["true"])
|
||||
self.assertIn("root@127.0.0.1", " ".join(cmd2))
|
||||
|
||||
|
||||
# -- P1-05：flock ---------------------------------------------------------
|
||||
|
||||
class StateLockP1FixTest(unittest.TestCase):
|
||||
def test_thread_mutual_exclusion(self):
|
||||
with tempfile.TemporaryDirectory() as d:
|
||||
state_dir = os.path.join(d, "state")
|
||||
lock_a = StateLock(state_dir)
|
||||
lock_b = StateLock(state_dir)
|
||||
acquired = []
|
||||
lock_a.acquire()
|
||||
try:
|
||||
def _take():
|
||||
# RLock 须由加锁线程释放，故获取与释放同线程。
|
||||
lock_b.acquire()
|
||||
acquired.append(True)
|
||||
lock_b.release()
|
||||
|
||||
t = threading.Thread(target=_take)
|
||||
t.start()
|
||||
time.sleep(0.3)
|
||||
self.assertEqual(acquired, [])
|
||||
self.assertTrue(t.is_alive())
|
||||
finally:
|
||||
lock_a.release()
|
||||
t.join(timeout=5)
|
||||
self.assertEqual(acquired, [True])
|
||||
self.assertTrue(os.path.exists(
|
||||
lock_path_for_state_dir(state_dir)))
|
||||
|
||||
def test_cross_process_mutual_exclusion(self):
|
||||
with tempfile.TemporaryDirectory() as d:
|
||||
state_dir = os.path.join(d, "state")
|
||||
os.makedirs(state_dir, exist_ok=True)
|
||||
root = str(Path(__file__).resolve().parents[2])
|
||||
ready = os.path.join(state_dir, "child-locked")
|
||||
script = (
|
||||
"import sys, time; sys.path.insert(0, %r); "
|
||||
"from cloud.chv.locking import StateLock; "
|
||||
"lock = StateLock(%r); lock.acquire(); "
|
||||
"open(%r, 'w').write('locked'); "
|
||||
"time.sleep(10)" % (root, state_dir, ready))
|
||||
proc = subprocess.Popen(
|
||||
[sys.executable, "-c", script],
|
||||
stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
|
||||
try:
|
||||
deadline = time.time() + 10
|
||||
while not os.path.exists(ready) and time.time() < deadline:
|
||||
time.sleep(0.05)
|
||||
self.assertTrue(os.path.exists(ready))
|
||||
self.assertIsNone(proc.poll())
|
||||
lock = StateLock(state_dir)
|
||||
acquired = []
|
||||
|
||||
def _take():
|
||||
lock.acquire()
|
||||
acquired.append(True)
|
||||
lock.release()
|
||||
|
||||
t = threading.Thread(target=_take)
|
||||
t.start()
|
||||
time.sleep(0.5)
|
||||
# 子进程持锁期间，本进程的加锁必须阻塞。
|
||||
self.assertEqual(acquired, [])
|
||||
proc.terminate()
|
||||
proc.wait(timeout=5)
|
||||
t.join(timeout=5)
|
||||
self.assertEqual(acquired, [True])
|
||||
finally:
|
||||
if proc.poll() is None:
|
||||
proc.kill()
|
||||
proc.wait(timeout=5)
|
||||
|
||||
def test_purge_snapshot_traversal_under_concurrent_mutation(self):
|
||||
# 旧无锁 pop + 持锁遍历并发可致字典迭代异常；新实现
|
||||
# 快照键列表后逐项持锁删，并发变更加锁路径下不得抛错。
|
||||
with tempfile.TemporaryDirectory():
|
||||
case = P1FixCase()
|
||||
case.setUp()
|
||||
try:
|
||||
ex = case.make_exec()
|
||||
ex.state_dir = case.state_dir
|
||||
stop = threading.Event()
|
||||
|
||||
def mutate():
|
||||
i = 0
|
||||
while not stop.is_set():
|
||||
name = "chv-m%d" % (i % 5)
|
||||
with ex._vms_lock:
|
||||
ex._vms[name] = FakeProc(alive=False)
|
||||
i += 1
|
||||
|
||||
t = threading.Thread(target=mutate)
|
||||
t.start()
|
||||
try:
|
||||
for _ in range(50):
|
||||
ex._purge_dead_registry()
|
||||
finally:
|
||||
stop.set()
|
||||
t.join(timeout=5)
|
||||
finally:
|
||||
case.tearDown()
|
||||
|
||||
|
||||
class WakeSerializationP1FixTest(P1FixCase):
|
||||
def test_concurrent_wakes_serialized_by_state_lock(self):
|
||||
# 两路并发 wake：每路 api 往返睡 0.2 秒；串行则总耗时
|
||||
# 至少 0.35 秒，并行则约 0.2 秒。阈值留足余量防抖动。
|
||||
def sleeping_api(socket_path, method, path, payload=None):
|
||||
if path == "/vmm.ping":
|
||||
time.sleep(0.2)
|
||||
return (200, {})
|
||||
|
||||
ex1 = self.make_exec(api_client=sleeping_api)
|
||||
ex2 = self.make_exec(api_client=sleeping_api)
|
||||
results = []
|
||||
start = time.time()
|
||||
|
||||
def wake(ex, task_id):
|
||||
try:
|
||||
results.append(ex.wake_vm({"id": task_id})["state"])
|
||||
except Exception as exc: # noqa: BLE001 - 失败即测试失败
|
||||
results.append("error:%s" % exc)
|
||||
|
||||
t1 = threading.Thread(target=wake, args=(ex1, "s1"))
|
||||
t2 = threading.Thread(target=wake, args=(ex2, "s2"))
|
||||
t1.start()
|
||||
t2.start()
|
||||
t1.join(timeout=30)
|
||||
t2.join(timeout=30)
|
||||
elapsed = time.time() - start
|
||||
self.assertEqual(sorted(results), ["running", "running"])
|
||||
self.assertGreaterEqual(elapsed, 0.35)
|
||||
self.assertTrue(os.path.exists(
|
||||
lock_path_for_state_dir(self.state_dir)))
|
||||
|
||||
|
||||
# -- P1-06：单条快照超帽 ---------------------------------------------------
|
||||
|
||||
class SingleSnapshotCapP1FixTest(unittest.TestCase):
|
||||
def test_single_oversize_snapshot_rejected_without_side_effects(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
old = os.path.join(tmp, "vm1", "old.snap")
|
||||
os.makedirs(os.path.join(tmp, "vm1"), exist_ok=True)
|
||||
with open(old, "wb") as fh:
|
||||
fh.write(b"O" * 100)
|
||||
from cloud.chv.disk import _save_snapshots
|
||||
_save_snapshots(tmp, "vm1", [{
|
||||
"url": "file://" + old, "path": old, "size": 100,
|
||||
"created_at": 0}])
|
||||
new = os.path.join(tmp, "vm1", "new.snap")
|
||||
with open(new, "wb") as fh:
|
||||
fh.write(b"N" * 1000)
|
||||
# 总量帽 500 字节：新单条 1000 字节本身已超帽。
|
||||
with self.assertRaises(ChvFailure) as ctx:
|
||||
record_snapshot(tmp, "vm1", "file://" + new,
|
||||
500 / (1024 ** 3))
|
||||
self.assertEqual(ctx.exception.kind,
|
||||
"snapshot_quota_exceeded")
|
||||
# 拒绝无副作用：旧快照文件仍在、登记仍只有旧条目。
|
||||
self.assertTrue(os.path.exists(old))
|
||||
from cloud.chv.disk import _load_snapshots
|
||||
entries = _load_snapshots(tmp, "vm1")
|
||||
self.assertEqual(len(entries), 1)
|
||||
self.assertEqual(entries[0]["path"], old)
|
||||
|
||||
|
||||
# -- P1-07：>0 校验 ---------------------------------------------------------
|
||||
|
||||
class PositiveConfigP1FixTest(P1FixCase):
|
||||
def test_config_layer_rejects_non_positive(self):
|
||||
base_env = {}
|
||||
for env_extra, field in (
|
||||
({"CHV_CPU": "0"}, "cpu"),
|
||||
({"CHV_CPU": "-2"}, "cpu"),
|
||||
({"CHV_MEM": "0"}, "mem"),
|
||||
({"CHV_MEM": "0G"}, "mem"),
|
||||
({"CHV_TASK_TIMEOUT": "0"}, "timeout"),
|
||||
({"CHV_TASK_TIMEOUT": "-5"}, "timeout")):
|
||||
env = dict(base_env)
|
||||
env.update(env_extra)
|
||||
with self.assertRaises(ValueError, msg=field):
|
||||
build_chv_config({}, env)
|
||||
# 合法值与 None（不限）不受影响。
|
||||
cfg = build_chv_config({}, {"CHV_TASK_TIMEOUT": "30"})
|
||||
self.assertEqual(cfg.task_timeout, 30.0)
|
||||
cfg = build_chv_config({}, {})
|
||||
self.assertIsNone(cfg.task_timeout)
|
||||
|
||||
def test_preflight_rejects_tampered_attributes(self):
|
||||
for attr, value in (("cpus", "0"), ("cpus", "-1"),
|
||||
("memory", "0"), ("task_timeout", 0),
|
||||
("task_timeout", -3)):
|
||||
ex = self.make_exec()
|
||||
setattr(ex, attr, value)
|
||||
with self.assertRaises(ChvFailure) as ctx:
|
||||
ex.preflight()
|
||||
self.assertEqual(ctx.exception.kind, "config_invalid")
|
||||
|
||||
|
||||
# -- P1-08：剪枝审计与快照失败审计 -----------------------------------------
|
||||
|
||||
class SnapshotAuditP1FixTest(P1FixCase):
|
||||
def test_retention_prune_delete_failure_audited(self):
|
||||
ex = self.make_exec()
|
||||
vm_dir = os.path.join(self.state_dir, "chv-abc123")
|
||||
os.makedirs(vm_dir, exist_ok=True)
|
||||
old = os.path.join(self.state_dir, "old.snap")
|
||||
with open(old, "wb") as fh:
|
||||
fh.write(b"O" * 10)
|
||||
with open(os.path.join(vm_dir, "snapshots.json"), "w") as fh:
|
||||
json.dump([{"url": "file://" + old, "path": old,
|
||||
"size": 10, "created_at": 0}], fh)
|
||||
new = os.path.join(self.state_dir, "new.snap")
|
||||
with open(new, "wb") as fh:
|
||||
fh.write(b"N" * 10)
|
||||
real_remove = os.remove
|
||||
|
||||
def remove(path):
|
||||
if path == old:
|
||||
raise OSError("删不掉")
|
||||
return real_remove(path)
|
||||
|
||||
with mock.patch.object(os, "remove", side_effect=remove):
|
||||
out = ex.snapshot_vm(self._task(), "file://" + new)
|
||||
self.assertEqual(out, "file://" + new)
|
||||
self.assertIn("vm_snapshot_prune_failed",
|
||||
self._audit_events())
|
||||
self.assertTrue(os.path.exists(old))
|
||||
|
||||
def test_snapshot_failure_audited(self):
|
||||
def failing_runner(cmd, timeout=None):
|
||||
if cmd and cmd[0] == "ch-remote":
|
||||
raise RuntimeError("ch-remote 挂了")
|
||||
return FakeResult(0, "", "")
|
||||
|
||||
ex = self.make_exec(runner=failing_runner)
|
||||
# 目标须在 state_dir 内才过白名单（ch-remote 失败在后）。
|
||||
target = "file://" + os.path.join(self.state_dir, "x.snap")
|
||||
with self.assertRaises(RuntimeError):
|
||||
ex.snapshot_vm(self._task(), target)
|
||||
self.assertIn("vm_snapshot_failed", self._audit_events())
|
||||
|
||||
|
||||
# -- P1-09：4 处三元 else 分支 ----------------------------------------------
|
||||
|
||||
class BranchCoverP1FixTest(P1FixCase):
|
||||
def test_pause_resume_snapshot_str_forms(self):
|
||||
ex = self.make_exec()
|
||||
os.makedirs(self.state_dir, exist_ok=True)
|
||||
self.assertTrue(ex.pause_vm("chv-abc"))
|
||||
self.assertTrue(ex.resume_vm("chv-abc"))
|
||||
snap = os.path.join(self.state_dir, "snap.raw")
|
||||
with open(snap, "wb") as fh:
|
||||
fh.write(b"S" * 10)
|
||||
out = ex.snapshot_vm("chv-abc", "file://" + snap)
|
||||
self.assertEqual(out, "file://" + snap)
|
||||
self.assertIn("vm_pause", self._audit_events())
|
||||
self.assertIn("vm_resume", self._audit_events())
|
||||
self.assertIn("vm_snapshot", self._audit_events())
|
||||
|
||||
def test_spill_audit_non_dict_task(self):
|
||||
ex = self.make_exec()
|
||||
os.makedirs(self.state_dir, exist_ok=True)
|
||||
ex._spill_audit("ev", "plain-task-id", "detail", None)
|
||||
path = os.path.join(self.state_dir, "chv-audit-spill.jsonl")
|
||||
with open(path) as fh:
|
||||
row = json.loads(fh.readline())
|
||||
self.assertEqual(row["task_id"], "plain-task-id")
|
||||
self.assertEqual(row["event"], "ev")
|
||||
|
||||
|
||||
# -- 覆盖补齐：新分支的行覆盖（诚实口径，不凑数改代码） --------
|
||||
|
||||
class CoverageCompletionP1FixTest(P1FixCase):
|
||||
def test_prune_events_no_file_and_no_delete(self):
|
||||
from cloud.chv.disk import _save_snapshots
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
_save_snapshots(tmp, "vm1", [
|
||||
{"url": "file:///gone.snap", "path": "/gone.snap",
|
||||
"size": 5, "created_at": 0},
|
||||
{"url": "https://example.invalid/x", "path": None,
|
||||
"size": 5, "created_at": 0}])
|
||||
events = []
|
||||
kept, pruned = prune_snapshots(
|
||||
tmp, "vm1", retention_days=30, now=10 ** 9,
|
||||
events=events)
|
||||
self.assertEqual(kept, [])
|
||||
self.assertEqual(len(pruned), 2)
|
||||
kinds = [e[0] for e in events]
|
||||
self.assertEqual(kinds, ["pruned_no_file", "pruned_no_file"])
|
||||
events2 = []
|
||||
_save_snapshots(tmp, "vm1", [
|
||||
{"url": "file:///gone2.snap", "path": "/gone2.snap",
|
||||
"size": 5, "created_at": 0}])
|
||||
kept2, _ = prune_snapshots(
|
||||
tmp, "vm1", retention_days=30, now=10 ** 9,
|
||||
delete_files=False, events=events2)
|
||||
self.assertEqual(kept2, [])
|
||||
self.assertEqual([e[0] for e in events2],
|
||||
["pruned_no_file"])
|
||||
|
||||
def test_record_without_cap_and_tighten_delete_failure_event(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
entries, removed = record_snapshot(
|
||||
tmp, "vm1", "https://example.invalid/new", None)
|
||||
self.assertEqual(len(entries), 1)
|
||||
self.assertEqual(removed, [])
|
||||
old = os.path.join(tmp, "vm1", "old.snap")
|
||||
os.makedirs(os.path.join(tmp, "vm1"), exist_ok=True)
|
||||
with open(old, "wb") as fh:
|
||||
fh.write(b"O" * 100)
|
||||
from cloud.chv.disk import _save_snapshots
|
||||
_save_snapshots(tmp, "vm1", [
|
||||
{"url": "file://" + old, "path": old, "size": 100,
|
||||
"created_at": 0}])
|
||||
real_remove = os.remove
|
||||
|
||||
def remove(path):
|
||||
if path == old:
|
||||
raise OSError("删不掉")
|
||||
return real_remove(path)
|
||||
|
||||
events = []
|
||||
with mock.patch.object(os, "remove", side_effect=remove):
|
||||
entries2, removed2 = record_snapshot(
|
||||
tmp, "vm1", "https://example.invalid/new2",
|
||||
0.0, events=events)
|
||||
self.assertEqual(len(removed2), 1)
|
||||
self.assertEqual([e[0] for e in events], ["delete_failed"])
|
||||
self.assertTrue(os.path.exists(old))
|
||||
|
||||
def test_tighten_no_file_entry_event(self):
|
||||
from cloud.chv.disk import _save_snapshots
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
_save_snapshots(tmp, "vm1", [
|
||||
{"url": "https://example.invalid/old", "path": None,
|
||||
"size": 100, "created_at": 0}])
|
||||
events = []
|
||||
entries, removed = record_snapshot(
|
||||
tmp, "vm1", "https://example.invalid/new",
|
||||
0.0, events=events)
|
||||
self.assertEqual(len(removed), 1)
|
||||
self.assertEqual([e[0] for e in events],
|
||||
["pruned_no_file"])
|
||||
|
||||
def test_state_lock_ctx_refreshes_on_state_dir_change(self):
|
||||
ex = self.make_exec()
|
||||
old_lock = ex._state_lock
|
||||
new_dir = os.path.join(self.dir, "state2")
|
||||
ex.state_dir = new_dir
|
||||
self.assertIsNot(ex._state_lock_ctx(), old_lock)
|
||||
self.assertEqual(ex._state_lock.state_dir, new_dir)
|
||||
|
||||
def test_gc_and_secrets_listdir_failure_returns_empty(self):
|
||||
ex = self.make_exec()
|
||||
ex.state_dir = os.path.join(self.dir, "missing-dir")
|
||||
self.assertEqual(ex.gc_stale_disks(),
|
||||
{"deleted": [], "failed": []})
|
||||
self.assertEqual(ex.cleanup_stale_secrets(), [])
|
||||
|
||||
def test_cleanup_listdir_failure_returns_empty(self):
|
||||
ex = self.make_exec()
|
||||
os.makedirs(self.state_dir, exist_ok=True)
|
||||
real_listdir = os.listdir
|
||||
calls = {"n": 0}
|
||||
|
||||
def listdir(path):
|
||||
calls["n"] += 1
|
||||
if calls["n"] >= 3:
|
||||
raise OSError("listdir 挂了")
|
||||
return real_listdir(path)
|
||||
|
||||
with mock.patch.object(os, "listdir", side_effect=listdir):
|
||||
self.assertEqual(ex.cleanup_stale_vms(), [])
|
||||
|
||||
def test_orphan_entries_compat_wrapper_and_no_emit_executor(self):
|
||||
from cloud.chv.orphans import _orphan_entries
|
||||
data = {"a": {"state": "warm", "vm": "chv-a",
|
||||
"task_id": "a", "snapshot_url": None},
|
||||
"b": {"state": "running", "vm": "chv-b"}}
|
||||
self.assertEqual(_orphan_entries(data),
|
||||
[{"vm": "chv-a", "task_id": "a",
|
||||
"snapshot_url": None}])
|
||||
|
||||
class NoEmit:
|
||||
def __init__(self, state_dir):
|
||||
self.state_dir = state_dir
|
||||
self.shutdowns = []
|
||||
|
||||
def shutdown_vm(self, vm):
|
||||
self.shutdowns.append(vm)
|
||||
|
||||
with tempfile.TemporaryDirectory() as d:
|
||||
_write_registry(d, {
|
||||
"a": {"state": "warm", "vm": "chv-a"}})
|
||||
ex = NoEmit(d)
|
||||
self.assertEqual(reconcile_orphans_for_executor(ex),
|
||||
["chv-a"])
|
||||
|
||||
def test_state_lock_flock_failure_closes_and_releases(self):
|
||||
import fcntl
|
||||
with tempfile.TemporaryDirectory() as d:
|
||||
state_dir = os.path.join(d, "state")
|
||||
lock = StateLock(state_dir)
|
||||
with mock.patch.object(fcntl, "flock",
|
||||
side_effect=OSError("flock 挂了")):
|
||||
with self.assertRaises(OSError):
|
||||
lock.acquire()
|
||||
self.assertIsNone(lock._fd)
|
||||
# 失败后锁仍可用（RLock 已释放、fd 已关闭）。
|
||||
lock.acquire()
|
||||
lock.release()
|
||||
|
||||
def test_pool_reconcile_failed_emit_failure_swallowed(self):
|
||||
from cloud.chv.experimental_pool import WarmPool
|
||||
stub = _PoolStubForP1Fix()
|
||||
pool = WarmPool(stub, registry_path=None)
|
||||
pool._entries["old-1"] = {"state": "orphan", "vm": "vm-old",
|
||||
"task": {"id": "old-1"}}
|
||||
self.assertEqual(pool.reconcile_orphans(), [])
|
||||
self.assertIn("old-1", pool._entries)
|
||||
|
||||
|
||||
class _PoolStubForP1Fix:
|
||||
"""实验池失败审计吞错测试用假执行器。"""
|
||||
|
||||
def shutdown_vm(self, task_or_name):
|
||||
raise RuntimeError("shutdown 失败")
|
||||
|
||||
def _emit(self, event, task, detail="", kind=None):
|
||||
raise RuntimeError("audit 失败")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in new issue
Block a user