feat: CHV 执行器优化(拆模块/去重复/SSH 钉死/池持久化/中途死亡分类)
- 拆模块:CHV 从 cloud/sandbox.py 巨类拆到 cloud/chv/(executor/ pool/config/api_client/ssh_transport/disk/errors),sandbox 只留 兼容 re-export,旧导入路径与新路径是同一对象 - 去重复:goal 解析、host 顶层参数、runner 超时抽成共享 helper, Local 与 CHV 在矩阵输入下结论全等(Docker goal-resume 变体 有意保留内联版,见 PR 描述) - 安全:SSH host key 按 VM 钉死(state_dir/<vm>/known_hosts, ssh-keyscan 首次捕获 TOFU),废除 no + /dev/null;CHV_SSH_USER 空串与 CHV_SSH_PORT 越界直接拒绝,不静默 fallback - execute 语义:只读已运行 VM,未运行报 vm_not_running,不为 status 起 VM - 预热池:registry 落盘、重启载入只当孤儿、reconcile 清理、 损坏 quarantine;条目仍一次性,绝不复用脏 VM - 长任务:VM 中途死亡判 interrupted(断点可续跑);中途心跳 经评估不做(无客内进度信号可采) - #215 未合并:只列复用清单,不复制模块 测试:原 42 项 + 新 24 项全过;相关旧套件与 cloud 全量 1496 项 全过(skipped=4)。验证仍是 mock + 干跑 + 单元测试,未做真机、 压测、安全审计;堆叠于 #216 之上,base 为 feat-cloud-hypervisor。
This commit is contained in:
1 parent
c371bbdf1e
commit
fde7bf4c3e
14 files changed
+2706
-1408
No files matched your search
@@ -0,0 +1,586 @@
|
||||
"""Cloud Hypervisor 优化轨(feat-chv-optimize,堆叠于 #216 之上)测试。
|
||||
|
||||
只用标准库与 mock runner/launcher/api_client,绝不启动真实
|
||||
VM(与 test_cloud_hypervisor.py 同款边界)。覆盖:
|
||||
- 拆模块:cloud/chv/* 新路径与 cloud/sandbox 旧路径是同一对象
|
||||
- 去重复:Local 与 CHV 的 goal 解析在矩阵输入下结论全等;
|
||||
三执行器 host 顶层参数顺序全等;runner 超时 helper
|
||||
- 安全:每 VM known_hosts 钉死、ssh-keyscan 首次捕获(TOFU)、
|
||||
非 root 用户与端口显式配置与收紧校验
|
||||
- execute 语义:只读已运行 VM,未运行报 vm_not_running
|
||||
- 预热池:registry 持久化、重启载入只当孤儿、reconcile 清理、
|
||||
损坏 quarantine、一 次性不复用
|
||||
- 长任务:VM 中途死亡判 interrupted 而非 failed/抛异常
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
try:
|
||||
from cloud.chv import ChvFailure
|
||||
from cloud.chv import CloudHypervisorExecutor
|
||||
from cloud.chv import WarmPool
|
||||
from cloud.chv.api_client import _unix_socket_api as _chv_api
|
||||
from cloud.chv.config import _parse_memory_bytes as _chv_mem
|
||||
from cloud.sandbox import ChvFailure as _sb_failure
|
||||
from cloud.sandbox import CloudHypervisorExecutor as _sb_exec
|
||||
from cloud.sandbox import DockerExecutor
|
||||
from cloud.sandbox import LocalExecutor
|
||||
from cloud.sandbox import WarmPool as _sb_pool
|
||||
from cloud.sandbox import _call_runner_with_timeout
|
||||
from cloud.sandbox import _parse_memory_bytes as _sb_mem
|
||||
from cloud.sandbox import _unix_socket_api as _sb_api
|
||||
except ImportError: # discover -s cloud/tests 时把项目根目录补进 sys.path
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[2]))
|
||||
from cloud.chv import ChvFailure
|
||||
from cloud.chv import CloudHypervisorExecutor
|
||||
from cloud.chv import WarmPool
|
||||
from cloud.chv.api_client import _unix_socket_api as _chv_api
|
||||
from cloud.chv.config import _parse_memory_bytes as _chv_mem
|
||||
from cloud.sandbox import ChvFailure as _sb_failure
|
||||
from cloud.sandbox import CloudHypervisorExecutor as _sb_exec
|
||||
from cloud.sandbox import DockerExecutor
|
||||
from cloud.sandbox import LocalExecutor
|
||||
from cloud.sandbox import WarmPool as _sb_pool
|
||||
from cloud.sandbox import _call_runner_with_timeout
|
||||
from cloud.sandbox import _parse_memory_bytes as _sb_mem
|
||||
from cloud.sandbox import _unix_socket_api as _sb_api
|
||||
|
||||
|
||||
class FakeResult:
|
||||
def __init__(self, returncode=0, stdout="", stderr=""):
|
||||
self.returncode = returncode
|
||||
self.stdout = stdout
|
||||
self.stderr = stderr
|
||||
|
||||
|
||||
class FakeProc:
|
||||
def __init__(self, alive=True):
|
||||
self._alive = alive
|
||||
self.terminated = False
|
||||
|
||||
def poll(self):
|
||||
return None if self._alive else 1
|
||||
|
||||
def terminate(self):
|
||||
self.terminated = True
|
||||
self._alive = False
|
||||
|
||||
def wait(self, timeout=None):
|
||||
return 0
|
||||
|
||||
def kill(self):
|
||||
self._alive = False
|
||||
|
||||
|
||||
def _write(path, content=b"x"):
|
||||
with open(path, "wb") as fh:
|
||||
fh.write(content)
|
||||
os.chmod(path, 0o755)
|
||||
return path
|
||||
|
||||
|
||||
class ChvOptCase(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.tmp = tempfile.TemporaryDirectory()
|
||||
self.dir = self.tmp.name
|
||||
self.binary = _write(os.path.join(self.dir, "cloud-hypervisor"))
|
||||
self.ch_remote = _write(os.path.join(self.dir, "ch-remote"))
|
||||
self.kernel = _write(os.path.join(self.dir, "vmlinux"))
|
||||
self.rootfs = _write(os.path.join(self.dir, "disk.raw"),
|
||||
b"R" * 100)
|
||||
self.ssh_key = _write(os.path.join(self.dir, "id_chv"))
|
||||
self.kvm = _write(os.path.join(self.dir, "kvm"))
|
||||
self.cpuinfo = os.path.join(self.dir, "cpuinfo")
|
||||
with open(self.cpuinfo, "w") as fh:
|
||||
fh.write("flags : fpu vmx sse\n")
|
||||
self.state_dir = os.path.join(self.dir, "state")
|
||||
self.calls = []
|
||||
self.api_calls = []
|
||||
self.launches = []
|
||||
|
||||
def tearDown(self):
|
||||
self.tmp.cleanup()
|
||||
|
||||
def make_exec(self, runner=None, api_client=None, launcher=None,
|
||||
env_extra=None, **overrides):
|
||||
env = {"CHV_EGRESS_PROXY": "http://10.0.0.1:8888"}
|
||||
if env_extra:
|
||||
env.update(env_extra)
|
||||
if runner is None:
|
||||
runner = self._default_runner
|
||||
if api_client is None:
|
||||
api_client = self._default_api
|
||||
if launcher is None:
|
||||
launcher = self._default_launcher
|
||||
kwargs = dict(
|
||||
binary=self.binary, ch_remote=self.ch_remote,
|
||||
kernel=self.kernel, rootfs=self.rootfs,
|
||||
ssh_key=self.ssh_key, kvm_path=self.kvm,
|
||||
cpuinfo_path=self.cpuinfo, state_dir=self.state_dir,
|
||||
runner=runner, launcher=launcher, api_client=api_client,
|
||||
sleeper=lambda s: None, env=env)
|
||||
kwargs.update(overrides)
|
||||
return CloudHypervisorExecutor(**kwargs)
|
||||
|
||||
def _default_runner(self, cmd, timeout=None):
|
||||
self.calls.append(list(cmd))
|
||||
if cmd[0] == "ssh":
|
||||
if cmd[-1] == "true":
|
||||
return FakeResult(0, "", "")
|
||||
if "dispatch" in cmd:
|
||||
return FakeResult(0, json.dumps(
|
||||
{"status": "completed", "text": "ok-dispatch"}), "")
|
||||
if "goal" in cmd:
|
||||
return FakeResult(0, "goal met after rounds\n", "")
|
||||
if "status" in cmd:
|
||||
return FakeResult(0, json.dumps(
|
||||
{"status": "completed", "text": "ok-status"}), "")
|
||||
return FakeResult(0, "", "")
|
||||
return FakeResult(0, "", "")
|
||||
|
||||
def _default_api(self, socket_path, method, path, payload=None):
|
||||
self.api_calls.append((method, path, payload))
|
||||
return (200, {})
|
||||
|
||||
def _default_launcher(self, cmd):
|
||||
self.launches.append(list(cmd))
|
||||
return FakeProc(alive=True)
|
||||
|
||||
def _task(self, task_id="abc123"):
|
||||
return {"id": task_id, "prompt": "做一个加法函数",
|
||||
"project_id": "proj1"}
|
||||
|
||||
|
||||
# -- 拆模块 ---------------------------------------------------------------
|
||||
class SplitTest(unittest.TestCase):
|
||||
def test_new_and_old_import_paths_are_same_objects(self):
|
||||
self.assertIs(CloudHypervisorExecutor, _sb_exec)
|
||||
self.assertIs(WarmPool, _sb_pool)
|
||||
self.assertIs(ChvFailure, _sb_failure)
|
||||
self.assertIs(_chv_mem, _sb_mem)
|
||||
self.assertIs(_chv_api, _sb_api)
|
||||
|
||||
def test_classes_live_in_chv_modules(self):
|
||||
self.assertEqual(CloudHypervisorExecutor.__module__,
|
||||
"cloud.chv.executor")
|
||||
self.assertEqual(WarmPool.__module__, "cloud.chv.pool")
|
||||
self.assertEqual(ChvFailure.__module__, "cloud.chv.errors")
|
||||
|
||||
|
||||
# -- 去重复:goal 解析与 host 命令 ------------------------------------------
|
||||
class SharedGoalTest(unittest.TestCase):
|
||||
CASES = [
|
||||
# (returncode, stdout, stderr)
|
||||
(0, "goal met after rounds\n", ""),
|
||||
(0, "round 1 verify_out=bad\ngoal unmet after rounds\n", ""),
|
||||
(1, "goal interrupted after rounds: boom\n", ""),
|
||||
(1, "KeyError: 'x'\n", ""),
|
||||
(1, "some odd failure\n", ""),
|
||||
(0, "nothing conclusive\n", ""),
|
||||
(1, "goal-resume 拒绝:断点丢失\n", ""),
|
||||
(1, "任务已存在\n", ""),
|
||||
(1, "goal met after rounds\n", ""),
|
||||
(1, "goal unmet after rounds\n", ""),
|
||||
(1, "", "HTTP 503 Service Unavailable\n"),
|
||||
]
|
||||
|
||||
def test_local_and_chv_goal_conclusions_match(self):
|
||||
local = LocalExecutor("/tmp/host.py")
|
||||
chv = ChvOptCase()
|
||||
chv.setUp()
|
||||
try:
|
||||
ex = chv.make_exec()
|
||||
for rc, out, err in self.CASES:
|
||||
res = FakeResult(rc, out, err)
|
||||
a = local._goal_result_from_runner("t1", res)
|
||||
b = ex._goal_result("t1", res)
|
||||
self.assertEqual(
|
||||
(a["status"], a["goal_met"]),
|
||||
(b["status"], b["goal_met"]),
|
||||
"rc=%s stdout=%r" % (rc, out))
|
||||
self.assertEqual(a["executor"], "local")
|
||||
self.assertEqual(b["executor"], "cloud-hypervisor")
|
||||
finally:
|
||||
chv.tearDown()
|
||||
|
||||
def test_already_exists_message_points_to_continue(self):
|
||||
chv = ChvOptCase()
|
||||
chv.setUp()
|
||||
try:
|
||||
ex = chv.make_exec()
|
||||
out = ex._goal_result(
|
||||
"t1", FakeResult(1, "任务已存在\n", ""))
|
||||
self.assertEqual(out["status"], "failed")
|
||||
self.assertIn("继续", out["goal_error"])
|
||||
finally:
|
||||
chv.tearDown()
|
||||
|
||||
def test_host_command_options_identical_across_executors(self):
|
||||
chv = ChvOptCase()
|
||||
chv.setUp()
|
||||
try:
|
||||
ex = chv.make_exec()
|
||||
local = LocalExecutor("/tmp/host.py")
|
||||
docker = DockerExecutor()
|
||||
for kwargs in (
|
||||
{"model": "m1", "effort": "low", "approval": "ask"},
|
||||
{"model": "m1", "effort": "low", "approval": "full",
|
||||
"sandbox": "read-only",
|
||||
"model_provider": "sub2api"},
|
||||
):
|
||||
lcmd = local._base_command(**kwargs)
|
||||
dcmd = docker._inner_host_command("status", **kwargs)
|
||||
ccmd = ex._guest_host_command("status", **kwargs)
|
||||
self.assertEqual(lcmd[2:], dcmd[2:-1])
|
||||
self.assertEqual(lcmd[2:], ccmd[2:-1])
|
||||
finally:
|
||||
chv.tearDown()
|
||||
|
||||
def test_runner_timeout_helper(self):
|
||||
seen = []
|
||||
|
||||
def only_cmd(cmd):
|
||||
seen.append(("cmd-only", list(cmd)))
|
||||
return FakeResult(0, "", "")
|
||||
|
||||
out = _call_runner_with_timeout(only_cmd, ["x"], timeout=5)
|
||||
self.assertEqual(out.returncode, 0)
|
||||
self.assertEqual(seen, [("cmd-only", ["x"])])
|
||||
|
||||
def with_timeout(cmd, timeout=None):
|
||||
seen.append(("with-timeout", timeout))
|
||||
return FakeResult(0, "", "")
|
||||
|
||||
_call_runner_with_timeout(with_timeout, ["x"], timeout=7)
|
||||
self.assertEqual(seen[-1], ("with-timeout", 7))
|
||||
_call_runner_with_timeout(with_timeout, ["x"], timeout=None)
|
||||
self.assertEqual(seen[-1], ("with-timeout", None))
|
||||
|
||||
|
||||
# -- 安全:known_hosts 钉死 -------------------------------------------------
|
||||
class SshPinningTest(ChvOptCase):
|
||||
def test_known_hosts_path_is_per_vm(self):
|
||||
ex = self.make_exec()
|
||||
a = ex.known_hosts_path({"id": "t-a"})
|
||||
b = ex.known_hosts_path({"id": "t-b"})
|
||||
self.assertNotEqual(a, b)
|
||||
self.assertEqual(a, os.path.join(self.state_dir, "chv-t-a",
|
||||
"known_hosts"))
|
||||
|
||||
def test_capture_host_key_writes_pinned_file_0600(self):
|
||||
def keyscan(cmd, timeout=None):
|
||||
self.calls.append(list(cmd))
|
||||
return FakeResult(
|
||||
0, "[127.0.0.1]:2222 ssh-ed25519 AAAAB3test\n"
|
||||
"evil.example.com ssh-rsa AAAAEvil\n"
|
||||
"# comment\n", "")
|
||||
ex = self.make_exec(runner=keyscan)
|
||||
self.assertTrue(ex.capture_host_key(self._task()))
|
||||
path = ex.known_hosts_path(self._task())
|
||||
with open(path) as fh:
|
||||
content = fh.read()
|
||||
self.assertIn("[127.0.0.1]:2222 ssh-ed25519 AAAAB3test",
|
||||
content)
|
||||
self.assertNotIn("evil.example.com", content)
|
||||
self.assertEqual(os.stat(path).st_mode & 0o777, 0o600)
|
||||
|
||||
def test_capture_host_key_empty_is_false_not_error(self):
|
||||
ex = self.make_exec(
|
||||
runner=lambda cmd, timeout=None: FakeResult(0, "", ""))
|
||||
self.assertFalse(ex.capture_host_key(self._task()))
|
||||
self.assertFalse(os.path.exists(
|
||||
ex.known_hosts_path(self._task())))
|
||||
|
||||
def test_wait_guest_ready_captures_before_ssh(self):
|
||||
def runner(cmd, timeout=None):
|
||||
self.calls.append(list(cmd))
|
||||
if cmd[0] == "ssh-keyscan":
|
||||
return FakeResult(
|
||||
0, "[127.0.0.1]:2222 ssh-ed25519 AAAAB3test\n",
|
||||
"")
|
||||
if cmd[0] == "ssh" and cmd[-1] == "true":
|
||||
return FakeResult(0, "", "")
|
||||
return FakeResult(0, "", "")
|
||||
ex = self.make_exec(runner=runner)
|
||||
self.assertTrue(ex.wait_guest_ready(self._task()))
|
||||
self.assertTrue(os.path.exists(
|
||||
ex.known_hosts_path(self._task())))
|
||||
keyscan_idx = next(i for i, c in enumerate(self.calls)
|
||||
if c[0] == "ssh-keyscan")
|
||||
ssh_idx = next(i for i, c in enumerate(self.calls)
|
||||
if c[0] == "ssh" and c[-1] == "true")
|
||||
self.assertLess(keyscan_idx, ssh_idx)
|
||||
|
||||
def test_ssh_user_port_explicit_config(self):
|
||||
ex = self.make_exec(ssh_user="codex", ssh_port=2201)
|
||||
cmd = ex.build_ssh_command(self._task(), ["true"])
|
||||
joined = " ".join(cmd)
|
||||
self.assertIn("codex@127.0.0.1", joined)
|
||||
self.assertNotIn("root@127.0.0.1", joined)
|
||||
self.assertIn("2201", joined)
|
||||
|
||||
def test_ssh_user_port_tightened_validation(self):
|
||||
with self.assertRaises(ValueError):
|
||||
self.make_exec(ssh_user="")
|
||||
with self.assertRaises(ValueError):
|
||||
self.make_exec(ssh_port=0)
|
||||
with self.assertRaises(ValueError):
|
||||
self.make_exec(ssh_port=70000)
|
||||
with self.assertRaises(ValueError):
|
||||
self.make_exec(ssh_port="abc")
|
||||
with self.assertRaises(ValueError):
|
||||
self.make_exec(env_extra={"CHV_SSH_PORT": "0"})
|
||||
|
||||
|
||||
# -- execute 语义 -------------------------------------------------------------
|
||||
class ExecuteGateTest(ChvOptCase):
|
||||
def test_execute_without_running_vm_is_classified(self):
|
||||
ex = self.make_exec()
|
||||
with self.assertRaises(ChvFailure) as ctx:
|
||||
ex.execute(self._task())
|
||||
self.assertEqual(ctx.exception.kind, "vm_not_running")
|
||||
self.assertEqual(self.launches, [])
|
||||
self.assertFalse(any(c[0] == "ssh" for c in self.calls))
|
||||
|
||||
def test_execute_with_dead_proc_and_no_socket_is_classified(self):
|
||||
ex = self.make_exec()
|
||||
ex._vms["chv-abc123"] = FakeProc(alive=False)
|
||||
with self.assertRaises(ChvFailure) as ctx:
|
||||
ex.execute(self._task())
|
||||
self.assertEqual(ctx.exception.kind, "vm_not_running")
|
||||
|
||||
def test_execute_with_socket_but_no_registry_proceeds(self):
|
||||
ex = self.make_exec()
|
||||
os.makedirs(self.state_dir, exist_ok=True)
|
||||
with open(os.path.join(self.state_dir,
|
||||
"chv-abc123.sock"), "w") as fh:
|
||||
fh.write("")
|
||||
out = ex.execute(self._task())
|
||||
self.assertIn("ok-status", out["stdout"])
|
||||
self.assertEqual(self.launches, [])
|
||||
|
||||
|
||||
# -- 预热池持久化与孤儿清理 ----------------------------------------------------
|
||||
class StubExecutor:
|
||||
"""WarmPool 测试替身:只实现池子调用的鸭子接口。"""
|
||||
|
||||
def __init__(self, state_dir):
|
||||
self.state_dir = state_dir
|
||||
self.warm_pool_size = 2
|
||||
self.snapshot_url = "file:///snap/base"
|
||||
self.wake_calls = []
|
||||
self.pause_calls = []
|
||||
self.shutdown_calls = []
|
||||
self.audit = []
|
||||
|
||||
def vm_name(self, task):
|
||||
raw = task.get("id") if isinstance(task, dict) else str(task)
|
||||
return raw if raw.startswith("chv-") else "chv-" + raw
|
||||
|
||||
def wake_vm(self, task):
|
||||
self.wake_calls.append(task["id"])
|
||||
return {"vm": self.vm_name(task), "state": "running"}
|
||||
|
||||
def pause_vm(self, task):
|
||||
self.pause_calls.append(task["id"])
|
||||
return True
|
||||
|
||||
def shutdown_vm(self, task_or_name):
|
||||
name = task_or_name if isinstance(task_or_name, str) \
|
||||
else self.vm_name(task_or_name)
|
||||
self.shutdown_calls.append(name)
|
||||
return True
|
||||
|
||||
def _emit(self, event, task, detail="", kind=None):
|
||||
self.audit.append(event)
|
||||
|
||||
|
||||
class PoolRegistryTest(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.tmp = tempfile.TemporaryDirectory()
|
||||
self.state_dir = os.path.join(self.tmp.name, "state")
|
||||
os.makedirs(self.state_dir, exist_ok=True)
|
||||
self.registry = os.path.join(self.state_dir, "warm-pool.json")
|
||||
|
||||
def tearDown(self):
|
||||
self.tmp.cleanup()
|
||||
|
||||
def test_persist_load_orphan_reconcile(self):
|
||||
stub = StubExecutor(self.state_dir)
|
||||
pool = WarmPool(stub, size=2, snapshot_url="file:///snap/base")
|
||||
added = pool.prewarm()
|
||||
self.assertEqual(len(added), 2)
|
||||
self.assertTrue(os.path.exists(self.registry))
|
||||
with open(self.registry) as fh:
|
||||
data = json.load(fh)
|
||||
self.assertEqual(len(data), 2)
|
||||
# 新池实例载入:上一世条目只当孤儿,绝不发给任务:
|
||||
stub2 = StubExecutor(self.state_dir)
|
||||
pool2 = WarmPool(stub2, size=2,
|
||||
snapshot_url="file:///snap/base")
|
||||
self.assertEqual(pool2.warm_count(), 0)
|
||||
self.assertEqual(pool2.orphan_count(), 2)
|
||||
self.assertEqual(pool2.acquire({"id": "t1"})["mode"], "cold")
|
||||
cleaned = pool2.reconcile_orphans()
|
||||
self.assertEqual(len(cleaned), 2)
|
||||
self.assertEqual(sorted(stub2.shutdown_calls), sorted(cleaned))
|
||||
self.assertEqual(pool2.orphan_count(), 0)
|
||||
with open(self.registry) as fh:
|
||||
self.assertEqual(json.load(fh), {})
|
||||
# 第三实例:无残留:
|
||||
pool3 = WarmPool(StubExecutor(self.state_dir), size=2,
|
||||
snapshot_url="file:///snap/base")
|
||||
self.assertEqual(pool3.orphan_count(), 0)
|
||||
|
||||
def test_corrupt_registry_quarantines_and_starts_empty(self):
|
||||
with open(self.registry, "w") as fh:
|
||||
fh.write("{not json")
|
||||
pool = WarmPool(StubExecutor(self.state_dir), size=2,
|
||||
snapshot_url="file:///snap/base")
|
||||
self.assertTrue(pool.registry_corrupt)
|
||||
self.assertEqual(pool.warm_count(), 0)
|
||||
self.assertEqual(pool.orphan_count(), 0)
|
||||
self.assertTrue(os.path.exists(self.registry + ".corrupt"))
|
||||
self.assertFalse(os.path.exists(self.registry))
|
||||
# 损坏后仍可正常预热(registry 重建):
|
||||
self.assertEqual(len(pool.prewarm()), 2)
|
||||
self.assertTrue(os.path.exists(self.registry))
|
||||
|
||||
def test_assigned_entry_on_load_is_never_served(self):
|
||||
with open(self.registry, "w") as fh:
|
||||
json.dump({"warm-1": {"state": "assigned",
|
||||
"vm": "chv-warm-1",
|
||||
"task_id": "old-task",
|
||||
"snapshot_url": "file:///snap/base"}},
|
||||
fh)
|
||||
pool = WarmPool(StubExecutor(self.state_dir), size=2,
|
||||
snapshot_url="file:///snap/base")
|
||||
self.assertEqual(pool.warm_count(), 0)
|
||||
self.assertEqual(pool.acquire({"id": "t1"})["mode"], "cold")
|
||||
self.assertEqual(pool.reconcile_orphans(), ["chv-warm-1"])
|
||||
|
||||
def test_release_never_rewarms_and_registry_clears(self):
|
||||
stub = StubExecutor(self.state_dir)
|
||||
pool = WarmPool(stub, size=1, snapshot_url="file:///snap/base")
|
||||
added = pool.prewarm()
|
||||
got = pool.acquire({"id": "t1"})
|
||||
self.assertEqual(got["mode"], "warm")
|
||||
self.assertTrue(pool.release({"id": "t1"}, pool_vm=added[0]))
|
||||
self.assertEqual(pool.warm_count(), 0)
|
||||
with open(self.registry) as fh:
|
||||
self.assertEqual(json.load(fh), {})
|
||||
|
||||
|
||||
# -- 长任务中途死亡 --------------------------------------------------------------
|
||||
class MidDeathTest(ChvOptCase):
|
||||
def _dispatch_with(self, runner):
|
||||
ex = self.make_exec(runner=runner)
|
||||
return ex.dispatch_task(self._task())
|
||||
|
||||
def test_failed_result_with_dead_vm_is_interrupted(self):
|
||||
proc_holder = []
|
||||
|
||||
def launcher(cmd):
|
||||
proc = FakeProc(alive=True)
|
||||
proc_holder.append(proc)
|
||||
self.launches.append(list(cmd))
|
||||
return proc
|
||||
|
||||
def runner(cmd, timeout=None):
|
||||
self.calls.append(list(cmd))
|
||||
if cmd[0] == "ssh" and "dispatch" in cmd:
|
||||
proc_holder[0]._alive = False
|
||||
return FakeResult(1, "",
|
||||
"Connection closed by remote host")
|
||||
if cmd[0] == "ssh":
|
||||
return FakeResult(0, "", "")
|
||||
return FakeResult(0, "", "")
|
||||
|
||||
ex = self.make_exec(runner=runner, launcher=launcher)
|
||||
out = ex.dispatch_task(self._task())
|
||||
self.assertEqual(out["status"], "interrupted")
|
||||
self.assertTrue(out["vm_dead"])
|
||||
|
||||
def test_failed_result_with_live_vm_stays_failed(self):
|
||||
def runner(cmd, timeout=None):
|
||||
self.calls.append(list(cmd))
|
||||
if cmd[0] == "ssh" and "dispatch" in cmd:
|
||||
return FakeResult(1, "", "host dispatch failed")
|
||||
if cmd[0] == "ssh":
|
||||
return FakeResult(0, "", "")
|
||||
return FakeResult(0, "", "")
|
||||
|
||||
ex = self.make_exec(runner=runner)
|
||||
out = ex.dispatch_task(self._task())
|
||||
self.assertEqual(out["status"], "failed")
|
||||
self.assertNotIn("vm_dead", out)
|
||||
|
||||
def test_exception_with_dead_vm_is_interrupted_not_raised(self):
|
||||
proc_holder = []
|
||||
|
||||
def launcher(cmd):
|
||||
proc = FakeProc(alive=True)
|
||||
proc_holder.append(proc)
|
||||
self.launches.append(list(cmd))
|
||||
return proc
|
||||
|
||||
def runner(cmd, timeout=None):
|
||||
self.calls.append(list(cmd))
|
||||
if cmd[0] == "ssh" and "dispatch" in cmd:
|
||||
proc_holder[0]._alive = False
|
||||
raise RuntimeError("ssh transport broken")
|
||||
if cmd[0] == "ssh":
|
||||
return FakeResult(0, "", "")
|
||||
return FakeResult(0, "", "")
|
||||
|
||||
ex = self.make_exec(runner=runner, launcher=launcher)
|
||||
out = ex.dispatch_task(self._task())
|
||||
self.assertEqual(out["status"], "interrupted")
|
||||
self.assertTrue(out["vm_dead"])
|
||||
|
||||
def test_exception_with_live_vm_propagates(self):
|
||||
def runner(cmd, timeout=None):
|
||||
self.calls.append(list(cmd))
|
||||
if cmd[0] == "ssh" and "dispatch" in cmd:
|
||||
raise RuntimeError("ssh transport broken")
|
||||
if cmd[0] == "ssh":
|
||||
return FakeResult(0, "", "")
|
||||
return FakeResult(0, "", "")
|
||||
|
||||
ex = self.make_exec(runner=runner)
|
||||
with self.assertRaises(RuntimeError):
|
||||
ex.dispatch_task(self._task())
|
||||
|
||||
def test_goal_exception_with_dead_vm_is_interrupted(self):
|
||||
proc_holder = []
|
||||
|
||||
def launcher(cmd):
|
||||
proc = FakeProc(alive=True)
|
||||
proc_holder.append(proc)
|
||||
self.launches.append(list(cmd))
|
||||
return proc
|
||||
|
||||
def runner(cmd, timeout=None):
|
||||
self.calls.append(list(cmd))
|
||||
if cmd[0] == "ssh" and "goal" in cmd:
|
||||
proc_holder[0]._alive = False
|
||||
raise RuntimeError("ssh transport broken")
|
||||
if cmd[0] == "ssh":
|
||||
return FakeResult(0, "", "")
|
||||
return FakeResult(0, "", "")
|
||||
|
||||
ex = self.make_exec(runner=runner, launcher=launcher)
|
||||
out = ex.goal_task({"id": "g1", "prompt": "做目标",
|
||||
"goal_mode": True})
|
||||
self.assertEqual(out["status"], "interrupted")
|
||||
self.assertIsNone(out["goal_met"])
|
||||
self.assertTrue(out["vm_dead"])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -279,6 +279,14 @@ class ConfigCommandTest(ChvCase):
|
||||
self.assertNotIn("SECRET", joined)
|
||||
# egress 代理地址在 env 前缀里(不是密钥),且只在代理已设时出现:
|
||||
self.assertIn("HTTPS_PROXY=http://10.0.0.1:8888", joined)
|
||||
# 本轨安全默认变化(#216 的 no + /dev/null 已废除):
|
||||
# host key 钉死在每 VM 的 known_hosts 文件上:
|
||||
self.assertIn("StrictHostKeyChecking=yes", joined)
|
||||
self.assertNotIn("StrictHostKeyChecking=no", joined)
|
||||
self.assertNotIn("/dev/null", joined)
|
||||
self.assertIn(os.path.join(self.state_dir,
|
||||
"chv-abc123", "known_hosts"),
|
||||
joined)
|
||||
|
||||
def test_isolation_names_and_disks_differ(self):
|
||||
ex = self.make_exec()
|
||||
@@ -534,6 +542,10 @@ class DispatchTest(ChvCase):
|
||||
|
||||
def test_execute_is_ssh_status_only(self):
|
||||
ex = self.make_exec()
|
||||
# 本轨 execute 语义门禁:只读已运行 VM。先登记一个活
|
||||
# 的 VMM 进程表示 VM 已醒(未登记时 execute 报
|
||||
# vm_not_running,见 test_chv_optimize.py):
|
||||
ex._vms["chv-abc123"] = FakeProc(alive=True)
|
||||
out = ex.execute(self._task())
|
||||
self.assertEqual(out["executor"], "cloud-hypervisor")
|
||||
self.assertIn("ok-status", out["stdout"])
|
||||
|
||||
Reference in new issue
Block a user