host
9771ed9e8c
P0：seed 磁盘 CIDATA ISO 挂进 VmConfig 第二块只读盘； restore provisioned 下复用钉死 known_hosts、缺失 fail-fast 绝不 keyscan；preflight 拦截 CHV_DISK_MODE=reuse（显式 实验二次确认 + 审计放行）；SSH 目标选客机 IP 方案（显式 host > net 静态 ip > 显式转发宣称回环，三者皆无 fail-fast）。 P1：网关 extra_env 经 SSH env 前缀注入、worker 守卫收窄到 Local/CHV；取消/清理/看门狗接 CHV 通用路径；WarmPool acquire 内 resume；容量门持久化计数；磁盘配额总量+余量 与已存在磁盘校验重拷；API 按 Content-Length 验完整性； 快照 file:// 验落盘再 shutdown；execute 死端口归类 vm_not_running。 P2 全部排除未做（用户范围收窄令，留待以后一轨）。 测试：原 84 + 新增 29 = 113 项，mock，无真 VM。
590 lines
23 KiB
Python
590 lines
23 KiB
Python
"""Cloud Hypervisor 优化轨（feat-chv-optimize，堆叠于 #216 之上）测试。
|
||
|
||
只用标准库与 mock runner/launcher/api_client，绝不启动真实
|
||
VM（与 test_cloud_hypervisor.py 同款边界）。覆盖：
|
||
- 拆模块：cloud/chv/* 新路径与 cloud/sandbox 旧路径是同一对象
|
||
- 去重复：Local 与 CHV 的 goal 解析在矩阵输入下结论全等；
|
||
三执行器 host 顶层参数顺序全等；runner 超时 helper
|
||
- 安全：每 VM known_hosts 钉死、ssh-keyscan 首次捕获（TOFU）、
|
||
非 root 用户与端口显式配置与收紧校验
|
||
- execute 语义：只读已运行 VM，未运行报 vm_not_running
|
||
- 预热池：registry 持久化、重启载入只当孤儿、reconcile 清理、
|
||
损坏 quarantine、一 次性不复用
|
||
- 长任务：VM 中途死亡判 interrupted 而非 failed/抛异常
|
||
"""
|
||
|
||
import json
|
||
import os
|
||
import sys
|
||
import tempfile
|
||
import unittest
|
||
from pathlib import Path
|
||
|
||
try:
|
||
from cloud.chv import ChvFailure
|
||
from cloud.chv import CloudHypervisorExecutor
|
||
from cloud.chv import WarmPool
|
||
from cloud.chv.api_client import _unix_socket_api as _chv_api
|
||
from cloud.chv.config import _parse_memory_bytes as _chv_mem
|
||
from cloud.sandbox import ChvFailure as _sb_failure
|
||
from cloud.sandbox import CloudHypervisorExecutor as _sb_exec
|
||
from cloud.sandbox import DockerExecutor
|
||
from cloud.sandbox import LocalExecutor
|
||
from cloud.sandbox import WarmPool as _sb_pool
|
||
from cloud.sandbox import _call_runner_with_timeout
|
||
from cloud.sandbox import _parse_memory_bytes as _sb_mem
|
||
from cloud.sandbox import _unix_socket_api as _sb_api
|
||
except ImportError: # discover -s cloud/tests 时把项目根目录补进 sys.path
|
||
sys.path.insert(0, str(Path(__file__).resolve().parents[2]))
|
||
from cloud.chv import ChvFailure
|
||
from cloud.chv import CloudHypervisorExecutor
|
||
from cloud.chv import WarmPool
|
||
from cloud.chv.api_client import _unix_socket_api as _chv_api
|
||
from cloud.chv.config import _parse_memory_bytes as _chv_mem
|
||
from cloud.sandbox import ChvFailure as _sb_failure
|
||
from cloud.sandbox import CloudHypervisorExecutor as _sb_exec
|
||
from cloud.sandbox import DockerExecutor
|
||
from cloud.sandbox import LocalExecutor
|
||
from cloud.sandbox import WarmPool as _sb_pool
|
||
from cloud.sandbox import _call_runner_with_timeout
|
||
from cloud.sandbox import _parse_memory_bytes as _sb_mem
|
||
from cloud.sandbox import _unix_socket_api as _sb_api
|
||
|
||
|
||
class FakeResult:
|
||
def __init__(self, returncode=0, stdout="", stderr=""):
|
||
self.returncode = returncode
|
||
self.stdout = stdout
|
||
self.stderr = stderr
|
||
|
||
|
||
class FakeProc:
|
||
def __init__(self, alive=True):
|
||
self._alive = alive
|
||
self.terminated = False
|
||
|
||
def poll(self):
|
||
return None if self._alive else 1
|
||
|
||
def terminate(self):
|
||
self.terminated = True
|
||
self._alive = False
|
||
|
||
def wait(self, timeout=None):
|
||
return 0
|
||
|
||
def kill(self):
|
||
self._alive = False
|
||
|
||
|
||
def _write(path, content=b"x"):
|
||
with open(path, "wb") as fh:
|
||
fh.write(content)
|
||
os.chmod(path, 0o755)
|
||
return path
|
||
|
||
|
||
class ChvOptCase(unittest.TestCase):
|
||
def setUp(self):
|
||
self.tmp = tempfile.TemporaryDirectory()
|
||
self.dir = self.tmp.name
|
||
self.binary = _write(os.path.join(self.dir, "cloud-hypervisor"))
|
||
self.ch_remote = _write(os.path.join(self.dir, "ch-remote"))
|
||
self.kernel = _write(os.path.join(self.dir, "vmlinux"))
|
||
self.rootfs = _write(os.path.join(self.dir, "disk.raw"),
|
||
b"R" * 100)
|
||
self.ssh_key = _write(os.path.join(self.dir, "id_chv"))
|
||
self.kvm = _write(os.path.join(self.dir, "kvm"))
|
||
self.cpuinfo = os.path.join(self.dir, "cpuinfo")
|
||
with open(self.cpuinfo, "w") as fh:
|
||
fh.write("flags : fpu vmx sse\n")
|
||
self.state_dir = os.path.join(self.dir, "state")
|
||
self.calls = []
|
||
self.api_calls = []
|
||
self.launches = []
|
||
|
||
def tearDown(self):
|
||
self.tmp.cleanup()
|
||
|
||
def make_exec(self, runner=None, api_client=None, launcher=None,
|
||
env_extra=None, **overrides):
|
||
env = {"CHV_EGRESS_PROXY": "http://10.0.0.1:8888",
|
||
# 实验室宣称：mock runner 假装回环端口已转发到客机
|
||
#（feat-chv-fix 起，执行器不再无条件假设回环能通）。
|
||
"CHV_SSH_FORWARD": "1"}
|
||
if env_extra:
|
||
env.update(env_extra)
|
||
if runner is None:
|
||
runner = self._default_runner
|
||
if api_client is None:
|
||
api_client = self._default_api
|
||
if launcher is None:
|
||
launcher = self._default_launcher
|
||
kwargs = dict(
|
||
binary=self.binary, ch_remote=self.ch_remote,
|
||
kernel=self.kernel, rootfs=self.rootfs,
|
||
ssh_key=self.ssh_key, kvm_path=self.kvm,
|
||
cpuinfo_path=self.cpuinfo, state_dir=self.state_dir,
|
||
runner=runner, launcher=launcher, api_client=api_client,
|
||
sleeper=lambda s: None, env=env)
|
||
kwargs.update(overrides)
|
||
return CloudHypervisorExecutor(**kwargs)
|
||
|
||
def _default_runner(self, cmd, timeout=None):
|
||
self.calls.append(list(cmd))
|
||
if cmd[0] == "ssh":
|
||
if cmd[-1] == "true":
|
||
return FakeResult(0, "", "")
|
||
if "dispatch" in cmd:
|
||
return FakeResult(0, json.dumps(
|
||
{"status": "completed", "text": "ok-dispatch"}), "")
|
||
if "goal" in cmd:
|
||
return FakeResult(0, "goal met after rounds\n", "")
|
||
if "status" in cmd:
|
||
return FakeResult(0, json.dumps(
|
||
{"status": "completed", "text": "ok-status"}), "")
|
||
return FakeResult(0, "", "")
|
||
return FakeResult(0, "", "")
|
||
|
||
def _default_api(self, socket_path, method, path, payload=None):
|
||
self.api_calls.append((method, path, payload))
|
||
return (200, {})
|
||
|
||
def _default_launcher(self, cmd):
|
||
self.launches.append(list(cmd))
|
||
return FakeProc(alive=True)
|
||
|
||
def _task(self, task_id="abc123"):
|
||
return {"id": task_id, "prompt": "做一个加法函数",
|
||
"project_id": "proj1"}
|
||
|
||
|
||
# -- 拆模块 ---------------------------------------------------------------
|
||
class SplitTest(unittest.TestCase):
|
||
def test_new_and_old_import_paths_are_same_objects(self):
|
||
self.assertIs(CloudHypervisorExecutor, _sb_exec)
|
||
self.assertIs(WarmPool, _sb_pool)
|
||
self.assertIs(ChvFailure, _sb_failure)
|
||
self.assertIs(_chv_mem, _sb_mem)
|
||
self.assertIs(_chv_api, _sb_api)
|
||
|
||
def test_classes_live_in_chv_modules(self):
|
||
self.assertEqual(CloudHypervisorExecutor.__module__,
|
||
"cloud.chv.executor")
|
||
self.assertEqual(WarmPool.__module__, "cloud.chv.pool")
|
||
self.assertEqual(ChvFailure.__module__, "cloud.chv.errors")
|
||
|
||
|
||
# -- 去重复：goal 解析与 host 命令 ------------------------------------------
|
||
class SharedGoalTest(unittest.TestCase):
|
||
CASES = [
|
||
# (returncode, stdout, stderr)
|
||
(0, "goal met after rounds\n", ""),
|
||
(0, "round 1 verify_out=bad\ngoal unmet after rounds\n", ""),
|
||
(1, "goal interrupted after rounds: boom\n", ""),
|
||
(1, "KeyError: 'x'\n", ""),
|
||
(1, "some odd failure\n", ""),
|
||
(0, "nothing conclusive\n", ""),
|
||
(1, "goal-resume 拒绝：断点丢失\n", ""),
|
||
(1, "任务已存在\n", ""),
|
||
(1, "goal met after rounds\n", ""),
|
||
(1, "goal unmet after rounds\n", ""),
|
||
(1, "", "HTTP 503 Service Unavailable\n"),
|
||
]
|
||
|
||
def test_local_and_chv_goal_conclusions_match(self):
|
||
local = LocalExecutor("/tmp/host.py")
|
||
chv = ChvOptCase()
|
||
chv.setUp()
|
||
try:
|
||
ex = chv.make_exec()
|
||
for rc, out, err in self.CASES:
|
||
res = FakeResult(rc, out, err)
|
||
a = local._goal_result_from_runner("t1", res)
|
||
b = ex._goal_result("t1", res)
|
||
self.assertEqual(
|
||
(a["status"], a["goal_met"]),
|
||
(b["status"], b["goal_met"]),
|
||
"rc=%s stdout=%r" % (rc, out))
|
||
self.assertEqual(a["executor"], "local")
|
||
self.assertEqual(b["executor"], "cloud-hypervisor")
|
||
finally:
|
||
chv.tearDown()
|
||
|
||
def test_already_exists_message_points_to_continue(self):
|
||
chv = ChvOptCase()
|
||
chv.setUp()
|
||
try:
|
||
ex = chv.make_exec()
|
||
out = ex._goal_result(
|
||
"t1", FakeResult(1, "任务已存在\n", ""))
|
||
self.assertEqual(out["status"], "failed")
|
||
self.assertIn("继续", out["goal_error"])
|
||
finally:
|
||
chv.tearDown()
|
||
|
||
def test_host_command_options_identical_across_executors(self):
|
||
chv = ChvOptCase()
|
||
chv.setUp()
|
||
try:
|
||
ex = chv.make_exec()
|
||
local = LocalExecutor("/tmp/host.py")
|
||
docker = DockerExecutor()
|
||
for kwargs in (
|
||
{"model": "m1", "effort": "low", "approval": "ask"},
|
||
{"model": "m1", "effort": "low", "approval": "full",
|
||
"sandbox": "read-only",
|
||
"model_provider": "sub2api"},
|
||
):
|
||
lcmd = local._base_command(**kwargs)
|
||
dcmd = docker._inner_host_command("status", **kwargs)
|
||
ccmd = ex._guest_host_command("status", **kwargs)
|
||
self.assertEqual(lcmd[2:], dcmd[2:-1])
|
||
self.assertEqual(lcmd[2:], ccmd[2:-1])
|
||
finally:
|
||
chv.tearDown()
|
||
|
||
def test_runner_timeout_helper(self):
|
||
seen = []
|
||
|
||
def only_cmd(cmd):
|
||
seen.append(("cmd-only", list(cmd)))
|
||
return FakeResult(0, "", "")
|
||
|
||
out = _call_runner_with_timeout(only_cmd, ["x"], timeout=5)
|
||
self.assertEqual(out.returncode, 0)
|
||
self.assertEqual(seen, [("cmd-only", ["x"])])
|
||
|
||
def with_timeout(cmd, timeout=None):
|
||
seen.append(("with-timeout", timeout))
|
||
return FakeResult(0, "", "")
|
||
|
||
_call_runner_with_timeout(with_timeout, ["x"], timeout=7)
|
||
self.assertEqual(seen[-1], ("with-timeout", 7))
|
||
_call_runner_with_timeout(with_timeout, ["x"], timeout=None)
|
||
self.assertEqual(seen[-1], ("with-timeout", None))
|
||
|
||
|
||
# -- 安全：known_hosts 钉死 -------------------------------------------------
|
||
class SshPinningTest(ChvOptCase):
|
||
def test_known_hosts_path_is_per_vm(self):
|
||
ex = self.make_exec()
|
||
a = ex.known_hosts_path({"id": "t-a"})
|
||
b = ex.known_hosts_path({"id": "t-b"})
|
||
self.assertNotEqual(a, b)
|
||
self.assertEqual(a, os.path.join(self.state_dir, "chv-t-a",
|
||
"known_hosts"))
|
||
|
||
def test_capture_host_key_writes_pinned_file_0600(self):
|
||
def keyscan(cmd, timeout=None):
|
||
self.calls.append(list(cmd))
|
||
return FakeResult(
|
||
0, "[127.0.0.1]:2222 ssh-ed25519 AAAAB3test\n"
|
||
"evil.example.com ssh-rsa AAAAEvil\n"
|
||
"# comment\n", "")
|
||
ex = self.make_exec(runner=keyscan)
|
||
self.assertTrue(ex.capture_host_key(self._task()))
|
||
path = ex.known_hosts_path(self._task())
|
||
with open(path) as fh:
|
||
content = fh.read()
|
||
self.assertIn("[127.0.0.1]:2222 ssh-ed25519 AAAAB3test",
|
||
content)
|
||
self.assertNotIn("evil.example.com", content)
|
||
self.assertEqual(os.stat(path).st_mode & 0o777, 0o600)
|
||
|
||
def test_capture_host_key_empty_is_false_not_error(self):
|
||
ex = self.make_exec(
|
||
runner=lambda cmd, timeout=None: FakeResult(0, "", ""))
|
||
self.assertFalse(ex.capture_host_key(self._task()))
|
||
self.assertFalse(os.path.exists(
|
||
ex.known_hosts_path(self._task())))
|
||
|
||
def test_wait_guest_ready_captures_before_ssh(self):
|
||
def runner(cmd, timeout=None):
|
||
self.calls.append(list(cmd))
|
||
if cmd[0] == "ssh-keyscan":
|
||
return FakeResult(
|
||
0, "[127.0.0.1]:2222 ssh-ed25519 AAAAB3test\n",
|
||
"")
|
||
if cmd[0] == "ssh" and cmd[-1] == "true":
|
||
return FakeResult(0, "", "")
|
||
return FakeResult(0, "", "")
|
||
ex = self.make_exec(runner=runner)
|
||
self.assertTrue(ex.wait_guest_ready(self._task()))
|
||
self.assertTrue(os.path.exists(
|
||
ex.known_hosts_path(self._task())))
|
||
keyscan_idx = next(i for i, c in enumerate(self.calls)
|
||
if c[0] == "ssh-keyscan")
|
||
ssh_idx = next(i for i, c in enumerate(self.calls)
|
||
if c[0] == "ssh" and c[-1] == "true")
|
||
self.assertLess(keyscan_idx, ssh_idx)
|
||
|
||
def test_ssh_user_port_explicit_config(self):
|
||
ex = self.make_exec(ssh_user="codex", ssh_port=2201)
|
||
cmd = ex.build_ssh_command(self._task(), ["true"])
|
||
joined = " ".join(cmd)
|
||
self.assertIn("codex@127.0.0.1", joined)
|
||
self.assertNotIn("root@127.0.0.1", joined)
|
||
self.assertIn("2201", joined)
|
||
|
||
def test_ssh_user_port_tightened_validation(self):
|
||
with self.assertRaises(ValueError):
|
||
self.make_exec(ssh_user="")
|
||
with self.assertRaises(ValueError):
|
||
self.make_exec(ssh_port=0)
|
||
with self.assertRaises(ValueError):
|
||
self.make_exec(ssh_port=70000)
|
||
with self.assertRaises(ValueError):
|
||
self.make_exec(ssh_port="abc")
|
||
with self.assertRaises(ValueError):
|
||
self.make_exec(env_extra={"CHV_SSH_PORT": "0"})
|
||
|
||
|
||
# -- execute 语义 -------------------------------------------------------------
|
||
class ExecuteGateTest(ChvOptCase):
|
||
def test_execute_without_running_vm_is_classified(self):
|
||
ex = self.make_exec()
|
||
with self.assertRaises(ChvFailure) as ctx:
|
||
ex.execute(self._task())
|
||
self.assertEqual(ctx.exception.kind, "vm_not_running")
|
||
self.assertEqual(self.launches, [])
|
||
self.assertFalse(any(c[0] == "ssh" for c in self.calls))
|
||
|
||
def test_execute_with_dead_proc_and_no_socket_is_classified(self):
|
||
ex = self.make_exec()
|
||
ex._vms["chv-abc123"] = FakeProc(alive=False)
|
||
with self.assertRaises(ChvFailure) as ctx:
|
||
ex.execute(self._task())
|
||
self.assertEqual(ctx.exception.kind, "vm_not_running")
|
||
|
||
def test_execute_with_socket_but_no_registry_proceeds(self):
|
||
ex = self.make_exec()
|
||
os.makedirs(self.state_dir, exist_ok=True)
|
||
with open(os.path.join(self.state_dir,
|
||
"chv-abc123.sock"), "w") as fh:
|
||
fh.write("")
|
||
out = ex.execute(self._task())
|
||
self.assertIn("ok-status", out["stdout"])
|
||
self.assertEqual(self.launches, [])
|
||
|
||
|
||
# -- 预热池持久化与孤儿清理 ----------------------------------------------------
|
||
class StubExecutor:
|
||
"""WarmPool 测试替身：只实现池子调用的鸭子接口。"""
|
||
|
||
def __init__(self, state_dir):
|
||
self.state_dir = state_dir
|
||
self.warm_pool_size = 2
|
||
self.snapshot_url = "file:///snap/base"
|
||
self.wake_calls = []
|
||
self.pause_calls = []
|
||
self.shutdown_calls = []
|
||
self.audit = []
|
||
|
||
def vm_name(self, task):
|
||
raw = task.get("id") if isinstance(task, dict) else str(task)
|
||
return raw if raw.startswith("chv-") else "chv-" + raw
|
||
|
||
def wake_vm(self, task):
|
||
self.wake_calls.append(task["id"])
|
||
return {"vm": self.vm_name(task), "state": "running"}
|
||
|
||
def pause_vm(self, task):
|
||
self.pause_calls.append(task["id"])
|
||
return True
|
||
|
||
def shutdown_vm(self, task_or_name):
|
||
name = task_or_name if isinstance(task_or_name, str) \
|
||
else self.vm_name(task_or_name)
|
||
self.shutdown_calls.append(name)
|
||
return True
|
||
|
||
def _emit(self, event, task, detail="", kind=None):
|
||
self.audit.append(event)
|
||
|
||
|
||
class PoolRegistryTest(unittest.TestCase):
|
||
def setUp(self):
|
||
self.tmp = tempfile.TemporaryDirectory()
|
||
self.state_dir = os.path.join(self.tmp.name, "state")
|
||
os.makedirs(self.state_dir, exist_ok=True)
|
||
self.registry = os.path.join(self.state_dir, "warm-pool.json")
|
||
|
||
def tearDown(self):
|
||
self.tmp.cleanup()
|
||
|
||
def test_persist_load_orphan_reconcile(self):
|
||
stub = StubExecutor(self.state_dir)
|
||
pool = WarmPool(stub, size=2, snapshot_url="file:///snap/base")
|
||
added = pool.prewarm()
|
||
self.assertEqual(len(added), 2)
|
||
self.assertTrue(os.path.exists(self.registry))
|
||
with open(self.registry) as fh:
|
||
data = json.load(fh)
|
||
self.assertEqual(len(data), 2)
|
||
# 新池实例载入：上一世条目只当孤儿，绝不发给任务：
|
||
stub2 = StubExecutor(self.state_dir)
|
||
pool2 = WarmPool(stub2, size=2,
|
||
snapshot_url="file:///snap/base")
|
||
self.assertEqual(pool2.warm_count(), 0)
|
||
self.assertEqual(pool2.orphan_count(), 2)
|
||
self.assertEqual(pool2.acquire({"id": "t1"})["mode"], "cold")
|
||
cleaned = pool2.reconcile_orphans()
|
||
self.assertEqual(len(cleaned), 2)
|
||
self.assertEqual(sorted(stub2.shutdown_calls), sorted(cleaned))
|
||
self.assertEqual(pool2.orphan_count(), 0)
|
||
with open(self.registry) as fh:
|
||
self.assertEqual(json.load(fh), {})
|
||
# 第三实例：无残留：
|
||
pool3 = WarmPool(StubExecutor(self.state_dir), size=2,
|
||
snapshot_url="file:///snap/base")
|
||
self.assertEqual(pool3.orphan_count(), 0)
|
||
|
||
def test_corrupt_registry_quarantines_and_starts_empty(self):
|
||
with open(self.registry, "w") as fh:
|
||
fh.write("{not json")
|
||
pool = WarmPool(StubExecutor(self.state_dir), size=2,
|
||
snapshot_url="file:///snap/base")
|
||
self.assertTrue(pool.registry_corrupt)
|
||
self.assertEqual(pool.warm_count(), 0)
|
||
self.assertEqual(pool.orphan_count(), 0)
|
||
self.assertTrue(os.path.exists(self.registry + ".corrupt"))
|
||
self.assertFalse(os.path.exists(self.registry))
|
||
# 损坏后仍可正常预热（registry 重建）：
|
||
self.assertEqual(len(pool.prewarm()), 2)
|
||
self.assertTrue(os.path.exists(self.registry))
|
||
|
||
def test_assigned_entry_on_load_is_never_served(self):
|
||
with open(self.registry, "w") as fh:
|
||
json.dump({"warm-1": {"state": "assigned",
|
||
"vm": "chv-warm-1",
|
||
"task_id": "old-task",
|
||
"snapshot_url": "file:///snap/base"}},
|
||
fh)
|
||
pool = WarmPool(StubExecutor(self.state_dir), size=2,
|
||
snapshot_url="file:///snap/base")
|
||
self.assertEqual(pool.warm_count(), 0)
|
||
self.assertEqual(pool.acquire({"id": "t1"})["mode"], "cold")
|
||
self.assertEqual(pool.reconcile_orphans(), ["chv-warm-1"])
|
||
|
||
def test_release_never_rewarms_and_registry_clears(self):
|
||
stub = StubExecutor(self.state_dir)
|
||
pool = WarmPool(stub, size=1, snapshot_url="file:///snap/base")
|
||
added = pool.prewarm()
|
||
got = pool.acquire({"id": "t1"})
|
||
self.assertEqual(got["mode"], "warm")
|
||
self.assertTrue(pool.release({"id": "t1"}, pool_vm=added[0]))
|
||
self.assertEqual(pool.warm_count(), 0)
|
||
with open(self.registry) as fh:
|
||
self.assertEqual(json.load(fh), {})
|
||
|
||
|
||
# -- 长任务中途死亡 --------------------------------------------------------------
|
||
class MidDeathTest(ChvOptCase):
|
||
def _dispatch_with(self, runner):
|
||
ex = self.make_exec(runner=runner)
|
||
return ex.dispatch_task(self._task())
|
||
|
||
def test_failed_result_with_dead_vm_is_interrupted(self):
|
||
proc_holder = []
|
||
|
||
def launcher(cmd):
|
||
proc = FakeProc(alive=True)
|
||
proc_holder.append(proc)
|
||
self.launches.append(list(cmd))
|
||
return proc
|
||
|
||
def runner(cmd, timeout=None):
|
||
self.calls.append(list(cmd))
|
||
if cmd[0] == "ssh" and "dispatch" in cmd:
|
||
proc_holder[0]._alive = False
|
||
return FakeResult(1, "",
|
||
"Connection closed by remote host")
|
||
if cmd[0] == "ssh":
|
||
return FakeResult(0, "", "")
|
||
return FakeResult(0, "", "")
|
||
|
||
ex = self.make_exec(runner=runner, launcher=launcher)
|
||
out = ex.dispatch_task(self._task())
|
||
self.assertEqual(out["status"], "interrupted")
|
||
self.assertTrue(out["vm_dead"])
|
||
|
||
def test_failed_result_with_live_vm_stays_failed(self):
|
||
def runner(cmd, timeout=None):
|
||
self.calls.append(list(cmd))
|
||
if cmd[0] == "ssh" and "dispatch" in cmd:
|
||
return FakeResult(1, "", "host dispatch failed")
|
||
if cmd[0] == "ssh":
|
||
return FakeResult(0, "", "")
|
||
return FakeResult(0, "", "")
|
||
|
||
ex = self.make_exec(runner=runner)
|
||
out = ex.dispatch_task(self._task())
|
||
self.assertEqual(out["status"], "failed")
|
||
self.assertNotIn("vm_dead", out)
|
||
|
||
def test_exception_with_dead_vm_is_interrupted_not_raised(self):
|
||
proc_holder = []
|
||
|
||
def launcher(cmd):
|
||
proc = FakeProc(alive=True)
|
||
proc_holder.append(proc)
|
||
self.launches.append(list(cmd))
|
||
return proc
|
||
|
||
def runner(cmd, timeout=None):
|
||
self.calls.append(list(cmd))
|
||
if cmd[0] == "ssh" and "dispatch" in cmd:
|
||
proc_holder[0]._alive = False
|
||
raise RuntimeError("ssh transport broken")
|
||
if cmd[0] == "ssh":
|
||
return FakeResult(0, "", "")
|
||
return FakeResult(0, "", "")
|
||
|
||
ex = self.make_exec(runner=runner, launcher=launcher)
|
||
out = ex.dispatch_task(self._task())
|
||
self.assertEqual(out["status"], "interrupted")
|
||
self.assertTrue(out["vm_dead"])
|
||
|
||
def test_exception_with_live_vm_propagates(self):
|
||
def runner(cmd, timeout=None):
|
||
self.calls.append(list(cmd))
|
||
if cmd[0] == "ssh" and "dispatch" in cmd:
|
||
raise RuntimeError("ssh transport broken")
|
||
if cmd[0] == "ssh":
|
||
return FakeResult(0, "", "")
|
||
return FakeResult(0, "", "")
|
||
|
||
ex = self.make_exec(runner=runner)
|
||
with self.assertRaises(RuntimeError):
|
||
ex.dispatch_task(self._task())
|
||
|
||
def test_goal_exception_with_dead_vm_is_interrupted(self):
|
||
proc_holder = []
|
||
|
||
def launcher(cmd):
|
||
proc = FakeProc(alive=True)
|
||
proc_holder.append(proc)
|
||
self.launches.append(list(cmd))
|
||
return proc
|
||
|
||
def runner(cmd, timeout=None):
|
||
self.calls.append(list(cmd))
|
||
if cmd[0] == "ssh" and "goal" in cmd:
|
||
proc_holder[0]._alive = False
|
||
raise RuntimeError("ssh transport broken")
|
||
if cmd[0] == "ssh":
|
||
return FakeResult(0, "", "")
|
||
return FakeResult(0, "", "")
|
||
|
||
ex = self.make_exec(runner=runner, launcher=launcher)
|
||
out = ex.goal_task({"id": "g1", "prompt": "做目标",
|
||
"goal_mode": True})
|
||
self.assertEqual(out["status"], "interrupted")
|
||
self.assertIsNone(out["goal_met"])
|
||
self.assertTrue(out["vm_dead"])
|
||
|
||
|
||
if __name__ == "__main__":
|
||
unittest.main()
|