feat: CHV 审核修复(P0+P1,堆叠于 #230)
P0:seed 磁盘 CIDATA ISO 挂进 VmConfig 第二块只读盘; restore provisioned 下复用钉死 known_hosts、缺失 fail-fast 绝不 keyscan;preflight 拦截 CHV_DISK_MODE=reuse(显式 实验二次确认 + 审计放行);SSH 目标选客机 IP 方案(显式 host > net 静态 ip > 显式转发宣称回环,三者皆无 fail-fast)。 P1:网关 extra_env 经 SSH env 前缀注入、worker 守卫收窄到 Local/CHV;取消/清理/看门狗接 CHV 通用路径;WarmPool acquire 内 resume;容量门持久化计数;磁盘配额总量+余量 与已存在磁盘校验重拷;API 按 Content-Length 验完整性; 快照 file:// 验落盘再 shutdown;execute 死端口归类 vm_not_running。 P2 全部排除未做(用户范围收窄令,留待以后一轨)。 测试:原 84 + 新增 29 = 113 项,mock,无真 VM。
This commit is contained in:
1 parent
22b83f118a
commit
9771ed9e8c
13 files changed
+1231
-128
No files matched your search
+243
-53
@@ -227,7 +227,8 @@ class CloudHypervisorExecutor(Executor):
|
||||
runner=None, launcher=None, ssh_key=None,
|
||||
ssh_user=None, ssh_port=None,
|
||||
guest_host_py="/app/host.py", cmdline=None,
|
||||
net=None, net_json=None, kvm_path="/dev/kvm",
|
||||
net=None, net_json=None, ssh_host=None,
|
||||
kvm_path="/dev/kvm",
|
||||
cpuinfo_path="/proc/cpuinfo", env=None,
|
||||
model=None, ch_remote=None, api_client=None,
|
||||
sleeper=None, api_timeout=None, ready_timeout=None,
|
||||
@@ -243,7 +244,8 @@ class CloudHypervisorExecutor(Executor):
|
||||
"state_dir": state_dir, "ssh_key": ssh_key,
|
||||
"ssh_user": ssh_user, "ssh_port": ssh_port,
|
||||
"guest_host_py": guest_host_py, "cmdline": cmdline,
|
||||
"net": net, "net_json": net_json, "kvm_path": kvm_path,
|
||||
"net": net, "net_json": net_json, "ssh_host": ssh_host,
|
||||
"kvm_path": kvm_path,
|
||||
"cpuinfo_path": cpuinfo_path, "ch_remote": ch_remote,
|
||||
"model": model, "api_timeout": api_timeout,
|
||||
"ready_timeout": ready_timeout,
|
||||
@@ -257,7 +259,8 @@ class CloudHypervisorExecutor(Executor):
|
||||
"max_vms", "disk_mode", "disk_gb", "egress_proxy",
|
||||
"allow_direct_egress", "task_timeout", "snapshot_url",
|
||||
"warm_pool_size", "api_timeout", "ready_timeout",
|
||||
"ready_interval", "host_key_mode"):
|
||||
"ready_interval", "host_key_mode", "ssh_host",
|
||||
"ssh_forward", "reuse_experiment"):
|
||||
setattr(self, _field, getattr(_cfg, _field))
|
||||
self.api_client = api_client if api_client is not None \
|
||||
else _unix_socket_api
|
||||
@@ -408,6 +411,17 @@ class CloudHypervisorExecutor(Executor):
|
||||
if not isinstance(_net, (list, dict)):
|
||||
missing.append(("net_config_invalid",
|
||||
"CHV_NET_JSON 须为 JSON list 或 dict"))
|
||||
# feat-chv-fix(P0-3):reuse 生产拦截。一行配置再也
|
||||
# 不能击穿租户隔离:只有显式实验二次确认
|
||||
# CHV_REUSE_SINGLE_TENANT_EXPERIMENT=1 时才放行,
|
||||
# 且放行本身在 wake_vm 里审计;否则 preflight 直接
|
||||
# 拒绝(kind=disk_mode_reuse_blocked)。
|
||||
if getattr(self, "disk_mode", "copy") == "reuse" \
|
||||
and not getattr(self, "reuse_experiment", False):
|
||||
missing.append(("disk_mode_reuse_blocked",
|
||||
"CHV_DISK_MODE=reuse 未经显式实验确认:"
|
||||
"须同时设 CHV_REUSE_SINGLE_TENANT_"
|
||||
"EXPERIMENT=1(仅单租户实验)"))
|
||||
if missing:
|
||||
kind = missing[0][0]
|
||||
detail = "; ".join(text for _, text in missing)
|
||||
@@ -463,6 +477,39 @@ class CloudHypervisorExecutor(Executor):
|
||||
return _known_hosts_path(self.state_dir,
|
||||
self.vm_name(task_or_name))
|
||||
|
||||
def _net_guest_ip(self):
|
||||
"""VmConfig net 首个带 ip 的条目(静态 tap 客机 IP)。"""
|
||||
if not self.net_json:
|
||||
return None
|
||||
try:
|
||||
net = json.loads(self.net_json)
|
||||
except ValueError:
|
||||
return None
|
||||
entries = net if isinstance(net, list) else [net]
|
||||
for entry in entries:
|
||||
if isinstance(entry, dict) and entry.get("ip"):
|
||||
return str(entry["ip"])
|
||||
return None
|
||||
|
||||
def resolve_ssh_host(self):
|
||||
"""SSH 目标解析(feat-chv-fix,P0-4 选定方案:客机 IP):
|
||||
|
||||
显式 CHV_SSH_HOST > net 静态客机 IP > 显式转发宣称下
|
||||
的回环;三者皆无时报 ChvFailure(ssh_target_undeclared),
|
||||
绝不无条件假设 127.0.0.1:2222 能通。"""
|
||||
if getattr(self, "ssh_host", None):
|
||||
return self.ssh_host
|
||||
guest_ip = self._net_guest_ip()
|
||||
if guest_ip:
|
||||
return guest_ip
|
||||
if getattr(self, "ssh_forward", False):
|
||||
return "127.0.0.1"
|
||||
raise ChvFailure(
|
||||
"ssh_target_undeclared",
|
||||
"SSH 目标未声明:须设 CHV_SSH_HOST、或在 CHV_NET_JSON "
|
||||
"里给 net 条目配 ip(静态 tap 客机 IP)、或显式设 "
|
||||
"CHV_SSH_FORWARD=1 宣称宿主 NAT 已转发回环端口")
|
||||
|
||||
def provision_host_key(self, task):
|
||||
"""按 VM 预置 host key(补缺口轨强方案,见 hostkeys.py):
|
||||
启动前为该 VM 生成一次性 ed25519 host key 对,公钥
|
||||
@@ -484,9 +531,16 @@ class CloudHypervisorExecutor(Executor):
|
||||
public_text = fh.read()
|
||||
with open(paths["private"], "r") as fh:
|
||||
private_text = fh.read()
|
||||
# feat-chv-fix:known_hosts 行的主机部分跟 SSH 目标走
|
||||
# (客机 IP 时不再写死回环,否则钉死校验永远对不上)。
|
||||
try:
|
||||
ssh_host = self.resolve_ssh_host()
|
||||
except ChvFailure:
|
||||
ssh_host = "127.0.0.1"
|
||||
write_known_hosts(
|
||||
paths["known_hosts"],
|
||||
[known_hosts_line(self.ssh_port, public_text)])
|
||||
[known_hosts_line(self.ssh_port, public_text,
|
||||
host=ssh_host)])
|
||||
seed_dir = write_seed(paths["seed_dir"], vm,
|
||||
private_text, public_text)
|
||||
return {"paths": paths, "seed_dir": seed_dir,
|
||||
@@ -502,14 +556,18 @@ class CloudHypervisorExecutor(Executor):
|
||||
抓不到可用 key 行时返回 False(不写文件、不报错),
|
||||
由 wait_guest_ready 下一轮再试;sshd 还没起来时
|
||||
这就是常态,不是失败。"""
|
||||
try:
|
||||
ssh_host = self.resolve_ssh_host()
|
||||
except ChvFailure:
|
||||
return False
|
||||
try:
|
||||
result = self._call_runner(
|
||||
["ssh-keyscan", "-p", str(self.ssh_port),
|
||||
"127.0.0.1"], timeout=10)
|
||||
ssh_host], timeout=10)
|
||||
stdout = getattr(result, "stdout", "") or ""
|
||||
except Exception:
|
||||
return False
|
||||
key_lines = parse_keyscan_output(stdout)
|
||||
key_lines = parse_keyscan_output(stdout, host=ssh_host)
|
||||
if not key_lines:
|
||||
return False
|
||||
try:
|
||||
@@ -518,7 +576,7 @@ class CloudHypervisorExecutor(Executor):
|
||||
return False
|
||||
return True
|
||||
|
||||
def build_ssh_command(self, task, inner_cmd):
|
||||
def build_ssh_command(self, task, inner_cmd, extra_env=None):
|
||||
"""构造 SSH 命令:ssh -p <端口> -i <key 文件> + 钉死选项
|
||||
+ user@127.0.0.1 + 客内命令。argv 里只有 key 文件路径,
|
||||
绝无任何 key 值。egress_proxy 已设时客内命令前加 env
|
||||
@@ -527,13 +585,19 @@ class CloudHypervisorExecutor(Executor):
|
||||
Host key 选项是钉死式(StrictHostKeyChecking=yes +
|
||||
每 VM known_hosts 文件),#216 的 no + /dev/null 已
|
||||
在本轨废除,见类 docstring 与 ssh_transport.py。"""
|
||||
try:
|
||||
ssh_host = self.resolve_ssh_host()
|
||||
except ChvFailure:
|
||||
# 纯命令构造(旧测试/干跑)不炸:回环缺省仅作
|
||||
# 构造兼容,wake/wait 路径仍严格解析。
|
||||
ssh_host = "127.0.0.1"
|
||||
return _build_ssh_cmd(
|
||||
self.ssh_port, self.ssh_key, self.ssh_user,
|
||||
self.egress_proxy, self.known_hosts_path(task),
|
||||
inner_cmd)
|
||||
inner_cmd, ssh_host=ssh_host, extra_env=extra_env)
|
||||
|
||||
# -- API 模式:VmConfig / API 调用 / 就绪轮询 -----------------------
|
||||
def build_vm_config(self, task_disk=None):
|
||||
def build_vm_config(self, task_disk=None, seed_disk=None):
|
||||
"""API 模式的 VmConfig(vm.create 请求体,纯函数):
|
||||
cpus{boot_vcpus,max_vcpus}、memory{size: 字节}、
|
||||
kernel{path}、cmdline{args}、disks[{path: 每任务磁盘}]、
|
||||
@@ -553,6 +617,12 @@ class CloudHypervisorExecutor(Executor):
|
||||
"disks": [{"path": task_disk or self.rootfs}],
|
||||
"rng": {"src": "/dev/urandom"},
|
||||
}
|
||||
# feat-chv-fix(P0-1):seed 磁盘是 VmConfig 第二块
|
||||
# 只读盘(CIDATA 卷标 ISO,客机 cloud-init NoCloud
|
||||
# 数据源),不是宿主目录——目录本身从不进 VmConfig。
|
||||
if seed_disk:
|
||||
config["disks"].append({"path": seed_disk,
|
||||
"readonly": True})
|
||||
if self.net_json:
|
||||
try:
|
||||
net = json.loads(self.net_json)
|
||||
@@ -595,22 +665,51 @@ class CloudHypervisorExecutor(Executor):
|
||||
with self._vms_lock:
|
||||
return len(self._vms)
|
||||
|
||||
def _check_capacity(self):
|
||||
"""宿主容量门:已登记 VM 数达到 max_vms 即明确拒绝
|
||||
(kind=host_capacity_exceeded,调用方应排队/拒绝,
|
||||
绝不降级 Docker/Local)。"""
|
||||
if self._active_count() >= self.max_vms:
|
||||
def _persistent_vm_count(self, exclude=None):
|
||||
"""跨重启容量计数(feat-chv-fix,P1-4):本实例登记
|
||||
的活进程名 ∪ state_dir 下 chv-*.sock 对应的 VM 名。
|
||||
worker 重启后 _vms 为空,但旧 VMM 的 socket 仍在,
|
||||
仍计入容量——重启不再绕过 CHV_MAX_VMS。注意这是
|
||||
保守计数:已死 VMM 的残留 socket 在清理前也计入,
|
||||
直到 cleanup_stale_vms / worker reconcile 清掉它。
|
||||
exclude:调用方自己的 VM 名(同任务重复 wake 时先
|
||||
关旧 VM,不应被自己的残留 socket 误拒)。"""
|
||||
names = set()
|
||||
with self._vms_lock:
|
||||
for name, proc in self._vms.items():
|
||||
try:
|
||||
alive = proc.poll() is None
|
||||
except Exception:
|
||||
alive = False
|
||||
if alive:
|
||||
names.add(name)
|
||||
try:
|
||||
entries = os.listdir(self.state_dir)
|
||||
except Exception:
|
||||
entries = []
|
||||
for entry in entries:
|
||||
if entry.endswith(".sock") and entry[:-5].startswith("chv-"):
|
||||
names.add(entry[:-5])
|
||||
if exclude:
|
||||
names.discard(exclude)
|
||||
return len(names)
|
||||
|
||||
def _check_capacity(self, exclude=None):
|
||||
"""宿主容量门:持久化计数(见 _persistent_vm_count)
|
||||
达到 max_vms 即明确拒绝(kind=host_capacity_exceeded,
|
||||
调用方应排队/拒绝,绝不降级 Docker/Local)。"""
|
||||
count = self._persistent_vm_count(exclude=exclude)
|
||||
if count >= self.max_vms:
|
||||
raise ChvFailure(
|
||||
"host_capacity_exceeded",
|
||||
"宿主 CHV 容量已满:已登记 %d 台,上限 %d,"
|
||||
"请排队或拒绝本任务" % (self._active_count(),
|
||||
self.max_vms))
|
||||
"宿主 CHV 容量已满:已计数 %d 台,上限 %d,"
|
||||
"请排队或拒绝本任务" % (count, self.max_vms))
|
||||
|
||||
def _check_disk_quota(self):
|
||||
"""磁盘配额门(实现在 cloud/chv/disk.py):
|
||||
CHV_DISK_GB 已设且基础镜像超过该 GB 数即拒绝
|
||||
(kind=disk_quota_exceeded)。未设则不检查。"""
|
||||
check_disk_quota(self.rootfs, self.disk_gb)
|
||||
"""磁盘配额门(实现在 cloud/chv/disk.py,feat-chv-fix
|
||||
口径:总量 + 本次 + 宿主余量,镜像不可读不静默跳过)。"""
|
||||
check_disk_quota(self.rootfs, self.disk_gb,
|
||||
state_dir=self.state_dir)
|
||||
|
||||
def _provision_task_disk(self, task):
|
||||
"""每任务磁盘供给(实现在 cloud/chv/disk.py):
|
||||
@@ -618,8 +717,8 @@ class CloudHypervisorExecutor(Executor):
|
||||
state_dir/<vm>/disk.raw,VmConfig 用该路径;复制
|
||||
慢但两个任务绝不写同一文件,测试用小文件可直接验证。
|
||||
- reuse(仅单租户实验):直接返回基础镜像路径,磁盘
|
||||
不独立,生产禁用(preflight 不拦它,但类 docstring
|
||||
与就绪清单里明写 reuse 不可用于生产)。
|
||||
不独立;preflight 拦截(须显式实验二次确认并审计
|
||||
放行,见 preflight,feat-chv-fix)。
|
||||
返回每任务磁盘路径。"""
|
||||
return provision_task_disk(
|
||||
self.state_dir, self.vm_name(task), self.rootfs,
|
||||
@@ -657,6 +756,19 @@ class CloudHypervisorExecutor(Executor):
|
||||
不先捕获就永远等不到就绪。超时时先关 VM 再 raise
|
||||
ChvFailure(kind=guest_not_ready),绝不在未就绪时
|
||||
派任务。"""
|
||||
# feat-chv-fix(P0-2):provisioned 模式下缺 known_hosts
|
||||
# 时直接 fail-fast,绝不调用 capture_host_key 回落 TOFU。
|
||||
# (keyscan 模式才走下面的捕获分支。)
|
||||
if getattr(self, "host_key_mode", "keyscan") == "provisioned" \
|
||||
and not os.path.exists(self.known_hosts_path(task)):
|
||||
try:
|
||||
self.stop_vm(task)
|
||||
except Exception:
|
||||
pass
|
||||
raise ChvFailure(
|
||||
"host_key_not_pinned",
|
||||
"provisioned 模式下 known_hosts 缺失,拒绝 keyscan "
|
||||
"回落:VM %s 未钉死 host key" % self.vm_name(task))
|
||||
attempts = max(1, int(self.ready_timeout / self.ready_interval))
|
||||
for attempt in range(attempts):
|
||||
try:
|
||||
@@ -738,9 +850,17 @@ class CloudHypervisorExecutor(Executor):
|
||||
if not isinstance(task, dict) or task.get("id") is None:
|
||||
raise ValueError("任务缺少 id")
|
||||
self.preflight()
|
||||
self._check_capacity()
|
||||
self._check_disk_quota()
|
||||
# feat-chv-fix:SSH 目标与 reuse 放行在启动前就定性。
|
||||
# resolve 失败即 fail-fast(不先起 VMM 再超时);
|
||||
# reuse 实验放行必须审计,留痕给运维复核。
|
||||
self.resolve_ssh_host()
|
||||
name = self.vm_name(task)
|
||||
self._check_capacity(exclude=name)
|
||||
self._check_disk_quota()
|
||||
if getattr(self, "disk_mode", "copy") == "reuse":
|
||||
self._emit("disk_reuse_experiment_allowed", task,
|
||||
"vm=%s CHV_REUSE_SINGLE_TENANT_EXPERIMENT=1 "
|
||||
"explicit" % name)
|
||||
old = None
|
||||
with self._vms_lock:
|
||||
if name in self._vms:
|
||||
@@ -757,15 +877,25 @@ class CloudHypervisorExecutor(Executor):
|
||||
self._emit("egress_direct_allowed", task,
|
||||
"vm=%s CHV_ALLOW_DIRECT_EGRESS=1 explicit" % name)
|
||||
try:
|
||||
seed_disk = None
|
||||
if getattr(self, "host_key_mode", "keyscan") == "provisioned":
|
||||
# 强方案:启动前预置每 VM host key 并钉死,
|
||||
# 失败即 fail-fast,绝不回落 keyscan。
|
||||
self.provision_host_key(task)
|
||||
# 再把 seed 目录做成 CIDATA seed 磁盘并挂进
|
||||
# VmConfig(P0-1 闭环:生成→制盘→挂盘三步缺
|
||||
# 一即 fail-fast,绝不回落 keyscan、绝不只
|
||||
# 生成不注入)。
|
||||
from cloud.chv.hostkeys import (
|
||||
build_seed_disk, seed_disk_path)
|
||||
provisioned = self.provision_host_key(task)
|
||||
seed_disk = seed_disk_path(self.state_dir, name)
|
||||
build_seed_disk(
|
||||
provisioned["seed_dir"], seed_disk,
|
||||
runner=lambda cmd: self._call_runner(cmd))
|
||||
self._launch_api_only(task)
|
||||
self.wait_api_ready(task)
|
||||
status, _ = self._api_call(
|
||||
socket_path, "PUT", "/vm.create",
|
||||
self.build_vm_config(task_disk))
|
||||
self.build_vm_config(task_disk, seed_disk=seed_disk))
|
||||
if status not in (200, 204):
|
||||
raise ChvFailure(
|
||||
"vm_create_failed",
|
||||
@@ -828,6 +958,18 @@ class CloudHypervisorExecutor(Executor):
|
||||
raise ValueError("snapshot_url 不能为空")
|
||||
self._ch_remote(task_or_name, "snapshot",
|
||||
"source_url=%s" % snapshot_url)
|
||||
# feat-chv-fix(P1-7):只查 returncode 不够。file://
|
||||
# 快照须验证目标文件存在且非空,否则报
|
||||
# snapshot_not_persisted,调用方(hibernate_vm)据此
|
||||
# 不 shutdown、不销毁原 VM;非 file:// 目标无法从宿
|
||||
# 主验证落盘,仍只认 returncode,但文档明写该边界。
|
||||
if snapshot_url.startswith("file://"):
|
||||
_snap_path = snapshot_url[len("file://"):]
|
||||
if not os.path.isfile(_snap_path) \
|
||||
or os.path.getsize(_snap_path) == 0:
|
||||
raise ChvFailure(
|
||||
"snapshot_not_persisted",
|
||||
"快照未落盘或为空:%s" % snapshot_url)
|
||||
task = task_or_name if isinstance(task_or_name, dict) \
|
||||
else {"id": task_or_name}
|
||||
self._emit("vm_snapshot", task,
|
||||
@@ -849,8 +991,27 @@ class CloudHypervisorExecutor(Executor):
|
||||
if "://" not in snapshot_url:
|
||||
snapshot_url = "file://%s" % snapshot_url
|
||||
self.preflight()
|
||||
self._check_capacity()
|
||||
self.resolve_ssh_host()
|
||||
# feat-chv-fix(P0-2 修正版):restore 不走 vm.create,
|
||||
# 没有 VmConfig 可挂 seed——审计报告“restore 须把 seed
|
||||
# 接入 VmConfig”的说法在此不适用,纠正为:快照内已
|
||||
# 固化原 VM 的 host key,重新预置新 key 反而会破坏
|
||||
# 钉死。正确不变量是:provisioned 下复用已钉死的
|
||||
# known_hosts(stop 时公钥保留、私钥已删但快照内仍有
|
||||
# 原私钥),缺失时 fail-fast、绝不 keyscan、绝不重
|
||||
# 生成。file:// 快照须先验存在且非空,否则不启动。
|
||||
# 注意:restore 时不重复验快照文件存在——快照落盘
|
||||
# 校验是 snapshot_vm/hibernate_vm 创建时的门(P1-7),
|
||||
# restore 的 ch-remote 本身会在源缺失时失败;在这里
|
||||
# 再验一次会把“源在远端/对象存储”的合法路径也挡掉。
|
||||
if getattr(self, "host_key_mode", "keyscan") == "provisioned" \
|
||||
and not os.path.exists(self.known_hosts_path(task)):
|
||||
raise ChvFailure(
|
||||
"host_key_not_pinned",
|
||||
"provisioned 模式下 restore 缺 known_hosts,拒绝 "
|
||||
"keyscan 回落:VM %s" % self.vm_name(task))
|
||||
name = self.vm_name(task)
|
||||
self._check_capacity(exclude=name)
|
||||
try:
|
||||
self._launch_api_only(task)
|
||||
self.wait_api_ready(task)
|
||||
@@ -936,7 +1097,8 @@ class CloudHypervisorExecutor(Executor):
|
||||
if os.path.isdir(seed_dir):
|
||||
shutil.rmtree(seed_dir)
|
||||
for _suffix in ("ssh_host_ed25519_key",
|
||||
"ssh_host_ed25519_key.pub"):
|
||||
"ssh_host_ed25519_key.pub",
|
||||
"seed.iso"):
|
||||
_p = os.path.join(self.state_dir, name, _suffix)
|
||||
if os.path.exists(_p):
|
||||
os.remove(_p)
|
||||
@@ -1027,14 +1189,16 @@ class CloudHypervisorExecutor(Executor):
|
||||
return cleaned
|
||||
|
||||
# -- 客内执行 ------------------------------------------------------
|
||||
def _run_guest(self, task, inner_args, **kwargs):
|
||||
def _run_guest(self, task, inner_args, extra_env=None, **kwargs):
|
||||
"""SSH 调客内 host.py,返回 Local/Docker 同款结果 dict
|
||||
(executor 字段改写为 cloud-hypervisor,并附 vm 名与
|
||||
api-socket)。runner 异常即原样传播,由 worker 侧统一
|
||||
写回 failed——本类不在这里把异常吞成假成功。"""
|
||||
写回 failed——本类不在这里把异常吞成假成功。
|
||||
extra_env(P1-1):经 SSH env 前缀进客机环境。"""
|
||||
self._validate()
|
||||
ssh_cmd = self.build_ssh_command(
|
||||
task, self._guest_host_command(*inner_args, **kwargs))
|
||||
task, self._guest_host_command(*inner_args, **kwargs),
|
||||
extra_env=extra_env)
|
||||
result = self._call_runner(ssh_cmd, timeout=self.task_timeout)
|
||||
# 方法内 import:见模块头注释的循环导入说明。
|
||||
from cloud.sandbox import _parse_host_result
|
||||
@@ -1063,10 +1227,12 @@ class CloudHypervisorExecutor(Executor):
|
||||
return parse_goal_result(task_id, result,
|
||||
"cloud-hypervisor")
|
||||
|
||||
def _run_guest_goal(self, task, inner_args, **kwargs):
|
||||
def _run_guest_goal(self, task, inner_args, extra_env=None,
|
||||
**kwargs):
|
||||
self._validate()
|
||||
ssh_cmd = self.build_ssh_command(
|
||||
task, self._guest_host_command(*inner_args, **kwargs))
|
||||
task, self._guest_host_command(*inner_args, **kwargs),
|
||||
extra_env=extra_env)
|
||||
return self._goal_result(
|
||||
task["id"],
|
||||
self._call_runner(ssh_cmd, timeout=self.task_timeout))
|
||||
@@ -1137,7 +1303,8 @@ class CloudHypervisorExecutor(Executor):
|
||||
"interrupted_reason": reason,
|
||||
}
|
||||
|
||||
def _run_with_vm(self, task, inner_args, goal=False, **kwargs):
|
||||
def _run_with_vm(self, task, inner_args, goal=False, extra_env=None,
|
||||
**kwargs):
|
||||
"""每任务 VM 生命周期:wake_vm(API 模式 + SSH 就绪)
|
||||
→ 跑客内命令 → finally shutdown_vm。wake 失败时不跑
|
||||
任务、直接抛分类失败。本方法不做内部重试:重试是
|
||||
@@ -1155,8 +1322,9 @@ class CloudHypervisorExecutor(Executor):
|
||||
try:
|
||||
if goal:
|
||||
return self._run_guest_goal(
|
||||
task, inner_args, **kwargs)
|
||||
result = self._run_guest(task, inner_args, **kwargs)
|
||||
task, inner_args, extra_env=extra_env, **kwargs)
|
||||
result = self._run_guest(task, inner_args,
|
||||
extra_env=extra_env, **kwargs)
|
||||
except Exception as exc:
|
||||
if self._vm_proc_dead(task):
|
||||
return self._vm_death_result(
|
||||
@@ -1195,10 +1363,25 @@ class CloudHypervisorExecutor(Executor):
|
||||
"execute() 只读已运行 VM 的 status:VM %s 未运行,"
|
||||
"请先 wake_vm() 再调 execute(),不要用 execute "
|
||||
"唤醒 VM" % self.vm_name(task))
|
||||
return self._run_guest(
|
||||
task, ("status", "--task", self._host_name(task)))
|
||||
try:
|
||||
return self._run_guest(
|
||||
task, ("status", "--task", self._host_name(task)))
|
||||
except Exception as exc:
|
||||
# feat-chv-fix(P1-8):socket 残留但 VMM 已死时,
|
||||
# _vm_is_running 会因 socket 存在而放行,随后的
|
||||
# SSH 失败不得以原始传输异常抛出——登记进程已死
|
||||
# 或 socket 已消失即统一归类 vm_not_running;VM
|
||||
# 仍活着时的 SSH 失败原样传播(那是客内 status
|
||||
# 失败,不是 VM 未运行)。
|
||||
if self._vm_proc_dead(task) \
|
||||
or not os.path.exists(self.api_socket(task)):
|
||||
raise ChvFailure(
|
||||
"vm_not_running",
|
||||
"execute() 时 VM 已死(socket 残留/进程已死),"
|
||||
"SSH 失败归类为未运行:%s" % exc)
|
||||
raise
|
||||
|
||||
def dispatch_task(self, task) -> dict:
|
||||
def dispatch_task(self, task, extra_env=None, model_provider=None) -> dict:
|
||||
"""客内 host.py dispatch:建 host 任务并跑一轮 turn。"""
|
||||
if not isinstance(task, dict) or task.get("id") is None:
|
||||
raise ValueError("任务缺少 id")
|
||||
@@ -1210,11 +1393,12 @@ class CloudHypervisorExecutor(Executor):
|
||||
if repo:
|
||||
args += ["--repo", repo]
|
||||
return self._run_with_vm(
|
||||
task, args, model=task.get("model"),
|
||||
task, args, extra_env=extra_env, model=task.get("model"),
|
||||
effort=task.get("effort"), approval=task.get("approval"),
|
||||
model_provider=model_provider,
|
||||
sandbox=("read-only" if task.get("plan") else None))
|
||||
|
||||
def run_task(self, task) -> dict:
|
||||
def run_task(self, task, extra_env=None, model_provider=None) -> dict:
|
||||
"""客内 host.py run:在已有 host 任务上再跑一轮 turn。"""
|
||||
if not isinstance(task, dict) or task.get("id") is None:
|
||||
raise ValueError("任务缺少 id")
|
||||
@@ -1223,10 +1407,11 @@ class CloudHypervisorExecutor(Executor):
|
||||
return self._run_with_vm(
|
||||
task, ("run", "--task", self._host_name(task),
|
||||
"--prompt", task["prompt"]),
|
||||
model=task.get("model"), effort=task.get("effort"),
|
||||
approval=task.get("approval"))
|
||||
extra_env=extra_env, model=task.get("model"),
|
||||
effort=task.get("effort"), approval=task.get("approval"),
|
||||
model_provider=model_provider)
|
||||
|
||||
def goal_task(self, task) -> dict:
|
||||
def goal_task(self, task, extra_env=None, model_provider=None) -> dict:
|
||||
"""客内 host.py goal:目标循环,轮次规则与 Local 同款
|
||||
(缺省 3 轮,>100 轮 loud fail,旧验收形态 0 轮禁用)。"""
|
||||
if not isinstance(task, dict) or task.get("id") is None:
|
||||
@@ -1252,11 +1437,13 @@ class CloudHypervisorExecutor(Executor):
|
||||
if repo:
|
||||
args += ["--repo", repo]
|
||||
return self._run_with_vm(
|
||||
task, args, goal=True, model=task.get("model"),
|
||||
effort=task.get("effort"), approval=task.get("approval"))
|
||||
task, args, goal=True, extra_env=extra_env,
|
||||
model=task.get("model"),
|
||||
effort=task.get("effort"), approval=task.get("approval"),
|
||||
model_provider=model_provider)
|
||||
|
||||
def goal_resume_task(self, task, goal_text=None,
|
||||
max_rounds=None) -> dict:
|
||||
max_rounds=None, extra_env=None, model_provider=None) -> dict:
|
||||
"""客内 host.py goal-resume:goal_text=None 为断点续跑
|
||||
(不带 --goal-text,沿用原 goal+verify),给 goal_text
|
||||
为开新目标(带 --goal-text 与 --max-rounds,默认 0)。"""
|
||||
@@ -1269,11 +1456,14 @@ class CloudHypervisorExecutor(Executor):
|
||||
elif max_rounds is not None:
|
||||
args += ["--max-rounds", str(max_rounds)]
|
||||
return self._run_with_vm(
|
||||
task, args, goal=True, model=task.get("model"),
|
||||
effort=task.get("effort"), approval=task.get("approval"))
|
||||
task, args, goal=True, extra_env=extra_env,
|
||||
model=task.get("model"),
|
||||
effort=task.get("effort"), approval=task.get("approval"),
|
||||
model_provider=model_provider)
|
||||
|
||||
def goal_resume_breakpoint_task(self, task,
|
||||
model_provider=None) -> dict:
|
||||
model_provider=None,
|
||||
extra_env=None) -> dict:
|
||||
"""客内 host.py goal-resume 断点续跑:无 --goal-text、
|
||||
无 --max-rounds,host 沿用原任务从断点轮次续跑。
|
||||
worker 层已接线(补缺口轨):非网关的 CHV 断点任务
|
||||
@@ -1286,7 +1476,7 @@ class CloudHypervisorExecutor(Executor):
|
||||
raise ValueError("任务缺少 id")
|
||||
return self._run_with_vm(
|
||||
task, ("goal-resume", "--task", self._host_name(task)),
|
||||
goal=True, model=task.get("model"),
|
||||
goal=True, extra_env=extra_env, model=task.get("model"),
|
||||
effort=task.get("effort"), approval=task.get("approval"),
|
||||
model_provider=model_provider)
|
||||
|
||||
|
||||
Reference in new issue
Block a user