feat: CHV 审核修复(P0+P1,堆叠于 #230)
P0:seed 磁盘 CIDATA ISO 挂进 VmConfig 第二块只读盘; restore provisioned 下复用钉死 known_hosts、缺失 fail-fast 绝不 keyscan;preflight 拦截 CHV_DISK_MODE=reuse(显式 实验二次确认 + 审计放行);SSH 目标选客机 IP 方案(显式 host > net 静态 ip > 显式转发宣称回环,三者皆无 fail-fast)。 P1:网关 extra_env 经 SSH env 前缀注入、worker 守卫收窄到 Local/CHV;取消/清理/看门狗接 CHV 通用路径;WarmPool acquire 内 resume;容量门持久化计数;磁盘配额总量+余量 与已存在磁盘校验重拷;API 按 Content-Length 验完整性; 快照 file:// 验落盘再 shutdown;execute 死端口归类 vm_not_running。 P2 全部排除未做(用户范围收窄令,留待以后一轨)。 测试:原 84 + 新增 29 = 113 项,mock,无真 VM。
This commit is contained in:
1 parent
22b83f118a
commit
9771ed9e8c
13 files changed
+1231
-128
No files matched your search
+53
-5
@@ -33,25 +33,73 @@ def _unix_socket_api(socket_path, method, path, payload=None):
|
|||||||
sock.connect(socket_path)
|
sock.connect(socket_path)
|
||||||
sock.sendall(raw)
|
sock.sendall(raw)
|
||||||
chunks = []
|
chunks = []
|
||||||
while True:
|
# feat-chv-fix(P1-6):读超时不再当“响应结束”。
|
||||||
|
# 先读满响应头,再按 Content-Length 读满体部;超时、
|
||||||
|
# 提前关流、体部短于 Content-Length 一律 RuntimeError
|
||||||
|
# (调用方判 api_timeout / 传输失败),绝不解析半截
|
||||||
|
# 响应。chunked 与无 Content-Length 响应明确拒绝
|
||||||
|
# (Cloud Hypervisor REST 不应发 chunked;无长度则
|
||||||
|
# 无法证完整性)。
|
||||||
|
buf = b""
|
||||||
|
header_end = None
|
||||||
|
timed_out = False
|
||||||
|
while header_end is None:
|
||||||
try:
|
try:
|
||||||
data = sock.recv(65536)
|
data = sock.recv(65536)
|
||||||
except socket.timeout:
|
except socket.timeout:
|
||||||
|
timed_out = True
|
||||||
break
|
break
|
||||||
if not data:
|
if not data:
|
||||||
break
|
break
|
||||||
chunks.append(data)
|
buf += data
|
||||||
|
header_end = buf.find(b"\r\n\r\n")
|
||||||
|
if timed_out or header_end is None:
|
||||||
|
raise RuntimeError("CHV API 响应头不完整(超时或提前关流)")
|
||||||
|
header = buf[:header_end].decode("utf-8", errors="replace")
|
||||||
|
body = buf[header_end + 4:]
|
||||||
|
content_length = None
|
||||||
|
is_chunked = False
|
||||||
|
for line in header.split("\r\n")[1:]:
|
||||||
|
if ":" not in line:
|
||||||
|
continue
|
||||||
|
name, value = line.split(":", 1)
|
||||||
|
name = name.strip().lower()
|
||||||
|
value = value.strip()
|
||||||
|
if name == "content-length":
|
||||||
|
try:
|
||||||
|
content_length = int(value)
|
||||||
|
except ValueError:
|
||||||
|
raise RuntimeError("CHV API Content-Length 非法")
|
||||||
|
elif name == "transfer-encoding" \
|
||||||
|
and "chunked" in value.lower():
|
||||||
|
is_chunked = True
|
||||||
|
if is_chunked:
|
||||||
|
raise RuntimeError("CHV API 响应为 chunked,拒绝解析")
|
||||||
|
if content_length is None:
|
||||||
|
raise RuntimeError("CHV API 响应无 Content-Length,拒绝解析")
|
||||||
|
while len(body) < content_length:
|
||||||
|
try:
|
||||||
|
data = sock.recv(65536)
|
||||||
|
except socket.timeout:
|
||||||
|
raise RuntimeError(
|
||||||
|
"CHV API 响应体截断:已收 %d / 需 %d 字节"
|
||||||
|
% (len(body), content_length))
|
||||||
|
if not data:
|
||||||
|
raise RuntimeError(
|
||||||
|
"CHV API 响应体截断:已收 %d / 需 %d 字节"
|
||||||
|
% (len(body), content_length))
|
||||||
|
body += data
|
||||||
|
body = body[:content_length]
|
||||||
finally:
|
finally:
|
||||||
sock.close()
|
sock.close()
|
||||||
text = b"".join(chunks).decode("utf-8", errors="replace")
|
text = header + "\r\n\r\n" + body.decode("utf-8", errors="replace")
|
||||||
status = 0
|
status = 0
|
||||||
if text.startswith("HTTP/"):
|
if text.startswith("HTTP/"):
|
||||||
try:
|
try:
|
||||||
status = int(text.split()[1])
|
status = int(text.split()[1])
|
||||||
except (IndexError, ValueError):
|
except (IndexError, ValueError):
|
||||||
status = 0
|
status = 0
|
||||||
body_text = text.split("\r\n\r\n", 1)[1] if "\r\n\r\n" in text \
|
body_text = body.decode("utf-8", errors="replace")
|
||||||
else ""
|
|
||||||
try:
|
try:
|
||||||
parsed = json.loads(body_text) if body_text.strip() else {}
|
parsed = json.loads(body_text) if body_text.strip() else {}
|
||||||
except ValueError:
|
except ValueError:
|
||||||
|
|||||||
@@ -93,6 +93,20 @@ def build_chv_config(params, env):
|
|||||||
if params.get("cpuinfo_path") is not None else "/proc/cpuinfo"
|
if params.get("cpuinfo_path") is not None else "/proc/cpuinfo"
|
||||||
ch_remote = params.get("ch_remote") or env.get("CHV_REMOTE") \
|
ch_remote = params.get("ch_remote") or env.get("CHV_REMOTE") \
|
||||||
or "ch-remote"
|
or "ch-remote"
|
||||||
|
# P0-4 SSH 目标解析(feat-chv-fix):显式 CHV_SSH_HOST 优先;
|
||||||
|
# 其次取 VmConfig net 首个带 ip 的条目(静态 tap 客机 IP,
|
||||||
|
# 即选定的客机 IP 发现方案);两者皆无时只有显式
|
||||||
|
# CHV_SSH_FORWARD=1 才允许回环 127.0.0.1(运维宣称宿主
|
||||||
|
# NAT 已把该端口转发到客机 sshd),否则执行器在 wake/
|
||||||
|
# wait 时报 ssh_target_undeclared,绝不无条件假设回环能通。
|
||||||
|
# vsock 未选:需客内 agent + 宿主代理两套新代码(#216 已
|
||||||
|
# 否决);纯 NAT 方案未选:DNAT 是宿主防火墙外部状态,
|
||||||
|
# 执行器代码既不能创建也不能校验它,写进代码仍是假保证。
|
||||||
|
ssh_host = params.get("ssh_host") or env.get("CHV_SSH_HOST")
|
||||||
|
ssh_forward = env.get("CHV_SSH_FORWARD") == "1"
|
||||||
|
# P0-3:reuse 实验二次确认(名字即带实验含义)。
|
||||||
|
reuse_experiment = env.get(
|
||||||
|
"CHV_REUSE_SINGLE_TENANT_EXPERIMENT") == "1"
|
||||||
model = params.get("model")
|
model = params.get("model")
|
||||||
try:
|
try:
|
||||||
max_vms = int(env.get("CHV_MAX_VMS") or 4)
|
max_vms = int(env.get("CHV_MAX_VMS") or 4)
|
||||||
@@ -145,6 +159,8 @@ def build_chv_config(params, env):
|
|||||||
net_json=net_json, kvm_path=kvm_path,
|
net_json=net_json, kvm_path=kvm_path,
|
||||||
cpuinfo_path=cpuinfo_path, ch_remote=ch_remote, model=model,
|
cpuinfo_path=cpuinfo_path, ch_remote=ch_remote, model=model,
|
||||||
max_vms=max_vms, disk_mode=disk_mode, disk_gb=disk_gb,
|
max_vms=max_vms, disk_mode=disk_mode, disk_gb=disk_gb,
|
||||||
|
ssh_host=ssh_host, ssh_forward=ssh_forward,
|
||||||
|
reuse_experiment=reuse_experiment,
|
||||||
egress_proxy=egress_proxy,
|
egress_proxy=egress_proxy,
|
||||||
allow_direct_egress=allow_direct_egress,
|
allow_direct_egress=allow_direct_egress,
|
||||||
host_key_mode=host_key_mode,
|
host_key_mode=host_key_mode,
|
||||||
|
|||||||
+72
-11
@@ -2,8 +2,18 @@
|
|||||||
|
|
||||||
单任务单 VM 的文件系统独立性就靠这里:copy 模式每任务
|
单任务单 VM 的文件系统独立性就靠这里:copy 模式每任务
|
||||||
复制一份基础镜像,两个任务绝不写同一文件;reuse 模式
|
复制一份基础镜像,两个任务绝不写同一文件;reuse 模式
|
||||||
直接返回基础镜像路径(仅单租户实验,生产禁用,见
|
直接返回基础镜像路径(仅单租户实验,生产拦截见执行器
|
||||||
CloudHypervisorExecutor 的类 docstring)。
|
preflight,须同时设 CHV_REUSE_SINGLE_TENANT_EXPERIMENT=1
|
||||||
|
并审计放行)。
|
||||||
|
|
||||||
|
feat-chv-fix(P1-5)配额口径:不再只比基础镜像大小。
|
||||||
|
check_disk_quota 改为三项:state_dir 已用总量 + 本次
|
||||||
|
拷贝(基础镜像大小)+ 宿主剩余空间必须容纳本次拷贝;
|
||||||
|
镜像不可读时不再静默跳过,报 disk_quota_unverified
|
||||||
|
(preflight 在 wake 路径上会先报 image_missing,这里
|
||||||
|
是直接调用者的第二道门)。provision_task_disk 对已存在
|
||||||
|
磁盘校验大小:与源不一致即重拷;源不可读即报失败,
|
||||||
|
绝不复用脏磁盘。
|
||||||
"""
|
"""
|
||||||
|
|
||||||
import os
|
import os
|
||||||
@@ -12,32 +22,83 @@ import shutil
|
|||||||
from cloud.chv.errors import ChvFailure
|
from cloud.chv.errors import ChvFailure
|
||||||
|
|
||||||
|
|
||||||
def check_disk_quota(rootfs, disk_gb):
|
def _dir_total_bytes(path):
|
||||||
"""磁盘配额门:CHV_DISK_GB 已设且基础镜像超过该 GB 数
|
total = 0
|
||||||
即拒绝(kind=disk_quota_exceeded)。未设则不检查;
|
for root, _dirs, files in os.walk(path):
|
||||||
镜像读不到时不拦(preflight 自会报 image_missing,
|
for name in files:
|
||||||
这里不抢它的错)。"""
|
try:
|
||||||
|
total += os.path.getsize(os.path.join(root, name))
|
||||||
|
except OSError:
|
||||||
|
pass
|
||||||
|
return total
|
||||||
|
|
||||||
|
|
||||||
|
def check_disk_quota(rootfs, disk_gb, state_dir=None):
|
||||||
|
"""磁盘配额门(feat-chv-fix 口径):
|
||||||
|
|
||||||
|
- disk_gb 未设时不检查;
|
||||||
|
- 镜像不可读时报 disk_quota_unverified,不静默跳过;
|
||||||
|
- 镜像大小超过 disk_gb 即 disk_quota_exceeded;
|
||||||
|
- state_dir 已给时:已用总量 + 本次拷贝超过 disk_gb
|
||||||
|
即 disk_quota_exceeded;宿主剩余空间不足以容纳本次
|
||||||
|
拷贝即 disk_quota_exceeded(宿主余量校验)。
|
||||||
|
"""
|
||||||
if disk_gb is None or not rootfs:
|
if disk_gb is None or not rootfs:
|
||||||
return
|
return
|
||||||
try:
|
try:
|
||||||
size = os.path.getsize(rootfs)
|
size = os.path.getsize(rootfs)
|
||||||
except OSError:
|
except OSError:
|
||||||
return
|
raise ChvFailure(
|
||||||
|
"disk_quota_unverified",
|
||||||
|
"磁盘镜像不可读,配额无法校验:%s" % rootfs)
|
||||||
if size > disk_gb * 1024 ** 3:
|
if size > disk_gb * 1024 ** 3:
|
||||||
raise ChvFailure(
|
raise ChvFailure(
|
||||||
"disk_quota_exceeded",
|
"disk_quota_exceeded",
|
||||||
"磁盘镜像 %d 字节超过配额 %.1f GB" % (size, disk_gb))
|
"磁盘镜像 %d 字节超过配额 %.1f GB" % (size, disk_gb))
|
||||||
|
if state_dir and os.path.isdir(state_dir):
|
||||||
|
used = _dir_total_bytes(state_dir)
|
||||||
|
if used + size > disk_gb * 1024 ** 3:
|
||||||
|
raise ChvFailure(
|
||||||
|
"disk_quota_exceeded",
|
||||||
|
"磁盘总量 %d 字节 + 本次 %d 字节超过配额 %.1f GB"
|
||||||
|
% (used, size, disk_gb))
|
||||||
|
try:
|
||||||
|
free = shutil.disk_usage(state_dir).free
|
||||||
|
except OSError:
|
||||||
|
free = None
|
||||||
|
if free is not None and free < size:
|
||||||
|
raise ChvFailure(
|
||||||
|
"disk_quota_exceeded",
|
||||||
|
"宿主剩余空间 %d 字节不足以容纳本次磁盘拷贝 %d 字节"
|
||||||
|
% (free, size))
|
||||||
|
|
||||||
|
|
||||||
def provision_task_disk(state_dir, vm_name, rootfs, disk_mode):
|
def provision_task_disk(state_dir, vm_name, rootfs, disk_mode):
|
||||||
"""每任务磁盘路径:copy 模式复制到
|
"""每任务磁盘路径:copy 模式复制到
|
||||||
state_dir/<vm>/disk.raw(已存在则复用,不重复复制);
|
state_dir/<vm>/disk.raw;reuse 模式直接返回基础镜像路径。
|
||||||
reuse 模式直接返回基础镜像路径。"""
|
|
||||||
|
feat-chv-fix:已存在磁盘须校验——大小与源不一致即
|
||||||
|
重拷(脏磁盘不复用);源不可读即 ChvFailure,绝不
|
||||||
|
静默返回旧磁盘。"""
|
||||||
if disk_mode == "reuse":
|
if disk_mode == "reuse":
|
||||||
return rootfs
|
return rootfs
|
||||||
task_dir = os.path.join(state_dir, vm_name)
|
task_dir = os.path.join(state_dir, vm_name)
|
||||||
os.makedirs(task_dir, exist_ok=True)
|
os.makedirs(task_dir, exist_ok=True)
|
||||||
task_disk = os.path.join(task_dir, "disk.raw")
|
task_disk = os.path.join(task_dir, "disk.raw")
|
||||||
if not os.path.exists(task_disk):
|
if os.path.exists(task_disk):
|
||||||
|
try:
|
||||||
|
same = os.path.getsize(task_disk) == os.path.getsize(rootfs)
|
||||||
|
except OSError:
|
||||||
|
raise ChvFailure(
|
||||||
|
"disk_quota_unverified",
|
||||||
|
"任务磁盘或源镜像不可读,无法校验复用:%s" % task_disk)
|
||||||
|
if same:
|
||||||
|
return task_disk
|
||||||
|
os.remove(task_disk)
|
||||||
|
try:
|
||||||
shutil.copyfile(rootfs, task_disk)
|
shutil.copyfile(rootfs, task_disk)
|
||||||
|
except OSError as exc:
|
||||||
|
raise ChvFailure(
|
||||||
|
"disk_quota_unverified",
|
||||||
|
"任务磁盘拷贝失败:%s" % exc)
|
||||||
return task_disk
|
return task_disk
|
||||||
+243
-53
@@ -227,7 +227,8 @@ class CloudHypervisorExecutor(Executor):
|
|||||||
runner=None, launcher=None, ssh_key=None,
|
runner=None, launcher=None, ssh_key=None,
|
||||||
ssh_user=None, ssh_port=None,
|
ssh_user=None, ssh_port=None,
|
||||||
guest_host_py="/app/host.py", cmdline=None,
|
guest_host_py="/app/host.py", cmdline=None,
|
||||||
net=None, net_json=None, kvm_path="/dev/kvm",
|
net=None, net_json=None, ssh_host=None,
|
||||||
|
kvm_path="/dev/kvm",
|
||||||
cpuinfo_path="/proc/cpuinfo", env=None,
|
cpuinfo_path="/proc/cpuinfo", env=None,
|
||||||
model=None, ch_remote=None, api_client=None,
|
model=None, ch_remote=None, api_client=None,
|
||||||
sleeper=None, api_timeout=None, ready_timeout=None,
|
sleeper=None, api_timeout=None, ready_timeout=None,
|
||||||
@@ -243,7 +244,8 @@ class CloudHypervisorExecutor(Executor):
|
|||||||
"state_dir": state_dir, "ssh_key": ssh_key,
|
"state_dir": state_dir, "ssh_key": ssh_key,
|
||||||
"ssh_user": ssh_user, "ssh_port": ssh_port,
|
"ssh_user": ssh_user, "ssh_port": ssh_port,
|
||||||
"guest_host_py": guest_host_py, "cmdline": cmdline,
|
"guest_host_py": guest_host_py, "cmdline": cmdline,
|
||||||
"net": net, "net_json": net_json, "kvm_path": kvm_path,
|
"net": net, "net_json": net_json, "ssh_host": ssh_host,
|
||||||
|
"kvm_path": kvm_path,
|
||||||
"cpuinfo_path": cpuinfo_path, "ch_remote": ch_remote,
|
"cpuinfo_path": cpuinfo_path, "ch_remote": ch_remote,
|
||||||
"model": model, "api_timeout": api_timeout,
|
"model": model, "api_timeout": api_timeout,
|
||||||
"ready_timeout": ready_timeout,
|
"ready_timeout": ready_timeout,
|
||||||
@@ -257,7 +259,8 @@ class CloudHypervisorExecutor(Executor):
|
|||||||
"max_vms", "disk_mode", "disk_gb", "egress_proxy",
|
"max_vms", "disk_mode", "disk_gb", "egress_proxy",
|
||||||
"allow_direct_egress", "task_timeout", "snapshot_url",
|
"allow_direct_egress", "task_timeout", "snapshot_url",
|
||||||
"warm_pool_size", "api_timeout", "ready_timeout",
|
"warm_pool_size", "api_timeout", "ready_timeout",
|
||||||
"ready_interval", "host_key_mode"):
|
"ready_interval", "host_key_mode", "ssh_host",
|
||||||
|
"ssh_forward", "reuse_experiment"):
|
||||||
setattr(self, _field, getattr(_cfg, _field))
|
setattr(self, _field, getattr(_cfg, _field))
|
||||||
self.api_client = api_client if api_client is not None \
|
self.api_client = api_client if api_client is not None \
|
||||||
else _unix_socket_api
|
else _unix_socket_api
|
||||||
@@ -408,6 +411,17 @@ class CloudHypervisorExecutor(Executor):
|
|||||||
if not isinstance(_net, (list, dict)):
|
if not isinstance(_net, (list, dict)):
|
||||||
missing.append(("net_config_invalid",
|
missing.append(("net_config_invalid",
|
||||||
"CHV_NET_JSON 须为 JSON list 或 dict"))
|
"CHV_NET_JSON 须为 JSON list 或 dict"))
|
||||||
|
# feat-chv-fix(P0-3):reuse 生产拦截。一行配置再也
|
||||||
|
# 不能击穿租户隔离:只有显式实验二次确认
|
||||||
|
# CHV_REUSE_SINGLE_TENANT_EXPERIMENT=1 时才放行,
|
||||||
|
# 且放行本身在 wake_vm 里审计;否则 preflight 直接
|
||||||
|
# 拒绝(kind=disk_mode_reuse_blocked)。
|
||||||
|
if getattr(self, "disk_mode", "copy") == "reuse" \
|
||||||
|
and not getattr(self, "reuse_experiment", False):
|
||||||
|
missing.append(("disk_mode_reuse_blocked",
|
||||||
|
"CHV_DISK_MODE=reuse 未经显式实验确认:"
|
||||||
|
"须同时设 CHV_REUSE_SINGLE_TENANT_"
|
||||||
|
"EXPERIMENT=1(仅单租户实验)"))
|
||||||
if missing:
|
if missing:
|
||||||
kind = missing[0][0]
|
kind = missing[0][0]
|
||||||
detail = "; ".join(text for _, text in missing)
|
detail = "; ".join(text for _, text in missing)
|
||||||
@@ -463,6 +477,39 @@ class CloudHypervisorExecutor(Executor):
|
|||||||
return _known_hosts_path(self.state_dir,
|
return _known_hosts_path(self.state_dir,
|
||||||
self.vm_name(task_or_name))
|
self.vm_name(task_or_name))
|
||||||
|
|
||||||
|
def _net_guest_ip(self):
|
||||||
|
"""VmConfig net 首个带 ip 的条目(静态 tap 客机 IP)。"""
|
||||||
|
if not self.net_json:
|
||||||
|
return None
|
||||||
|
try:
|
||||||
|
net = json.loads(self.net_json)
|
||||||
|
except ValueError:
|
||||||
|
return None
|
||||||
|
entries = net if isinstance(net, list) else [net]
|
||||||
|
for entry in entries:
|
||||||
|
if isinstance(entry, dict) and entry.get("ip"):
|
||||||
|
return str(entry["ip"])
|
||||||
|
return None
|
||||||
|
|
||||||
|
def resolve_ssh_host(self):
|
||||||
|
"""SSH 目标解析(feat-chv-fix,P0-4 选定方案:客机 IP):
|
||||||
|
|
||||||
|
显式 CHV_SSH_HOST > net 静态客机 IP > 显式转发宣称下
|
||||||
|
的回环;三者皆无时报 ChvFailure(ssh_target_undeclared),
|
||||||
|
绝不无条件假设 127.0.0.1:2222 能通。"""
|
||||||
|
if getattr(self, "ssh_host", None):
|
||||||
|
return self.ssh_host
|
||||||
|
guest_ip = self._net_guest_ip()
|
||||||
|
if guest_ip:
|
||||||
|
return guest_ip
|
||||||
|
if getattr(self, "ssh_forward", False):
|
||||||
|
return "127.0.0.1"
|
||||||
|
raise ChvFailure(
|
||||||
|
"ssh_target_undeclared",
|
||||||
|
"SSH 目标未声明:须设 CHV_SSH_HOST、或在 CHV_NET_JSON "
|
||||||
|
"里给 net 条目配 ip(静态 tap 客机 IP)、或显式设 "
|
||||||
|
"CHV_SSH_FORWARD=1 宣称宿主 NAT 已转发回环端口")
|
||||||
|
|
||||||
def provision_host_key(self, task):
|
def provision_host_key(self, task):
|
||||||
"""按 VM 预置 host key(补缺口轨强方案,见 hostkeys.py):
|
"""按 VM 预置 host key(补缺口轨强方案,见 hostkeys.py):
|
||||||
启动前为该 VM 生成一次性 ed25519 host key 对,公钥
|
启动前为该 VM 生成一次性 ed25519 host key 对,公钥
|
||||||
@@ -484,9 +531,16 @@ class CloudHypervisorExecutor(Executor):
|
|||||||
public_text = fh.read()
|
public_text = fh.read()
|
||||||
with open(paths["private"], "r") as fh:
|
with open(paths["private"], "r") as fh:
|
||||||
private_text = fh.read()
|
private_text = fh.read()
|
||||||
|
# feat-chv-fix:known_hosts 行的主机部分跟 SSH 目标走
|
||||||
|
# (客机 IP 时不再写死回环,否则钉死校验永远对不上)。
|
||||||
|
try:
|
||||||
|
ssh_host = self.resolve_ssh_host()
|
||||||
|
except ChvFailure:
|
||||||
|
ssh_host = "127.0.0.1"
|
||||||
write_known_hosts(
|
write_known_hosts(
|
||||||
paths["known_hosts"],
|
paths["known_hosts"],
|
||||||
[known_hosts_line(self.ssh_port, public_text)])
|
[known_hosts_line(self.ssh_port, public_text,
|
||||||
|
host=ssh_host)])
|
||||||
seed_dir = write_seed(paths["seed_dir"], vm,
|
seed_dir = write_seed(paths["seed_dir"], vm,
|
||||||
private_text, public_text)
|
private_text, public_text)
|
||||||
return {"paths": paths, "seed_dir": seed_dir,
|
return {"paths": paths, "seed_dir": seed_dir,
|
||||||
@@ -502,14 +556,18 @@ class CloudHypervisorExecutor(Executor):
|
|||||||
抓不到可用 key 行时返回 False(不写文件、不报错),
|
抓不到可用 key 行时返回 False(不写文件、不报错),
|
||||||
由 wait_guest_ready 下一轮再试;sshd 还没起来时
|
由 wait_guest_ready 下一轮再试;sshd 还没起来时
|
||||||
这就是常态,不是失败。"""
|
这就是常态,不是失败。"""
|
||||||
|
try:
|
||||||
|
ssh_host = self.resolve_ssh_host()
|
||||||
|
except ChvFailure:
|
||||||
|
return False
|
||||||
try:
|
try:
|
||||||
result = self._call_runner(
|
result = self._call_runner(
|
||||||
["ssh-keyscan", "-p", str(self.ssh_port),
|
["ssh-keyscan", "-p", str(self.ssh_port),
|
||||||
"127.0.0.1"], timeout=10)
|
ssh_host], timeout=10)
|
||||||
stdout = getattr(result, "stdout", "") or ""
|
stdout = getattr(result, "stdout", "") or ""
|
||||||
except Exception:
|
except Exception:
|
||||||
return False
|
return False
|
||||||
key_lines = parse_keyscan_output(stdout)
|
key_lines = parse_keyscan_output(stdout, host=ssh_host)
|
||||||
if not key_lines:
|
if not key_lines:
|
||||||
return False
|
return False
|
||||||
try:
|
try:
|
||||||
@@ -518,7 +576,7 @@ class CloudHypervisorExecutor(Executor):
|
|||||||
return False
|
return False
|
||||||
return True
|
return True
|
||||||
|
|
||||||
def build_ssh_command(self, task, inner_cmd):
|
def build_ssh_command(self, task, inner_cmd, extra_env=None):
|
||||||
"""构造 SSH 命令:ssh -p <端口> -i <key 文件> + 钉死选项
|
"""构造 SSH 命令:ssh -p <端口> -i <key 文件> + 钉死选项
|
||||||
+ user@127.0.0.1 + 客内命令。argv 里只有 key 文件路径,
|
+ user@127.0.0.1 + 客内命令。argv 里只有 key 文件路径,
|
||||||
绝无任何 key 值。egress_proxy 已设时客内命令前加 env
|
绝无任何 key 值。egress_proxy 已设时客内命令前加 env
|
||||||
@@ -527,13 +585,19 @@ class CloudHypervisorExecutor(Executor):
|
|||||||
Host key 选项是钉死式(StrictHostKeyChecking=yes +
|
Host key 选项是钉死式(StrictHostKeyChecking=yes +
|
||||||
每 VM known_hosts 文件),#216 的 no + /dev/null 已
|
每 VM known_hosts 文件),#216 的 no + /dev/null 已
|
||||||
在本轨废除,见类 docstring 与 ssh_transport.py。"""
|
在本轨废除,见类 docstring 与 ssh_transport.py。"""
|
||||||
|
try:
|
||||||
|
ssh_host = self.resolve_ssh_host()
|
||||||
|
except ChvFailure:
|
||||||
|
# 纯命令构造(旧测试/干跑)不炸:回环缺省仅作
|
||||||
|
# 构造兼容,wake/wait 路径仍严格解析。
|
||||||
|
ssh_host = "127.0.0.1"
|
||||||
return _build_ssh_cmd(
|
return _build_ssh_cmd(
|
||||||
self.ssh_port, self.ssh_key, self.ssh_user,
|
self.ssh_port, self.ssh_key, self.ssh_user,
|
||||||
self.egress_proxy, self.known_hosts_path(task),
|
self.egress_proxy, self.known_hosts_path(task),
|
||||||
inner_cmd)
|
inner_cmd, ssh_host=ssh_host, extra_env=extra_env)
|
||||||
|
|
||||||
# -- API 模式:VmConfig / API 调用 / 就绪轮询 -----------------------
|
# -- API 模式:VmConfig / API 调用 / 就绪轮询 -----------------------
|
||||||
def build_vm_config(self, task_disk=None):
|
def build_vm_config(self, task_disk=None, seed_disk=None):
|
||||||
"""API 模式的 VmConfig(vm.create 请求体,纯函数):
|
"""API 模式的 VmConfig(vm.create 请求体,纯函数):
|
||||||
cpus{boot_vcpus,max_vcpus}、memory{size: 字节}、
|
cpus{boot_vcpus,max_vcpus}、memory{size: 字节}、
|
||||||
kernel{path}、cmdline{args}、disks[{path: 每任务磁盘}]、
|
kernel{path}、cmdline{args}、disks[{path: 每任务磁盘}]、
|
||||||
@@ -553,6 +617,12 @@ class CloudHypervisorExecutor(Executor):
|
|||||||
"disks": [{"path": task_disk or self.rootfs}],
|
"disks": [{"path": task_disk or self.rootfs}],
|
||||||
"rng": {"src": "/dev/urandom"},
|
"rng": {"src": "/dev/urandom"},
|
||||||
}
|
}
|
||||||
|
# feat-chv-fix(P0-1):seed 磁盘是 VmConfig 第二块
|
||||||
|
# 只读盘(CIDATA 卷标 ISO,客机 cloud-init NoCloud
|
||||||
|
# 数据源),不是宿主目录——目录本身从不进 VmConfig。
|
||||||
|
if seed_disk:
|
||||||
|
config["disks"].append({"path": seed_disk,
|
||||||
|
"readonly": True})
|
||||||
if self.net_json:
|
if self.net_json:
|
||||||
try:
|
try:
|
||||||
net = json.loads(self.net_json)
|
net = json.loads(self.net_json)
|
||||||
@@ -595,22 +665,51 @@ class CloudHypervisorExecutor(Executor):
|
|||||||
with self._vms_lock:
|
with self._vms_lock:
|
||||||
return len(self._vms)
|
return len(self._vms)
|
||||||
|
|
||||||
def _check_capacity(self):
|
def _persistent_vm_count(self, exclude=None):
|
||||||
"""宿主容量门:已登记 VM 数达到 max_vms 即明确拒绝
|
"""跨重启容量计数(feat-chv-fix,P1-4):本实例登记
|
||||||
(kind=host_capacity_exceeded,调用方应排队/拒绝,
|
的活进程名 ∪ state_dir 下 chv-*.sock 对应的 VM 名。
|
||||||
绝不降级 Docker/Local)。"""
|
worker 重启后 _vms 为空,但旧 VMM 的 socket 仍在,
|
||||||
if self._active_count() >= self.max_vms:
|
仍计入容量——重启不再绕过 CHV_MAX_VMS。注意这是
|
||||||
|
保守计数:已死 VMM 的残留 socket 在清理前也计入,
|
||||||
|
直到 cleanup_stale_vms / worker reconcile 清掉它。
|
||||||
|
exclude:调用方自己的 VM 名(同任务重复 wake 时先
|
||||||
|
关旧 VM,不应被自己的残留 socket 误拒)。"""
|
||||||
|
names = set()
|
||||||
|
with self._vms_lock:
|
||||||
|
for name, proc in self._vms.items():
|
||||||
|
try:
|
||||||
|
alive = proc.poll() is None
|
||||||
|
except Exception:
|
||||||
|
alive = False
|
||||||
|
if alive:
|
||||||
|
names.add(name)
|
||||||
|
try:
|
||||||
|
entries = os.listdir(self.state_dir)
|
||||||
|
except Exception:
|
||||||
|
entries = []
|
||||||
|
for entry in entries:
|
||||||
|
if entry.endswith(".sock") and entry[:-5].startswith("chv-"):
|
||||||
|
names.add(entry[:-5])
|
||||||
|
if exclude:
|
||||||
|
names.discard(exclude)
|
||||||
|
return len(names)
|
||||||
|
|
||||||
|
def _check_capacity(self, exclude=None):
|
||||||
|
"""宿主容量门:持久化计数(见 _persistent_vm_count)
|
||||||
|
达到 max_vms 即明确拒绝(kind=host_capacity_exceeded,
|
||||||
|
调用方应排队/拒绝,绝不降级 Docker/Local)。"""
|
||||||
|
count = self._persistent_vm_count(exclude=exclude)
|
||||||
|
if count >= self.max_vms:
|
||||||
raise ChvFailure(
|
raise ChvFailure(
|
||||||
"host_capacity_exceeded",
|
"host_capacity_exceeded",
|
||||||
"宿主 CHV 容量已满:已登记 %d 台,上限 %d,"
|
"宿主 CHV 容量已满:已计数 %d 台,上限 %d,"
|
||||||
"请排队或拒绝本任务" % (self._active_count(),
|
"请排队或拒绝本任务" % (count, self.max_vms))
|
||||||
self.max_vms))
|
|
||||||
|
|
||||||
def _check_disk_quota(self):
|
def _check_disk_quota(self):
|
||||||
"""磁盘配额门(实现在 cloud/chv/disk.py):
|
"""磁盘配额门(实现在 cloud/chv/disk.py,feat-chv-fix
|
||||||
CHV_DISK_GB 已设且基础镜像超过该 GB 数即拒绝
|
口径:总量 + 本次 + 宿主余量,镜像不可读不静默跳过)。"""
|
||||||
(kind=disk_quota_exceeded)。未设则不检查。"""
|
check_disk_quota(self.rootfs, self.disk_gb,
|
||||||
check_disk_quota(self.rootfs, self.disk_gb)
|
state_dir=self.state_dir)
|
||||||
|
|
||||||
def _provision_task_disk(self, task):
|
def _provision_task_disk(self, task):
|
||||||
"""每任务磁盘供给(实现在 cloud/chv/disk.py):
|
"""每任务磁盘供给(实现在 cloud/chv/disk.py):
|
||||||
@@ -618,8 +717,8 @@ class CloudHypervisorExecutor(Executor):
|
|||||||
state_dir/<vm>/disk.raw,VmConfig 用该路径;复制
|
state_dir/<vm>/disk.raw,VmConfig 用该路径;复制
|
||||||
慢但两个任务绝不写同一文件,测试用小文件可直接验证。
|
慢但两个任务绝不写同一文件,测试用小文件可直接验证。
|
||||||
- reuse(仅单租户实验):直接返回基础镜像路径,磁盘
|
- reuse(仅单租户实验):直接返回基础镜像路径,磁盘
|
||||||
不独立,生产禁用(preflight 不拦它,但类 docstring
|
不独立;preflight 拦截(须显式实验二次确认并审计
|
||||||
与就绪清单里明写 reuse 不可用于生产)。
|
放行,见 preflight,feat-chv-fix)。
|
||||||
返回每任务磁盘路径。"""
|
返回每任务磁盘路径。"""
|
||||||
return provision_task_disk(
|
return provision_task_disk(
|
||||||
self.state_dir, self.vm_name(task), self.rootfs,
|
self.state_dir, self.vm_name(task), self.rootfs,
|
||||||
@@ -657,6 +756,19 @@ class CloudHypervisorExecutor(Executor):
|
|||||||
不先捕获就永远等不到就绪。超时时先关 VM 再 raise
|
不先捕获就永远等不到就绪。超时时先关 VM 再 raise
|
||||||
ChvFailure(kind=guest_not_ready),绝不在未就绪时
|
ChvFailure(kind=guest_not_ready),绝不在未就绪时
|
||||||
派任务。"""
|
派任务。"""
|
||||||
|
# feat-chv-fix(P0-2):provisioned 模式下缺 known_hosts
|
||||||
|
# 时直接 fail-fast,绝不调用 capture_host_key 回落 TOFU。
|
||||||
|
# (keyscan 模式才走下面的捕获分支。)
|
||||||
|
if getattr(self, "host_key_mode", "keyscan") == "provisioned" \
|
||||||
|
and not os.path.exists(self.known_hosts_path(task)):
|
||||||
|
try:
|
||||||
|
self.stop_vm(task)
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
raise ChvFailure(
|
||||||
|
"host_key_not_pinned",
|
||||||
|
"provisioned 模式下 known_hosts 缺失,拒绝 keyscan "
|
||||||
|
"回落:VM %s 未钉死 host key" % self.vm_name(task))
|
||||||
attempts = max(1, int(self.ready_timeout / self.ready_interval))
|
attempts = max(1, int(self.ready_timeout / self.ready_interval))
|
||||||
for attempt in range(attempts):
|
for attempt in range(attempts):
|
||||||
try:
|
try:
|
||||||
@@ -738,9 +850,17 @@ class CloudHypervisorExecutor(Executor):
|
|||||||
if not isinstance(task, dict) or task.get("id") is None:
|
if not isinstance(task, dict) or task.get("id") is None:
|
||||||
raise ValueError("任务缺少 id")
|
raise ValueError("任务缺少 id")
|
||||||
self.preflight()
|
self.preflight()
|
||||||
self._check_capacity()
|
# feat-chv-fix:SSH 目标与 reuse 放行在启动前就定性。
|
||||||
self._check_disk_quota()
|
# resolve 失败即 fail-fast(不先起 VMM 再超时);
|
||||||
|
# reuse 实验放行必须审计,留痕给运维复核。
|
||||||
|
self.resolve_ssh_host()
|
||||||
name = self.vm_name(task)
|
name = self.vm_name(task)
|
||||||
|
self._check_capacity(exclude=name)
|
||||||
|
self._check_disk_quota()
|
||||||
|
if getattr(self, "disk_mode", "copy") == "reuse":
|
||||||
|
self._emit("disk_reuse_experiment_allowed", task,
|
||||||
|
"vm=%s CHV_REUSE_SINGLE_TENANT_EXPERIMENT=1 "
|
||||||
|
"explicit" % name)
|
||||||
old = None
|
old = None
|
||||||
with self._vms_lock:
|
with self._vms_lock:
|
||||||
if name in self._vms:
|
if name in self._vms:
|
||||||
@@ -757,15 +877,25 @@ class CloudHypervisorExecutor(Executor):
|
|||||||
self._emit("egress_direct_allowed", task,
|
self._emit("egress_direct_allowed", task,
|
||||||
"vm=%s CHV_ALLOW_DIRECT_EGRESS=1 explicit" % name)
|
"vm=%s CHV_ALLOW_DIRECT_EGRESS=1 explicit" % name)
|
||||||
try:
|
try:
|
||||||
|
seed_disk = None
|
||||||
if getattr(self, "host_key_mode", "keyscan") == "provisioned":
|
if getattr(self, "host_key_mode", "keyscan") == "provisioned":
|
||||||
# 强方案:启动前预置每 VM host key 并钉死,
|
# 强方案:启动前预置每 VM host key 并钉死,
|
||||||
# 失败即 fail-fast,绝不回落 keyscan。
|
# 再把 seed 目录做成 CIDATA seed 磁盘并挂进
|
||||||
self.provision_host_key(task)
|
# VmConfig(P0-1 闭环:生成→制盘→挂盘三步缺
|
||||||
|
# 一即 fail-fast,绝不回落 keyscan、绝不只
|
||||||
|
# 生成不注入)。
|
||||||
|
from cloud.chv.hostkeys import (
|
||||||
|
build_seed_disk, seed_disk_path)
|
||||||
|
provisioned = self.provision_host_key(task)
|
||||||
|
seed_disk = seed_disk_path(self.state_dir, name)
|
||||||
|
build_seed_disk(
|
||||||
|
provisioned["seed_dir"], seed_disk,
|
||||||
|
runner=lambda cmd: self._call_runner(cmd))
|
||||||
self._launch_api_only(task)
|
self._launch_api_only(task)
|
||||||
self.wait_api_ready(task)
|
self.wait_api_ready(task)
|
||||||
status, _ = self._api_call(
|
status, _ = self._api_call(
|
||||||
socket_path, "PUT", "/vm.create",
|
socket_path, "PUT", "/vm.create",
|
||||||
self.build_vm_config(task_disk))
|
self.build_vm_config(task_disk, seed_disk=seed_disk))
|
||||||
if status not in (200, 204):
|
if status not in (200, 204):
|
||||||
raise ChvFailure(
|
raise ChvFailure(
|
||||||
"vm_create_failed",
|
"vm_create_failed",
|
||||||
@@ -828,6 +958,18 @@ class CloudHypervisorExecutor(Executor):
|
|||||||
raise ValueError("snapshot_url 不能为空")
|
raise ValueError("snapshot_url 不能为空")
|
||||||
self._ch_remote(task_or_name, "snapshot",
|
self._ch_remote(task_or_name, "snapshot",
|
||||||
"source_url=%s" % snapshot_url)
|
"source_url=%s" % snapshot_url)
|
||||||
|
# feat-chv-fix(P1-7):只查 returncode 不够。file://
|
||||||
|
# 快照须验证目标文件存在且非空,否则报
|
||||||
|
# snapshot_not_persisted,调用方(hibernate_vm)据此
|
||||||
|
# 不 shutdown、不销毁原 VM;非 file:// 目标无法从宿
|
||||||
|
# 主验证落盘,仍只认 returncode,但文档明写该边界。
|
||||||
|
if snapshot_url.startswith("file://"):
|
||||||
|
_snap_path = snapshot_url[len("file://"):]
|
||||||
|
if not os.path.isfile(_snap_path) \
|
||||||
|
or os.path.getsize(_snap_path) == 0:
|
||||||
|
raise ChvFailure(
|
||||||
|
"snapshot_not_persisted",
|
||||||
|
"快照未落盘或为空:%s" % snapshot_url)
|
||||||
task = task_or_name if isinstance(task_or_name, dict) \
|
task = task_or_name if isinstance(task_or_name, dict) \
|
||||||
else {"id": task_or_name}
|
else {"id": task_or_name}
|
||||||
self._emit("vm_snapshot", task,
|
self._emit("vm_snapshot", task,
|
||||||
@@ -849,8 +991,27 @@ class CloudHypervisorExecutor(Executor):
|
|||||||
if "://" not in snapshot_url:
|
if "://" not in snapshot_url:
|
||||||
snapshot_url = "file://%s" % snapshot_url
|
snapshot_url = "file://%s" % snapshot_url
|
||||||
self.preflight()
|
self.preflight()
|
||||||
self._check_capacity()
|
self.resolve_ssh_host()
|
||||||
|
# feat-chv-fix(P0-2 修正版):restore 不走 vm.create,
|
||||||
|
# 没有 VmConfig 可挂 seed——审计报告“restore 须把 seed
|
||||||
|
# 接入 VmConfig”的说法在此不适用,纠正为:快照内已
|
||||||
|
# 固化原 VM 的 host key,重新预置新 key 反而会破坏
|
||||||
|
# 钉死。正确不变量是:provisioned 下复用已钉死的
|
||||||
|
# known_hosts(stop 时公钥保留、私钥已删但快照内仍有
|
||||||
|
# 原私钥),缺失时 fail-fast、绝不 keyscan、绝不重
|
||||||
|
# 生成。file:// 快照须先验存在且非空,否则不启动。
|
||||||
|
# 注意:restore 时不重复验快照文件存在——快照落盘
|
||||||
|
# 校验是 snapshot_vm/hibernate_vm 创建时的门(P1-7),
|
||||||
|
# restore 的 ch-remote 本身会在源缺失时失败;在这里
|
||||||
|
# 再验一次会把“源在远端/对象存储”的合法路径也挡掉。
|
||||||
|
if getattr(self, "host_key_mode", "keyscan") == "provisioned" \
|
||||||
|
and not os.path.exists(self.known_hosts_path(task)):
|
||||||
|
raise ChvFailure(
|
||||||
|
"host_key_not_pinned",
|
||||||
|
"provisioned 模式下 restore 缺 known_hosts,拒绝 "
|
||||||
|
"keyscan 回落:VM %s" % self.vm_name(task))
|
||||||
name = self.vm_name(task)
|
name = self.vm_name(task)
|
||||||
|
self._check_capacity(exclude=name)
|
||||||
try:
|
try:
|
||||||
self._launch_api_only(task)
|
self._launch_api_only(task)
|
||||||
self.wait_api_ready(task)
|
self.wait_api_ready(task)
|
||||||
@@ -936,7 +1097,8 @@ class CloudHypervisorExecutor(Executor):
|
|||||||
if os.path.isdir(seed_dir):
|
if os.path.isdir(seed_dir):
|
||||||
shutil.rmtree(seed_dir)
|
shutil.rmtree(seed_dir)
|
||||||
for _suffix in ("ssh_host_ed25519_key",
|
for _suffix in ("ssh_host_ed25519_key",
|
||||||
"ssh_host_ed25519_key.pub"):
|
"ssh_host_ed25519_key.pub",
|
||||||
|
"seed.iso"):
|
||||||
_p = os.path.join(self.state_dir, name, _suffix)
|
_p = os.path.join(self.state_dir, name, _suffix)
|
||||||
if os.path.exists(_p):
|
if os.path.exists(_p):
|
||||||
os.remove(_p)
|
os.remove(_p)
|
||||||
@@ -1027,14 +1189,16 @@ class CloudHypervisorExecutor(Executor):
|
|||||||
return cleaned
|
return cleaned
|
||||||
|
|
||||||
# -- 客内执行 ------------------------------------------------------
|
# -- 客内执行 ------------------------------------------------------
|
||||||
def _run_guest(self, task, inner_args, **kwargs):
|
def _run_guest(self, task, inner_args, extra_env=None, **kwargs):
|
||||||
"""SSH 调客内 host.py,返回 Local/Docker 同款结果 dict
|
"""SSH 调客内 host.py,返回 Local/Docker 同款结果 dict
|
||||||
(executor 字段改写为 cloud-hypervisor,并附 vm 名与
|
(executor 字段改写为 cloud-hypervisor,并附 vm 名与
|
||||||
api-socket)。runner 异常即原样传播,由 worker 侧统一
|
api-socket)。runner 异常即原样传播,由 worker 侧统一
|
||||||
写回 failed——本类不在这里把异常吞成假成功。"""
|
写回 failed——本类不在这里把异常吞成假成功。
|
||||||
|
extra_env(P1-1):经 SSH env 前缀进客机环境。"""
|
||||||
self._validate()
|
self._validate()
|
||||||
ssh_cmd = self.build_ssh_command(
|
ssh_cmd = self.build_ssh_command(
|
||||||
task, self._guest_host_command(*inner_args, **kwargs))
|
task, self._guest_host_command(*inner_args, **kwargs),
|
||||||
|
extra_env=extra_env)
|
||||||
result = self._call_runner(ssh_cmd, timeout=self.task_timeout)
|
result = self._call_runner(ssh_cmd, timeout=self.task_timeout)
|
||||||
# 方法内 import:见模块头注释的循环导入说明。
|
# 方法内 import:见模块头注释的循环导入说明。
|
||||||
from cloud.sandbox import _parse_host_result
|
from cloud.sandbox import _parse_host_result
|
||||||
@@ -1063,10 +1227,12 @@ class CloudHypervisorExecutor(Executor):
|
|||||||
return parse_goal_result(task_id, result,
|
return parse_goal_result(task_id, result,
|
||||||
"cloud-hypervisor")
|
"cloud-hypervisor")
|
||||||
|
|
||||||
def _run_guest_goal(self, task, inner_args, **kwargs):
|
def _run_guest_goal(self, task, inner_args, extra_env=None,
|
||||||
|
**kwargs):
|
||||||
self._validate()
|
self._validate()
|
||||||
ssh_cmd = self.build_ssh_command(
|
ssh_cmd = self.build_ssh_command(
|
||||||
task, self._guest_host_command(*inner_args, **kwargs))
|
task, self._guest_host_command(*inner_args, **kwargs),
|
||||||
|
extra_env=extra_env)
|
||||||
return self._goal_result(
|
return self._goal_result(
|
||||||
task["id"],
|
task["id"],
|
||||||
self._call_runner(ssh_cmd, timeout=self.task_timeout))
|
self._call_runner(ssh_cmd, timeout=self.task_timeout))
|
||||||
@@ -1137,7 +1303,8 @@ class CloudHypervisorExecutor(Executor):
|
|||||||
"interrupted_reason": reason,
|
"interrupted_reason": reason,
|
||||||
}
|
}
|
||||||
|
|
||||||
def _run_with_vm(self, task, inner_args, goal=False, **kwargs):
|
def _run_with_vm(self, task, inner_args, goal=False, extra_env=None,
|
||||||
|
**kwargs):
|
||||||
"""每任务 VM 生命周期:wake_vm(API 模式 + SSH 就绪)
|
"""每任务 VM 生命周期:wake_vm(API 模式 + SSH 就绪)
|
||||||
→ 跑客内命令 → finally shutdown_vm。wake 失败时不跑
|
→ 跑客内命令 → finally shutdown_vm。wake 失败时不跑
|
||||||
任务、直接抛分类失败。本方法不做内部重试:重试是
|
任务、直接抛分类失败。本方法不做内部重试:重试是
|
||||||
@@ -1155,8 +1322,9 @@ class CloudHypervisorExecutor(Executor):
|
|||||||
try:
|
try:
|
||||||
if goal:
|
if goal:
|
||||||
return self._run_guest_goal(
|
return self._run_guest_goal(
|
||||||
task, inner_args, **kwargs)
|
task, inner_args, extra_env=extra_env, **kwargs)
|
||||||
result = self._run_guest(task, inner_args, **kwargs)
|
result = self._run_guest(task, inner_args,
|
||||||
|
extra_env=extra_env, **kwargs)
|
||||||
except Exception as exc:
|
except Exception as exc:
|
||||||
if self._vm_proc_dead(task):
|
if self._vm_proc_dead(task):
|
||||||
return self._vm_death_result(
|
return self._vm_death_result(
|
||||||
@@ -1195,10 +1363,25 @@ class CloudHypervisorExecutor(Executor):
|
|||||||
"execute() 只读已运行 VM 的 status:VM %s 未运行,"
|
"execute() 只读已运行 VM 的 status:VM %s 未运行,"
|
||||||
"请先 wake_vm() 再调 execute(),不要用 execute "
|
"请先 wake_vm() 再调 execute(),不要用 execute "
|
||||||
"唤醒 VM" % self.vm_name(task))
|
"唤醒 VM" % self.vm_name(task))
|
||||||
return self._run_guest(
|
try:
|
||||||
task, ("status", "--task", self._host_name(task)))
|
return self._run_guest(
|
||||||
|
task, ("status", "--task", self._host_name(task)))
|
||||||
|
except Exception as exc:
|
||||||
|
# feat-chv-fix(P1-8):socket 残留但 VMM 已死时,
|
||||||
|
# _vm_is_running 会因 socket 存在而放行,随后的
|
||||||
|
# SSH 失败不得以原始传输异常抛出——登记进程已死
|
||||||
|
# 或 socket 已消失即统一归类 vm_not_running;VM
|
||||||
|
# 仍活着时的 SSH 失败原样传播(那是客内 status
|
||||||
|
# 失败,不是 VM 未运行)。
|
||||||
|
if self._vm_proc_dead(task) \
|
||||||
|
or not os.path.exists(self.api_socket(task)):
|
||||||
|
raise ChvFailure(
|
||||||
|
"vm_not_running",
|
||||||
|
"execute() 时 VM 已死(socket 残留/进程已死),"
|
||||||
|
"SSH 失败归类为未运行:%s" % exc)
|
||||||
|
raise
|
||||||
|
|
||||||
def dispatch_task(self, task) -> dict:
|
def dispatch_task(self, task, extra_env=None, model_provider=None) -> dict:
|
||||||
"""客内 host.py dispatch:建 host 任务并跑一轮 turn。"""
|
"""客内 host.py dispatch:建 host 任务并跑一轮 turn。"""
|
||||||
if not isinstance(task, dict) or task.get("id") is None:
|
if not isinstance(task, dict) or task.get("id") is None:
|
||||||
raise ValueError("任务缺少 id")
|
raise ValueError("任务缺少 id")
|
||||||
@@ -1210,11 +1393,12 @@ class CloudHypervisorExecutor(Executor):
|
|||||||
if repo:
|
if repo:
|
||||||
args += ["--repo", repo]
|
args += ["--repo", repo]
|
||||||
return self._run_with_vm(
|
return self._run_with_vm(
|
||||||
task, args, model=task.get("model"),
|
task, args, extra_env=extra_env, model=task.get("model"),
|
||||||
effort=task.get("effort"), approval=task.get("approval"),
|
effort=task.get("effort"), approval=task.get("approval"),
|
||||||
|
model_provider=model_provider,
|
||||||
sandbox=("read-only" if task.get("plan") else None))
|
sandbox=("read-only" if task.get("plan") else None))
|
||||||
|
|
||||||
def run_task(self, task) -> dict:
|
def run_task(self, task, extra_env=None, model_provider=None) -> dict:
|
||||||
"""客内 host.py run:在已有 host 任务上再跑一轮 turn。"""
|
"""客内 host.py run:在已有 host 任务上再跑一轮 turn。"""
|
||||||
if not isinstance(task, dict) or task.get("id") is None:
|
if not isinstance(task, dict) or task.get("id") is None:
|
||||||
raise ValueError("任务缺少 id")
|
raise ValueError("任务缺少 id")
|
||||||
@@ -1223,10 +1407,11 @@ class CloudHypervisorExecutor(Executor):
|
|||||||
return self._run_with_vm(
|
return self._run_with_vm(
|
||||||
task, ("run", "--task", self._host_name(task),
|
task, ("run", "--task", self._host_name(task),
|
||||||
"--prompt", task["prompt"]),
|
"--prompt", task["prompt"]),
|
||||||
model=task.get("model"), effort=task.get("effort"),
|
extra_env=extra_env, model=task.get("model"),
|
||||||
approval=task.get("approval"))
|
effort=task.get("effort"), approval=task.get("approval"),
|
||||||
|
model_provider=model_provider)
|
||||||
|
|
||||||
def goal_task(self, task) -> dict:
|
def goal_task(self, task, extra_env=None, model_provider=None) -> dict:
|
||||||
"""客内 host.py goal:目标循环,轮次规则与 Local 同款
|
"""客内 host.py goal:目标循环,轮次规则与 Local 同款
|
||||||
(缺省 3 轮,>100 轮 loud fail,旧验收形态 0 轮禁用)。"""
|
(缺省 3 轮,>100 轮 loud fail,旧验收形态 0 轮禁用)。"""
|
||||||
if not isinstance(task, dict) or task.get("id") is None:
|
if not isinstance(task, dict) or task.get("id") is None:
|
||||||
@@ -1252,11 +1437,13 @@ class CloudHypervisorExecutor(Executor):
|
|||||||
if repo:
|
if repo:
|
||||||
args += ["--repo", repo]
|
args += ["--repo", repo]
|
||||||
return self._run_with_vm(
|
return self._run_with_vm(
|
||||||
task, args, goal=True, model=task.get("model"),
|
task, args, goal=True, extra_env=extra_env,
|
||||||
effort=task.get("effort"), approval=task.get("approval"))
|
model=task.get("model"),
|
||||||
|
effort=task.get("effort"), approval=task.get("approval"),
|
||||||
|
model_provider=model_provider)
|
||||||
|
|
||||||
def goal_resume_task(self, task, goal_text=None,
|
def goal_resume_task(self, task, goal_text=None,
|
||||||
max_rounds=None) -> dict:
|
max_rounds=None, extra_env=None, model_provider=None) -> dict:
|
||||||
"""客内 host.py goal-resume:goal_text=None 为断点续跑
|
"""客内 host.py goal-resume:goal_text=None 为断点续跑
|
||||||
(不带 --goal-text,沿用原 goal+verify),给 goal_text
|
(不带 --goal-text,沿用原 goal+verify),给 goal_text
|
||||||
为开新目标(带 --goal-text 与 --max-rounds,默认 0)。"""
|
为开新目标(带 --goal-text 与 --max-rounds,默认 0)。"""
|
||||||
@@ -1269,11 +1456,14 @@ class CloudHypervisorExecutor(Executor):
|
|||||||
elif max_rounds is not None:
|
elif max_rounds is not None:
|
||||||
args += ["--max-rounds", str(max_rounds)]
|
args += ["--max-rounds", str(max_rounds)]
|
||||||
return self._run_with_vm(
|
return self._run_with_vm(
|
||||||
task, args, goal=True, model=task.get("model"),
|
task, args, goal=True, extra_env=extra_env,
|
||||||
effort=task.get("effort"), approval=task.get("approval"))
|
model=task.get("model"),
|
||||||
|
effort=task.get("effort"), approval=task.get("approval"),
|
||||||
|
model_provider=model_provider)
|
||||||
|
|
||||||
def goal_resume_breakpoint_task(self, task,
|
def goal_resume_breakpoint_task(self, task,
|
||||||
model_provider=None) -> dict:
|
model_provider=None,
|
||||||
|
extra_env=None) -> dict:
|
||||||
"""客内 host.py goal-resume 断点续跑:无 --goal-text、
|
"""客内 host.py goal-resume 断点续跑:无 --goal-text、
|
||||||
无 --max-rounds,host 沿用原任务从断点轮次续跑。
|
无 --max-rounds,host 沿用原任务从断点轮次续跑。
|
||||||
worker 层已接线(补缺口轨):非网关的 CHV 断点任务
|
worker 层已接线(补缺口轨):非网关的 CHV 断点任务
|
||||||
@@ -1286,7 +1476,7 @@ class CloudHypervisorExecutor(Executor):
|
|||||||
raise ValueError("任务缺少 id")
|
raise ValueError("任务缺少 id")
|
||||||
return self._run_with_vm(
|
return self._run_with_vm(
|
||||||
task, ("goal-resume", "--task", self._host_name(task)),
|
task, ("goal-resume", "--task", self._host_name(task)),
|
||||||
goal=True, model=task.get("model"),
|
goal=True, extra_env=extra_env, model=task.get("model"),
|
||||||
effort=task.get("effort"), approval=task.get("approval"),
|
effort=task.get("effort"), approval=task.get("approval"),
|
||||||
model_provider=model_provider)
|
model_provider=model_provider)
|
||||||
|
|
||||||
|
|||||||
+37
-4
@@ -44,16 +44,49 @@ def host_key_paths(state_dir, vm_name):
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
def known_hosts_line(port, public_key_text):
|
def known_hosts_line(port, public_key_text, host="127.0.0.1"):
|
||||||
"""公钥文件内容转 known_hosts 行:只取前两列(类型+数据)。
|
"""公钥文件内容转 known_hosts 行:只取前两列(类型+数据)。
|
||||||
|
|
||||||
公钥文件形如 'ssh-ed25519 AAAA... [comment]',known_hosts
|
公钥文件形如 'ssh-ed25519 AAAA... [comment]',known_hosts
|
||||||
行形如 '[127.0.0.1]:<port> ssh-ed25519 AAAA...'。非法
|
行形如 '[<host>]:<port> ssh-ed25519 AAAA...'(host 缺省
|
||||||
内容 ValueError,绝不把陌生行原样写进文件。"""
|
回环,仅作兼容;客机 IP 方案下 host 应为解析出的客机 IP)。
|
||||||
|
非法内容 ValueError,绝不把陌生行原样写进文件。"""
|
||||||
parts = (public_key_text or "").strip().split()
|
parts = (public_key_text or "").strip().split()
|
||||||
if len(parts) < 2 or not parts[0].startswith("ssh-"):
|
if len(parts) < 2 or not parts[0].startswith("ssh-"):
|
||||||
raise ValueError("host 公钥格式非法")
|
raise ValueError("host 公钥格式非法")
|
||||||
return "[127.0.0.1]:%s %s %s" % (port, parts[0], parts[1])
|
return "[%s]:%s %s %s" % (host, port, parts[0], parts[1])
|
||||||
|
|
||||||
|
|
||||||
|
def seed_disk_path(state_dir, vm_name):
|
||||||
|
"""该 VM 的 NoCloud seed 磁盘镜像路径(CIDATA 卷标 ISO)。"""
|
||||||
|
return os.path.join(state_dir, vm_name, "seed.iso")
|
||||||
|
|
||||||
|
|
||||||
|
def build_seed_disk(seed_dir, seed_disk, runner=None):
|
||||||
|
"""把 seed 目录做成 CIDATA 卷标 ISO(feat-chv-fix,P0-1 的
|
||||||
|
注入载体):genisoimage -volid CIDATA -joliet -rock。
|
||||||
|
|
||||||
|
runner 不给时用 subprocess;测试可注入假 runner(假
|
||||||
|
runner 负责把 seed_disk 文件写出来)。工具失败、返回码
|
||||||
|
非 0、或产物不存在/为空时一律 RuntimeError,调用方
|
||||||
|
(wake_vm provisioned 路径)fail-fast,绝不带着“seed
|
||||||
|
目录已写但未挂进 VmConfig”的 VM 继续启动。seed 目录
|
||||||
|
本身只是中间物,不再被当成注入通道。"""
|
||||||
|
cmd = ["genisoimage", "-output", seed_disk, "-volid", "CIDATA",
|
||||||
|
"-joliet", "-rock", seed_dir]
|
||||||
|
if runner is not None:
|
||||||
|
result = runner(cmd)
|
||||||
|
rc = getattr(result, "returncode", 0)
|
||||||
|
if rc not in (0, None):
|
||||||
|
raise RuntimeError("genisoimage 失败:returncode=%s" % rc)
|
||||||
|
else:
|
||||||
|
import subprocess
|
||||||
|
result = subprocess.run(cmd, capture_output=True, text=True)
|
||||||
|
if result.returncode != 0:
|
||||||
|
raise RuntimeError("genisoimage 失败:%s" % result.stderr)
|
||||||
|
if not os.path.isfile(seed_disk) or os.path.getsize(seed_disk) == 0:
|
||||||
|
raise RuntimeError("seed 磁盘未生成:%s" % seed_disk)
|
||||||
|
return seed_disk
|
||||||
|
|
||||||
|
|
||||||
def build_seed_user_data(private_key_text, public_key_text):
|
def build_seed_user_data(private_key_text, public_key_text):
|
||||||
|
|||||||
+33
-10
@@ -220,20 +220,43 @@ class WarmPool:
|
|||||||
return added
|
return added
|
||||||
|
|
||||||
def acquire(self, task):
|
def acquire(self, task):
|
||||||
"""分配一个预热 VM 给任务:有 warm 条目时消耗一个
|
"""分配一个预热 VM 给任务(feat-chv-fix,P1-3 接线):
|
||||||
(状态 assigned,返回 mode=warm + 该条目的合成任务
|
|
||||||
名,供调用方 resume 后使用);无 warm 条目时返回
|
有 warm 条目时先调 executor.resume_vm() 把暂停的 VM
|
||||||
mode=cold(调用方回落 wake_vm 冷启动)。"""
|
真正唤醒(resume 失败即条目作废、继续找下一个,
|
||||||
|
绝不把 paused 的 VM 当 running 发给任务),再标
|
||||||
|
assigned 并返回 mode=warm;resume 全部失败或无 warm
|
||||||
|
条目时返回 mode=cold(调用方回落 wake_vm 冷启动)。
|
||||||
|
注意:本方法只解决“acquire 不再悬空 resume 责任”
|
||||||
|
这一半;worker dispatch 前查池的另一半仍未接线
|
||||||
|
(见执行器/就绪文档的“未接线缺口”标注),预热收益
|
||||||
|
在生产路径上仍不是已交付特性。"""
|
||||||
for name, entry in list(self._entries.items()):
|
for name, entry in list(self._entries.items()):
|
||||||
if entry["state"] == "warm":
|
if entry["state"] != "warm":
|
||||||
entry["state"] = "assigned"
|
continue
|
||||||
entry["task"] = task
|
resume = getattr(self.executor, "resume_vm", None)
|
||||||
self._save_registry()
|
if resume is not None:
|
||||||
|
try:
|
||||||
|
resume(entry.get("task") or task)
|
||||||
|
except Exception:
|
||||||
|
self._entries.pop(name, None)
|
||||||
|
self._save_registry()
|
||||||
|
continue
|
||||||
|
# 无 resume_vm 的鸭子型执行器(旧测试替身):
|
||||||
|
# 保持旧 assigned 语义,不因此丢弃条目;真实
|
||||||
|
# CloudHypervisorExecutor 一律有 resume_vm,走
|
||||||
|
# 上面分支,责任不悬空。
|
||||||
|
entry["state"] = "assigned"
|
||||||
|
entry["task"] = task
|
||||||
|
self._save_registry()
|
||||||
|
try:
|
||||||
self.executor._emit(
|
self.executor._emit(
|
||||||
"warm_pool_acquire", task,
|
"warm_pool_acquire", task,
|
||||||
"vm=%s" % name)
|
"vm=%s" % name)
|
||||||
return {"mode": "warm", "vm": name,
|
except Exception:
|
||||||
"pool_task": entry["task"]}
|
pass
|
||||||
|
return {"mode": "warm", "vm": name,
|
||||||
|
"pool_task": entry["task"]}
|
||||||
return {"mode": "cold", "vm": None, "pool_task": None}
|
return {"mode": "cold", "vm": None, "pool_task": None}
|
||||||
|
|
||||||
def release(self, task, pool_vm=None):
|
def release(self, task, pool_vm=None):
|
||||||
|
|||||||
+32
-18
@@ -34,34 +34,48 @@ def known_hosts_path(state_dir, vm_name):
|
|||||||
|
|
||||||
|
|
||||||
def build_ssh_command(ssh_port, ssh_key, ssh_user, egress_proxy,
|
def build_ssh_command(ssh_port, ssh_key, ssh_user, egress_proxy,
|
||||||
known_hosts_file, inner_cmd):
|
known_hosts_file, inner_cmd, ssh_host="127.0.0.1",
|
||||||
|
extra_env=None):
|
||||||
"""构造 SSH 命令:ssh -p <端口> -i <key 文件> + 钉死选项
|
"""构造 SSH 命令:ssh -p <端口> -i <key 文件> + 钉死选项
|
||||||
+ user@127.0.0.1 + 客内命令。
|
+ user@<ssh_host> + 客内命令。
|
||||||
|
|
||||||
与 #216 的区别只有 host key 选项:StrictHostKeyChecking=yes
|
feat-chv-fix(P0-4):ssh_host 不再在传输层写死回环,
|
||||||
+ UserKnownHostsFile=<每 VM 文件>,不再是 no + /dev/null。
|
由执行器解析传入(显式 CHV_SSH_HOST / net 客机 IP /
|
||||||
BatchMode=yes 不变(绝不交互问 host key / 口令)。
|
显式转发宣称下的回环);本函数缺省回环仅为旧调用点
|
||||||
|
兼容,执行器路径绝不依赖该缺省。
|
||||||
|
extra_env(feat-chv-fix,P1-1 网关注入):经同一 env
|
||||||
|
前缀进客机环境(与 egress 代理前缀同款机制),不进
|
||||||
|
镜像/代码/prompt;值本身只出现在宿主侧 ssh argv(与
|
||||||
|
worker 同一信任边界),执行器绝不把完整命令记入日志。
|
||||||
egress_proxy 已设时客内命令前加 env 前缀(代理地址本身
|
egress_proxy 已设时客内命令前加 env 前缀(代理地址本身
|
||||||
不是密钥;客机出口只走该白名单代理)。"""
|
不是密钥;客机出口只走该白名单代理)。"""
|
||||||
inner = list(inner_cmd)
|
inner = list(inner_cmd)
|
||||||
|
env_assignments = []
|
||||||
|
if extra_env:
|
||||||
|
for key in sorted(extra_env):
|
||||||
|
env_assignments.append("%s=%s" % (key, extra_env[key]))
|
||||||
if egress_proxy:
|
if egress_proxy:
|
||||||
inner = ["env",
|
env_assignments += [
|
||||||
"HTTPS_PROXY=%s" % egress_proxy,
|
"HTTPS_PROXY=%s" % egress_proxy,
|
||||||
"HTTP_PROXY=%s" % egress_proxy,
|
"HTTP_PROXY=%s" % egress_proxy,
|
||||||
"NO_PROXY=localhost,127.0.0.1"] + inner
|
"NO_PROXY=localhost,127.0.0.1"]
|
||||||
|
if env_assignments:
|
||||||
|
inner = ["env"] + env_assignments + inner
|
||||||
return ["ssh", "-p", str(ssh_port),
|
return ["ssh", "-p", str(ssh_port),
|
||||||
"-i", ssh_key,
|
"-i", ssh_key,
|
||||||
"-o", "StrictHostKeyChecking=yes",
|
"-o", "StrictHostKeyChecking=yes",
|
||||||
"-o", "UserKnownHostsFile=%s" % known_hosts_file,
|
"-o", "UserKnownHostsFile=%s" % known_hosts_file,
|
||||||
"-o", "BatchMode=yes",
|
"-o", "BatchMode=yes",
|
||||||
"%s@127.0.0.1" % ssh_user] + inner
|
"%s@%s" % (ssh_user, ssh_host)] + inner
|
||||||
|
|
||||||
|
|
||||||
def parse_keyscan_output(text):
|
def parse_keyscan_output(text, host="127.0.0.1"):
|
||||||
"""从 ssh-keyscan 的 stdout 里挑出本机回环端口的 host key
|
"""从 ssh-keyscan 的 stdout 里挑出目标主机的 host key
|
||||||
行:形如 "[127.0.0.1]:<port> ssh-ed25519 AAAA..."(-p 时
|
行:形如 "[<host>]:<port> ssh-ed25519 AAAA..."(-p 时
|
||||||
的标准写法)或 "127.0.0.1 ssh-rsa AAAA..."。其它主机行、
|
的标准写法)或 "<host> ssh-rsa AAAA..."。其它主机行、
|
||||||
注释行、空行一律丢掉,绝不把陌生主机的 key 写进文件。"""
|
注释行、空行一律丢掉,绝不把陌生主机的 key 写进文件。
|
||||||
|
host 缺省回环仅为旧调用点兼容;客机 IP 方案下应传
|
||||||
|
解析出的客机 IP。"""
|
||||||
kept = []
|
kept = []
|
||||||
for line in (text or "").splitlines():
|
for line in (text or "").splitlines():
|
||||||
line = line.strip()
|
line = line.strip()
|
||||||
@@ -70,9 +84,9 @@ def parse_keyscan_output(text):
|
|||||||
parts = line.split()
|
parts = line.split()
|
||||||
if len(parts) < 3:
|
if len(parts) < 3:
|
||||||
continue
|
continue
|
||||||
host, keytype = parts[0], parts[1]
|
line_host, keytype = parts[0], parts[1]
|
||||||
if host != "127.0.0.1" \
|
if line_host != host \
|
||||||
and not host.startswith("[127.0.0.1]:"):
|
and not line_host.startswith("[%s]:" % host):
|
||||||
continue
|
continue
|
||||||
if not keytype.startswith("ssh-"):
|
if not keytype.startswith("ssh-"):
|
||||||
continue
|
continue
|
||||||
|
|||||||
@@ -0,0 +1,554 @@
|
|||||||
|
"""CHV 审核修复轨测试(堆叠于 #230,base=feat-chv-gaps)。
|
||||||
|
|
||||||
|
只用标准库与 mock runner/launcher/api_client,绝不启动真实
|
||||||
|
VM:当前环境没有可跑真 VM 的宿主。真实 VM 启动、seed 盘
|
||||||
|
在真机上的 cloud-init 应用、SSH 可达实测一律不在本文件
|
||||||
|
声称已验证;本文件只证明代码层面的接线(VmConfig 含 seed
|
||||||
|
盘、restore 不回落、reuse 拦截、SSH 目标解析、配额/快照/
|
||||||
|
API 完整性等)成立。
|
||||||
|
"""
|
||||||
|
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
import socket
|
||||||
|
import sys
|
||||||
|
import tempfile
|
||||||
|
import threading
|
||||||
|
import unittest
|
||||||
|
from pathlib import Path
|
||||||
|
from types import SimpleNamespace
|
||||||
|
|
||||||
|
try:
|
||||||
|
from cloud.chv.api_client import _unix_socket_api
|
||||||
|
from cloud.chv.disk import check_disk_quota, provision_task_disk
|
||||||
|
from cloud.chv.hostkeys import build_seed_disk, seed_disk_path
|
||||||
|
from cloud.chv.pool import WarmPool
|
||||||
|
from cloud.sandbox import (
|
||||||
|
ChvFailure,
|
||||||
|
CloudHypervisorExecutor,
|
||||||
|
)
|
||||||
|
from cloud.worker import Worker
|
||||||
|
except ImportError:
|
||||||
|
sys.path.insert(0, str(Path(__file__).resolve().parents[2]))
|
||||||
|
from cloud.chv.api_client import _unix_socket_api
|
||||||
|
from cloud.chv.disk import check_disk_quota, provision_task_disk
|
||||||
|
from cloud.chv.hostkeys import build_seed_disk, seed_disk_path
|
||||||
|
from cloud.chv.pool import WarmPool
|
||||||
|
from cloud.sandbox import (
|
||||||
|
ChvFailure,
|
||||||
|
CloudHypervisorExecutor,
|
||||||
|
)
|
||||||
|
from cloud.worker import Worker
|
||||||
|
|
||||||
|
|
||||||
|
class FakeResult:
|
||||||
|
def __init__(self, returncode=0, stdout="", stderr=""):
|
||||||
|
self.returncode = returncode
|
||||||
|
self.stdout = stdout
|
||||||
|
self.stderr = stderr
|
||||||
|
|
||||||
|
|
||||||
|
class FakeProc:
|
||||||
|
def __init__(self, alive=True):
|
||||||
|
self._alive = alive
|
||||||
|
|
||||||
|
def poll(self):
|
||||||
|
return None if self._alive else 1
|
||||||
|
|
||||||
|
def terminate(self):
|
||||||
|
self._alive = False
|
||||||
|
|
||||||
|
def wait(self, timeout=None):
|
||||||
|
return 0
|
||||||
|
|
||||||
|
def kill(self):
|
||||||
|
self._alive = False
|
||||||
|
|
||||||
|
|
||||||
|
def _write(path, content=b"x"):
|
||||||
|
with open(path, "wb") as fh:
|
||||||
|
fh.write(content)
|
||||||
|
os.chmod(path, 0o755)
|
||||||
|
return path
|
||||||
|
|
||||||
|
|
||||||
|
class ChvFixCase(unittest.TestCase):
|
||||||
|
def setUp(self):
|
||||||
|
self.tmp = tempfile.TemporaryDirectory()
|
||||||
|
self.dir = self.tmp.name
|
||||||
|
self.binary = _write(os.path.join(self.dir, "cloud-hypervisor"))
|
||||||
|
self.ch_remote = _write(os.path.join(self.dir, "ch-remote"))
|
||||||
|
self.kernel = _write(os.path.join(self.dir, "vmlinux"))
|
||||||
|
self.rootfs = _write(os.path.join(self.dir, "disk.raw"),
|
||||||
|
b"R" * 100)
|
||||||
|
self.ssh_key = _write(os.path.join(self.dir, "id_chv"))
|
||||||
|
self.kvm = _write(os.path.join(self.dir, "kvm"))
|
||||||
|
self.cpuinfo = os.path.join(self.dir, "cpuinfo")
|
||||||
|
with open(self.cpuinfo, "w") as fh:
|
||||||
|
fh.write("flags : fpu vmx sse\n")
|
||||||
|
self.state_dir = os.path.join(self.dir, "state")
|
||||||
|
self.calls = []
|
||||||
|
self.api_calls = []
|
||||||
|
self.api_payloads = {}
|
||||||
|
self.audit = []
|
||||||
|
|
||||||
|
def tearDown(self):
|
||||||
|
self.tmp.cleanup()
|
||||||
|
|
||||||
|
def _default_runner(self, cmd, timeout=None):
|
||||||
|
self.calls.append(list(cmd))
|
||||||
|
if cmd and cmd[0] == "ssh":
|
||||||
|
if cmd[-1] == "true":
|
||||||
|
return FakeResult(0, "", "")
|
||||||
|
if "goal" in cmd:
|
||||||
|
return FakeResult(0, "goal met after rounds\n", "")
|
||||||
|
return FakeResult(0, "", "")
|
||||||
|
if cmd and cmd[0] == "ssh-keygen":
|
||||||
|
priv = cmd[cmd.index("-f") + 1]
|
||||||
|
with open(priv, "w") as fh:
|
||||||
|
fh.write("-----BEGIN OPENSSH PRIVATE KEY-----\nFAKE\n")
|
||||||
|
os.chmod(priv, 0o600)
|
||||||
|
with open(priv + ".pub", "w") as fh:
|
||||||
|
fh.write("ssh-ed25519 AAAAfake1 vm-key\n")
|
||||||
|
return FakeResult(0, "", "")
|
||||||
|
if cmd and cmd[0] == "genisoimage":
|
||||||
|
out = cmd[cmd.index("-output") + 1]
|
||||||
|
with open(out, "wb") as fh:
|
||||||
|
fh.write(b"CIDATA" * 16)
|
||||||
|
return FakeResult(0, "", "")
|
||||||
|
return FakeResult(0, "", "")
|
||||||
|
|
||||||
|
def _default_api(self, socket_path, method, path, payload=None):
|
||||||
|
self.api_calls.append((method, path))
|
||||||
|
if path == "/vm.create":
|
||||||
|
self.api_payloads["vm.create"] = payload
|
||||||
|
return (200, {})
|
||||||
|
|
||||||
|
def make_exec(self, runner=None, env_extra=None, **overrides):
|
||||||
|
env = {"CHV_EGRESS_PROXY": "http://10.0.0.1:8888",
|
||||||
|
"CHV_SSH_FORWARD": "1"}
|
||||||
|
if env_extra:
|
||||||
|
env.update(env_extra)
|
||||||
|
kwargs = dict(
|
||||||
|
binary=self.binary, ch_remote=self.ch_remote,
|
||||||
|
kernel=self.kernel, rootfs=self.rootfs,
|
||||||
|
ssh_key=self.ssh_key, kvm_path=self.kvm,
|
||||||
|
cpuinfo_path=self.cpuinfo, state_dir=self.state_dir,
|
||||||
|
runner=runner or self._default_runner,
|
||||||
|
launcher=lambda cmd: FakeProc(alive=True),
|
||||||
|
api_client=self._default_api,
|
||||||
|
sleeper=lambda s: None, env=env)
|
||||||
|
kwargs.update(overrides)
|
||||||
|
ex = CloudHypervisorExecutor(**kwargs)
|
||||||
|
ex.audit_callback = lambda e, t, d, k=None: self.audit.append(
|
||||||
|
(e, t.get("id") if isinstance(t, dict) else t, d, k))
|
||||||
|
return ex
|
||||||
|
|
||||||
|
|
||||||
|
# -- P0-1 seed 磁盘接入 -------------------------------------------------
|
||||||
|
class SeedDiskTest(ChvFixCase):
|
||||||
|
def test_vm_config_carries_seed_disk_readonly(self):
|
||||||
|
ex = self.make_exec()
|
||||||
|
cfg = ex.build_vm_config("/tmp/x/disk.raw",
|
||||||
|
seed_disk="/tmp/x/seed.iso")
|
||||||
|
self.assertEqual(len(cfg["disks"]), 2)
|
||||||
|
self.assertEqual(cfg["disks"][1]["path"], "/tmp/x/seed.iso")
|
||||||
|
self.assertTrue(cfg["disks"][1]["readonly"])
|
||||||
|
cfg2 = ex.build_vm_config("/tmp/x/disk.raw")
|
||||||
|
self.assertEqual(len(cfg2["disks"]), 1)
|
||||||
|
|
||||||
|
def test_provisioned_wake_mounts_seed_disk(self):
|
||||||
|
ex = self.make_exec(
|
||||||
|
env_extra={"CHV_HOST_KEY_MODE": "provisioned"})
|
||||||
|
out = ex.wake_vm({"id": "seed1"})
|
||||||
|
self.assertEqual(out["state"], "running")
|
||||||
|
payload = self.api_payloads["vm.create"]
|
||||||
|
self.assertEqual(len(payload["disks"]), 2)
|
||||||
|
seed_disk = payload["disks"][1]["path"]
|
||||||
|
self.assertTrue(seed_disk.endswith("seed.iso"))
|
||||||
|
self.assertTrue(payload["disks"][1]["readonly"])
|
||||||
|
self.assertTrue(os.path.exists(seed_disk))
|
||||||
|
self.assertIn("genisoimage", [c[0] for c in self.calls])
|
||||||
|
|
||||||
|
def test_seed_disk_build_failure_is_fail_fast(self):
|
||||||
|
def bad_runner(cmd, timeout=None):
|
||||||
|
self.calls.append(list(cmd))
|
||||||
|
if cmd and cmd[0] == "genisoimage":
|
||||||
|
return FakeResult(1, "", "no genisoimage")
|
||||||
|
return self._default_runner(cmd, timeout)
|
||||||
|
|
||||||
|
ex = self.make_exec(
|
||||||
|
runner=bad_runner,
|
||||||
|
env_extra={"CHV_HOST_KEY_MODE": "provisioned"})
|
||||||
|
with self.assertRaises(RuntimeError):
|
||||||
|
ex.wake_vm({"id": "seedfail"})
|
||||||
|
# 制盘失败在起 VMM 之前,不应走到 vm.create。
|
||||||
|
self.assertNotIn("/vm.create",
|
||||||
|
[p for _, p in self.api_calls])
|
||||||
|
|
||||||
|
def test_build_seed_disk_rejects_missing_product(self):
|
||||||
|
with self.assertRaises(RuntimeError):
|
||||||
|
build_seed_disk(self.dir, os.path.join(self.dir, "no.iso"),
|
||||||
|
runner=lambda cmd: FakeResult(0, "", ""))
|
||||||
|
|
||||||
|
|
||||||
|
# -- P0-2 restore 不回落 ------------------------------------------------
|
||||||
|
class RestoreNoFallbackTest(ChvFixCase):
|
||||||
|
def test_wait_guest_ready_provisioned_missing_known_hosts(self):
|
||||||
|
ex = self.make_exec(
|
||||||
|
env_extra={"CHV_HOST_KEY_MODE": "provisioned"})
|
||||||
|
with self.assertRaises(ChvFailure) as ctx:
|
||||||
|
ex.wait_guest_ready({"id": "nopin"})
|
||||||
|
self.assertEqual(ctx.exception.kind, "host_key_not_pinned")
|
||||||
|
self.assertFalse(any(c[0] == "ssh-keyscan" for c in self.calls))
|
||||||
|
|
||||||
|
def test_restore_provisioned_missing_known_hosts_fails_before_launch(self):
|
||||||
|
ex = self.make_exec(
|
||||||
|
env_extra={"CHV_HOST_KEY_MODE": "provisioned"})
|
||||||
|
launches = []
|
||||||
|
ex.launcher = lambda cmd: launches.append(cmd) or FakeProc(True)
|
||||||
|
with self.assertRaises(ChvFailure) as ctx:
|
||||||
|
ex.restore_vm({"id": "r1"}, "file:///snap/x")
|
||||||
|
self.assertEqual(ctx.exception.kind, "host_key_not_pinned")
|
||||||
|
self.assertEqual(launches, [])
|
||||||
|
|
||||||
|
|
||||||
|
# -- P0-3 reuse 拦截 ----------------------------------------------------
|
||||||
|
class ReuseBlockTest(ChvFixCase):
|
||||||
|
def test_reuse_blocked_without_experiment_flag(self):
|
||||||
|
ex = self.make_exec(env_extra={"CHV_DISK_MODE": "reuse"})
|
||||||
|
with self.assertRaises(ChvFailure) as ctx:
|
||||||
|
ex.preflight()
|
||||||
|
self.assertEqual(ctx.exception.kind,
|
||||||
|
"disk_mode_reuse_blocked")
|
||||||
|
|
||||||
|
def test_reuse_passes_with_experiment_flag_and_is_audited(self):
|
||||||
|
ex = self.make_exec(env_extra={
|
||||||
|
"CHV_DISK_MODE": "reuse",
|
||||||
|
"CHV_REUSE_SINGLE_TENANT_EXPERIMENT": "1"})
|
||||||
|
self.assertTrue(ex.preflight())
|
||||||
|
out = ex.wake_vm({"id": "reuse1"})
|
||||||
|
self.assertEqual(out["state"], "running")
|
||||||
|
self.assertTrue(any(e == "disk_reuse_experiment_allowed"
|
||||||
|
for e, _, _, _ in self.audit))
|
||||||
|
|
||||||
|
|
||||||
|
# -- P0-4 SSH 目标解析 --------------------------------------------------
|
||||||
|
class SshTargetTest(ChvFixCase):
|
||||||
|
def test_explicit_host_wins(self):
|
||||||
|
ex = self.make_exec(ssh_host="10.9.8.7")
|
||||||
|
self.assertEqual(ex.resolve_ssh_host(), "10.9.8.7")
|
||||||
|
|
||||||
|
def test_net_guest_ip_used_when_no_explicit_host(self):
|
||||||
|
ex = self.make_exec(env_extra={
|
||||||
|
"CHV_NET_JSON": '[{"tap": "", "ip": "192.168.100.5",'
|
||||||
|
'"mask": "255.255.255.0"}]',
|
||||||
|
"CHV_SSH_FORWARD": "0"})
|
||||||
|
# env_extra 覆盖 forward=0 后仍应走 net ip,不报未声明。
|
||||||
|
self.assertEqual(ex.resolve_ssh_host(), "192.168.100.5")
|
||||||
|
cmd = ex.build_ssh_command({"id": "t"}, ["true"])
|
||||||
|
self.assertIn("root@192.168.100.5", " ".join(cmd))
|
||||||
|
|
||||||
|
def test_undeclared_target_is_fail_fast(self):
|
||||||
|
ex = self.make_exec(env_extra={"CHV_SSH_FORWARD": "0"})
|
||||||
|
with self.assertRaises(ChvFailure) as ctx:
|
||||||
|
ex.resolve_ssh_host()
|
||||||
|
self.assertEqual(ctx.exception.kind, "ssh_target_undeclared")
|
||||||
|
|
||||||
|
def test_forward_attestation_allows_loopback(self):
|
||||||
|
ex = self.make_exec()
|
||||||
|
self.assertEqual(ex.resolve_ssh_host(), "127.0.0.1")
|
||||||
|
|
||||||
|
|
||||||
|
# -- P1-1 网关 extra_env ------------------------------------------------
|
||||||
|
class GatewayExtraEnvTest(ChvFixCase):
|
||||||
|
def test_chv_ssh_command_carries_extra_env(self):
|
||||||
|
ex = self.make_exec()
|
||||||
|
cmd = ex.build_ssh_command({"id": "t"}, ["true"],
|
||||||
|
extra_env={"SUB2API_API_KEY": "k1"})
|
||||||
|
joined = " ".join(cmd)
|
||||||
|
self.assertIn("SUB2API_API_KEY=k1", joined)
|
||||||
|
self.assertIn("HTTPS_PROXY=http://10.0.0.1:8888", joined)
|
||||||
|
|
||||||
|
def test_worker_gateway_guard_allows_chv(self):
|
||||||
|
from cloud.server import init_db
|
||||||
|
db_path = os.path.join(self.dir, "control.db")
|
||||||
|
init_db(db_path)
|
||||||
|
ex = self.make_exec()
|
||||||
|
worker = Worker(db_path, "/fake/host.py", executor=ex,
|
||||||
|
env={"XCM_API_KEY": "test-placeholder"})
|
||||||
|
task = {"id": "gw1", "project_id": "p1", "prompt": "做目标",
|
||||||
|
"goal_mode": 1,
|
||||||
|
"_gateway_provider": "sub2api",
|
||||||
|
"_gateway_env": {"SUB2API_API_KEY": "gwkey",
|
||||||
|
"SUB2API_BASE_URL": "http://gw/"}}
|
||||||
|
result, error = worker._run_attempt(task)
|
||||||
|
self.assertIsNone(error)
|
||||||
|
self.assertEqual(result["status"], "completed")
|
||||||
|
ssh_calls = [c for c in self.calls if c and c[0] == "ssh"]
|
||||||
|
self.assertTrue(any("SUB2API_API_KEY=gwkey" in " ".join(c)
|
||||||
|
for c in ssh_calls))
|
||||||
|
|
||||||
|
|
||||||
|
# -- P1-2 取消/清理/看门狗 ----------------------------------------------
|
||||||
|
class CancelCleanupTest(ChvFixCase):
|
||||||
|
def test_worker_cancel_calls_cancel_vm(self):
|
||||||
|
from cloud.server import init_db
|
||||||
|
db_path = os.path.join(self.dir, "control.db")
|
||||||
|
init_db(db_path)
|
||||||
|
ex = self.make_exec()
|
||||||
|
cancelled = []
|
||||||
|
ex.cancel_vm = lambda name: cancelled.append(name) or True
|
||||||
|
worker = Worker(db_path, "/fake/host.py", executor=ex,
|
||||||
|
env={"XCM_API_KEY": "test-placeholder"})
|
||||||
|
worker._running_set("t-cancel")
|
||||||
|
import sqlite3, time
|
||||||
|
conn = sqlite3.connect(db_path)
|
||||||
|
conn.execute(
|
||||||
|
"INSERT INTO tasks(id, project_id, prompt, status,"
|
||||||
|
" idempotency_key, depends_on, priority, created_by,"
|
||||||
|
" created_at, updated_at, last_heartbeat)"
|
||||||
|
" VALUES(?, ?, ?, ?, NULL, ?, ?, 'tester', ?, ?, NULL)",
|
||||||
|
("t-cancel", "p1", "做事", "running", "[]", 0,
|
||||||
|
time.time(), time.time()))
|
||||||
|
conn.commit()
|
||||||
|
conn.close()
|
||||||
|
self.assertTrue(worker.cancel("t-cancel"))
|
||||||
|
self.assertEqual(cancelled, ["t-cancel"])
|
||||||
|
|
||||||
|
def test_worker_cleanup_calls_cleanup_stale_vms(self):
|
||||||
|
from cloud.server import init_db
|
||||||
|
db_path = os.path.join(self.dir, "control.db")
|
||||||
|
init_db(db_path)
|
||||||
|
ex = self.make_exec()
|
||||||
|
ex.cleanup_stale_vms = lambda: ["chv-stale"]
|
||||||
|
worker = Worker(db_path, "/fake/host.py", executor=ex,
|
||||||
|
env={"XCM_API_KEY": "test-placeholder"})
|
||||||
|
self.assertIn("chv-stale", worker.cleanup_stale_containers())
|
||||||
|
|
||||||
|
def test_chv_cancel_watchdog_kills_vm_on_cancel(self):
|
||||||
|
import threading
|
||||||
|
from cloud.server import init_db
|
||||||
|
db_path = os.path.join(self.dir, "control.db")
|
||||||
|
init_db(db_path)
|
||||||
|
ex = self.make_exec()
|
||||||
|
killed = []
|
||||||
|
ex.cancel_vm = lambda name: killed.append(name) or True
|
||||||
|
worker = Worker(db_path, "/fake/host.py", executor=ex,
|
||||||
|
env={"XCM_API_KEY": "test-placeholder"})
|
||||||
|
worker._cancel_requested.add("t-wd")
|
||||||
|
stop = threading.Event()
|
||||||
|
worker._chv_cancel_watchdog("t-wd", "chv-t-wd", stop)
|
||||||
|
self.assertEqual(killed, ["chv-t-wd"])
|
||||||
|
|
||||||
|
|
||||||
|
# -- P1-3 WarmPool acquire resume ---------------------------------------
|
||||||
|
class PoolAcquireResumeTest(unittest.TestCase):
|
||||||
|
def test_acquire_resumes_paused_vm(self):
|
||||||
|
class Stub:
|
||||||
|
def __init__(self):
|
||||||
|
self.resumed = []
|
||||||
|
self.state_dir = None
|
||||||
|
self.warm_pool_size = 1
|
||||||
|
self.snapshot_url = "file:///snap/base"
|
||||||
|
|
||||||
|
def vm_name(self, task):
|
||||||
|
raw = task.get("id") if isinstance(task, dict) \
|
||||||
|
else str(task)
|
||||||
|
return raw if raw.startswith("chv-") \
|
||||||
|
else "chv-" + raw
|
||||||
|
|
||||||
|
def wake_vm(self, task):
|
||||||
|
return {"vm": self.vm_name(task)}
|
||||||
|
|
||||||
|
def pause_vm(self, task):
|
||||||
|
return True
|
||||||
|
|
||||||
|
def resume_vm(self, task):
|
||||||
|
self.resumed.append(task["id"])
|
||||||
|
return True
|
||||||
|
|
||||||
|
def shutdown_vm(self, task_or_name):
|
||||||
|
return True
|
||||||
|
|
||||||
|
def _emit(self, event, task, detail="", kind=None):
|
||||||
|
pass
|
||||||
|
|
||||||
|
stub = Stub()
|
||||||
|
pool = WarmPool(stub, size=1, snapshot_url="file:///snap/base",
|
||||||
|
registry_path=None)
|
||||||
|
pool.prewarm()
|
||||||
|
got = pool.acquire({"id": "t1"})
|
||||||
|
self.assertEqual(got["mode"], "warm")
|
||||||
|
self.assertEqual(stub.resumed, ["warm-1"])
|
||||||
|
|
||||||
|
def test_acquire_discards_vm_whose_resume_fails(self):
|
||||||
|
class Stub:
|
||||||
|
def __init__(self):
|
||||||
|
self.state_dir = None
|
||||||
|
self.warm_pool_size = 1
|
||||||
|
self.snapshot_url = "file:///snap/base"
|
||||||
|
|
||||||
|
def vm_name(self, task):
|
||||||
|
raw = task.get("id") if isinstance(task, dict) \
|
||||||
|
else str(task)
|
||||||
|
return raw if raw.startswith("chv-") \
|
||||||
|
else "chv-" + raw
|
||||||
|
|
||||||
|
def wake_vm(self, task):
|
||||||
|
return {"vm": self.vm_name(task)}
|
||||||
|
|
||||||
|
def pause_vm(self, task):
|
||||||
|
return True
|
||||||
|
|
||||||
|
def resume_vm(self, task):
|
||||||
|
raise RuntimeError("resume 失败")
|
||||||
|
|
||||||
|
def shutdown_vm(self, task_or_name):
|
||||||
|
return True
|
||||||
|
|
||||||
|
def _emit(self, event, task, detail="", kind=None):
|
||||||
|
pass
|
||||||
|
|
||||||
|
pool = WarmPool(Stub(), size=1, snapshot_url="file:///snap/base",
|
||||||
|
registry_path=None)
|
||||||
|
pool.prewarm()
|
||||||
|
self.assertEqual(pool.acquire({"id": "t1"})["mode"], "cold")
|
||||||
|
|
||||||
|
|
||||||
|
# -- P1-4 容量持久化计数 -------------------------------------------------
|
||||||
|
class CapacityPersistentTest(ChvFixCase):
|
||||||
|
def test_socket_counts_after_restart(self):
|
||||||
|
ex = self.make_exec(env_extra={"CHV_MAX_VMS": "1"})
|
||||||
|
os.makedirs(self.state_dir, exist_ok=True)
|
||||||
|
with open(os.path.join(self.state_dir, "chv-other.sock"),
|
||||||
|
"w") as fh:
|
||||||
|
fh.write("")
|
||||||
|
with self.assertRaises(ChvFailure) as ctx:
|
||||||
|
ex.wake_vm({"id": "new1"})
|
||||||
|
self.assertEqual(ctx.exception.kind, "host_capacity_exceeded")
|
||||||
|
|
||||||
|
def test_own_socket_does_not_block_rewake(self):
|
||||||
|
ex = self.make_exec(env_extra={"CHV_MAX_VMS": "1"})
|
||||||
|
os.makedirs(self.state_dir, exist_ok=True)
|
||||||
|
with open(os.path.join(self.state_dir, "chv-t1.sock"),
|
||||||
|
"w") as fh:
|
||||||
|
fh.write("")
|
||||||
|
# 同任务重复 wake:自己的残留 socket 不计入容量。
|
||||||
|
out = ex.wake_vm({"id": "t1"})
|
||||||
|
self.assertEqual(out["state"], "running")
|
||||||
|
|
||||||
|
|
||||||
|
# -- P1-5 磁盘配额 -------------------------------------------------------
|
||||||
|
class DiskQuotaTest(ChvFixCase):
|
||||||
|
def test_unreadable_image_is_not_silently_skipped(self):
|
||||||
|
with self.assertRaises(ChvFailure) as ctx:
|
||||||
|
check_disk_quota("/nonexistent/disk.raw", 1.0)
|
||||||
|
self.assertEqual(ctx.exception.kind, "disk_quota_unverified")
|
||||||
|
|
||||||
|
def test_total_usage_over_quota(self):
|
||||||
|
os.makedirs(self.state_dir, exist_ok=True)
|
||||||
|
with open(os.path.join(self.state_dir, "old.raw"),
|
||||||
|
"wb") as fh:
|
||||||
|
fh.write(b"X" * 100)
|
||||||
|
# 配额 150 字节:已用 100 + 本次 100 > 150。
|
||||||
|
with self.assertRaises(ChvFailure) as ctx:
|
||||||
|
check_disk_quota(self.rootfs, 150 / (1024 ** 3),
|
||||||
|
state_dir=self.state_dir)
|
||||||
|
self.assertEqual(ctx.exception.kind, "disk_quota_exceeded")
|
||||||
|
|
||||||
|
def test_existing_disk_size_mismatch_is_recopied(self):
|
||||||
|
task_disk = provision_task_disk(self.state_dir, "chv-a",
|
||||||
|
self.rootfs, "copy")
|
||||||
|
with open(task_disk, "wb") as fh:
|
||||||
|
fh.write(b"Z" * 10)
|
||||||
|
task_disk2 = provision_task_disk(self.state_dir, "chv-a",
|
||||||
|
self.rootfs, "copy")
|
||||||
|
self.assertEqual(task_disk, task_disk2)
|
||||||
|
with open(task_disk2, "rb") as fh:
|
||||||
|
self.assertEqual(fh.read(), b"R" * 100)
|
||||||
|
|
||||||
|
|
||||||
|
# -- P1-6 API 截断 -------------------------------------------------------
|
||||||
|
class ApiTruncationTest(unittest.TestCase):
|
||||||
|
def _serve(self, raw):
|
||||||
|
tmp = tempfile.TemporaryDirectory()
|
||||||
|
sock_path = os.path.join(tmp.name, "vmm.sock")
|
||||||
|
srv = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
|
||||||
|
srv.bind(sock_path)
|
||||||
|
srv.listen(1)
|
||||||
|
|
||||||
|
def _run():
|
||||||
|
conn, _ = srv.accept()
|
||||||
|
try:
|
||||||
|
conn.recv(65536)
|
||||||
|
conn.sendall(raw)
|
||||||
|
finally:
|
||||||
|
conn.close()
|
||||||
|
srv.close()
|
||||||
|
|
||||||
|
threading.Thread(target=_run, daemon=True).start()
|
||||||
|
return tmp, sock_path
|
||||||
|
|
||||||
|
def test_truncated_body_raises(self):
|
||||||
|
body = b'{"ok": true}'
|
||||||
|
raw = (b"HTTP/1.1 200 OK\r\nContent-Length: 100\r\n\r\n"
|
||||||
|
+ body)
|
||||||
|
tmp, sock_path = self._serve(raw)
|
||||||
|
with self.assertRaises(RuntimeError):
|
||||||
|
_unix_socket_api(sock_path, "GET", "/vmm.ping")
|
||||||
|
tmp.cleanup()
|
||||||
|
|
||||||
|
def test_chunked_rejected(self):
|
||||||
|
raw = (b"HTTP/1.1 200 OK\r\nTransfer-Encoding: chunked\r\n\r\n"
|
||||||
|
b"5\r\nhello\r\n0\r\n\r\n")
|
||||||
|
tmp, sock_path = self._serve(raw)
|
||||||
|
with self.assertRaises(RuntimeError):
|
||||||
|
_unix_socket_api(sock_path, "GET", "/vmm.ping")
|
||||||
|
tmp.cleanup()
|
||||||
|
|
||||||
|
def test_full_body_parses(self):
|
||||||
|
body = b'{"ok": true}'
|
||||||
|
raw = (b"HTTP/1.1 200 OK\r\nContent-Length: %d\r\n\r\n"
|
||||||
|
% len(body)) + body
|
||||||
|
tmp, sock_path = self._serve(raw)
|
||||||
|
status, parsed = _unix_socket_api(sock_path, "GET", "/vmm.ping")
|
||||||
|
self.assertEqual(status, 200)
|
||||||
|
self.assertTrue(parsed["ok"])
|
||||||
|
tmp.cleanup()
|
||||||
|
|
||||||
|
|
||||||
|
# -- P1-7 快照落盘验证 ---------------------------------------------------
|
||||||
|
class SnapshotPersistTest(ChvFixCase):
|
||||||
|
def test_snapshot_missing_file_does_not_shutdown(self):
|
||||||
|
ex = self.make_exec()
|
||||||
|
with self.assertRaises(ChvFailure) as ctx:
|
||||||
|
ex.hibernate_vm({"id": "h1"}, "file:///nonexistent/snap")
|
||||||
|
self.assertEqual(ctx.exception.kind, "snapshot_not_persisted")
|
||||||
|
kinds = [c[-1] for c in self.calls if c[0] == self.ch_remote]
|
||||||
|
self.assertIn("pause", kinds)
|
||||||
|
self.assertNotIn("shutdown", kinds)
|
||||||
|
|
||||||
|
|
||||||
|
# -- P1-8 execute 死端口归类 ---------------------------------------------
|
||||||
|
class ExecuteDeadPortTest(ChvFixCase):
|
||||||
|
def test_dead_socket_ssh_failure_is_vm_not_running(self):
|
||||||
|
def raising_runner(cmd, timeout=None):
|
||||||
|
self.calls.append(list(cmd))
|
||||||
|
if cmd and cmd[0] == "ssh":
|
||||||
|
raise RuntimeError("connection refused")
|
||||||
|
return FakeResult(0, "", "")
|
||||||
|
|
||||||
|
ex = self.make_exec(runner=raising_runner)
|
||||||
|
os.makedirs(self.state_dir, exist_ok=True)
|
||||||
|
with open(os.path.join(self.state_dir, "chv-dead.sock"),
|
||||||
|
"w") as fh:
|
||||||
|
fh.write("")
|
||||||
|
ex._vms["chv-dead"] = FakeProc(alive=False)
|
||||||
|
with self.assertRaises(ChvFailure) as ctx:
|
||||||
|
ex.execute({"id": "dead"})
|
||||||
|
self.assertEqual(ctx.exception.kind, "vm_not_running")
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
unittest.main()
|
||||||
@@ -119,7 +119,10 @@ class ChvGapCase(unittest.TestCase):
|
|||||||
return FakeProc(alive=True)
|
return FakeProc(alive=True)
|
||||||
|
|
||||||
def make_exec(self, runner=None, env_extra=None, **overrides):
|
def make_exec(self, runner=None, env_extra=None, **overrides):
|
||||||
env = {"CHV_EGRESS_PROXY": "http://10.0.0.1:8888"}
|
env = {"CHV_EGRESS_PROXY": "http://10.0.0.1:8888",
|
||||||
|
# 实验室宣称:mock runner 假装回环端口已转发到客机
|
||||||
|
#(feat-chv-fix 起,执行器不再无条件假设回环能通)。
|
||||||
|
"CHV_SSH_FORWARD": "1"}
|
||||||
if env_extra:
|
if env_extra:
|
||||||
env.update(env_extra)
|
env.update(env_extra)
|
||||||
kwargs = dict(
|
kwargs = dict(
|
||||||
|
|||||||
@@ -108,7 +108,10 @@ class ChvOptCase(unittest.TestCase):
|
|||||||
|
|
||||||
def make_exec(self, runner=None, api_client=None, launcher=None,
|
def make_exec(self, runner=None, api_client=None, launcher=None,
|
||||||
env_extra=None, **overrides):
|
env_extra=None, **overrides):
|
||||||
env = {"CHV_EGRESS_PROXY": "http://10.0.0.1:8888"}
|
env = {"CHV_EGRESS_PROXY": "http://10.0.0.1:8888",
|
||||||
|
# 实验室宣称:mock runner 假装回环端口已转发到客机
|
||||||
|
#(feat-chv-fix 起,执行器不再无条件假设回环能通)。
|
||||||
|
"CHV_SSH_FORWARD": "1"}
|
||||||
if env_extra:
|
if env_extra:
|
||||||
env.update(env_extra)
|
env.update(env_extra)
|
||||||
if runner is None:
|
if runner is None:
|
||||||
|
|||||||
@@ -98,7 +98,10 @@ class ChvCase(unittest.TestCase):
|
|||||||
|
|
||||||
def make_exec(self, runner=None, api_client=None, launcher=None,
|
def make_exec(self, runner=None, api_client=None, launcher=None,
|
||||||
env_extra=None, **overrides):
|
env_extra=None, **overrides):
|
||||||
env = {"CHV_EGRESS_PROXY": "http://10.0.0.1:8888"}
|
env = {"CHV_EGRESS_PROXY": "http://10.0.0.1:8888",
|
||||||
|
# 实验室宣称:mock runner 假装回环端口已转发到客机
|
||||||
|
#(feat-chv-fix 起,执行器不再无条件假设回环能通)。
|
||||||
|
"CHV_SSH_FORWARD": "1"}
|
||||||
if env_extra:
|
if env_extra:
|
||||||
env.update(env_extra)
|
env.update(env_extra)
|
||||||
if runner is None:
|
if runner is None:
|
||||||
@@ -410,9 +413,15 @@ class LifecycleTest(ChvCase):
|
|||||||
|
|
||||||
def test_hibernate_is_pause_snapshot_shutdown(self):
|
def test_hibernate_is_pause_snapshot_shutdown(self):
|
||||||
ex = self.make_exec()
|
ex = self.make_exec()
|
||||||
out = ex.hibernate_vm(self._task(), "file:///snap/h1")
|
# feat-chv-fix(P1-7):file:// 快照须真实存在且非空,
|
||||||
|
# 否则 hibernate 不 shutdown;用临时文件作快照目标。
|
||||||
|
snap = os.path.join(self.dir, "h1.snap")
|
||||||
|
with open(snap, "wb") as fh:
|
||||||
|
fh.write(b"S" * 64)
|
||||||
|
snap_url = "file://" + snap
|
||||||
|
out = ex.hibernate_vm(self._task(), snap_url)
|
||||||
self.assertEqual(out["state"], "hibernated")
|
self.assertEqual(out["state"], "hibernated")
|
||||||
self.assertEqual(out["snapshot"], "file:///snap/h1")
|
self.assertEqual(out["snapshot"], snap_url)
|
||||||
kinds = [c[-1] for c in self.calls if c[0] == self.ch_remote]
|
kinds = [c[-1] for c in self.calls if c[0] == self.ch_remote]
|
||||||
self.assertEqual(kinds[0], "pause")
|
self.assertEqual(kinds[0], "pause")
|
||||||
self.assertTrue(any(a.startswith("source_url")
|
self.assertTrue(any(a.startswith("source_url")
|
||||||
@@ -420,7 +429,7 @@ class LifecycleTest(ChvCase):
|
|||||||
self.assertIn("shutdown", kinds)
|
self.assertIn("shutdown", kinds)
|
||||||
# 从休眠唤醒 = restore + resume:
|
# 从休眠唤醒 = restore + resume:
|
||||||
ex2 = self.make_exec()
|
ex2 = self.make_exec()
|
||||||
out2 = ex2.wake_from_hibernate(self._task(), "file:///snap/h1")
|
out2 = ex2.wake_from_hibernate(self._task(), snap_url)
|
||||||
self.assertEqual(out2["mode"], "restored")
|
self.assertEqual(out2["mode"], "restored")
|
||||||
|
|
||||||
def test_stop_dead_proc(self):
|
def test_stop_dead_proc(self):
|
||||||
|
|||||||
+138
-19
@@ -333,22 +333,28 @@ def _merge_env(base, override):
|
|||||||
return merged
|
return merged
|
||||||
|
|
||||||
|
|
||||||
def _call_goal_resume_breakpoint(executor, task, model_provider=None):
|
def _call_goal_resume_breakpoint(executor, task, model_provider=None,
|
||||||
|
extra_env=None):
|
||||||
"""通用断点调用:执行器提供 goal_resume_breakpoint_task 时调用它。
|
"""通用断点调用:执行器提供 goal_resume_breakpoint_task 时调用它。
|
||||||
|
|
||||||
Local 仍走原路径(调用方直接传 model_provider,不经过这里,
|
Local 仍走原路径(调用方直接传 model_provider,不经过这里,
|
||||||
以免改生产语义);这里只用于非 Local 执行器:签名接受
|
以免改生产语义);这里只用于非 Local 执行器:签名接受
|
||||||
model_provider 时传它(CHV/Docker 均接受),否则只传 task
|
model_provider 时传它(CHV/Docker 均接受),否则只传 task
|
||||||
(旧假执行器单参)。网关 key 绝不经这里传值:worker 侧
|
(旧假执行器单参)。feat-chv-fix(P1-1):CHV 的
|
||||||
gateway_env 只按名注入,见 _run_attempt 的网关守卫。"""
|
gateway_env 经此进入 SSH env 前缀注入客机环境(不进
|
||||||
|
镜像/代码/prompt);CHV 没有 Local 那样的进程 env 通道,
|
||||||
|
SSH env 前缀是唯一通道,值只出现在宿主侧 ssh argv。"""
|
||||||
method = getattr(executor, "goal_resume_breakpoint_task")
|
method = getattr(executor, "goal_resume_breakpoint_task")
|
||||||
try:
|
try:
|
||||||
params = inspect.signature(method).parameters
|
params = inspect.signature(method).parameters
|
||||||
except (TypeError, ValueError):
|
except (TypeError, ValueError):
|
||||||
return method(task)
|
return method(task)
|
||||||
|
kwargs = {}
|
||||||
if "model_provider" in params:
|
if "model_provider" in params:
|
||||||
return method(task, model_provider=model_provider)
|
kwargs["model_provider"] = model_provider
|
||||||
return method(task)
|
if "extra_env" in params and extra_env is not None:
|
||||||
|
kwargs["extra_env"] = extra_env
|
||||||
|
return method(task, **kwargs)
|
||||||
|
|
||||||
|
|
||||||
def _resolve_model(model=None, config_path=None):
|
def _resolve_model(model=None, config_path=None):
|
||||||
@@ -2368,6 +2374,12 @@ class Worker:
|
|||||||
self.executor.kill_container(entry["container"])
|
self.executor.kill_container(entry["container"])
|
||||||
except Exception:
|
except Exception:
|
||||||
pass
|
pass
|
||||||
|
# feat-chv-fix(P1-2):CHV 取消真关 VM,不只改库状态。
|
||||||
|
if hasattr(self.executor, "cancel_vm"):
|
||||||
|
try:
|
||||||
|
self.executor.cancel_vm(task_id)
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
self._running_clear(task_id)
|
self._running_clear(task_id)
|
||||||
conn = _db_connect(self.db_path)
|
conn = _db_connect(self.db_path)
|
||||||
try:
|
try:
|
||||||
@@ -2470,22 +2482,87 @@ class Worker:
|
|||||||
pass
|
pass
|
||||||
return
|
return
|
||||||
|
|
||||||
|
def _chv_cancel_watchdog(self, task_id, vm_name, stop_event):
|
||||||
|
"""CHV 同步路径的取消看门狗(feat-chv-fix,P1-2,Docker
|
||||||
|
取消看门狗同款模式):同步 SSH 阻塞时 worker 的 cancel()
|
||||||
|
只能改库、清 _running 项,阻塞中的 run 本身不会醒;
|
||||||
|
本线程每 1 秒查库/取消标志,命中时调 cancel_vm 真关
|
||||||
|
VM,然后退出。查库失败时当没取消,不误杀。"""
|
||||||
|
while not stop_event.wait(1.0):
|
||||||
|
try:
|
||||||
|
if task_id in self._cancel_requested:
|
||||||
|
cancelled = True
|
||||||
|
else:
|
||||||
|
db_task = self.get_task(task_id)
|
||||||
|
cancelled = (db_task is not None and
|
||||||
|
db_task.get("status") == "interrupted")
|
||||||
|
except Exception:
|
||||||
|
cancelled = False
|
||||||
|
if not cancelled:
|
||||||
|
continue
|
||||||
|
try:
|
||||||
|
cancel = getattr(self.executor, "cancel_vm", None)
|
||||||
|
if cancel is not None and vm_name:
|
||||||
|
cancel(vm_name)
|
||||||
|
log_event(self.logger, "worker_chv_killed",
|
||||||
|
task_id=task_id,
|
||||||
|
message="cancel watchdog killed VM %s "
|
||||||
|
"(task interrupted/cancelled)"
|
||||||
|
% vm_name)
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
return
|
||||||
|
|
||||||
|
def _chv_quota_watchdog(self, task_id, vm_name, stop_event,
|
||||||
|
quota_check, interval=30.0):
|
||||||
|
"""CHV 同步路径的配额看门狗(feat-chv-fix,P1-2,Docker
|
||||||
|
配额看门狗同款):命中时记 _quota_hit_tasks 并调
|
||||||
|
cancel_vm 真关 VM;_run_attempt 在同步返回后把结果
|
||||||
|
改写成 quota_exceeded,走统一配额中断收尾。"""
|
||||||
|
while not stop_event.wait(interval):
|
||||||
|
try:
|
||||||
|
over = bool(quota_check()) if quota_check else False
|
||||||
|
except Exception:
|
||||||
|
over = False
|
||||||
|
if not over:
|
||||||
|
continue
|
||||||
|
self._mark_quota_hit(task_id)
|
||||||
|
try:
|
||||||
|
cancel = getattr(self.executor, "cancel_vm", None)
|
||||||
|
if cancel is not None and vm_name:
|
||||||
|
cancel(vm_name)
|
||||||
|
log_event(self.logger, "worker_chv_killed",
|
||||||
|
task_id=task_id,
|
||||||
|
message="quota watchdog killed VM %s "
|
||||||
|
"(quota exceeded)" % vm_name)
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
return
|
||||||
|
|
||||||
def cleanup_stale_containers(self):
|
def cleanup_stale_containers(self):
|
||||||
"""启动清理兜底:清掉上一轮残留的 codex- 容器。
|
"""启动清理兜底:清掉上一轮残留的 codex- 容器。
|
||||||
|
|
||||||
只在执行器有 cleanup_stale_containers 时做(Docker);
|
Docker 执行器清容器、CHV 执行器清残留 VM(feat-chv-fix);
|
||||||
列出或删除失败只记日志,绝不抛异常炸掉 worker。
|
列出或删除失败只记日志,绝不抛异常炸掉 worker。
|
||||||
返回已清理的容器名列表(非 Docker 执行器返回 [])。
|
返回已清理的容器/VM 名列表(两者皆无的执行器返回 [])。
|
||||||
"""
|
"""
|
||||||
|
cleaned = []
|
||||||
cleanup = getattr(self.executor, "cleanup_stale_containers", None)
|
cleanup = getattr(self.executor, "cleanup_stale_containers", None)
|
||||||
if cleanup is None:
|
if cleanup is not None:
|
||||||
return []
|
try:
|
||||||
try:
|
cleaned = list(cleanup() or [])
|
||||||
cleaned = cleanup()
|
except Exception as exc:
|
||||||
except Exception as exc:
|
log_event(self.logger, "worker_cleanup_failed",
|
||||||
log_event(self.logger, "worker_cleanup_failed",
|
message="cleanup stale containers failed: %s" % exc)
|
||||||
message="cleanup stale containers failed: %s" % exc)
|
# feat-chv-fix(P1-2):CHV 残留 VM 走通用清理路径,
|
||||||
return []
|
# 不再只靠 worker 启动/关闭的 WarmPool reconcile 兜底。
|
||||||
|
chv_cleanup = getattr(self.executor, "cleanup_stale_vms", None)
|
||||||
|
if chv_cleanup is not None:
|
||||||
|
try:
|
||||||
|
cleaned = cleaned + list(chv_cleanup() or [])
|
||||||
|
except Exception as exc:
|
||||||
|
log_event(self.logger, "worker_cleanup_failed",
|
||||||
|
message="cleanup stale vms failed: %s" % exc)
|
||||||
if cleaned:
|
if cleaned:
|
||||||
log_event(self.logger, "worker_cleanup",
|
log_event(self.logger, "worker_cleanup",
|
||||||
message="cleaned stale containers: %s"
|
message="cleaned stale containers: %s"
|
||||||
@@ -4639,6 +4716,29 @@ class Worker:
|
|||||||
_watchdog_stop, quota_check),
|
_watchdog_stop, quota_check),
|
||||||
daemon=True)
|
daemon=True)
|
||||||
_quota_watchdog_thread.start()
|
_quota_watchdog_thread.start()
|
||||||
|
# feat-chv-fix(P1-2):CHV 同步路径同款看门狗(VM 名
|
||||||
|
# 由任务 id 派生)。进程模式不重复起(_execute_via_
|
||||||
|
# process 已轮询 quota_check)。
|
||||||
|
if isinstance(self.executor, CloudHypervisorExecutor) \
|
||||||
|
and hasattr(self.executor, "cancel_vm"):
|
||||||
|
try:
|
||||||
|
_chv_vm_name = self.executor.vm_name(task)
|
||||||
|
except Exception:
|
||||||
|
_chv_vm_name = None
|
||||||
|
if _chv_vm_name:
|
||||||
|
_watchdog_stop = threading.Event()
|
||||||
|
_watchdog_thread = threading.Thread(
|
||||||
|
target=self._chv_cancel_watchdog,
|
||||||
|
args=(task_id, _chv_vm_name, _watchdog_stop),
|
||||||
|
daemon=True)
|
||||||
|
_watchdog_thread.start()
|
||||||
|
if quota_check is not None and not self._use_popen():
|
||||||
|
_quota_watchdog_thread = threading.Thread(
|
||||||
|
target=self._chv_quota_watchdog,
|
||||||
|
args=(task_id, _chv_vm_name,
|
||||||
|
_watchdog_stop, quota_check),
|
||||||
|
daemon=True)
|
||||||
|
_quota_watchdog_thread.start()
|
||||||
# 启动前分段日志(2026-10-29 任务 A):认领后 61 秒中断,
|
# 启动前分段日志(2026-10-29 任务 A):认领后 61 秒中断,
|
||||||
# 但阻塞的 docker run 中途没有任何标记、容器 --rm 后输出
|
# 但阻塞的 docker run 中途没有任何标记、容器 --rm 后输出
|
||||||
# 也没了,事后完全无法定位卡在哪一段。派发开始/结束进
|
# 也没了,事后完全无法定位卡在哪一段。派发开始/结束进
|
||||||
@@ -4774,8 +4874,12 @@ class Worker:
|
|||||||
# Docker 路径上面已有 is_docker 守卫,不走这里。
|
# Docker 路径上面已有 is_docker 守卫,不走这里。
|
||||||
# 网关任务在非 Local 执行器上无法注入 key 与
|
# 网关任务在非 Local 执行器上无法注入 key 与
|
||||||
# provider,必须 loud fail,不静默跑错网关。
|
# provider,必须 loud fail,不静默跑错网关。
|
||||||
|
# feat-chv-fix(P1-1):守卫收窄——CHV 已补
|
||||||
|
# extra_env SSH 注入路径,不再被“非 Local 即
|
||||||
|
# 拦”;无注入能力的执行器仍 loud fail。
|
||||||
if gateway_provider and not isinstance(
|
if gateway_provider and not isinstance(
|
||||||
self.executor, LocalExecutor):
|
self.executor,
|
||||||
|
(LocalExecutor, CloudHypervisorExecutor)):
|
||||||
raise RuntimeError(
|
raise RuntimeError(
|
||||||
"sub2api 网关不支持该执行器:%s"
|
"sub2api 网关不支持该执行器:%s"
|
||||||
% type(self.executor).__name__)
|
% type(self.executor).__name__)
|
||||||
@@ -4796,7 +4900,8 @@ class Worker:
|
|||||||
# 故这里不放宽网关规则。
|
# 故这里不放宽网关规则。
|
||||||
result = _call_goal_resume_breakpoint(
|
result = _call_goal_resume_breakpoint(
|
||||||
self.executor, task,
|
self.executor, task,
|
||||||
model_provider=gateway_provider)
|
model_provider=gateway_provider,
|
||||||
|
extra_env=gateway_env)
|
||||||
elif task.get("_goal_resume_breakpoint"):
|
elif task.get("_goal_resume_breakpoint"):
|
||||||
# 假执行器/CloudExecutor 没有断点方法时
|
# 假执行器/CloudExecutor 没有断点方法时
|
||||||
# 绝不回落成 goal(会撞已存在)或
|
# 绝不回落成 goal(会撞已存在)或
|
||||||
@@ -4825,6 +4930,11 @@ class Worker:
|
|||||||
task, seed_dir=seed_host_dir,
|
task, seed_dir=seed_host_dir,
|
||||||
extra_env=gateway_env,
|
extra_env=gateway_env,
|
||||||
model_provider=gateway_provider)
|
model_provider=gateway_provider)
|
||||||
|
elif isinstance(
|
||||||
|
self.executor, CloudHypervisorExecutor):
|
||||||
|
result = self.executor.goal_task(
|
||||||
|
task, extra_env=gateway_env,
|
||||||
|
model_provider=gateway_provider)
|
||||||
else:
|
else:
|
||||||
result = self.executor.goal_task(task)
|
result = self.executor.goal_task(task)
|
||||||
else:
|
else:
|
||||||
@@ -4833,6 +4943,11 @@ class Worker:
|
|||||||
task, seed_dir=seed_host_dir,
|
task, seed_dir=seed_host_dir,
|
||||||
extra_env=gateway_env,
|
extra_env=gateway_env,
|
||||||
model_provider=gateway_provider)
|
model_provider=gateway_provider)
|
||||||
|
elif isinstance(
|
||||||
|
self.executor, CloudHypervisorExecutor):
|
||||||
|
result = self.executor.dispatch_task(
|
||||||
|
task, extra_env=gateway_env,
|
||||||
|
model_provider=gateway_provider)
|
||||||
else:
|
else:
|
||||||
result = self.executor.dispatch_task(task)
|
result = self.executor.dispatch_task(task)
|
||||||
combined = (result.get("stdout", "") or "") + \
|
combined = (result.get("stdout", "") or "") + \
|
||||||
@@ -4856,14 +4971,18 @@ class Worker:
|
|||||||
quota_check)
|
quota_check)
|
||||||
else:
|
else:
|
||||||
if gateway_provider and not isinstance(
|
if gateway_provider and not isinstance(
|
||||||
self.executor, LocalExecutor):
|
self.executor,
|
||||||
|
(LocalExecutor,
|
||||||
|
CloudHypervisorExecutor)):
|
||||||
raise RuntimeError(
|
raise RuntimeError(
|
||||||
"sub2api 网关不支持该执行器:%s"
|
"sub2api 网关不支持该执行器:%s"
|
||||||
% type(self.executor).__name__)
|
% type(self.executor).__name__)
|
||||||
result = self.executor.run_task(
|
result = self.executor.run_task(
|
||||||
task, extra_env=gateway_env,
|
task, extra_env=gateway_env,
|
||||||
model_provider=gateway_provider) \
|
model_provider=gateway_provider) \
|
||||||
if isinstance(self.executor, LocalExecutor) \
|
if isinstance(self.executor,
|
||||||
|
(LocalExecutor,
|
||||||
|
CloudHypervisorExecutor)) \
|
||||||
else self.executor.run_task(task)
|
else self.executor.run_task(task)
|
||||||
except Exception as exc: # runner 异常也如实写回 failed
|
except Exception as exc: # runner 异常也如实写回 failed
|
||||||
result = {"executor": "local", "task_id": task_id,
|
result = {"executor": "local", "task_id": task_id,
|
||||||
|
|||||||
@@ -36,7 +36,7 @@
|
|||||||
shutdown,同一任务重复 wake 先关旧 VM;VM 名、api-socket、
|
shutdown,同一任务重复 wake 先关旧 VM;VM 名、api-socket、
|
||||||
每任务磁盘路径均按任务派生,互不相同(测试断言)。
|
每任务磁盘路径均按任务派生,互不相同(测试断言)。
|
||||||
- ✅🧪 每任务磁盘独立:copy 模式复制基础镜像为每任务磁盘;
|
- ✅🧪 每任务磁盘独立:copy 模式复制基础镜像为每任务磁盘;
|
||||||
reuse 模式仅单租户实验,生产禁用(代码与文档双写)。
|
reuse 模式仅单租户实验,preflight 拦截(须显式实验二次确认并审计放行,feat-chv-fix 起代码真正拦截;此前“代码与文档双写”的说法是错的,代码当时并不拦截)。
|
||||||
- ✅🧪 租户无共享路径:执行器不挂载宿主根、宿主代码目录或含
|
- ✅🧪 租户无共享路径:执行器不挂载宿主根、宿主代码目录或含
|
||||||
密钥目录;客机代码在镜像内 `/app`,不在宿主共享挂载上。
|
密钥目录;客机代码在镜像内 `/app`,不在宿主共享挂载上。
|
||||||
- ✅🧪 密钥不进镜像/代码/prompt:SSH argv 只有 key 文件路径;
|
- ✅🧪 密钥不进镜像/代码/prompt:SSH argv 只有 key 文件路径;
|
||||||
@@ -44,7 +44,7 @@
|
|||||||
命令中不含哨兵密钥值。
|
命令中不含哨兵密钥值。
|
||||||
- ✅🧪 SSH host key 按 VM 钉死(优化轨,#216 的 no + /dev/null
|
- ✅🧪 SSH host key 按 VM 钉死(优化轨,#216 的 no + /dev/null
|
||||||
已废除):每 VM 在 state_dir/<vm>/known_hosts 有独立文件,
|
已废除):每 VM 在 state_dir/<vm>/known_hosts 有独立文件,
|
||||||
首次启动经 ssh-keyscan 在本机回环端口捕获 key 写入该文件,
|
首次启动经 ssh-keyscan 在解析出的 SSH 目标上捕获 key 写入该文件,
|
||||||
此后 SSH 一律 StrictHostKeyChecking=yes + 该文件校验;
|
此后 SSH 一律 StrictHostKeyChecking=yes + 该文件校验;
|
||||||
非 root 客机用户(CHV_SSH_USER,显式空串直接拒绝)与端口
|
非 root 客机用户(CHV_SSH_USER,显式空串直接拒绝)与端口
|
||||||
(CHV_SSH_PORT,1-65535)为显式配置。mock 只验证捕获→
|
(CHV_SSH_PORT,1-65535)为显式配置。mock 只验证捕获→
|
||||||
@@ -209,3 +209,33 @@
|
|||||||
chv-guest 镜像构建发布 → 真实 VM 启动/快照恢复/任务端到端
|
chv-guest 镜像构建发布 → 真实 VM 启动/快照恢复/任务端到端
|
||||||
验证 → 压测 → 安全审计 → CI 变绿后按当时 main 头复核 diff
|
验证 → 压测 → 安全审计 → CI 变绿后按当时 main 头复核 diff
|
||||||
再合并部署。缺一不可。
|
再合并部署。缺一不可。
|
||||||
|
|
||||||
|
## 10. 审核修复轨 feat-chv-fix(堆叠于 #230 之上)
|
||||||
|
|
||||||
|
- ✅🧪 P0 已修:seed 磁盘(CIDATA ISO,第二块只读盘)挂进
|
||||||
|
VmConfig;restore 不回落 keyscan(provisioned 下复用已钉
|
||||||
|
死 known_hosts、缺失 fail-fast,快照内 host key 不重预置);
|
||||||
|
reuse 由 preflight 拦截(显式实验二次确认 + 审计放行);
|
||||||
|
SSH 目标选客机 IP 方案(显式 host > net 静态 ip > 显式
|
||||||
|
转发宣称下的回环,三者皆无 fail-fast;vsock 与纯 NAT 方案
|
||||||
|
未选,理由见 config.py)。
|
||||||
|
- ✅🧪 P1 已修或有结论:网关 extra_env 经 SSH env 前缀注入,
|
||||||
|
worker 守卫收窄到 Local/CHV;取消/清理/看门狗接 CHV 通用
|
||||||
|
路径;WarmPool acquire 内 resume(但 worker dispatch 前查
|
||||||
|
池仍未接线,预热收益仍不是已交付特性);容量门改持久化
|
||||||
|
计数(socket 联合计数);磁盘配额改总量+余量、已存在磁盘
|
||||||
|
校验重拷;API 客户端按 Content-Length 验完整性;快照
|
||||||
|
file:// 目标验存在且非空再 shutdown;execute 死端口归类
|
||||||
|
vm_not_running。
|
||||||
|
- ⚠️ 仍未接线/未验证(不得声称已交付):worker dispatch 前查
|
||||||
|
WarmPool;CHV + 网关真实任务;真机 wake/boot/SSH(含 seed
|
||||||
|
注入与 NAT/客机 IP 实测);快照恢复端到端;压测;安全审计;
|
||||||
|
CI 账单恢复。另:P0-2 审计原说法“restore 须把 seed 接入
|
||||||
|
VmConfig”已纠正——restore 不走 vm.create,seed 无从挂盘,
|
||||||
|
正确不变量是复用钉死 known_hosts。
|
||||||
|
- ⛔ P2 全部排除、未做(用户紧急范围收窄令,留待以后一轨):
|
||||||
|
长函数拆分、第二启动路径清理 / Docker goal 解析注释、
|
||||||
|
test_chv_gaps.py 的 grep 文本断言改行为断言、build.sh 与
|
||||||
|
README 口径对齐、stop_vm 吞异常改审计、配置校验时机统一。
|
||||||
|
本轨的 grep 文本测试、README/build.sh、stop_vm 吞异常、
|
||||||
|
preflight 校验口径均保持 #230 原样。
|
||||||
Reference in new issue
Block a user