feat(chv): 架构与代码质量 9.5+ 提升——长函数清零、WarmPool facade、#215 对齐、审计计数、只读 VM 字段、ADR-8~11

This commit is contained in:
host committed 2026-10-31 06:20:12 +00:00
1 parent c908a0147a
commit 42de20da68
15 files changed
+840 -345

No files matched your search

+93 -72
View File
@@ -9,12 +9,100 @@ import os
import socket
def _read_header(sock):
"""读满响应头:返回 (header_text, 已收体部前缀)。
超时或提前关流即 RuntimeError,绝不把半截头当结束
(feat-chv-fix P1-6 的语义,纯移动)。"""
buf = b""
header_end = None
timed_out = False
while header_end is None:
try:
data = sock.recv(65536)
except socket.timeout:
timed_out = True
break
if not data:
break
buf += data
header_end = buf.find(b"\r\n\r\n")
if timed_out or header_end is None:
raise RuntimeError("CHV API 响应头不完整(超时或提前关流)")
header = buf[:header_end].decode("utf-8", errors="replace")
return header, buf[header_end + 4:]
def _parse_content_length(header):
"""从响应头取 Content-Length:chunked 或无长度即拒绝。
chunked 明确拒绝(CHV REST 不应发 chunked);无长度则
无法证完整性,同样拒绝。"""
content_length = None
is_chunked = False
for line in header.split("\r\n")[1:]:
if ":" not in line:
continue
name, value = line.split(":", 1)
name = name.strip().lower()
value = value.strip()
if name == "content-length":
try:
content_length = int(value)
except ValueError:
raise RuntimeError("CHV API Content-Length 非法")
elif name == "transfer-encoding" \
and "chunked" in value.lower():
is_chunked = True
if is_chunked:
raise RuntimeError("CHV API 响应为 chunked,拒绝解析")
if content_length is None:
raise RuntimeError("CHV API 响应无 Content-Length,拒绝解析")
return content_length
def _read_body(sock, body, content_length):
"""按 Content-Length 读满体部:超时或提前关流即截断错误。"""
while len(body) < content_length:
try:
data = sock.recv(65536)
except socket.timeout:
raise RuntimeError(
"CHV API 响应体截断:已收 %d / 需 %d 字节"
% (len(body), content_length))
if not data:
raise RuntimeError(
"CHV API 响应体截断:已收 %d / 需 %d 字节"
% (len(body), content_length))
body += data
return body[:content_length]
def _parse_response(header, body):
"""从已读满的头与体部解析 (status, body):能解析 JSON
时为 dict,否则为原文本;状态行解析失败时 status 为 0。"""
text = header + "\r\n\r\n" + body.decode("utf-8", errors="replace")
status = 0
if text.startswith("HTTP/"):
try:
status = int(text.split()[1])
except (IndexError, ValueError):
status = 0
body_text = body.decode("utf-8", errors="replace")
try:
parsed = json.loads(body_text) if body_text.strip() else {}
except ValueError:
parsed = body_text
return status, parsed
def _unix_socket_api(socket_path, method, path, payload=None):
"""Cloud Hypervisor REST API 的默认传输:标准库 raw HTTP
over Unix socket(无第三方依赖)。返回 (status, body),
body 能解析 JSON 时为 dict,否则为原文本。socket 不存在
或连接失败时 raise RuntimeError(调用方据此判 api 超时
/失败,不编造 200)。"""
/失败,不编造 200)。读满语义见 _read_header /
_parse_content_length / _read_body(P1-6,纯移动)。"""
if not os.path.exists(socket_path):
raise RuntimeError("api socket 不存在:%s" % socket_path)
body = b""
@@ -32,79 +120,12 @@ def _unix_socket_api(socket_path, method, path, payload=None):
try:
sock.connect(socket_path)
sock.sendall(raw)
chunks = []
# feat-chv-fix(P1-6):读超时不再当“响应结束”。
# 先读满响应头,再按 Content-Length 读满体部;超时、
# 提前关流、体部短于 Content-Length 一律 RuntimeError
# (调用方判 api_timeout / 传输失败),绝不解析半截
# 响应。chunked 与无 Content-Length 响应明确拒绝
# (Cloud Hypervisor REST 不应发 chunked;无长度则
# 无法证完整性)。
buf = b""
header_end = None
timed_out = False
while header_end is None:
try:
data = sock.recv(65536)
except socket.timeout:
timed_out = True
break
if not data:
break
buf += data
header_end = buf.find(b"\r\n\r\n")
if timed_out or header_end is None:
raise RuntimeError("CHV API 响应头不完整(超时或提前关流)")
header = buf[:header_end].decode("utf-8", errors="replace")
body = buf[header_end + 4:]
content_length = None
is_chunked = False
for line in header.split("\r\n")[1:]:
if ":" not in line:
continue
name, value = line.split(":", 1)
name = name.strip().lower()
value = value.strip()
if name == "content-length":
try:
content_length = int(value)
except ValueError:
raise RuntimeError("CHV API Content-Length 非法")
elif name == "transfer-encoding" \
and "chunked" in value.lower():
is_chunked = True
if is_chunked:
raise RuntimeError("CHV API 响应为 chunked,拒绝解析")
if content_length is None:
raise RuntimeError("CHV API 响应无 Content-Length,拒绝解析")
while len(body) < content_length:
try:
data = sock.recv(65536)
except socket.timeout:
raise RuntimeError(
"CHV API 响应体截断:已收 %d / 需 %d 字节"
% (len(body), content_length))
if not data:
raise RuntimeError(
"CHV API 响应体截断:已收 %d / 需 %d 字节"
% (len(body), content_length))
body += data
body = body[:content_length]
header, body = _read_header(sock)
content_length = _parse_content_length(header)
body = _read_body(sock, body, content_length)
finally:
sock.close()
text = header + "\r\n\r\n" + body.decode("utf-8", errors="replace")
status = 0
if text.startswith("HTTP/"):
try:
status = int(text.split()[1])
except (IndexError, ValueError):
status = 0
body_text = body.decode("utf-8", errors="replace")
try:
parsed = json.loads(body_text) if body_text.strip() else {}
except ValueError:
parsed = body_text
return status, parsed
return _parse_response(header, body)
def normalize_api_result(result) -> tuple:
+59
View File
@@ -0,0 +1,59 @@
"""#215(feat-arch-converge)接口对齐层,CHV 本地实现。
背景:#215 未合并,本基线里没有 cloud/timeouts.py、
cloud/gates.py、cloud/quota.py、cloud/sandbox_policy.py。
本模块绝不复制 #215 的文件,只把 CHV 真正能映射的
接口写成与 #215 相同的签名,#215 合并后替换为 import
即是机械的(见 docs/chv-architecture.md ADR-8)。
映射表(逐个核对过 #215 远程分支的真实函数,不是凭空编的):
- gates.resolve_inside(base, rel):目录门,CHV 快照/磁盘
路径白名单正好需要它——本模块实现同语义并已在
hardening.validate_snapshot_url 里使用,#215 合并后
改为 from cloud.gates import resolve_inside 即可。
- sandbox_policy.tmpfs_isolated(executor_kind, path):CHV
客机 /tmp 落在客机磁盘上,不是宿主 tmpfs,故对
cloud-hypervisor 一律返回 (False, ...)——fail-closed,
绝不宣称隔离。未知执行器同样 fail-closed。
- timeouts.resolve_module_timeout / quota.decide:无映射。
前者是审批/队列超时(None=不过期、0=不过期),CHV
的 api/ready 超时是秒数且 0 无效,语义不同,硬套会
改行为;后者是按月美元配额,CHV 的配额是 VM 数/磁盘
GB/快照数,单位不同。两者在本模块里 deliberately
不提供,测试钉住“CHV 不 import 它们”。
本模块只依赖标准库,是叶子模块,不 import cloud.sandbox。
"""
import os
def resolve_inside(base, rel):
"""目录门:base 内的相对路径安全解析,越界回 None。
空串回 base 本身;绝对路径、.. 逃逸、符号链接逃逸、
commonpath 跨盘符 ValueError 一律 None。签名与 #215
的 gates.resolve_inside 一致,合并后可直接替换为 import。"""
if not rel:
return base
if os.path.isabs(rel):
return None
real_base = os.path.realpath(base)
target = os.path.realpath(os.path.join(real_base, rel))
try:
if os.path.commonpath([real_base, target]) != real_base:
return None
except ValueError:
return None
return target
def tmpfs_isolated(executor_kind, path):
"""某执行器下某路径是否走 tmpfs 隔离:(isolated, reason)。
cloud-hypervisor:客机 /tmp 落在客机磁盘上,不走宿主
tmpfs,一律 (False, ...);未知执行器 fail-closed 同样
False。签名与 #215 的 sandbox_policy.tmpfs_isolated 一致。"""
if executor_kind == "cloud-hypervisor":
return False, "CHV 客机 /tmp 落在客机磁盘上,不走宿主 tmpfs"
return False, "未知执行器,按不隔离处理(fail-closed 口径)"
+40 -35
View File
@@ -170,45 +170,21 @@ def _timeout_fragment(params: dict, env: dict) -> dict:
}
def build_chv_config(params: dict, env: dict):
"""把显式构造参数(params,None=未给)与环境变量(env,
默认为 os.environ 的调用方传入 dict)解析成 ChvConfig。
def _assemble_chv_config(base: dict, ssh: dict, quota: dict,
timeouts: dict):
"""把基础字段 + SSH/配额/超时三个片段组装成 ChvConfig。
显式参数只有 __init__ 本来就有的那些;max_vms、磁盘模式、
磁盘配额、egress、快照、预热池目标数本来就没有显式参数,
仍只走环境变量(与 #216 一致,不悄悄加新参数面)。
片段化(feat-chv-arch95):SSH 面、配额面、超时面分别
由 _ssh_fragment/_quota_fragment/_timeout_fragment 解析,
错误文案与校验顺序逐字一致;build_chv_config 只组装。"""
env = env if env is not None else {}
binary = params.get("binary") or env.get("CHV_BINARY") \
or "cloud-hypervisor"
kernel = params.get("kernel") or env.get("CHV_KERNEL")
rootfs = params.get("rootfs") or params.get("image") \
or env.get("CHV_ROOTFS") or env.get("CHV_IMAGE")
cpus = str(params.get("cpus") or env.get("CHV_CPU") or "2")
memory = params.get("memory") or env.get("CHV_MEM") or "2G"
state_dir = params.get("state_dir") or env.get("CHV_STATE_DIR") \
or os.path.join(os.path.expanduser("~"),
".local", "share", "codex-app-server", "chv-vms")
kvm_path = params.get("kvm_path") \
if params.get("kvm_path") is not None else "/dev/kvm"
cpuinfo_path = params.get("cpuinfo_path") \
if params.get("cpuinfo_path") is not None else "/proc/cpuinfo"
ch_remote = params.get("ch_remote") or env.get("CHV_REMOTE") \
or "ch-remote"
model = params.get("model")
ssh = _ssh_fragment(params, env)
quota = _quota_fragment(env)
timeouts = _timeout_fragment(params, env)
纯移动:字段名与取值逐字不变,build_chv_config 只解析。"""
return ChvConfig(
binary=binary, kernel=kernel, rootfs=rootfs, cpus=cpus,
memory=memory, state_dir=state_dir, ssh_key=ssh["ssh_key"],
ssh_user=ssh["ssh_user"], ssh_port=ssh["ssh_port"],
binary=base["binary"], kernel=base["kernel"],
rootfs=base["rootfs"], cpus=base["cpus"],
memory=base["memory"], state_dir=base["state_dir"],
ssh_key=ssh["ssh_key"], ssh_user=ssh["ssh_user"],
ssh_port=ssh["ssh_port"],
guest_host_py=ssh["guest_host_py"], cmdline=ssh["cmdline"],
net=ssh["net"], net_json=ssh["net_json"],
kvm_path=kvm_path, cpuinfo_path=cpuinfo_path,
ch_remote=ch_remote, model=model,
kvm_path=base["kvm_path"], cpuinfo_path=base["cpuinfo_path"],
ch_remote=base["ch_remote"], model=base["model"],
max_vms=quota["max_vms"], disk_mode=quota["disk_mode"],
disk_gb=quota["disk_gb"], ssh_host=ssh["ssh_host"],
ssh_forward=ssh["ssh_forward"],
@@ -228,3 +204,32 @@ def build_chv_config(params: dict, env: dict):
snapshot_retention_days=quota["snapshot_retention_days"],
snapshot_dir=quota["snapshot_dir"],
allow_remote_snapshot=quota["allow_remote_snapshot"])
def build_chv_config(params: dict, env: dict):
"""把显式构造参数与环境变量解析成 ChvConfig(见
_assemble_chv_config 的组装段;片段化与校验顺序不变)。"""
env = env if env is not None else {}
base = {
"binary": params.get("binary") or env.get("CHV_BINARY")
or "cloud-hypervisor",
"kernel": params.get("kernel") or env.get("CHV_KERNEL"),
"rootfs": params.get("rootfs") or params.get("image")
or env.get("CHV_ROOTFS") or env.get("CHV_IMAGE"),
"cpus": str(params.get("cpus") or env.get("CHV_CPU") or "2"),
"memory": params.get("memory") or env.get("CHV_MEM") or "2G",
"state_dir": params.get("state_dir")
or env.get("CHV_STATE_DIR")
or os.path.join(os.path.expanduser("~"),
".local", "share", "codex-app-server", "chv-vms"),
"kvm_path": params.get("kvm_path")
if params.get("kvm_path") is not None else "/dev/kvm",
"cpuinfo_path": params.get("cpuinfo_path")
if params.get("cpuinfo_path") is not None else "/proc/cpuinfo",
"ch_remote": params.get("ch_remote") or env.get("CHV_REMOTE")
or "ch-remote",
"model": params.get("model"),
}
return _assemble_chv_config(
base, _ssh_fragment(params, env), _quota_fragment(env),
_timeout_fragment(params, env))
+215 -195
View File
@@ -289,6 +289,7 @@ class CloudHypervisorExecutor(Executor):
else _unix_socket_api
self.sleeper = sleeper if sleeper is not None else time.sleep
self.audit_callback = None
self.audit_failure_count = 0
self._vms_lock = threading.Lock()
if runner is not None:
self.runner = runner
@@ -602,9 +603,15 @@ class CloudHypervisorExecutor(Executor):
feat-chv-secfix(M5):审计回调失败不再双重静默
pass——记宿主 error 日志(审计完整性本身没有保证,
但失败至少可见;任务仍不因此炸,审计失败不改变
任务状态是 worker 层的既定语义)。审计 detail 里
绝不含密钥值(调用方负责:本类所有 _emit 的 detail
只含 VM 名、路径、URL、kind,不含任何 key 值)。"""
任务状态是 worker 层的既定语义)。feat-chv-arch95plus
取舍(见 ADR-9):不改成“审计失败即中断任务”——
审计是可观测性不是门禁,DB 短暂不可用时硬中断会把
一次审计失败放大成任务失败,且与 worker 全局
best-effort 语义冲突;改为“失败记日志 + 计数 + 不
中断”,计数见 self.audit_failure_count(宿主可观测,
非任务状态)。审计 detail 里绝不含密钥值(调用方
负责:本类所有 _emit 的 detail 只含 VM 名、路径、
URL、kind,不含任何 key 值)。"""
try:
if self.audit_callback is not None:
self.audit_callback(event, task, detail, kind)
@@ -612,6 +619,8 @@ class CloudHypervisorExecutor(Executor):
_LOGGER.info("chv_audit event=%s kind=%s %s",
event, kind, detail)
except Exception as exc:
self.audit_failure_count = getattr(
self, "audit_failure_count", 0) + 1
_LOGGER.error("chv_audit_failed event=%s kind=%s error=%s",
event, kind, exc)
@@ -981,21 +990,37 @@ class CloudHypervisorExecutor(Executor):
"vmm.ping 超时:%d 次内 API 未通,socket=%s"
% (attempts, socket_path))
def wait_guest_ready(self, task):
"""等 guest SSH 就绪(只选 SSH,见类 docstring):
轮询 ssh … "true",returncode 0 即就绪;次数 =
ready_timeout/ready_interval 取整,至少 1 次。
每轮先尽力捕获 host key(known_hosts 文件不存在时;
sshd 未起时抓不到是常态,下轮再试),再发 ssh
"true"——钉死式 SSH 在文件缺失时会直接拒绝未知主机,
不先捕获就永远等不到就绪。超时时先关 VM 再 raise
ChvFailure(kind=guest_not_ready),绝不在未就绪时
派任务。"""
# feat-chv-fix(P0-2):provisioned 模式下缺 known_hosts
# 时直接 fail-fast,绝不调用 capture_host_key 回落 TOFU。
# (keyscan 模式才走下面的捕获分支。)
def _provisioned_key_guard(self, task):
"""provisioned 下缺 known_hosts 即 fail-fast,绝不 keyscan。
返回 True 表示已拦截(调用方须先关 VM 再抛
host_key_not_pinned);False 表示可继续轮询。"""
if getattr(self, "host_key_mode", "provisioned") == "provisioned" \
and not os.path.exists(self.known_hosts_path(task)):
return True
return False
def _check_host_key_mismatch(self, task, result):
"""SSH stderr 含钉死拒绝标记时审计并返回 True(须 fail-fast)。
MITM 尝试是独立安全事件,不轮询成普通超时;审计
detail 只含 VM 名与标记,不含 key 值。"""
stderr = getattr(result, "stderr", "") or ""
if "Host key verification failed" in stderr \
or "REMOTE HOST IDENTIFICATION" in stderr:
self._emit("host_key_mismatch", task,
"vm=%s ssh host key 被拒绝(钉死不匹配)"
% self.vm_name(task), "host_key_mismatch")
return True
return False
def wait_guest_ready(self, task):
"""等 guest SSH 就绪:轮询 ssh "true",returncode 0 即就绪;
每轮先尽力捕获 host key(keyscan 模式),超时先关 VM
再 raise guest_not_ready,绝不在未就绪时派任务。"""
# P0-2:provisioned 下缺 known_hosts 即 fail-fast,绝不
# 调用 capture_host_key 回落 TOFU(keyscan 才走捕获分支)。
if self._provisioned_key_guard(task):
try:
self.stop_vm(task)
except Exception:
@@ -1013,17 +1038,9 @@ class CloudHypervisorExecutor(Executor):
self.build_ssh_command(task, ["true"]))
if getattr(result, "returncode", 1) == 0:
return True
# feat-chv-secfix(M5):host key 不匹配是独立
# 安全事件,不是“sshd 没起”。SSH stderr 含
# 钉死拒绝标记时立即审计 host_key_mismatch 并
# fail-fast(不把 MITM 尝试轮询成普通超时),
# 审计 detail 只含 VM 名与标记,不含 key 值。
stderr = getattr(result, "stderr", "") or ""
if "Host key verification failed" in stderr \
or "REMOTE HOST IDENTIFICATION" in stderr:
self._emit("host_key_mismatch", task,
"vm=%s ssh host key 被拒绝(钉死不匹配)"
% self.vm_name(task), "host_key_mismatch")
# M5:host key 不匹配是独立安全事件,见
# _check_host_key_mismatch(审计 + fail-fast)。
if self._check_host_key_mismatch(task, result):
try:
self.stop_vm(task)
except Exception:
@@ -1394,58 +1411,60 @@ class CloudHypervisorExecutor(Executor):
% (self.vm_name(task_or_name), snapshot_url))
return snapshot_url
def restore_vm(self, task, snapshot_url):
"""快照恢复唤醒:preflight → 只起 VMM(--api-socket)
→ 等 vmm.ping → ch-remote restore → 显式 resume →
等 guest SSH 就绪。恢复后的 VM 默认是 paused,不
resume 就不算醒——本方法绝不省略 resume 那一步。
任一步失败都关 VM 再抛。成功返回 {vm, api_socket,
state: "running", ready: True, mode: "restored"}。"""
def _validate_restore(self, task, snapshot_url):
"""restore 前置校验:返回归一化后的 snapshot_url。
restore 不走 vm.create、没有 VmConfig 可挂 seed:
快照内已固化原 VM host key,重预置反而破坏钉死,
故 provisioned 下复用已钉死的 known_hosts、缺失
fail-fast、绝不 keyscan、绝不重生成。restore 时不
重复验快照文件存在(远端/对象存储源合法,ch-remote
本身会在源缺失时失败)。"""
if not isinstance(task, dict) or task.get("id") is None:
raise ValueError("任务缺少 id")
if not snapshot_url:
raise ValueError("snapshot_url 不能为空")
if "://" not in snapshot_url:
snapshot_url = "file://%s" % snapshot_url
# feat-chv-secfix(L2):恢复源同样过 URL 白名单
# (file:// 须在白名单目录内,远端须显式放行),
# 在 preflight/起 VMM 之前就拒绝。
validate_snapshot_url(
snapshot_url, self.state_dir,
snapshot_dir=getattr(self, "snapshot_dir", None),
allow_remote=getattr(self, "allow_remote_snapshot", False))
self.preflight()
self.resolve_ssh_host()
# feat-chv-fix(P0-2 修正版):restore 不走 vm.create,
# 没有 VmConfig 可挂 seed——审计报告“restore 须把 seed
# 接入 VmConfig”的说法在此不适用,纠正为:快照内已
# 固化原 VM 的 host key,重新预置新 key 反而会破坏
# 钉死。正确不变量是:provisioned 下复用已钉死的
# known_hosts(stop 时公钥保留、私钥已删但快照内仍有
# 原私钥),缺失时 fail-fast、绝不 keyscan、绝不重
# 生成。file:// 快照须先验存在且非空,否则不启动。
# 注意:restore 时不重复验快照文件存在——快照落盘
# 校验是 snapshot_vm/hibernate_vm 创建时的门(P1-7),
# restore 的 ch-remote 本身会在源缺失时失败;在这里
# 再验一次会把“源在远端/对象存储”的合法路径也挡掉。
if getattr(self, "host_key_mode", "provisioned") == "provisioned" \
and not os.path.exists(self.known_hosts_path(task)):
if self._provisioned_key_guard(task):
raise ChvFailure(
"host_key_not_pinned",
"provisioned 模式下 restore 缺 known_hosts,拒绝 "
"keyscan 回落:VM %s" % self.vm_name(task))
return snapshot_url
def _restore_boot(self, task, snapshot_url):
"""restore 的启动段:起 VMM → 等 ping → restore →
显式 resume → 等 SSH 就绪(恢复后默认 paused,不
resume 就不算醒)。"""
name = self.vm_name(task)
self._launch_api_only(task)
self.wait_api_ready(task)
self._check_api_socket_permissions(self.api_socket(task))
self._ch_remote(task, "restore",
"source_url=%s" % snapshot_url)
self._emit("vm_restore", task,
"vm=%s snapshot=%s" % (name, snapshot_url))
self.resume_vm(task)
self.wait_guest_ready(task)
def restore_vm(self, task, snapshot_url):
"""快照恢复唤醒:校验 → 只起 VMM → restore → 显式
resume → 等 SSH 就绪。恢复后默认 paused,不 resume
就不算醒。任一步失败都关 VM 再抛;成功返回
{vm, api_socket, state: "running", ready: True,
mode: "restored"}。"""
snapshot_url = self._validate_restore(task, snapshot_url)
name = self.vm_name(task)
self._check_capacity(exclude=name)
try:
self._launch_api_only(task)
self.wait_api_ready(task)
self._check_api_socket_permissions(self.api_socket(task))
self._ch_remote(task, "restore",
"source_url=%s" % snapshot_url)
self._emit("vm_restore", task,
"vm=%s snapshot=%s" % (name, snapshot_url))
self.resume_vm(task)
self.wait_guest_ready(task)
self._restore_boot(task, snapshot_url)
except Exception:
try:
self.stop_vm(task)
@@ -1476,58 +1495,26 @@ class CloudHypervisorExecutor(Executor):
说明来源是休眠快照。"""
return self.restore_vm(task, snapshot_url)
def stop_vm(self, task_or_name):
"""停止 VM(尽力而为,不抛异常;三种情况):
- 已死进程:登记进程 poll() 非 None,跳过 terminate
等待,直接删 socket、清登记,返回 True。
- 已关 VM:无登记进程、无 socket,ch-remote 仍尽力
调过一次,返回 False(本来就没有什么可清)。
- socket 残留:有 socket、无登记进程,删 socket 并
返回 True。
ch-remote shutdown 失败不炸(VM 可能已死),但
ch-remote 命令本身仍会发出——测试据此断言关闭路径
确实用了 ch-remote shutdown,而不是只杀进程。"""
name = self.vm_name(task_or_name)
socket_path = os.path.join(self.state_dir, name + ".sock")
def _stop_registered_proc(self, name, proc):
"""清登记进程:已死直接清,活着先杀再清(纯移动)。"""
if proc is None:
return
with self._vms_lock:
had_proc = name in self._vms
proc = self._vms.get(name)
had = had_proc or os.path.exists(socket_path)
self._vms.pop(name, None)
try:
self.runner([self.ch_remote,
"--api-socket=%s" % socket_path, "shutdown"])
alive = proc.poll() is None
except Exception:
pass
if proc is not None:
with self._vms_lock:
self._vms.pop(name, None)
try:
alive = proc.poll() is None
except Exception:
alive = False
if alive:
self._kill_proc(proc)
try:
if os.path.exists(socket_path):
os.remove(socket_path)
except Exception:
pass
# 补缺口轨:per-VM seed 目录与 host 私钥随 VM
# 关闭而删,绝不留给下一个 VM 复用;known_hosts
# 公钥文件保留供事后审计,删失败不炸关闭。下次
# 同名 VM 预置时若私钥已删即重新生成并覆盖
# known_hosts,不会用旧钉死去校验新 VM。
# feat-chv-secfix(H3/M2):清理分三块独立记账——
# seed/私钥块、任务磁盘块、net 登记块,任一块失败
# 都记对应审计 + 写脏标记(清理失败即拒起),绝不
# 再以“一块失败、整段吞掉”收场;任务磁盘块是 H3
# 的主修复:stop 即删 disk.raw(保留期 0;需取证
# 须在 shutdown 前经 SSH 拷出,见 recycle_vm 文档)。
# net.json 是 live 预约(非审计 artifact),随 VM
# 关闭而删,tap/mac/ip 预约随即释放。
_task = task_or_name if isinstance(task_or_name, dict) \
else {"id": task_or_name}
_cleanup_failed = False
alive = False
if alive:
self._kill_proc(proc)
def _cleanup_vm_files(self, name, task):
"""stop 的三块文件清理:seed/私钥、任务磁盘、net 登记。
任一块失败记对应审计并返回 True(脏标记由调用方写);
seed/私钥与磁盘删除的审计名与文案逐字不变。"""
from cloud.chv.hostkeys import dirty_marker_path # noqa: F401
failed = False
try:
seed_dir = os.path.join(self.state_dir, name, "seed")
if os.path.isdir(seed_dir):
@@ -1539,45 +1526,69 @@ class CloudHypervisorExecutor(Executor):
if os.path.exists(_p):
os.remove(_p)
except Exception as _cleanup_exc:
_cleanup_failed = True
self._emit("vm_key_cleanup_failed", _task,
failed = True
self._emit("vm_key_cleanup_failed", task,
"vm=%s error=%s" % (name, _cleanup_exc))
try:
if delete_task_disk(self.state_dir, name):
self._emit("vm_disk_deleted", _task,
self._emit("vm_disk_deleted", task,
"vm=%s disk deleted on stop" % name)
except Exception as _disk_exc:
_cleanup_failed = True
self._emit("vm_disk_delete_failed", _task,
failed = True
self._emit("vm_disk_delete_failed", task,
"vm=%s error=%s" % (name, _disk_exc))
try:
_net_path = os.path.join(self.state_dir, name, "net.json")
if os.path.exists(_net_path):
os.remove(_net_path)
except Exception:
_cleanup_failed = True
failed = True
self._guest_env_delivered.pop(name, None)
if _cleanup_failed:
# M2 根治:脏标记。标记本身写失败也不炸关闭,
# 但下次 provision_host_key 仍会因残留文件复用
# 旧 key——该残留情形由 cleanup_stale_secrets 的
# 扫描兜底(见 cleanup_stale_vms),文档明写。
try:
from cloud.chv.hostkeys import dirty_marker_path
_marker = dirty_marker_path(self.state_dir, name)
return failed
def _write_dirty_marker(self, name, failed):
"""M2 脏标记:清理失败写标记、成功清标记;写失败不炸关闭。"""
from cloud.chv.hostkeys import dirty_marker_path
try:
_marker = dirty_marker_path(self.state_dir, name)
if failed:
os.makedirs(os.path.dirname(_marker), exist_ok=True)
with open(_marker, "w") as fh:
fh.write("cleanup failed\n")
except Exception:
pass
else:
try:
from cloud.chv.hostkeys import dirty_marker_path
_marker = dirty_marker_path(self.state_dir, name)
if os.path.exists(_marker):
os.remove(_marker)
except Exception:
pass
elif os.path.exists(_marker):
os.remove(_marker)
except Exception:
pass
def stop_vm(self, task_or_name):
"""停止 VM(尽力而为,不抛异常):已死进程直接清登记,
已关 VM 仍尽力调一次 ch-remote(返回 False),socket
残留删 socket(返回 True)。ch-remote shutdown 失败
不炸,但命令本身仍会发出(测试据此断言关闭路径)。"""
name = self.vm_name(task_or_name)
socket_path = os.path.join(self.state_dir, name + ".sock")
with self._vms_lock:
had_proc = name in self._vms
proc = self._vms.get(name)
had = had_proc or os.path.exists(socket_path)
try:
self.runner([self.ch_remote,
"--api-socket=%s" % socket_path, "shutdown"])
except Exception:
pass
self._stop_registered_proc(name, proc)
try:
if os.path.exists(socket_path):
os.remove(socket_path)
except Exception:
pass
# per-VM seed 目录与 host 私钥随 VM 关闭而删,绝不留给
# 下一个 VM 复用;known_hosts 公钥保留供事后审计。
# 三块清理独立记账 + 脏标记(H3/M2,见 _cleanup_vm_files)。
_task = task_or_name if isinstance(task_or_name, dict) \
else {"id": task_or_name}
_cleanup_failed = self._cleanup_vm_files(name, _task)
self._write_dirty_marker(name, _cleanup_failed)
task = task_or_name if isinstance(task_or_name, dict) \
else {"id": task_or_name}
if had:
@@ -1624,12 +1635,49 @@ class CloudHypervisorExecutor(Executor):
"vm=%s gc error=%s" % (entry, exc))
return {"deleted": deleted, "failed": failed}
def _is_live_vm(self, entry):
"""VM 是否活着:socket 存在或登记进程未死即活。"""
if os.path.exists(os.path.join(self.state_dir, entry + ".sock")):
return True
with self._vms_lock:
proc = self._vms.get(entry)
if proc is not None:
try:
return proc.poll() is None
except Exception:
pass
return False
def _clean_one_vm_secrets(self, entry, vm_dir):
"""单 VM 残留清理:私钥一律删(live VM 私钥在 seed 盘
建好后即删,残留只可能是崩溃遗留);非 live VM 的
seed.iso/seed 目录同理删。返回 (删私钥, 删 seed)。"""
removed_priv = False
priv = os.path.join(vm_dir, "ssh_host_ed25519_key")
if os.path.exists(priv):
try:
os.remove(priv)
removed_priv = True
except Exception:
pass
removed_seed = False
if not self._is_live_vm(entry):
for _suffix in ("seed.iso", "seed"):
_p = os.path.join(vm_dir, _suffix)
try:
if os.path.isdir(_p):
shutil.rmtree(_p)
removed_seed = True
elif os.path.exists(_p):
os.remove(_p)
removed_seed = True
except Exception:
pass
return removed_priv, removed_seed
def cleanup_stale_secrets(self):
"""崩溃残留扫描(H5):state_dir 下的 host 私钥文件
在本轨不变量下只可能是残留——live VM 的私钥在 seed
盘建好后即删(见 wake_vm),所以任何残留私钥文件
一律删除并审计;非 live VM 的 seed.iso/seed 目录同
理。返回已清理的 VM 名列表。worker 启动与
"""崩溃残留扫描(H5):私钥一律删、非 live VM 的 seed
同理删并审计;返回已清理 VM 名。worker 启动与
cleanup_stale_vms 时调它,不依赖 socket 是否存在。"""
cleaned = []
try:
@@ -1642,36 +1690,8 @@ class CloudHypervisorExecutor(Executor):
vm_dir = os.path.join(self.state_dir, entry)
if not os.path.isdir(vm_dir):
continue
priv = os.path.join(vm_dir, "ssh_host_ed25519_key")
removed_priv = False
if os.path.exists(priv):
try:
os.remove(priv)
removed_priv = True
except Exception:
pass
live = os.path.exists(
os.path.join(self.state_dir, entry + ".sock"))
with self._vms_lock:
proc = self._vms.get(entry)
if proc is not None:
try:
live = live or proc.poll() is None
except Exception:
pass
removed_seed = False
if not live:
for _suffix in ("seed.iso", "seed"):
_p = os.path.join(vm_dir, _suffix)
try:
if os.path.isdir(_p):
shutil.rmtree(_p)
removed_seed = True
elif os.path.exists(_p):
os.remove(_p)
removed_seed = True
except Exception:
pass
removed_priv, removed_seed = \
self._clean_one_vm_secrets(entry, vm_dir)
if removed_priv or removed_seed:
cleaned.append(entry)
self._emit("host_secret_residue_cleaned",
@@ -1712,32 +1732,11 @@ class CloudHypervisorExecutor(Executor):
"lost": ["guest /tmp", "guest worktree temp files"],
}
def cleanup_stale_vms(self):
"""清理残留 VM:只清 socket 残留与已死进程对应的 VM。
def _stop_dead_socket_vms(self, entries):
"""socket 扫描:登记进程仍活的跳过,其余走 stop_vm。
feat-chv-secfix(H3/H5):先跑残留盘 GC 与残留
密钥扫描(不依赖 socket 是否存在,覆盖 worker 崩溃
后 socket 已删、文件仍在的情形),再走原 socket 扫描。
扫描 state_dir 下 *.sock:登记进程仍活(poll() 为 None)
的跳过;其余一律走 stop_vm。返回已清理的 VM 名列表;
state_dir 不存在时返回 [],绝不抛异常。基础设施或
其它执行器的 socket 不在 state_dir 里,不会被误删;
同目录下非 chv- 前缀的 socket 也跳过(只认本执行器
的 chv- 命名)。
"""
返回已清理 VM 名;非 chv- 前缀 socket 一律跳过。"""
cleaned = []
try:
self.cleanup_stale_secrets()
except Exception:
pass
try:
self.gc_stale_disks()
except Exception:
pass
try:
entries = os.listdir(self.state_dir)
except Exception:
return []
for entry in entries:
if not entry.endswith(".sock"):
continue
@@ -1754,9 +1753,11 @@ class CloudHypervisorExecutor(Executor):
continue
if self.stop_vm(name):
cleaned.append(name)
# 登记进程已死但 socket 文件已被删的情况:登记项本身
# 也是残留,顺手清掉登记,不计入 cleaned(没有 socket
# 可删,不算一次清理)。
return cleaned
def _purge_dead_registry(self):
"""登记进程已死但 socket 已被删的登记项顺手清掉
(没有 socket 可删,不计入清理数)。"""
for name in list(self._vms.keys()):
proc = self._vms.get(name)
try:
@@ -1765,6 +1766,25 @@ class CloudHypervisorExecutor(Executor):
dead = True
if dead:
self._vms.pop(name, None)
def cleanup_stale_vms(self):
"""清理残留 VM:先跑残留盘 GC 与残留密钥扫描,再走
socket 扫描与登记清理。只清 socket 残留与已死进程
对应的 VM;state_dir 不存在返回 [],绝不抛异常。"""
try:
self.cleanup_stale_secrets()
except Exception:
pass
try:
self.gc_stale_disks()
except Exception:
pass
try:
entries = os.listdir(self.state_dir)
except Exception:
return []
cleaned = self._stop_dead_socket_vms(entries)
self._purge_dead_registry()
return cleaned
# -- 客内执行 ------------------------------------------------------
+42 -26
View File
@@ -99,33 +99,47 @@ def validate_net_entries(entries: list, existing_entries: list) -> list:
vm})的 tap/mac/ip 任一重复即 net_tap_conflict,
拒绝启动并由调用方审计。
返回归一化后的条目 list。"""
seen_taps, seen_macs, seen_ips = set(), set(), set()
normalized = []
for entry in entries:
unknown = set(entry) - _NET_FIELDS
if unknown:
raise ChvFailure(
"net_config_invalid",
"CHV_NET_JSON 条目含未知字段:%s(白名单:%s)"
% (",".join(sorted(unknown)),
",".join(sorted(_NET_FIELDS))))
tap = str(entry.get("tap") or "").strip()
mac = str(entry.get("mac") or "").strip().lower()
ip = str(entry.get("ip") or "").strip()
for kind, value, seen in (("tap", tap, seen_taps),
("mac", mac, seen_macs),
("ip", ip, seen_ips)):
normalized = [_normalize_net_entry(e) for e in entries]
_check_net_duplicates(normalized)
_check_net_conflicts(normalized, existing_entries)
return normalized
def _normalize_net_entry(entry: dict) -> dict:
"""单条 net 条目归一化:字段白名单 + tap/mac/ip 归一化。
空 tap 视为未指定,不参与 tap 冲突比较(见
validate_net_entries 的说明,纯移动)。"""
unknown = set(entry) - _NET_FIELDS
if unknown:
raise ChvFailure(
"net_config_invalid",
"CHV_NET_JSON 条目含未知字段:%s(白名单:%s)"
% (",".join(sorted(unknown)),
",".join(sorted(_NET_FIELDS))))
return {"tap": str(entry.get("tap") or "").strip(),
"mac": str(entry.get("mac") or "").strip().lower(),
"ip": str(entry.get("ip") or "").strip(),
"mask": str(entry.get("mask") or "")}
def _check_net_duplicates(normalized: list) -> None:
"""本次条目内 tap/mac/ip 重复即 net_config_invalid。"""
seen = {"tap": set(), "mac": set(), "ip": set()}
for entry in normalized:
for kind in ("tap", "mac", "ip"):
value = entry.get(kind)
if not value:
continue
if kind == "tap" and not tap:
continue
if value in seen:
if value in seen[kind]:
raise ChvFailure(
"net_config_invalid",
"CHV_NET_JSON 内 %s 重复:%s" % (kind, value))
seen.add(value)
normalized.append({"tap": tap, "mac": mac, "ip": ip,
"mask": str(entry.get("mask") or "")})
seen[kind].add(value)
def _check_net_conflicts(normalized: list, existing_entries: list) -> None:
"""与已登记 VM 的 tap/mac/ip 任一重复即 net_tap_conflict。"""
for entry in normalized:
for other in existing_entries:
for kind in ("tap", "mac", "ip"):
@@ -136,7 +150,6 @@ def validate_net_entries(entries: list, existing_entries: list) -> list:
"net %s %s 已被 VM %s 使用,拒绝共享"
" tap/mac/ip:每 VM 必须独立"
% (kind, value, other.get("vm")))
return normalized
def validate_egress_proxy(egress_proxy: str) -> None:
@@ -162,9 +175,12 @@ def validate_egress_proxy(egress_proxy: str) -> None:
def _is_within(directory, path):
directory = os.path.realpath(directory)
path = os.path.realpath(path)
return path == directory or path.startswith(directory + os.sep)
"""目录内判定:经 compat215.resolve_inside(#215 目录门
同签名),越界回 None 即不在内。绝对路径先转相对路径
再过门,.. 逃逸与符号链接逃逸一律挡。"""
from cloud.chv.compat215 import resolve_inside
rel = os.path.relpath(path, directory)
return resolve_inside(directory, rel) is not None
def validate_snapshot_url(snapshot_url: str, state_dir: str, snapshot_dir=None,
+10
View File
@@ -304,3 +304,13 @@ class WarmPool:
"warm_pool_release", task,
"vm=%s destroyed=%s" % (pool_vm, destroyed))
return destroyed
def reconcile_orphans_for_executor(executor) -> list:
"""生产路径唯一的 WarmPool 入口:只做孤儿清理。
worker 启动/关闭调它,不构造 WarmPool 类、不调
prewarm/acquire/release——实验接口与生产路径彻底分离:
生产代码 import 本函数即可,acquire/prewarm 只存在于
实验用的 WarmPool 类与测试里。"""
return WarmPool(executor).reconcile_orphans()