feat(chv): 架构与代码质量 9.5+ 提升——长函数清零、WarmPool facade、#215 对齐、审计计数、只读 VM 字段、ADR-8~11
This commit is contained in:
1 parent
c908a0147a
commit
42de20da68
15 files changed
+840
-345
No files matched your search
+215
-195
@@ -289,6 +289,7 @@ class CloudHypervisorExecutor(Executor):
|
||||
else _unix_socket_api
|
||||
self.sleeper = sleeper if sleeper is not None else time.sleep
|
||||
self.audit_callback = None
|
||||
self.audit_failure_count = 0
|
||||
self._vms_lock = threading.Lock()
|
||||
if runner is not None:
|
||||
self.runner = runner
|
||||
@@ -602,9 +603,15 @@ class CloudHypervisorExecutor(Executor):
|
||||
feat-chv-secfix(M5):审计回调失败不再双重静默
|
||||
pass——记宿主 error 日志(审计完整性本身没有保证,
|
||||
但失败至少可见;任务仍不因此炸,审计失败不改变
|
||||
任务状态是 worker 层的既定语义)。审计 detail 里
|
||||
绝不含密钥值(调用方负责:本类所有 _emit 的 detail
|
||||
只含 VM 名、路径、URL、kind,不含任何 key 值)。"""
|
||||
任务状态是 worker 层的既定语义)。feat-chv-arch95plus
|
||||
取舍(见 ADR-9):不改成“审计失败即中断任务”——
|
||||
审计是可观测性不是门禁,DB 短暂不可用时硬中断会把
|
||||
一次审计失败放大成任务失败,且与 worker 全局
|
||||
best-effort 语义冲突;改为“失败记日志 + 计数 + 不
|
||||
中断”,计数见 self.audit_failure_count(宿主可观测,
|
||||
非任务状态)。审计 detail 里绝不含密钥值(调用方
|
||||
负责:本类所有 _emit 的 detail 只含 VM 名、路径、
|
||||
URL、kind,不含任何 key 值)。"""
|
||||
try:
|
||||
if self.audit_callback is not None:
|
||||
self.audit_callback(event, task, detail, kind)
|
||||
@@ -612,6 +619,8 @@ class CloudHypervisorExecutor(Executor):
|
||||
_LOGGER.info("chv_audit event=%s kind=%s %s",
|
||||
event, kind, detail)
|
||||
except Exception as exc:
|
||||
self.audit_failure_count = getattr(
|
||||
self, "audit_failure_count", 0) + 1
|
||||
_LOGGER.error("chv_audit_failed event=%s kind=%s error=%s",
|
||||
event, kind, exc)
|
||||
|
||||
@@ -981,21 +990,37 @@ class CloudHypervisorExecutor(Executor):
|
||||
"vmm.ping 超时:%d 次内 API 未通,socket=%s"
|
||||
% (attempts, socket_path))
|
||||
|
||||
def wait_guest_ready(self, task):
|
||||
"""等 guest SSH 就绪(只选 SSH,见类 docstring):
|
||||
轮询 ssh … "true",returncode 0 即就绪;次数 =
|
||||
ready_timeout/ready_interval 取整,至少 1 次。
|
||||
每轮先尽力捕获 host key(known_hosts 文件不存在时;
|
||||
sshd 未起时抓不到是常态,下轮再试),再发 ssh
|
||||
"true"——钉死式 SSH 在文件缺失时会直接拒绝未知主机,
|
||||
不先捕获就永远等不到就绪。超时时先关 VM 再 raise
|
||||
ChvFailure(kind=guest_not_ready),绝不在未就绪时
|
||||
派任务。"""
|
||||
# feat-chv-fix(P0-2):provisioned 模式下缺 known_hosts
|
||||
# 时直接 fail-fast,绝不调用 capture_host_key 回落 TOFU。
|
||||
# (keyscan 模式才走下面的捕获分支。)
|
||||
def _provisioned_key_guard(self, task):
|
||||
"""provisioned 下缺 known_hosts 即 fail-fast,绝不 keyscan。
|
||||
|
||||
返回 True 表示已拦截(调用方须先关 VM 再抛
|
||||
host_key_not_pinned);False 表示可继续轮询。"""
|
||||
if getattr(self, "host_key_mode", "provisioned") == "provisioned" \
|
||||
and not os.path.exists(self.known_hosts_path(task)):
|
||||
return True
|
||||
return False
|
||||
|
||||
def _check_host_key_mismatch(self, task, result):
|
||||
"""SSH stderr 含钉死拒绝标记时审计并返回 True(须 fail-fast)。
|
||||
|
||||
MITM 尝试是独立安全事件,不轮询成普通超时;审计
|
||||
detail 只含 VM 名与标记,不含 key 值。"""
|
||||
stderr = getattr(result, "stderr", "") or ""
|
||||
if "Host key verification failed" in stderr \
|
||||
or "REMOTE HOST IDENTIFICATION" in stderr:
|
||||
self._emit("host_key_mismatch", task,
|
||||
"vm=%s ssh host key 被拒绝(钉死不匹配)"
|
||||
% self.vm_name(task), "host_key_mismatch")
|
||||
return True
|
||||
return False
|
||||
|
||||
def wait_guest_ready(self, task):
|
||||
"""等 guest SSH 就绪:轮询 ssh "true",returncode 0 即就绪;
|
||||
每轮先尽力捕获 host key(keyscan 模式),超时先关 VM
|
||||
再 raise guest_not_ready,绝不在未就绪时派任务。"""
|
||||
# P0-2:provisioned 下缺 known_hosts 即 fail-fast,绝不
|
||||
# 调用 capture_host_key 回落 TOFU(keyscan 才走捕获分支)。
|
||||
if self._provisioned_key_guard(task):
|
||||
try:
|
||||
self.stop_vm(task)
|
||||
except Exception:
|
||||
@@ -1013,17 +1038,9 @@ class CloudHypervisorExecutor(Executor):
|
||||
self.build_ssh_command(task, ["true"]))
|
||||
if getattr(result, "returncode", 1) == 0:
|
||||
return True
|
||||
# feat-chv-secfix(M5):host key 不匹配是独立
|
||||
# 安全事件,不是“sshd 没起”。SSH stderr 含
|
||||
# 钉死拒绝标记时立即审计 host_key_mismatch 并
|
||||
# fail-fast(不把 MITM 尝试轮询成普通超时),
|
||||
# 审计 detail 只含 VM 名与标记,不含 key 值。
|
||||
stderr = getattr(result, "stderr", "") or ""
|
||||
if "Host key verification failed" in stderr \
|
||||
or "REMOTE HOST IDENTIFICATION" in stderr:
|
||||
self._emit("host_key_mismatch", task,
|
||||
"vm=%s ssh host key 被拒绝(钉死不匹配)"
|
||||
% self.vm_name(task), "host_key_mismatch")
|
||||
# M5:host key 不匹配是独立安全事件,见
|
||||
# _check_host_key_mismatch(审计 + fail-fast)。
|
||||
if self._check_host_key_mismatch(task, result):
|
||||
try:
|
||||
self.stop_vm(task)
|
||||
except Exception:
|
||||
@@ -1394,58 +1411,60 @@ class CloudHypervisorExecutor(Executor):
|
||||
% (self.vm_name(task_or_name), snapshot_url))
|
||||
return snapshot_url
|
||||
|
||||
def restore_vm(self, task, snapshot_url):
|
||||
"""快照恢复唤醒:preflight → 只起 VMM(--api-socket)
|
||||
→ 等 vmm.ping → ch-remote restore → 显式 resume →
|
||||
等 guest SSH 就绪。恢复后的 VM 默认是 paused,不
|
||||
resume 就不算醒——本方法绝不省略 resume 那一步。
|
||||
任一步失败都关 VM 再抛。成功返回 {vm, api_socket,
|
||||
state: "running", ready: True, mode: "restored"}。"""
|
||||
def _validate_restore(self, task, snapshot_url):
|
||||
"""restore 前置校验:返回归一化后的 snapshot_url。
|
||||
|
||||
restore 不走 vm.create、没有 VmConfig 可挂 seed:
|
||||
快照内已固化原 VM host key,重预置反而破坏钉死,
|
||||
故 provisioned 下复用已钉死的 known_hosts、缺失
|
||||
fail-fast、绝不 keyscan、绝不重生成。restore 时不
|
||||
重复验快照文件存在(远端/对象存储源合法,ch-remote
|
||||
本身会在源缺失时失败)。"""
|
||||
if not isinstance(task, dict) or task.get("id") is None:
|
||||
raise ValueError("任务缺少 id")
|
||||
if not snapshot_url:
|
||||
raise ValueError("snapshot_url 不能为空")
|
||||
if "://" not in snapshot_url:
|
||||
snapshot_url = "file://%s" % snapshot_url
|
||||
# feat-chv-secfix(L2):恢复源同样过 URL 白名单
|
||||
# (file:// 须在白名单目录内,远端须显式放行),
|
||||
# 在 preflight/起 VMM 之前就拒绝。
|
||||
validate_snapshot_url(
|
||||
snapshot_url, self.state_dir,
|
||||
snapshot_dir=getattr(self, "snapshot_dir", None),
|
||||
allow_remote=getattr(self, "allow_remote_snapshot", False))
|
||||
self.preflight()
|
||||
self.resolve_ssh_host()
|
||||
# feat-chv-fix(P0-2 修正版):restore 不走 vm.create,
|
||||
# 没有 VmConfig 可挂 seed——审计报告“restore 须把 seed
|
||||
# 接入 VmConfig”的说法在此不适用,纠正为:快照内已
|
||||
# 固化原 VM 的 host key,重新预置新 key 反而会破坏
|
||||
# 钉死。正确不变量是:provisioned 下复用已钉死的
|
||||
# known_hosts(stop 时公钥保留、私钥已删但快照内仍有
|
||||
# 原私钥),缺失时 fail-fast、绝不 keyscan、绝不重
|
||||
# 生成。file:// 快照须先验存在且非空,否则不启动。
|
||||
# 注意:restore 时不重复验快照文件存在——快照落盘
|
||||
# 校验是 snapshot_vm/hibernate_vm 创建时的门(P1-7),
|
||||
# restore 的 ch-remote 本身会在源缺失时失败;在这里
|
||||
# 再验一次会把“源在远端/对象存储”的合法路径也挡掉。
|
||||
if getattr(self, "host_key_mode", "provisioned") == "provisioned" \
|
||||
and not os.path.exists(self.known_hosts_path(task)):
|
||||
if self._provisioned_key_guard(task):
|
||||
raise ChvFailure(
|
||||
"host_key_not_pinned",
|
||||
"provisioned 模式下 restore 缺 known_hosts,拒绝 "
|
||||
"keyscan 回落:VM %s" % self.vm_name(task))
|
||||
return snapshot_url
|
||||
|
||||
def _restore_boot(self, task, snapshot_url):
|
||||
"""restore 的启动段:起 VMM → 等 ping → restore →
|
||||
显式 resume → 等 SSH 就绪(恢复后默认 paused,不
|
||||
resume 就不算醒)。"""
|
||||
name = self.vm_name(task)
|
||||
self._launch_api_only(task)
|
||||
self.wait_api_ready(task)
|
||||
self._check_api_socket_permissions(self.api_socket(task))
|
||||
self._ch_remote(task, "restore",
|
||||
"source_url=%s" % snapshot_url)
|
||||
self._emit("vm_restore", task,
|
||||
"vm=%s snapshot=%s" % (name, snapshot_url))
|
||||
self.resume_vm(task)
|
||||
self.wait_guest_ready(task)
|
||||
|
||||
def restore_vm(self, task, snapshot_url):
|
||||
"""快照恢复唤醒:校验 → 只起 VMM → restore → 显式
|
||||
resume → 等 SSH 就绪。恢复后默认 paused,不 resume
|
||||
就不算醒。任一步失败都关 VM 再抛;成功返回
|
||||
{vm, api_socket, state: "running", ready: True,
|
||||
mode: "restored"}。"""
|
||||
snapshot_url = self._validate_restore(task, snapshot_url)
|
||||
name = self.vm_name(task)
|
||||
self._check_capacity(exclude=name)
|
||||
try:
|
||||
self._launch_api_only(task)
|
||||
self.wait_api_ready(task)
|
||||
self._check_api_socket_permissions(self.api_socket(task))
|
||||
self._ch_remote(task, "restore",
|
||||
"source_url=%s" % snapshot_url)
|
||||
self._emit("vm_restore", task,
|
||||
"vm=%s snapshot=%s" % (name, snapshot_url))
|
||||
self.resume_vm(task)
|
||||
self.wait_guest_ready(task)
|
||||
self._restore_boot(task, snapshot_url)
|
||||
except Exception:
|
||||
try:
|
||||
self.stop_vm(task)
|
||||
@@ -1476,58 +1495,26 @@ class CloudHypervisorExecutor(Executor):
|
||||
说明来源是休眠快照。"""
|
||||
return self.restore_vm(task, snapshot_url)
|
||||
|
||||
def stop_vm(self, task_or_name):
|
||||
"""停止 VM(尽力而为,不抛异常;三种情况):
|
||||
- 已死进程:登记进程 poll() 非 None,跳过 terminate
|
||||
等待,直接删 socket、清登记,返回 True。
|
||||
- 已关 VM:无登记进程、无 socket,ch-remote 仍尽力
|
||||
调过一次,返回 False(本来就没有什么可清)。
|
||||
- socket 残留:有 socket、无登记进程,删 socket 并
|
||||
返回 True。
|
||||
ch-remote shutdown 失败不炸(VM 可能已死),但
|
||||
ch-remote 命令本身仍会发出——测试据此断言关闭路径
|
||||
确实用了 ch-remote shutdown,而不是只杀进程。"""
|
||||
name = self.vm_name(task_or_name)
|
||||
socket_path = os.path.join(self.state_dir, name + ".sock")
|
||||
def _stop_registered_proc(self, name, proc):
|
||||
"""清登记进程:已死直接清,活着先杀再清(纯移动)。"""
|
||||
if proc is None:
|
||||
return
|
||||
with self._vms_lock:
|
||||
had_proc = name in self._vms
|
||||
proc = self._vms.get(name)
|
||||
had = had_proc or os.path.exists(socket_path)
|
||||
self._vms.pop(name, None)
|
||||
try:
|
||||
self.runner([self.ch_remote,
|
||||
"--api-socket=%s" % socket_path, "shutdown"])
|
||||
alive = proc.poll() is None
|
||||
except Exception:
|
||||
pass
|
||||
if proc is not None:
|
||||
with self._vms_lock:
|
||||
self._vms.pop(name, None)
|
||||
try:
|
||||
alive = proc.poll() is None
|
||||
except Exception:
|
||||
alive = False
|
||||
if alive:
|
||||
self._kill_proc(proc)
|
||||
try:
|
||||
if os.path.exists(socket_path):
|
||||
os.remove(socket_path)
|
||||
except Exception:
|
||||
pass
|
||||
# 补缺口轨:per-VM seed 目录与 host 私钥随 VM
|
||||
# 关闭而删,绝不留给下一个 VM 复用;known_hosts
|
||||
# 公钥文件保留供事后审计,删失败不炸关闭。下次
|
||||
# 同名 VM 预置时若私钥已删即重新生成并覆盖
|
||||
# known_hosts,不会用旧钉死去校验新 VM。
|
||||
# feat-chv-secfix(H3/M2):清理分三块独立记账——
|
||||
# seed/私钥块、任务磁盘块、net 登记块,任一块失败
|
||||
# 都记对应审计 + 写脏标记(清理失败即拒起),绝不
|
||||
# 再以“一块失败、整段吞掉”收场;任务磁盘块是 H3
|
||||
# 的主修复:stop 即删 disk.raw(保留期 0;需取证
|
||||
# 须在 shutdown 前经 SSH 拷出,见 recycle_vm 文档)。
|
||||
# net.json 是 live 预约(非审计 artifact),随 VM
|
||||
# 关闭而删,tap/mac/ip 预约随即释放。
|
||||
_task = task_or_name if isinstance(task_or_name, dict) \
|
||||
else {"id": task_or_name}
|
||||
_cleanup_failed = False
|
||||
alive = False
|
||||
if alive:
|
||||
self._kill_proc(proc)
|
||||
|
||||
def _cleanup_vm_files(self, name, task):
|
||||
"""stop 的三块文件清理:seed/私钥、任务磁盘、net 登记。
|
||||
|
||||
任一块失败记对应审计并返回 True(脏标记由调用方写);
|
||||
seed/私钥与磁盘删除的审计名与文案逐字不变。"""
|
||||
from cloud.chv.hostkeys import dirty_marker_path # noqa: F401
|
||||
failed = False
|
||||
try:
|
||||
seed_dir = os.path.join(self.state_dir, name, "seed")
|
||||
if os.path.isdir(seed_dir):
|
||||
@@ -1539,45 +1526,69 @@ class CloudHypervisorExecutor(Executor):
|
||||
if os.path.exists(_p):
|
||||
os.remove(_p)
|
||||
except Exception as _cleanup_exc:
|
||||
_cleanup_failed = True
|
||||
self._emit("vm_key_cleanup_failed", _task,
|
||||
failed = True
|
||||
self._emit("vm_key_cleanup_failed", task,
|
||||
"vm=%s error=%s" % (name, _cleanup_exc))
|
||||
try:
|
||||
if delete_task_disk(self.state_dir, name):
|
||||
self._emit("vm_disk_deleted", _task,
|
||||
self._emit("vm_disk_deleted", task,
|
||||
"vm=%s disk deleted on stop" % name)
|
||||
except Exception as _disk_exc:
|
||||
_cleanup_failed = True
|
||||
self._emit("vm_disk_delete_failed", _task,
|
||||
failed = True
|
||||
self._emit("vm_disk_delete_failed", task,
|
||||
"vm=%s error=%s" % (name, _disk_exc))
|
||||
try:
|
||||
_net_path = os.path.join(self.state_dir, name, "net.json")
|
||||
if os.path.exists(_net_path):
|
||||
os.remove(_net_path)
|
||||
except Exception:
|
||||
_cleanup_failed = True
|
||||
failed = True
|
||||
self._guest_env_delivered.pop(name, None)
|
||||
if _cleanup_failed:
|
||||
# M2 根治:脏标记。标记本身写失败也不炸关闭,
|
||||
# 但下次 provision_host_key 仍会因残留文件复用
|
||||
# 旧 key——该残留情形由 cleanup_stale_secrets 的
|
||||
# 扫描兜底(见 cleanup_stale_vms),文档明写。
|
||||
try:
|
||||
from cloud.chv.hostkeys import dirty_marker_path
|
||||
_marker = dirty_marker_path(self.state_dir, name)
|
||||
return failed
|
||||
|
||||
def _write_dirty_marker(self, name, failed):
|
||||
"""M2 脏标记:清理失败写标记、成功清标记;写失败不炸关闭。"""
|
||||
from cloud.chv.hostkeys import dirty_marker_path
|
||||
try:
|
||||
_marker = dirty_marker_path(self.state_dir, name)
|
||||
if failed:
|
||||
os.makedirs(os.path.dirname(_marker), exist_ok=True)
|
||||
with open(_marker, "w") as fh:
|
||||
fh.write("cleanup failed\n")
|
||||
except Exception:
|
||||
pass
|
||||
else:
|
||||
try:
|
||||
from cloud.chv.hostkeys import dirty_marker_path
|
||||
_marker = dirty_marker_path(self.state_dir, name)
|
||||
if os.path.exists(_marker):
|
||||
os.remove(_marker)
|
||||
except Exception:
|
||||
pass
|
||||
elif os.path.exists(_marker):
|
||||
os.remove(_marker)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
def stop_vm(self, task_or_name):
|
||||
"""停止 VM(尽力而为,不抛异常):已死进程直接清登记,
|
||||
已关 VM 仍尽力调一次 ch-remote(返回 False),socket
|
||||
残留删 socket(返回 True)。ch-remote shutdown 失败
|
||||
不炸,但命令本身仍会发出(测试据此断言关闭路径)。"""
|
||||
name = self.vm_name(task_or_name)
|
||||
socket_path = os.path.join(self.state_dir, name + ".sock")
|
||||
with self._vms_lock:
|
||||
had_proc = name in self._vms
|
||||
proc = self._vms.get(name)
|
||||
had = had_proc or os.path.exists(socket_path)
|
||||
try:
|
||||
self.runner([self.ch_remote,
|
||||
"--api-socket=%s" % socket_path, "shutdown"])
|
||||
except Exception:
|
||||
pass
|
||||
self._stop_registered_proc(name, proc)
|
||||
try:
|
||||
if os.path.exists(socket_path):
|
||||
os.remove(socket_path)
|
||||
except Exception:
|
||||
pass
|
||||
# per-VM seed 目录与 host 私钥随 VM 关闭而删,绝不留给
|
||||
# 下一个 VM 复用;known_hosts 公钥保留供事后审计。
|
||||
# 三块清理独立记账 + 脏标记(H3/M2,见 _cleanup_vm_files)。
|
||||
_task = task_or_name if isinstance(task_or_name, dict) \
|
||||
else {"id": task_or_name}
|
||||
_cleanup_failed = self._cleanup_vm_files(name, _task)
|
||||
self._write_dirty_marker(name, _cleanup_failed)
|
||||
task = task_or_name if isinstance(task_or_name, dict) \
|
||||
else {"id": task_or_name}
|
||||
if had:
|
||||
@@ -1624,12 +1635,49 @@ class CloudHypervisorExecutor(Executor):
|
||||
"vm=%s gc error=%s" % (entry, exc))
|
||||
return {"deleted": deleted, "failed": failed}
|
||||
|
||||
def _is_live_vm(self, entry):
|
||||
"""VM 是否活着:socket 存在或登记进程未死即活。"""
|
||||
if os.path.exists(os.path.join(self.state_dir, entry + ".sock")):
|
||||
return True
|
||||
with self._vms_lock:
|
||||
proc = self._vms.get(entry)
|
||||
if proc is not None:
|
||||
try:
|
||||
return proc.poll() is None
|
||||
except Exception:
|
||||
pass
|
||||
return False
|
||||
|
||||
def _clean_one_vm_secrets(self, entry, vm_dir):
|
||||
"""单 VM 残留清理:私钥一律删(live VM 私钥在 seed 盘
|
||||
建好后即删,残留只可能是崩溃遗留);非 live VM 的
|
||||
seed.iso/seed 目录同理删。返回 (删私钥, 删 seed)。"""
|
||||
removed_priv = False
|
||||
priv = os.path.join(vm_dir, "ssh_host_ed25519_key")
|
||||
if os.path.exists(priv):
|
||||
try:
|
||||
os.remove(priv)
|
||||
removed_priv = True
|
||||
except Exception:
|
||||
pass
|
||||
removed_seed = False
|
||||
if not self._is_live_vm(entry):
|
||||
for _suffix in ("seed.iso", "seed"):
|
||||
_p = os.path.join(vm_dir, _suffix)
|
||||
try:
|
||||
if os.path.isdir(_p):
|
||||
shutil.rmtree(_p)
|
||||
removed_seed = True
|
||||
elif os.path.exists(_p):
|
||||
os.remove(_p)
|
||||
removed_seed = True
|
||||
except Exception:
|
||||
pass
|
||||
return removed_priv, removed_seed
|
||||
|
||||
def cleanup_stale_secrets(self):
|
||||
"""崩溃残留扫描(H5):state_dir 下的 host 私钥文件
|
||||
在本轨不变量下只可能是残留——live VM 的私钥在 seed
|
||||
盘建好后即删(见 wake_vm),所以任何残留私钥文件
|
||||
一律删除并审计;非 live VM 的 seed.iso/seed 目录同
|
||||
理。返回已清理的 VM 名列表。worker 启动与
|
||||
"""崩溃残留扫描(H5):私钥一律删、非 live VM 的 seed
|
||||
同理删并审计;返回已清理 VM 名。worker 启动与
|
||||
cleanup_stale_vms 时调它,不依赖 socket 是否存在。"""
|
||||
cleaned = []
|
||||
try:
|
||||
@@ -1642,36 +1690,8 @@ class CloudHypervisorExecutor(Executor):
|
||||
vm_dir = os.path.join(self.state_dir, entry)
|
||||
if not os.path.isdir(vm_dir):
|
||||
continue
|
||||
priv = os.path.join(vm_dir, "ssh_host_ed25519_key")
|
||||
removed_priv = False
|
||||
if os.path.exists(priv):
|
||||
try:
|
||||
os.remove(priv)
|
||||
removed_priv = True
|
||||
except Exception:
|
||||
pass
|
||||
live = os.path.exists(
|
||||
os.path.join(self.state_dir, entry + ".sock"))
|
||||
with self._vms_lock:
|
||||
proc = self._vms.get(entry)
|
||||
if proc is not None:
|
||||
try:
|
||||
live = live or proc.poll() is None
|
||||
except Exception:
|
||||
pass
|
||||
removed_seed = False
|
||||
if not live:
|
||||
for _suffix in ("seed.iso", "seed"):
|
||||
_p = os.path.join(vm_dir, _suffix)
|
||||
try:
|
||||
if os.path.isdir(_p):
|
||||
shutil.rmtree(_p)
|
||||
removed_seed = True
|
||||
elif os.path.exists(_p):
|
||||
os.remove(_p)
|
||||
removed_seed = True
|
||||
except Exception:
|
||||
pass
|
||||
removed_priv, removed_seed = \
|
||||
self._clean_one_vm_secrets(entry, vm_dir)
|
||||
if removed_priv or removed_seed:
|
||||
cleaned.append(entry)
|
||||
self._emit("host_secret_residue_cleaned",
|
||||
@@ -1712,32 +1732,11 @@ class CloudHypervisorExecutor(Executor):
|
||||
"lost": ["guest /tmp", "guest worktree temp files"],
|
||||
}
|
||||
|
||||
def cleanup_stale_vms(self):
|
||||
"""清理残留 VM:只清 socket 残留与已死进程对应的 VM。
|
||||
def _stop_dead_socket_vms(self, entries):
|
||||
"""socket 扫描:登记进程仍活的跳过,其余走 stop_vm。
|
||||
|
||||
feat-chv-secfix(H3/H5):先跑残留盘 GC 与残留
|
||||
密钥扫描(不依赖 socket 是否存在,覆盖 worker 崩溃
|
||||
后 socket 已删、文件仍在的情形),再走原 socket 扫描。
|
||||
扫描 state_dir 下 *.sock:登记进程仍活(poll() 为 None)
|
||||
的跳过;其余一律走 stop_vm。返回已清理的 VM 名列表;
|
||||
state_dir 不存在时返回 [],绝不抛异常。基础设施或
|
||||
其它执行器的 socket 不在 state_dir 里,不会被误删;
|
||||
同目录下非 chv- 前缀的 socket 也跳过(只认本执行器
|
||||
的 chv- 命名)。
|
||||
"""
|
||||
返回已清理 VM 名;非 chv- 前缀 socket 一律跳过。"""
|
||||
cleaned = []
|
||||
try:
|
||||
self.cleanup_stale_secrets()
|
||||
except Exception:
|
||||
pass
|
||||
try:
|
||||
self.gc_stale_disks()
|
||||
except Exception:
|
||||
pass
|
||||
try:
|
||||
entries = os.listdir(self.state_dir)
|
||||
except Exception:
|
||||
return []
|
||||
for entry in entries:
|
||||
if not entry.endswith(".sock"):
|
||||
continue
|
||||
@@ -1754,9 +1753,11 @@ class CloudHypervisorExecutor(Executor):
|
||||
continue
|
||||
if self.stop_vm(name):
|
||||
cleaned.append(name)
|
||||
# 登记进程已死但 socket 文件已被删的情况:登记项本身
|
||||
# 也是残留,顺手清掉登记,不计入 cleaned(没有 socket
|
||||
# 可删,不算一次清理)。
|
||||
return cleaned
|
||||
|
||||
def _purge_dead_registry(self):
|
||||
"""登记进程已死但 socket 已被删的登记项顺手清掉
|
||||
(没有 socket 可删,不计入清理数)。"""
|
||||
for name in list(self._vms.keys()):
|
||||
proc = self._vms.get(name)
|
||||
try:
|
||||
@@ -1765,6 +1766,25 @@ class CloudHypervisorExecutor(Executor):
|
||||
dead = True
|
||||
if dead:
|
||||
self._vms.pop(name, None)
|
||||
|
||||
def cleanup_stale_vms(self):
|
||||
"""清理残留 VM:先跑残留盘 GC 与残留密钥扫描,再走
|
||||
socket 扫描与登记清理。只清 socket 残留与已死进程
|
||||
对应的 VM;state_dir 不存在返回 [],绝不抛异常。"""
|
||||
try:
|
||||
self.cleanup_stale_secrets()
|
||||
except Exception:
|
||||
pass
|
||||
try:
|
||||
self.gc_stale_disks()
|
||||
except Exception:
|
||||
pass
|
||||
try:
|
||||
entries = os.listdir(self.state_dir)
|
||||
except Exception:
|
||||
return []
|
||||
cleaned = self._stop_dead_socket_vms(entries)
|
||||
self._purge_dead_registry()
|
||||
return cleaned
|
||||
|
||||
# -- 客内执行 ------------------------------------------------------
|
||||
|
||||
Reference in new issue
Block a user