feat(chv): 架构与代码质量 9.5+ 提升——长函数清零、WarmPool facade、#215 对齐、审计计数、只读 VM 字段、ADR-8~11

This commit is contained in:
host committed 2026-10-31 06:20:12 +00:00
1 parent c908a0147a
commit 42de20da68
15 files changed
+840 -345

No files matched your search

+215 -195
View File
@@ -289,6 +289,7 @@ class CloudHypervisorExecutor(Executor):
else _unix_socket_api
self.sleeper = sleeper if sleeper is not None else time.sleep
self.audit_callback = None
self.audit_failure_count = 0
self._vms_lock = threading.Lock()
if runner is not None:
self.runner = runner
@@ -602,9 +603,15 @@ class CloudHypervisorExecutor(Executor):
feat-chv-secfix(M5):审计回调失败不再双重静默
pass——记宿主 error 日志(审计完整性本身没有保证,
但失败至少可见;任务仍不因此炸,审计失败不改变
任务状态是 worker 层的既定语义)。审计 detail 里
绝不含密钥值(调用方负责:本类所有 _emit 的 detail
只含 VM 名、路径、URL、kind,不含任何 key 值)。"""
任务状态是 worker 层的既定语义)。feat-chv-arch95plus
取舍(见 ADR-9):不改成“审计失败即中断任务”——
审计是可观测性不是门禁,DB 短暂不可用时硬中断会把
一次审计失败放大成任务失败,且与 worker 全局
best-effort 语义冲突;改为“失败记日志 + 计数 + 不
中断”,计数见 self.audit_failure_count(宿主可观测,
非任务状态)。审计 detail 里绝不含密钥值(调用方
负责:本类所有 _emit 的 detail 只含 VM 名、路径、
URL、kind,不含任何 key 值)。"""
try:
if self.audit_callback is not None:
self.audit_callback(event, task, detail, kind)
@@ -612,6 +619,8 @@ class CloudHypervisorExecutor(Executor):
_LOGGER.info("chv_audit event=%s kind=%s %s",
event, kind, detail)
except Exception as exc:
self.audit_failure_count = getattr(
self, "audit_failure_count", 0) + 1
_LOGGER.error("chv_audit_failed event=%s kind=%s error=%s",
event, kind, exc)
@@ -981,21 +990,37 @@ class CloudHypervisorExecutor(Executor):
"vmm.ping 超时:%d 次内 API 未通,socket=%s"
% (attempts, socket_path))
def wait_guest_ready(self, task):
"""等 guest SSH 就绪(只选 SSH,见类 docstring):
轮询 ssh … "true",returncode 0 即就绪;次数 =
ready_timeout/ready_interval 取整,至少 1 次。
每轮先尽力捕获 host key(known_hosts 文件不存在时;
sshd 未起时抓不到是常态,下轮再试),再发 ssh
"true"——钉死式 SSH 在文件缺失时会直接拒绝未知主机,
不先捕获就永远等不到就绪。超时时先关 VM 再 raise
ChvFailure(kind=guest_not_ready),绝不在未就绪时
派任务。"""
# feat-chv-fix(P0-2):provisioned 模式下缺 known_hosts
# 时直接 fail-fast,绝不调用 capture_host_key 回落 TOFU。
# (keyscan 模式才走下面的捕获分支。)
def _provisioned_key_guard(self, task):
"""provisioned 下缺 known_hosts 即 fail-fast,绝不 keyscan。
返回 True 表示已拦截(调用方须先关 VM 再抛
host_key_not_pinned);False 表示可继续轮询。"""
if getattr(self, "host_key_mode", "provisioned") == "provisioned" \
and not os.path.exists(self.known_hosts_path(task)):
return True
return False
def _check_host_key_mismatch(self, task, result):
"""SSH stderr 含钉死拒绝标记时审计并返回 True(须 fail-fast)。
MITM 尝试是独立安全事件,不轮询成普通超时;审计
detail 只含 VM 名与标记,不含 key 值。"""
stderr = getattr(result, "stderr", "") or ""
if "Host key verification failed" in stderr \
or "REMOTE HOST IDENTIFICATION" in stderr:
self._emit("host_key_mismatch", task,
"vm=%s ssh host key 被拒绝(钉死不匹配)"
% self.vm_name(task), "host_key_mismatch")
return True
return False
def wait_guest_ready(self, task):
"""等 guest SSH 就绪:轮询 ssh "true",returncode 0 即就绪;
每轮先尽力捕获 host key(keyscan 模式),超时先关 VM
再 raise guest_not_ready,绝不在未就绪时派任务。"""
# P0-2:provisioned 下缺 known_hosts 即 fail-fast,绝不
# 调用 capture_host_key 回落 TOFU(keyscan 才走捕获分支)。
if self._provisioned_key_guard(task):
try:
self.stop_vm(task)
except Exception:
@@ -1013,17 +1038,9 @@ class CloudHypervisorExecutor(Executor):
self.build_ssh_command(task, ["true"]))
if getattr(result, "returncode", 1) == 0:
return True
# feat-chv-secfix(M5):host key 不匹配是独立
# 安全事件,不是“sshd 没起”。SSH stderr 含
# 钉死拒绝标记时立即审计 host_key_mismatch 并
# fail-fast(不把 MITM 尝试轮询成普通超时),
# 审计 detail 只含 VM 名与标记,不含 key 值。
stderr = getattr(result, "stderr", "") or ""
if "Host key verification failed" in stderr \
or "REMOTE HOST IDENTIFICATION" in stderr:
self._emit("host_key_mismatch", task,
"vm=%s ssh host key 被拒绝(钉死不匹配)"
% self.vm_name(task), "host_key_mismatch")
# M5:host key 不匹配是独立安全事件,见
# _check_host_key_mismatch(审计 + fail-fast)。
if self._check_host_key_mismatch(task, result):
try:
self.stop_vm(task)
except Exception:
@@ -1394,58 +1411,60 @@ class CloudHypervisorExecutor(Executor):
% (self.vm_name(task_or_name), snapshot_url))
return snapshot_url
def restore_vm(self, task, snapshot_url):
"""快照恢复唤醒:preflight → 只起 VMM(--api-socket)
→ 等 vmm.ping → ch-remote restore → 显式 resume →
等 guest SSH 就绪。恢复后的 VM 默认是 paused,不
resume 就不算醒——本方法绝不省略 resume 那一步。
任一步失败都关 VM 再抛。成功返回 {vm, api_socket,
state: "running", ready: True, mode: "restored"}。"""
def _validate_restore(self, task, snapshot_url):
"""restore 前置校验:返回归一化后的 snapshot_url。
restore 不走 vm.create、没有 VmConfig 可挂 seed:
快照内已固化原 VM host key,重预置反而破坏钉死,
故 provisioned 下复用已钉死的 known_hosts、缺失
fail-fast、绝不 keyscan、绝不重生成。restore 时不
重复验快照文件存在(远端/对象存储源合法,ch-remote
本身会在源缺失时失败)。"""
if not isinstance(task, dict) or task.get("id") is None:
raise ValueError("任务缺少 id")
if not snapshot_url:
raise ValueError("snapshot_url 不能为空")
if "://" not in snapshot_url:
snapshot_url = "file://%s" % snapshot_url
# feat-chv-secfix(L2):恢复源同样过 URL 白名单
# (file:// 须在白名单目录内,远端须显式放行),
# 在 preflight/起 VMM 之前就拒绝。
validate_snapshot_url(
snapshot_url, self.state_dir,
snapshot_dir=getattr(self, "snapshot_dir", None),
allow_remote=getattr(self, "allow_remote_snapshot", False))
self.preflight()
self.resolve_ssh_host()
# feat-chv-fix(P0-2 修正版):restore 不走 vm.create,
# 没有 VmConfig 可挂 seed——审计报告“restore 须把 seed
# 接入 VmConfig”的说法在此不适用,纠正为:快照内已
# 固化原 VM 的 host key,重新预置新 key 反而会破坏
# 钉死。正确不变量是:provisioned 下复用已钉死的
# known_hosts(stop 时公钥保留、私钥已删但快照内仍有
# 原私钥),缺失时 fail-fast、绝不 keyscan、绝不重
# 生成。file:// 快照须先验存在且非空,否则不启动。
# 注意:restore 时不重复验快照文件存在——快照落盘
# 校验是 snapshot_vm/hibernate_vm 创建时的门(P1-7),
# restore 的 ch-remote 本身会在源缺失时失败;在这里
# 再验一次会把“源在远端/对象存储”的合法路径也挡掉。
if getattr(self, "host_key_mode", "provisioned") == "provisioned" \
and not os.path.exists(self.known_hosts_path(task)):
if self._provisioned_key_guard(task):
raise ChvFailure(
"host_key_not_pinned",
"provisioned 模式下 restore 缺 known_hosts,拒绝 "
"keyscan 回落:VM %s" % self.vm_name(task))
return snapshot_url
def _restore_boot(self, task, snapshot_url):
"""restore 的启动段:起 VMM → 等 ping → restore →
显式 resume → 等 SSH 就绪(恢复后默认 paused,不
resume 就不算醒)。"""
name = self.vm_name(task)
self._launch_api_only(task)
self.wait_api_ready(task)
self._check_api_socket_permissions(self.api_socket(task))
self._ch_remote(task, "restore",
"source_url=%s" % snapshot_url)
self._emit("vm_restore", task,
"vm=%s snapshot=%s" % (name, snapshot_url))
self.resume_vm(task)
self.wait_guest_ready(task)
def restore_vm(self, task, snapshot_url):
"""快照恢复唤醒:校验 → 只起 VMM → restore → 显式
resume → 等 SSH 就绪。恢复后默认 paused,不 resume
就不算醒。任一步失败都关 VM 再抛;成功返回
{vm, api_socket, state: "running", ready: True,
mode: "restored"}。"""
snapshot_url = self._validate_restore(task, snapshot_url)
name = self.vm_name(task)
self._check_capacity(exclude=name)
try:
self._launch_api_only(task)
self.wait_api_ready(task)
self._check_api_socket_permissions(self.api_socket(task))
self._ch_remote(task, "restore",
"source_url=%s" % snapshot_url)
self._emit("vm_restore", task,
"vm=%s snapshot=%s" % (name, snapshot_url))
self.resume_vm(task)
self.wait_guest_ready(task)
self._restore_boot(task, snapshot_url)
except Exception:
try:
self.stop_vm(task)
@@ -1476,58 +1495,26 @@ class CloudHypervisorExecutor(Executor):
说明来源是休眠快照。"""
return self.restore_vm(task, snapshot_url)
def stop_vm(self, task_or_name):
"""停止 VM(尽力而为,不抛异常;三种情况):
- 已死进程:登记进程 poll() 非 None,跳过 terminate
等待,直接删 socket、清登记,返回 True。
- 已关 VM:无登记进程、无 socket,ch-remote 仍尽力
调过一次,返回 False(本来就没有什么可清)。
- socket 残留:有 socket、无登记进程,删 socket 并
返回 True。
ch-remote shutdown 失败不炸(VM 可能已死),但
ch-remote 命令本身仍会发出——测试据此断言关闭路径
确实用了 ch-remote shutdown,而不是只杀进程。"""
name = self.vm_name(task_or_name)
socket_path = os.path.join(self.state_dir, name + ".sock")
def _stop_registered_proc(self, name, proc):
"""清登记进程:已死直接清,活着先杀再清(纯移动)。"""
if proc is None:
return
with self._vms_lock:
had_proc = name in self._vms
proc = self._vms.get(name)
had = had_proc or os.path.exists(socket_path)
self._vms.pop(name, None)
try:
self.runner([self.ch_remote,
"--api-socket=%s" % socket_path, "shutdown"])
alive = proc.poll() is None
except Exception:
pass
if proc is not None:
with self._vms_lock:
self._vms.pop(name, None)
try:
alive = proc.poll() is None
except Exception:
alive = False
if alive:
self._kill_proc(proc)
try:
if os.path.exists(socket_path):
os.remove(socket_path)
except Exception:
pass
# 补缺口轨:per-VM seed 目录与 host 私钥随 VM
# 关闭而删,绝不留给下一个 VM 复用;known_hosts
# 公钥文件保留供事后审计,删失败不炸关闭。下次
# 同名 VM 预置时若私钥已删即重新生成并覆盖
# known_hosts,不会用旧钉死去校验新 VM。
# feat-chv-secfix(H3/M2):清理分三块独立记账——
# seed/私钥块、任务磁盘块、net 登记块,任一块失败
# 都记对应审计 + 写脏标记(清理失败即拒起),绝不
# 再以“一块失败、整段吞掉”收场;任务磁盘块是 H3
# 的主修复:stop 即删 disk.raw(保留期 0;需取证
# 须在 shutdown 前经 SSH 拷出,见 recycle_vm 文档)。
# net.json 是 live 预约(非审计 artifact),随 VM
# 关闭而删,tap/mac/ip 预约随即释放。
_task = task_or_name if isinstance(task_or_name, dict) \
else {"id": task_or_name}
_cleanup_failed = False
alive = False
if alive:
self._kill_proc(proc)
def _cleanup_vm_files(self, name, task):
"""stop 的三块文件清理:seed/私钥、任务磁盘、net 登记。
任一块失败记对应审计并返回 True(脏标记由调用方写);
seed/私钥与磁盘删除的审计名与文案逐字不变。"""
from cloud.chv.hostkeys import dirty_marker_path # noqa: F401
failed = False
try:
seed_dir = os.path.join(self.state_dir, name, "seed")
if os.path.isdir(seed_dir):
@@ -1539,45 +1526,69 @@ class CloudHypervisorExecutor(Executor):
if os.path.exists(_p):
os.remove(_p)
except Exception as _cleanup_exc:
_cleanup_failed = True
self._emit("vm_key_cleanup_failed", _task,
failed = True
self._emit("vm_key_cleanup_failed", task,
"vm=%s error=%s" % (name, _cleanup_exc))
try:
if delete_task_disk(self.state_dir, name):
self._emit("vm_disk_deleted", _task,
self._emit("vm_disk_deleted", task,
"vm=%s disk deleted on stop" % name)
except Exception as _disk_exc:
_cleanup_failed = True
self._emit("vm_disk_delete_failed", _task,
failed = True
self._emit("vm_disk_delete_failed", task,
"vm=%s error=%s" % (name, _disk_exc))
try:
_net_path = os.path.join(self.state_dir, name, "net.json")
if os.path.exists(_net_path):
os.remove(_net_path)
except Exception:
_cleanup_failed = True
failed = True
self._guest_env_delivered.pop(name, None)
if _cleanup_failed:
# M2 根治:脏标记。标记本身写失败也不炸关闭,
# 但下次 provision_host_key 仍会因残留文件复用
# 旧 key——该残留情形由 cleanup_stale_secrets 的
# 扫描兜底(见 cleanup_stale_vms),文档明写。
try:
from cloud.chv.hostkeys import dirty_marker_path
_marker = dirty_marker_path(self.state_dir, name)
return failed
def _write_dirty_marker(self, name, failed):
"""M2 脏标记:清理失败写标记、成功清标记;写失败不炸关闭。"""
from cloud.chv.hostkeys import dirty_marker_path
try:
_marker = dirty_marker_path(self.state_dir, name)
if failed:
os.makedirs(os.path.dirname(_marker), exist_ok=True)
with open(_marker, "w") as fh:
fh.write("cleanup failed\n")
except Exception:
pass
else:
try:
from cloud.chv.hostkeys import dirty_marker_path
_marker = dirty_marker_path(self.state_dir, name)
if os.path.exists(_marker):
os.remove(_marker)
except Exception:
pass
elif os.path.exists(_marker):
os.remove(_marker)
except Exception:
pass
def stop_vm(self, task_or_name):
"""停止 VM(尽力而为,不抛异常):已死进程直接清登记,
已关 VM 仍尽力调一次 ch-remote(返回 False),socket
残留删 socket(返回 True)。ch-remote shutdown 失败
不炸,但命令本身仍会发出(测试据此断言关闭路径)。"""
name = self.vm_name(task_or_name)
socket_path = os.path.join(self.state_dir, name + ".sock")
with self._vms_lock:
had_proc = name in self._vms
proc = self._vms.get(name)
had = had_proc or os.path.exists(socket_path)
try:
self.runner([self.ch_remote,
"--api-socket=%s" % socket_path, "shutdown"])
except Exception:
pass
self._stop_registered_proc(name, proc)
try:
if os.path.exists(socket_path):
os.remove(socket_path)
except Exception:
pass
# per-VM seed 目录与 host 私钥随 VM 关闭而删,绝不留给
# 下一个 VM 复用;known_hosts 公钥保留供事后审计。
# 三块清理独立记账 + 脏标记(H3/M2,见 _cleanup_vm_files)。
_task = task_or_name if isinstance(task_or_name, dict) \
else {"id": task_or_name}
_cleanup_failed = self._cleanup_vm_files(name, _task)
self._write_dirty_marker(name, _cleanup_failed)
task = task_or_name if isinstance(task_or_name, dict) \
else {"id": task_or_name}
if had:
@@ -1624,12 +1635,49 @@ class CloudHypervisorExecutor(Executor):
"vm=%s gc error=%s" % (entry, exc))
return {"deleted": deleted, "failed": failed}
def _is_live_vm(self, entry):
"""VM 是否活着:socket 存在或登记进程未死即活。"""
if os.path.exists(os.path.join(self.state_dir, entry + ".sock")):
return True
with self._vms_lock:
proc = self._vms.get(entry)
if proc is not None:
try:
return proc.poll() is None
except Exception:
pass
return False
def _clean_one_vm_secrets(self, entry, vm_dir):
"""单 VM 残留清理:私钥一律删(live VM 私钥在 seed 盘
建好后即删,残留只可能是崩溃遗留);非 live VM 的
seed.iso/seed 目录同理删。返回 (删私钥, 删 seed)。"""
removed_priv = False
priv = os.path.join(vm_dir, "ssh_host_ed25519_key")
if os.path.exists(priv):
try:
os.remove(priv)
removed_priv = True
except Exception:
pass
removed_seed = False
if not self._is_live_vm(entry):
for _suffix in ("seed.iso", "seed"):
_p = os.path.join(vm_dir, _suffix)
try:
if os.path.isdir(_p):
shutil.rmtree(_p)
removed_seed = True
elif os.path.exists(_p):
os.remove(_p)
removed_seed = True
except Exception:
pass
return removed_priv, removed_seed
def cleanup_stale_secrets(self):
"""崩溃残留扫描(H5):state_dir 下的 host 私钥文件
在本轨不变量下只可能是残留——live VM 的私钥在 seed
盘建好后即删(见 wake_vm),所以任何残留私钥文件
一律删除并审计;非 live VM 的 seed.iso/seed 目录同
理。返回已清理的 VM 名列表。worker 启动与
"""崩溃残留扫描(H5):私钥一律删、非 live VM 的 seed
同理删并审计;返回已清理 VM 名。worker 启动与
cleanup_stale_vms 时调它,不依赖 socket 是否存在。"""
cleaned = []
try:
@@ -1642,36 +1690,8 @@ class CloudHypervisorExecutor(Executor):
vm_dir = os.path.join(self.state_dir, entry)
if not os.path.isdir(vm_dir):
continue
priv = os.path.join(vm_dir, "ssh_host_ed25519_key")
removed_priv = False
if os.path.exists(priv):
try:
os.remove(priv)
removed_priv = True
except Exception:
pass
live = os.path.exists(
os.path.join(self.state_dir, entry + ".sock"))
with self._vms_lock:
proc = self._vms.get(entry)
if proc is not None:
try:
live = live or proc.poll() is None
except Exception:
pass
removed_seed = False
if not live:
for _suffix in ("seed.iso", "seed"):
_p = os.path.join(vm_dir, _suffix)
try:
if os.path.isdir(_p):
shutil.rmtree(_p)
removed_seed = True
elif os.path.exists(_p):
os.remove(_p)
removed_seed = True
except Exception:
pass
removed_priv, removed_seed = \
self._clean_one_vm_secrets(entry, vm_dir)
if removed_priv or removed_seed:
cleaned.append(entry)
self._emit("host_secret_residue_cleaned",
@@ -1712,32 +1732,11 @@ class CloudHypervisorExecutor(Executor):
"lost": ["guest /tmp", "guest worktree temp files"],
}
def cleanup_stale_vms(self):
"""清理残留 VM:只清 socket 残留与已死进程对应的 VM。
def _stop_dead_socket_vms(self, entries):
"""socket 扫描:登记进程仍活的跳过,其余走 stop_vm。
feat-chv-secfix(H3/H5):先跑残留盘 GC 与残留
密钥扫描(不依赖 socket 是否存在,覆盖 worker 崩溃
后 socket 已删、文件仍在的情形),再走原 socket 扫描。
扫描 state_dir 下 *.sock:登记进程仍活(poll() 为 None)
的跳过;其余一律走 stop_vm。返回已清理的 VM 名列表;
state_dir 不存在时返回 [],绝不抛异常。基础设施或
其它执行器的 socket 不在 state_dir 里,不会被误删;
同目录下非 chv- 前缀的 socket 也跳过(只认本执行器
的 chv- 命名)。
"""
返回已清理 VM 名;非 chv- 前缀 socket 一律跳过。"""
cleaned = []
try:
self.cleanup_stale_secrets()
except Exception:
pass
try:
self.gc_stale_disks()
except Exception:
pass
try:
entries = os.listdir(self.state_dir)
except Exception:
return []
for entry in entries:
if not entry.endswith(".sock"):
continue
@@ -1754,9 +1753,11 @@ class CloudHypervisorExecutor(Executor):
continue
if self.stop_vm(name):
cleaned.append(name)
# 登记进程已死但 socket 文件已被删的情况:登记项本身
# 也是残留,顺手清掉登记,不计入 cleaned(没有 socket
# 可删,不算一次清理)。
return cleaned
def _purge_dead_registry(self):
"""登记进程已死但 socket 已被删的登记项顺手清掉
(没有 socket 可删,不计入清理数)。"""
for name in list(self._vms.keys()):
proc = self._vms.get(name)
try:
@@ -1765,6 +1766,25 @@ class CloudHypervisorExecutor(Executor):
dead = True
if dead:
self._vms.pop(name, None)
def cleanup_stale_vms(self):
"""清理残留 VM:先跑残留盘 GC 与残留密钥扫描,再走
socket 扫描与登记清理。只清 socket 残留与已死进程
对应的 VM;state_dir 不存在返回 [],绝不抛异常。"""
try:
self.cleanup_stale_secrets()
except Exception:
pass
try:
self.gc_stale_disks()
except Exception:
pass
try:
entries = os.listdir(self.state_dir)
except Exception:
return []
cleaned = self._stop_dead_socket_vms(entries)
self._purge_dead_registry()
return cleaned
# -- 客内执行 ------------------------------------------------------