fix(chv): 生产级审核 P1-01~P1-10 修复(堆叠于 #270 之上)

This commit is contained in:
muse-agent committed 2026-11-02 03:25:49 +00:00
1 parent 8c8d62f70d
commit 971e054938
12 files changed
+1545 -120

No files matched your search

+32 -3
View File
@@ -80,15 +80,39 @@ class ChvConfig:
def _base_fragment(params: dict, env: dict) -> dict:
"""身份/资源基底片段:二进制、内核、根盘、CPU/内存、
state_dir、KVM/cpuinfo 路径、ch-remote、模型。无校验,
取值口径与旧 build_chv_config 内联 base 逐字一致。"""
取值口径与旧 build_chv_config 内联 base 逐字一致,
只有一处新增(feat-chv-p1fix,P1-07):能解析出的
CPU/内存非正值直接 ValueError,不再一路进 VmConfig
到 VMM 侧才炸;解析不出的格式仍留给 preflight 报
config_invalid(构造时不改旧报错时机分工)。"""
# 显式 cpus=0 不再经 or 链静默回落成 2:0 是非法值,
# 应报出来(P1-07);None 仍走 env/默认值。
_cpus_raw = params.get("cpus") if params.get("cpus") is not None \
else (env.get("CHV_CPU") or "2")
cpus = str(_cpus_raw)
# 解析不出的格式留给 preflight(旧时机分工不变):
# 只对“解析成功且 <=0”的值在这里报,格式错误不报。
try:
_cpus_parsed = int(cpus)
except (TypeError, ValueError):
_cpus_parsed = None
if _cpus_parsed is not None and _cpus_parsed <= 0:
raise ValueError("CHV_CPU 必须是大于 0 的整数")
memory = params.get("memory") or env.get("CHV_MEM") or "2G"
try:
_mem_parsed = _parse_memory_bytes(memory)
except ValueError:
_mem_parsed = None
if _mem_parsed is not None and _mem_parsed <= 0:
raise ValueError("CHV_MEM 必须是大于 0")
return {
"binary": params.get("binary") or env.get("CHV_BINARY")
or "cloud-hypervisor",
"kernel": params.get("kernel") or env.get("CHV_KERNEL"),
"rootfs": params.get("rootfs") or params.get("image")
or env.get("CHV_ROOTFS") or env.get("CHV_IMAGE"),
"cpus": str(params.get("cpus") or env.get("CHV_CPU") or "2"),
"memory": params.get("memory") or env.get("CHV_MEM") or "2G",
"cpus": cpus,
"memory": memory,
"state_dir": params.get("state_dir")
or env.get("CHV_STATE_DIR")
or os.path.join(os.path.expanduser("~"),
@@ -221,6 +245,11 @@ def _timeout_fragment(params: dict, env: dict) -> dict:
task_timeout_raw = env.get("CHV_TASK_TIMEOUT")
task_timeout = _to_float(task_timeout_raw) \
if task_timeout_raw else None
# feat-chv-p1fix(P1-07):显式设的任务超时非正即非法。
# None = 不限(旧语义,保留);0/负数过去一路进 VmConfig
# 相关路径(runner timeout=0/负),现在构造时即 ValueError。
if task_timeout is not None and task_timeout <= 0:
raise ValueError("CHV_TASK_TIMEOUT 必须是大于 0 的数字")
try:
api_timeout = _to_float(
params.get("api_timeout")
+78 -24
View File
@@ -165,10 +165,21 @@ def _save_snapshots(state_dir, vm_name, entries):
def prune_snapshots(state_dir: str, vm_name: str,
retention_days: float | None,
now: float | None = None,
delete_files: bool = True) -> tuple[list, list]:
delete_files: bool = True,
events: list | None = None) -> tuple[list, list]:
"""按保留期剪枝:删过期条目;delete_files 时同时删
对应的 file:// 快照文件(只删登记里记的 path,不猜
文件名)。返回(保留条目,剪掉条目)。"""
文件名)。返回(保留条目,剪掉条目)。
feat-chv-p1fix(P1-08):删文件动作不再静默——
events 非空列表时,每条剪枝追加一条事件:
("deleted", entry, None) / ("delete_failed", entry,
error) / ("pruned_no_file", entry, None)(登记条目
被剪、但无对应文件可删)。调用方(executor.snapshot_vm)
据此发 vm_snapshot_pruned / vm_snapshot_prune_failed
审计;events 为 None 时行为与旧版一致(仍删、仍
best-effort),不改旧调用点返回值。
"""
import time
now = time.time() if now is None else now
kept, pruned = [], []
@@ -181,8 +192,15 @@ def prune_snapshots(state_dir: str, vm_name: str,
try:
if os.path.exists(entry["path"]):
os.remove(entry["path"])
except OSError:
pass
if events is not None:
events.append(("deleted", entry, None))
elif events is not None:
events.append(("pruned_no_file", entry, None))
except OSError as exc:
if events is not None:
events.append(("delete_failed", entry, exc))
elif events is not None:
events.append(("pruned_no_file", entry, None))
else:
kept.append(entry)
if pruned:
@@ -192,13 +210,16 @@ def prune_snapshots(state_dir: str, vm_name: str,
def check_snapshot_limits(state_dir: str, vm_name: str, max_count: int,
max_total_gb: float,
retention_days: float | None) -> list:
retention_days: float | None,
events: list | None = None) -> list:
"""快照上限预检(snapshot_vm 调 ch-remote 之前调):
先按保留期剪枝,再查数量(已有数 + 本次 > 上限即
snapshot_quota_exceeded)与总量(已有总量已达上限即
拒绝;本次大小事前未知,总量在 record 时二次收紧,
见 record_snapshot)。返回剪枝后条目。"""
kept, _ = prune_snapshots(state_dir, vm_name, retention_days)
见 record_snapshot)。返回剪枝后条目。events 透传给
prune_snapshots(见 P1-08),供调用方审计剪枝动作。"""
kept, _ = prune_snapshots(state_dir, vm_name, retention_days,
events=events)
if len(kept) + 1 > max_count:
raise ChvFailure(
"snapshot_quota_exceeded",
@@ -221,11 +242,24 @@ def check_snapshot_limits(state_dir: str, vm_name: str, max_count: int,
return kept
def record_snapshot(state_dir: str, vm_name: str, snapshot_url: str, max_total_gb: float) -> tuple[list, list]:
def record_snapshot(state_dir: str, vm_name: str, snapshot_url: str, max_total_gb: float,
events: list | None = None) -> tuple[list, list]:
"""快照成功后登记;file:// 目标记实际大小。登记后若
总量超上限,按最旧优先删登记外的旧快照文件(总量硬
帽,删动作由调用方审计——本函数返回被删的旧条目)。
返回(全部条目,被总量帽删掉的旧条目)。"""
返回(全部条目,被总量帽删掉的旧条目)。
feat-chv-p1fix(P1-06):单条快照本身大于总量帽时,
在任何落盘/删旧动作之前直接拒绝
(snapshot_quota_exceeded),不悄悄保留超大快照——
旧循环 `while len > 1` 在单条超帽时直接停、超大快照
被保留。大小未知(size 为 None,如远端目标或文件
缺失)时无法判定,不拒绝,沿用旧口径。
feat-chv-p1fix(P1-08):总量帽删文件动作记 events
(("deleted"/"delete_failed"/"pruned_no_file", entry,
error)),供调用方审计;删失败时登记条目仍移除
(旧语义不变),但失败本身不再静默。
"""
import time
path = None
size = None
@@ -235,23 +269,43 @@ def record_snapshot(state_dir: str, vm_name: str, snapshot_url: str, max_total_g
size = os.path.getsize(path)
except OSError:
size = None
if max_total_gb is not None and size is not None \
and size > max_total_gb * 1024 ** 3:
raise ChvFailure(
"snapshot_quota_exceeded",
"单条快照 %d 字节已超过总量上限 %.1f GB(VM %s),"
"拒绝记录" % (size, max_total_gb, vm_name))
entries = _load_snapshots(state_dir, vm_name)
entries.append({"url": snapshot_url, "path": path,
"size": size, "created_at": time.time()})
_save_snapshots(state_dir, vm_name, entries)
removed = []
if max_total_gb is not None:
cap = max_total_gb * 1024 ** 3
total = sum(e.get("size") or 0 for e in entries)
while len(entries) > 1 and total > cap:
oldest = entries.pop(0)
if oldest.get("path") and os.path.exists(oldest["path"]):
try:
os.remove(oldest["path"])
except OSError:
pass
total -= oldest.get("size") or 0
removed.append(oldest)
if removed:
_save_snapshots(state_dir, vm_name, entries)
removed = _tighten_snapshot_cap(entries, max_total_gb, events)
if removed:
_save_snapshots(state_dir, vm_name, entries)
return entries, removed
def _tighten_snapshot_cap(entries, max_total_gb, events):
"""总量帽收紧(record_snapshot 提取,守 50 行门禁):
总量超帽时按最旧优先删登记外的旧快照文件,返回被删
的旧条目;删动作记 events(见 record_snapshot)。"""
removed = []
if max_total_gb is None:
return removed
cap = max_total_gb * 1024 ** 3
total = sum(e.get("size") or 0 for e in entries)
while len(entries) > 1 and total > cap:
oldest = entries.pop(0)
if oldest.get("path") and os.path.exists(oldest["path"]):
try:
os.remove(oldest["path"])
if events is not None:
events.append(("deleted", oldest, None))
except OSError as exc:
if events is not None:
events.append(("delete_failed", oldest, exc))
elif events is not None:
events.append(("pruned_no_file", oldest, None))
total -= oldest.get("size") or 0
removed.append(oldest)
return removed
+193 -47
View File
@@ -55,6 +55,7 @@ from cloud.chv.hardening import (
validate_net_entries,
validate_snapshot_url,
)
from cloud.chv.locking import StateLock
from cloud.chv.ssh_transport import build_ssh_command as _build_ssh_cmd
from cloud.chv.ssh_transport import known_hosts_path as _known_hosts_path
from cloud.chv.ssh_transport import (
@@ -299,6 +300,7 @@ class CloudHypervisorExecutor(Executor):
sleeper: "Callable[..., object]"
audit_callback: object
audit_failure_count: int
_state_lock: StateLock
_vms: dict
_guest_env_delivered: dict
@@ -365,6 +367,11 @@ class CloudHypervisorExecutor(Executor):
# 名 source 客内文件,未投递的密钥名一律拒绝(防止
# “wake 没投递、执行时却按名 source 一个旧文件”)。
self._guest_env_delivered = {}
# feat-chv-p1fix(P1-05):容量/net/磁盘 check→act
# 与 purge/cleanup 的跨进程锁(flock,见 locking.py)。
# 只在 wake/restore/cleanup 时加锁,不在构造时建锁
# 文件(构造本身不应有 state_dir 写副作用)。
self._state_lock = StateLock(self.state_dir)
self.api_client = api_client if api_client is not None \
else _unix_socket_api
self.sleeper = sleeper if sleeper is not None else time.sleep
@@ -465,16 +472,23 @@ class CloudHypervisorExecutor(Executor):
# P2-6:配置校验时机统一到 preflight。CPU 非整数、内存
# 字符串非法过去只在 build_vm_config 才炸,现在 preflight
# 一次报出(kind=config_invalid)。
# feat-chv-p1fix(P1-07):构造时校验拦不住“属性事后
# 改写/注入”的非法值,preflight 这里再验 >0——CPU
# 0/负、内存 0、任务超时 0/负一律干净 preflight 失败,
# 不进 VmConfig、不等 VMM 侧才炸。
try:
int(self.cpus)
if int(self.cpus) <= 0:
raise ValueError("CHV_CPU")
except (TypeError, ValueError):
missing.append(("config_invalid",
"CHV_CPU 必须是整数:%r" % (self.cpus,)))
"CHV_CPU 必须是大于 0 的整数:%r"
% (self.cpus,)))
try:
_parse_memory_bytes(self.memory)
if _parse_memory_bytes(self.memory) <= 0:
raise ValueError("CHV_MEM")
except ValueError:
missing.append(("config_invalid",
"CHV_MEM 格式非法:%r" % (self.memory,)))
"CHV_MEM 格式非法或为 0:%r" % (self.memory,)))
self._preflight_numeric_checks(missing)
def _preflight_numeric_checks(self, missing: list) -> None:
"""preflight 数值面:端口/配额/超时/快照统一校验,纯移动。
@@ -513,6 +527,7 @@ class CloudHypervisorExecutor(Executor):
except (TypeError, ValueError):
missing.append(("config_invalid",
"CHV_*_TIMEOUT/INTERVAL 必须是正数"))
self._preflight_task_timeout_check(missing)
try:
if int(getattr(self, "warm_pool_size", 0)) < 0:
raise ValueError("warm_pool_size")
@@ -520,6 +535,19 @@ class CloudHypervisorExecutor(Executor):
missing.append(("config_invalid",
"CHV_WARM_POOL_SIZE 必须是非负整数"))
self._preflight_snapshot_checks(missing)
def _preflight_task_timeout_check(self, missing: list) -> None:
"""preflight 任务超时面(feat-chv-p1fix,P1-07 提取:
保持 _preflight_numeric_checks 在 50 行硬门禁内)。
None = 不限(旧语义);显式 0/负数即 preflight 失败。"""
try:
_task_timeout = getattr(self, "task_timeout", None)
if _task_timeout is not None \
and float(_task_timeout) <= 0:
raise ValueError("task_timeout")
except (TypeError, ValueError):
missing.append(("config_invalid",
"CHV_TASK_TIMEOUT 必须是大于 0 的数字"))
def _preflight_snapshot_checks(self, missing: list) -> None:
"""preflight 快照/host-key 面:统一校验,纯移动。"""
try:
@@ -815,13 +843,15 @@ class CloudHypervisorExecutor(Executor):
public_text: str) -> None:
"""写预置 known_hosts(to10 提取)。主机部分跟 SSH
目标走(客机 IP 时不再写死回环,否则钉死校验永远
对不上);解析失败时回环仅作钉死写入,不放宽校验。"""
对不上)。feat-chv-p1fix(P1-04 同款):解析失败即
抛 ChvFailure,绝不回落回环写钉死——回环钉死会让
SSH 校验钉在一个“解析失败后猜出来的目标”上;
wake 路径在 provision 之前已在 _wake_prechecks 里
严格解析过,此处失败只发生在直接调用时,抛错是
正确语义。"""
from cloud.chv.hostkeys import known_hosts_line
from cloud.chv.ssh_transport import write_known_hosts
try:
ssh_host = self.resolve_ssh_host()
except ChvFailure:
ssh_host = "127.0.0.1"
ssh_host = self.resolve_ssh_host()
write_known_hosts(
paths["known_hosts"],
[known_hosts_line(self.ssh_port, public_text,
@@ -921,12 +951,14 @@ class CloudHypervisorExecutor(Executor):
即 CHV + 网关密钥现在要求 provisioned 模式(已是
默认),keyscan 模式下的网关任务 fail-closed,不再
把密钥值拼进 argv 将就跑。"""
try:
ssh_host = self.resolve_ssh_host()
except ChvFailure:
# 纯命令构造(旧测试/干跑)不炸:回环缺省仅作
# 构造兼容,wake/wait 路径仍严格解析。
ssh_host = "127.0.0.1"
# feat-chv-p1fix(P1-04):删除旧回环回落——构造失败
# 即抛 ssh_target_undeclared,调用方无法再把“显式
# 回环”与“解析失败后回落”混为一谈。显式回环仍可用:
# 设 CHV_SSH_HOST=127.0.0.1,或显式宣称
# CHV_SSH_FORWARD=1;wake/wait 路径本来就严格解析,
# 本改动只收紧公开构造方法,不放宽任何严格路径,
# 也没有新增 dry-run 例外参数(干跑同样不许猜目标)。
ssh_host = self.resolve_ssh_host()
secrets, plain = split_secret_env(extra_env)
guest_env_file = None
if secrets:
@@ -1065,6 +1097,13 @@ class CloudHypervisorExecutor(Executor):
names.discard(exclude)
return len(names)
def _state_lock_ctx(self) -> StateLock:
"""当前 state_dir 的 StateLock(state_dir 若被事后改写,
换用新路径的锁实例,避免锁在旧目录上)。"""
if self._state_lock.state_dir != self.state_dir:
self._state_lock = StateLock(self.state_dir)
return self._state_lock
def _check_capacity(self, exclude=None):
"""宿主容量门:持久化计数(见 _persistent_vm_count)
达到 max_vms 即明确拒绝(kind=host_capacity_exceeded,
@@ -1442,7 +1481,18 @@ class CloudHypervisorExecutor(Executor):
密钥子集在 provisioned 模式下写进 seed 客内文件,
keyscan 模式下有密钥子集即 fail-closed;起 VM 前的
preflight/容量/配额失败也发 vm_failed 审计(L3);
net 隔离在起 VMM 前校验(H7),成功后登记 net.json。"""
net 隔离在起 VMM 前校验(H7),成功后登记 net.json。
feat-chv-p1fix(P1-05):全程持 state_dir flock——
容量/net/磁盘的 check→act 只有在“查完立即占位、
占位前不放行”时才成立;锁提前释放会在 socket 尚
未绑定前留出计数窗口,仍可超限。串行化代价只落在
wake/restore/cleanup 之间,已运行 VM 的任务执行
(_run_guest)不持此锁。"""
with self._state_lock_ctx():
return self._wake_vm_locked(task, extra_env)
def _wake_vm_locked(self, task: dict, extra_env: dict | None = None) -> dict:
"""持 state 锁的 wake 本体(见 wake_vm)。"""
if not isinstance(task, dict) or task.get("id") is None:
raise ValueError("任务缺少 id")
name = self.vm_name(task)
@@ -1499,19 +1549,48 @@ class CloudHypervisorExecutor(Executor):
feat-chv-secfix(L2/M3):ch-remote 之前先过 URL
白名单(file:// 须在 state_dir/CHV_SNAPSHOT_DIR 内,
远端须显式放行)与数量/总量/保留期上限;成功后登
记并在总量超帽时按最旧优先剪枝(删动作审计)。"""
记并在总量超帽时按最旧优先剪枝(删动作审计)。
feat-chv-p1fix(P1-08):保留期剪枝与总量帽剪枝的
删文件成功/失败均发审计;失败路径统一发
vm_snapshot_failed 审计再抛(此前失败路径无任何
快照失败审计,只有 wake 前置失败才有 vm_failed)。
本体拆到 _snapshot_create 以守 50 行硬门禁。"""
if not snapshot_url:
raise ValueError("snapshot_url 不能为空")
task = task_or_name if isinstance(task_or_name, dict) \
else {"id": task_or_name}
vm_name = self.vm_name(task_or_name)
try:
self._snapshot_create(task_or_name, task, vm_name,
snapshot_url)
except Exception as exc:
kind = getattr(exc, "kind", None) or type(exc).__name__
self._emit("vm_snapshot_failed", task,
"vm=%s snapshot=%s kind=%s error=%s"
% (vm_name, snapshot_url, kind, exc), kind)
raise
self._emit("vm_snapshot", task,
"vm=%s snapshot=%s"
% (self.vm_name(task_or_name), snapshot_url))
return snapshot_url
def _snapshot_create(self, task_or_name, task, vm_name,
snapshot_url) -> None:
"""快照创建段:白名单→上限/保留期剪枝→ch-remote→
落盘校验→登记/总量剪枝(见 snapshot_vm)。"""
validate_snapshot_url(
snapshot_url, self.state_dir,
snapshot_dir=getattr(self, "snapshot_dir", None),
allow_remote=getattr(self, "allow_remote_snapshot", False))
vm_name = self.vm_name(task_or_name)
allow_remote=getattr(
self, "allow_remote_snapshot", False))
prune_events: list = []
check_snapshot_limits(
self.state_dir, vm_name,
getattr(self, "snapshot_max_count", 10),
getattr(self, "snapshot_max_total_gb", 20.0),
getattr(self, "snapshot_retention_days", 30.0))
getattr(self, "snapshot_retention_days", 30.0),
events=prune_events)
self._emit_prune_events(task, vm_name, prune_events, "保留期")
self._ch_remote(task_or_name, "snapshot",
"source_url=%s" % snapshot_url)
# feat-chv-fix(P1-7):只查 returncode 不够。file://
@@ -1526,19 +1605,39 @@ class CloudHypervisorExecutor(Executor):
raise ChvFailure(
"snapshot_not_persisted",
"快照未落盘或为空:%s" % snapshot_url)
task = task_or_name if isinstance(task_or_name, dict) \
else {"id": task_or_name}
_entries, _removed = record_snapshot(
record_events: list = []
record_snapshot(
self.state_dir, vm_name, snapshot_url,
getattr(self, "snapshot_max_total_gb", 20.0))
for _old in _removed:
self._emit("vm_snapshot_pruned", task,
"vm=%s pruned snapshot=%s(总量超帽,最旧优先)"
% (vm_name, _old.get("url")))
self._emit("vm_snapshot", task,
"vm=%s snapshot=%s"
% (self.vm_name(task_or_name), snapshot_url))
return snapshot_url
getattr(self, "snapshot_max_total_gb", 20.0),
events=record_events)
self._emit_prune_events(task, vm_name, record_events,
"总量超帽")
def _emit_prune_events(self, task, vm_name, events, reason) -> None:
"""剪枝事件转审计(feat-chv-p1fix,P1-08):成功删
文件与登记剪枝发 vm_snapshot_pruned,删失败发
vm_snapshot_prune_failed;无文件可删的登记剪枝只
发 pruned(删动作本身不存在,无失败可言)。"""
deleted = 0
failed = 0
for status, entry, error in events:
url = entry.get("url") if isinstance(entry, dict) else None
if status == "delete_failed":
failed += 1
self._emit("vm_snapshot_prune_failed", task,
"vm=%s pruned snapshot=%s reason=%s "
"error=%s" % (vm_name, url, reason, error))
else:
deleted += 1
self._emit("vm_snapshot_pruned", task,
"vm=%s pruned snapshot=%s reason=%s "
"deleted=%s" % (
vm_name, url, reason,
status == "deleted"))
if failed:
_LOGGER.warning("chv_snapshot_prune_failed vm=%s "
"reason=%s failed=%d", vm_name, reason,
failed)
def _validate_restore(self, task, snapshot_url):
"""restore 前置校验:返回归一化后的 snapshot_url。
@@ -1588,7 +1687,14 @@ class CloudHypervisorExecutor(Executor):
resume → 等 SSH 就绪。恢复后默认 paused,不 resume
就不算醒。任一步失败都关 VM 再抛;成功返回
{vm, api_socket, state: "running", ready: True,
mode: "restored"}。"""
mode: "restored"}。feat-chv-p1fix(P1-05):与 wake
同款,全程持 state 锁——restore 同样查容量、建 VM,
不持锁会与并发 wake 互相超限。"""
with self._state_lock_ctx():
return self._restore_vm_locked(task, snapshot_url)
def _restore_vm_locked(self, task: dict, snapshot_url: str) -> dict:
"""持 state 锁的 restore 本体(见 restore_vm)。"""
snapshot_url = self._validate_restore(task, snapshot_url)
name = self.vm_name(task)
self._check_capacity(exclude=name)
@@ -1781,15 +1887,22 @@ class CloudHypervisorExecutor(Executor):
def _clean_one_vm_secrets(self, entry, vm_dir):
"""单 VM 残留清理:私钥一律删(live VM 私钥在 seed 盘
建好后即删,残留只可能是崩溃遗留);非 live VM 的
seed.iso/seed 目录同理删。返回 (删私钥, 删 seed)。"""
seed.iso/seed 目录同理删。返回 (删私钥, 删 seed,
删失败)。
feat-chv-p1fix(P1-03):删失败不再静默 pass——
第三元组就是失败标志,调用方据此发审计、写脏标记,
删失败的 VM 绝不进 cleaned(调用方可区分成功与失败)。
"""
removed_priv = False
failed = False
priv = os.path.join(vm_dir, "ssh_host_ed25519_key")
if os.path.exists(priv):
try:
os.remove(priv)
removed_priv = True
except Exception:
pass
failed = True
removed_seed = False
if not self._is_live_vm(entry):
for _suffix in ("seed.iso", "seed"):
@@ -1802,13 +1915,18 @@ class CloudHypervisorExecutor(Executor):
os.remove(_p)
removed_seed = True
except Exception:
pass
return removed_priv, removed_seed
failed = True
return removed_priv, removed_seed, failed
def cleanup_stale_secrets(self) -> list:
"""崩溃残留扫描(H5):私钥一律删、非 live VM 的 seed
同理删并审计;返回已清理 VM 名。worker 启动与
cleanup_stale_vms 时调它,不依赖 socket 是否存在。"""
cleanup_stale_vms 时调它,不依赖 socket 是否存在。
feat-chv-p1fix(P1-03):删失败的 VM 不进 cleaned,
另发 host_secret_residue_delete_failed 审计并写脏
标记(下次 provision_host_key 见标记即拒起,见
vm_dirty_refused),直至人工清理——最敏感物残留
不再“重试前不可观测”。"""
cleaned: list = []
try:
entries = os.listdir(self.state_dir)
@@ -1820,9 +1938,18 @@ class CloudHypervisorExecutor(Executor):
vm_dir = os.path.join(self.state_dir, entry)
if not os.path.isdir(vm_dir):
continue
removed_priv, removed_seed = \
removed_priv, removed_seed, failed = \
self._clean_one_vm_secrets(entry, vm_dir)
if removed_priv or removed_seed:
if failed:
self._emit("host_secret_residue_delete_failed",
{"id": entry},
"vm=%s private_key_removed=%s "
"seed_removed=%s" % (
entry, removed_priv, removed_seed))
self._write_dirty_marker(entry, True)
# 删失败时即使部分删掉了,也不进 cleaned:残留仍在,
# “已清理”名单里绝不出现未清干净的 VM。
if not failed and (removed_priv or removed_seed):
cleaned.append(entry)
self._emit("host_secret_residue_cleaned",
{"id": entry},
@@ -1865,7 +1992,9 @@ class CloudHypervisorExecutor(Executor):
def _stop_dead_socket_vms(self, entries):
"""socket 扫描:登记进程仍活的跳过,其余走 stop_vm。
返回已清理 VM 名;非 chv- 前缀 socket 一律跳过。"""
返回已清理 VM 名;非 chv- 前缀 socket 一律跳过。
feat-chv-p1fix(P1-05):登记进程的读持 _vms_lock,
不再无锁读 dict(与 purge/注册并发会读到中间态)。"""
cleaned = []
for entry in entries:
if not entry.endswith(".sock"):
@@ -1873,7 +2002,8 @@ class CloudHypervisorExecutor(Executor):
name = entry[:-len(".sock")]
if not name.startswith("chv-"):
continue
proc = self._vms.get(name)
with self._vms_lock:
proc = self._vms.get(name)
if proc is not None:
try:
alive = proc.poll() is None
@@ -1887,20 +2017,36 @@ class CloudHypervisorExecutor(Executor):
def _purge_dead_registry(self):
"""登记进程已死但 socket 已被删的登记项顺手清掉
(没有 socket 可删,不计入清理数)。"""
for name in list(self._vms.keys()):
proc = self._vms.get(name)
(没有 socket 可删,不计入清理数)。
feat-chv-p1fix(P1-05):先在锁内快照键列表,再
逐项在锁内读/删——旧“无锁 pop 与持锁遍历并发”
会致 dictionary changed size during iteration,
且 pop 本身无锁。"""
with self._vms_lock:
names = list(self._vms.keys())
for name in names:
with self._vms_lock:
proc = self._vms.get(name)
try:
dead = proc is not None and proc.poll() is not None
except Exception:
dead = True
if dead:
self._vms.pop(name, None)
with self._vms_lock:
self._vms.pop(name, None)
def cleanup_stale_vms(self) -> list:
"""清理残留 VM:先跑残留盘 GC 与残留密钥扫描,再走
socket 扫描与登记清理。只清 socket 残留与已死进程
对应的 VM;state_dir 不存在返回 [],绝不抛异常。"""
对应的 VM;state_dir 不存在返回 [],绝不抛异常。
feat-chv-p1fix(P1-05):全程持 state 锁,与 wake/
restore 互斥——清理删盘/删 socket 时不许另一路
wake 同时为同名 VM 供盘、建 VM。"""
with self._state_lock_ctx():
return self._cleanup_stale_vms_locked()
def _cleanup_stale_vms_locked(self) -> list:
"""持 state 锁的 cleanup 本体(见 cleanup_stale_vms)。"""
try:
self.cleanup_stale_secrets()
except Exception:
+17 -2
View File
@@ -159,8 +159,14 @@ class WarmPool:
对每个条目尽力 shutdown_vm,条目清掉并落盘,返回
已清理的 VM 名列表。生产启动/关闭不走本方法,走
cloud.chv.orphans.reconcile_orphans_for_executor()。"""
cloud.chv.orphans.reconcile_orphans_for_executor()。
feat-chv-p1fix(P1-01 同款语义):shutdown 失败的
条目不进 cleaned、不从内存条目里 pop(下次 reconcile
可重试),并记 failed 审计;文件落盘仍只写
warm/assigned 态(见 P2-16,实验路径的文件语义
本轨不改,内存态保留即重试依据)。"""
cleaned = []
failed = []
for name, entry in list(self._entries.items()):
if entry["state"] not in ("orphan", "assigned"):
continue
@@ -168,7 +174,8 @@ class WarmPool:
try:
self.executor.shutdown_vm(vm)
except Exception:
pass
failed.append(vm)
continue
self._entries.pop(name, None)
cleaned.append(vm)
if cleaned:
@@ -179,6 +186,14 @@ class WarmPool:
"cleaned=%s" % ",".join(cleaned))
except Exception:
pass
if failed:
try:
self.executor._emit(
"warm_pool_reconcile_failed", {"id": None},
"failed=%s cleaned=%s" % (
",".join(failed), ",".join(cleaned)))
except Exception:
pass
return cleaned
# -- 池操作 ----------------------------------------------------------
+107
View File
@@ -0,0 +1,107 @@
"""CHV state_dir 跨进程锁(feat-chv-p1fix,P1-05)。
问题:容量 / net / 磁盘的 check→act 原来只靠进程内
threading.Lock 护 `_vms` dict;多 worker 进程共用同一
state_dir 时,两路 wake 可双双通过容量/磁盘/net 门并
双双建 VM(check-then-act 竞态),`_purge_dead_registry`
的无锁 pop 与持锁遍历并发还可致字典迭代异常。
解法:同一把锁护住 check→act 全程——文件锁(flock)
+ 进程内可重入锁的组合:
- 文件锁:state_dir/chv-state.lock 上的 `LOCK_EX`。
不同进程(多 worker)靠它互斥;同一进程内不同
executor 实例(各有自己的 StateLock 对象)同样靠它
互斥,因为 flock 按打开的文件描述符区分,两个实例
各自 open → 第二个会阻塞,直到第一个释放。
- 进程内可重入:同一 executor 实例内,wake 全程会
嵌套调用同样需要锁的方法(如 wake → 失败时 stop →
cleanup_stale_vms 等路径),因此 StateLock 自带
threading.RLock + 深度计数:同一线程重入时不再重复
flock,只在最外层加锁/释放,避免同一线程把自己锁死。
注意:同一线程经“不同 StateLock 实例”嵌套加锁不在
保证内——代码里不存在这条路径(wake/restore/cleanup/
reconcile 各自只用本实例或本 registry 的锁,且
reconcile 不在 wake 持锁期间被调用)。
持有范围(见 executor.py):wake_vm 全程(prechecks →
prepare → seed → boot 含 SSH 就绪等待)、restore_vm 全程、
cleanup_stale_vms 全程(含 gc/密钥扫描/socket 扫描/purge)、
orphans.reconcile 全程。锁内串行化 wake 是有意的代价:
容量门只有“查完立即占位、占位前不放行”才成立;把锁
提前释放到 vm.create 之后会在 socket 尚未绑定前留出
计数窗口,仍可超限。串行 wake 不影响已运行 VM 的任务
执行(_run_guest 不持此锁)。
本模块只依赖标准库,是叶子模块;fcntl 只在 POSIX 上
可用——本执行器只面向 Linux 宿主(KVM 路径本身即
Linux 专属),非 POSIX 平台 import 即失败,不静默回落
纯线程锁(回落会把多 worker 保证悄悄降格)。
"""
import fcntl
import os
import threading
LOCK_FILENAME = "chv-state.lock"
def lock_path_for_state_dir(state_dir: str) -> str:
"""state_dir 对应的锁文件路径。"""
return os.path.join(state_dir, LOCK_FILENAME)
class StateLock:
"""state_dir 跨进程 + 进程内可重入锁(上下文管理器)。
用法:`with executor_state_lock:`。acquire 时先拿进程内
RLock,再在深度为 0 时 open 锁文件并 flock LOCK_EX
(阻塞,直到其它进程/实例释放);release 时深度归零才
解锁并关闭文件描述符。锁文件本身永不删除(删文件会
破坏 flock 的互斥语义:两个进程可能各持一个已删文件
的锁而互不感知)。
"""
def __init__(self, state_dir: str) -> None:
self.state_dir = state_dir
self.lock_path = lock_path_for_state_dir(state_dir)
self._rlock = threading.RLock()
self._depth = 0
self._fd: int | None = None
def acquire(self) -> None:
self._rlock.acquire()
try:
if self._depth == 0:
os.makedirs(self.state_dir, exist_ok=True)
fd = os.open(self.lock_path,
os.O_RDWR | os.O_CREAT, 0o600)
try:
fcntl.flock(fd, fcntl.LOCK_EX)
except Exception:
os.close(fd)
raise
self._fd = fd
self._depth += 1
except Exception:
self._rlock.release()
raise
def release(self) -> None:
try:
self._depth -= 1
if self._depth == 0 and self._fd is not None:
try:
fcntl.flock(self._fd, fcntl.LOCK_UN)
finally:
os.close(self._fd)
self._fd = None
finally:
self._rlock.release()
def __enter__(self) -> "StateLock":
self.acquire()
return self
def __exit__(self, *exc_info: object) -> None:
self.release()
+110 -25
View File
@@ -17,12 +17,19 @@ Registry 语义与旧 WarmPool._load_registry 完全一致
VM,绝不发给任务,只能清理。
- 损坏 fail-safe:文件不是合法 JSON / 顶层不是 dict 时,
整个文件 quarantine 成 <path>.corrupt,清理从空开始;
损坏本身发审计(feat-chv-p1fix,P1-02),绝不静默
返回空——损坏文件的孤儿 VM 此前只能靠 socket 扫描。
单条目结构非法(无 vm、或 state 不在 warm/assigned)
时只丢该条,不连坐其它条目。
- 清理后落盘为空 dict(原子 tmp + rename);写失败只记
日志警告,不炸 worker 启动/关闭。shutdown_vm 抛错时该
条目仍计入已清理(旧 reconcile 语义:尽力清理、不因
单条失败卡住整批),_emit 失败同样 best-effort。
时只丢该条,不连坐其它条目;非法条目发审计并计数
(feat-chv-p1fix,P1-02),落盘时不再写回。
- 清理后落盘:全部成功时为空 dict;有失败项时只写回
失败项的原始条目(原子 tmp + rename),下次 reconcile
可重试,失败项绝不被“已清理”掩盖(feat-chv-p1fix,
P1-01)。写失败只记日志警告,不炸 worker 启动/关闭。
shutdown_vm 抛错的条目不进 cleaned、不触发“全清”,
而是记 failed 列表并经 executor._emit 发审计;
_emit 失败同样 best-effort。整个 reconcile 在
state_dir 的 flock 下跑(见 locking.py),与 wake 互斥。
"""
import json
@@ -91,52 +98,130 @@ def _orphan_entries(data: dict) -> list[dict]:
"""从原始 registry dict 里筛出可清理的上一世条目。
只留 state 为 warm/assigned 且有 vm 的条目;非法条目
静默丢弃(不连坐)。返回条目为 {"vm", "task_id",
不连坐其它条目(计数与审计见 reconcile 本体,不再
静默丢弃)。返回条目为 {"vm", "task_id",
"snapshot_url"} 的新 dict,不改原数据。"""
entries: list[dict] = []
for entry in data.values():
for _key, entry in _orphan_entries_with_keys(data)[0]:
entries.append(entry)
return entries
def _orphan_entries_with_keys(data: dict) -> tuple[list, list]:
"""筛条目并保留原始 registry 键:返回(可清理条目,
非法条目键)。
可清理条目为 (原始键, {"vm", "task_id",
"snapshot_url"});非法条目只记键(键本身不含密钥,
可进审计 detail),原始值不进审计。"""
entries: list = []
invalid_keys: list = []
for key, entry in data.items():
if not isinstance(entry, dict):
invalid_keys.append(key)
continue
if not entry.get("vm"):
invalid_keys.append(key)
continue
if entry.get("state") not in ("warm", "assigned"):
invalid_keys.append(key)
continue
entries.append({
entries.append((key, {
"vm": entry["vm"],
"task_id": entry.get("task_id"),
"snapshot_url": entry.get("snapshot_url"),
})
return entries
}))
return entries, invalid_keys
def _best_effort_emit(executor: object, event: str, detail: str) -> None:
"""经 executor._emit 发审计(best-effort,兼容只接受
(event, task, detail) 三参的假执行器;无 _emit 时跳过)。"""
emit = getattr(executor, "_emit", None)
if not callable(emit):
return
try:
emit(event, {"id": None}, detail)
except Exception:
pass
def reconcile_orphans_for_executor(executor: object) -> list[str]:
"""生产路径唯一的残留清理入口:返回已清理的 VM 名列表。
只做清理:读 registry → 每个上一世条目尽力
shutdown_vm → 落盘为空 → best-effort 审计事件。
shutdown_vm → 落盘 → best-effort 审计事件。
绝不构造 WarmPool、绝不调 prewarm/acquire/release,
绝不把任何条目发给任务。无 registry 路径时返回 []。"""
绝不把任何条目发给任务。无 registry 路径时返回 []。
feat-chv-p1fix(P1-01/P1-02/P1-05)语义:
- shutdown 失败的条目不进 cleaned,记 failed 并审计,
落盘时只写回失败项原始条目(可重试),绝不清库;
- registry 损坏时 quarantine 后仍发审计,不静默返回空;
- 非法条目发审计并计数,落盘时丢弃(已审计,不再
每次重启重复审计同一批非法条目);
- 全程持 state_dir flock,与 wake/restore/cleanup 互斥。
返回值仍只含 cleaned(调用方兼容不变);失败与
损坏/非法的可见性走 executor._emit(worker 接线后
即任务 timeline 审计),不靠返回值传递。
"""
path = registry_path_for_executor(executor)
if not path:
return []
data, _corrupt = _read_registry(path)
if data is None:
if not os.path.exists(path):
return []
entries = _orphan_entries(data)
state_dir = getattr(executor, "state_dir", None)
if state_dir:
from cloud.chv.locking import StateLock
lock = StateLock(state_dir)
else: # pragma: no cover - state_dir 缺失时 path 已为 None
from contextlib import nullcontext
lock = nullcontext() # type: ignore[assignment]
with lock:
return _reconcile_locked(executor, path)
def _reconcile_locked(executor: object, path: str) -> list[str]:
"""持锁的 reconcile 本体(见 reconcile_orphans_for_executor)。"""
data, corrupt = _read_registry(path)
if data is None:
if corrupt:
_best_effort_emit(
executor, "warm_pool_registry_corrupt",
"path=%s registry corrupt, quarantined" % path)
return []
entries, invalid_keys = _orphan_entries_with_keys(data)
if invalid_keys:
_best_effort_emit(
executor, "warm_pool_registry_invalid_entries",
"invalid=%d keys=%s"
% (len(invalid_keys), ",".join(str(k) for k in invalid_keys)))
cleaned: list[str] = []
for entry in entries:
failed: list[str] = []
failed_errors: list[str] = []
failed_keys: list[str] = []
for key, entry in entries:
vm = entry["vm"]
try:
executor.shutdown_vm(vm) # type: ignore[attr-defined]
except Exception:
pass
except Exception as exc:
failed.append(vm)
failed_keys.append(key)
failed_errors.append("%s:%s" % (vm, type(exc).__name__))
continue
cleaned.append(vm)
if cleaned or failed or invalid_keys:
remaining = {key: data[key] for key in failed_keys
if key in data}
_write_registry(path, remaining)
if cleaned:
_write_registry(path, {})
try:
executor._emit( # type: ignore[attr-defined]
"warm_pool_reconcile", {"id": None},
"cleaned=%s" % ",".join(cleaned))
except Exception:
pass
_best_effort_emit(
executor, "warm_pool_reconcile",
"cleaned=%s" % ",".join(cleaned))
if failed:
_best_effort_emit(
executor, "warm_pool_reconcile_failed",
"failed=%s cleaned=%s errors=%s"
% (",".join(failed), ",".join(cleaned),
";".join(failed_errors)))
return cleaned