fix(chv): 生产级审核 P1-01~P1-10 修复(堆叠于 #270 之上)
This commit is contained in:
1 parent
8c8d62f70d
commit
971e054938
12 files changed
+1528
-103
No files matched your search
+32
-3
@@ -80,15 +80,39 @@ class ChvConfig:
|
||||
def _base_fragment(params: dict, env: dict) -> dict:
|
||||
"""身份/资源基底片段:二进制、内核、根盘、CPU/内存、
|
||||
state_dir、KVM/cpuinfo 路径、ch-remote、模型。无校验,
|
||||
取值口径与旧 build_chv_config 内联 base 逐字一致。"""
|
||||
取值口径与旧 build_chv_config 内联 base 逐字一致,
|
||||
只有一处新增(feat-chv-p1fix,P1-07):能解析出的
|
||||
CPU/内存非正值直接 ValueError,不再一路进 VmConfig
|
||||
到 VMM 侧才炸;解析不出的格式仍留给 preflight 报
|
||||
config_invalid(构造时不改旧报错时机分工)。"""
|
||||
# 显式 cpus=0 不再经 or 链静默回落成 2:0 是非法值,
|
||||
# 应报出来(P1-07);None 仍走 env/默认值。
|
||||
_cpus_raw = params.get("cpus") if params.get("cpus") is not None \
|
||||
else (env.get("CHV_CPU") or "2")
|
||||
cpus = str(_cpus_raw)
|
||||
# 解析不出的格式留给 preflight(旧时机分工不变):
|
||||
# 只对“解析成功且 <=0”的值在这里报,格式错误不报。
|
||||
try:
|
||||
_cpus_parsed = int(cpus)
|
||||
except (TypeError, ValueError):
|
||||
_cpus_parsed = None
|
||||
if _cpus_parsed is not None and _cpus_parsed <= 0:
|
||||
raise ValueError("CHV_CPU 必须是大于 0 的整数")
|
||||
memory = params.get("memory") or env.get("CHV_MEM") or "2G"
|
||||
try:
|
||||
_mem_parsed = _parse_memory_bytes(memory)
|
||||
except ValueError:
|
||||
_mem_parsed = None
|
||||
if _mem_parsed is not None and _mem_parsed <= 0:
|
||||
raise ValueError("CHV_MEM 必须是大于 0")
|
||||
return {
|
||||
"binary": params.get("binary") or env.get("CHV_BINARY")
|
||||
or "cloud-hypervisor",
|
||||
"kernel": params.get("kernel") or env.get("CHV_KERNEL"),
|
||||
"rootfs": params.get("rootfs") or params.get("image")
|
||||
or env.get("CHV_ROOTFS") or env.get("CHV_IMAGE"),
|
||||
"cpus": str(params.get("cpus") or env.get("CHV_CPU") or "2"),
|
||||
"memory": params.get("memory") or env.get("CHV_MEM") or "2G",
|
||||
"cpus": cpus,
|
||||
"memory": memory,
|
||||
"state_dir": params.get("state_dir")
|
||||
or env.get("CHV_STATE_DIR")
|
||||
or os.path.join(os.path.expanduser("~"),
|
||||
@@ -221,6 +245,11 @@ def _timeout_fragment(params: dict, env: dict) -> dict:
|
||||
task_timeout_raw = env.get("CHV_TASK_TIMEOUT")
|
||||
task_timeout = _to_float(task_timeout_raw) \
|
||||
if task_timeout_raw else None
|
||||
# feat-chv-p1fix(P1-07):显式设的任务超时非正即非法。
|
||||
# None = 不限(旧语义,保留);0/负数过去一路进 VmConfig
|
||||
# 相关路径(runner timeout=0/负),现在构造时即 ValueError。
|
||||
if task_timeout is not None and task_timeout <= 0:
|
||||
raise ValueError("CHV_TASK_TIMEOUT 必须是大于 0 的数字")
|
||||
try:
|
||||
api_timeout = _to_float(
|
||||
params.get("api_timeout")
|
||||
|
||||
+69
-15
@@ -165,10 +165,21 @@ def _save_snapshots(state_dir, vm_name, entries):
|
||||
def prune_snapshots(state_dir: str, vm_name: str,
|
||||
retention_days: float | None,
|
||||
now: float | None = None,
|
||||
delete_files: bool = True) -> tuple[list, list]:
|
||||
delete_files: bool = True,
|
||||
events: list | None = None) -> tuple[list, list]:
|
||||
"""按保留期剪枝:删过期条目;delete_files 时同时删
|
||||
对应的 file:// 快照文件(只删登记里记的 path,不猜
|
||||
文件名)。返回(保留条目,剪掉条目)。"""
|
||||
文件名)。返回(保留条目,剪掉条目)。
|
||||
|
||||
feat-chv-p1fix(P1-08):删文件动作不再静默——
|
||||
events 非空列表时,每条剪枝追加一条事件:
|
||||
("deleted", entry, None) / ("delete_failed", entry,
|
||||
error) / ("pruned_no_file", entry, None)(登记条目
|
||||
被剪、但无对应文件可删)。调用方(executor.snapshot_vm)
|
||||
据此发 vm_snapshot_pruned / vm_snapshot_prune_failed
|
||||
审计;events 为 None 时行为与旧版一致(仍删、仍
|
||||
best-effort),不改旧调用点返回值。
|
||||
"""
|
||||
import time
|
||||
now = time.time() if now is None else now
|
||||
kept, pruned = [], []
|
||||
@@ -181,8 +192,15 @@ def prune_snapshots(state_dir: str, vm_name: str,
|
||||
try:
|
||||
if os.path.exists(entry["path"]):
|
||||
os.remove(entry["path"])
|
||||
except OSError:
|
||||
pass
|
||||
if events is not None:
|
||||
events.append(("deleted", entry, None))
|
||||
elif events is not None:
|
||||
events.append(("pruned_no_file", entry, None))
|
||||
except OSError as exc:
|
||||
if events is not None:
|
||||
events.append(("delete_failed", entry, exc))
|
||||
elif events is not None:
|
||||
events.append(("pruned_no_file", entry, None))
|
||||
else:
|
||||
kept.append(entry)
|
||||
if pruned:
|
||||
@@ -192,13 +210,16 @@ def prune_snapshots(state_dir: str, vm_name: str,
|
||||
|
||||
def check_snapshot_limits(state_dir: str, vm_name: str, max_count: int,
|
||||
max_total_gb: float,
|
||||
retention_days: float | None) -> list:
|
||||
retention_days: float | None,
|
||||
events: list | None = None) -> list:
|
||||
"""快照上限预检(snapshot_vm 调 ch-remote 之前调):
|
||||
先按保留期剪枝,再查数量(已有数 + 本次 > 上限即
|
||||
snapshot_quota_exceeded)与总量(已有总量已达上限即
|
||||
拒绝;本次大小事前未知,总量在 record 时二次收紧,
|
||||
见 record_snapshot)。返回剪枝后条目。"""
|
||||
kept, _ = prune_snapshots(state_dir, vm_name, retention_days)
|
||||
见 record_snapshot)。返回剪枝后条目。events 透传给
|
||||
prune_snapshots(见 P1-08),供调用方审计剪枝动作。"""
|
||||
kept, _ = prune_snapshots(state_dir, vm_name, retention_days,
|
||||
events=events)
|
||||
if len(kept) + 1 > max_count:
|
||||
raise ChvFailure(
|
||||
"snapshot_quota_exceeded",
|
||||
@@ -221,11 +242,24 @@ def check_snapshot_limits(state_dir: str, vm_name: str, max_count: int,
|
||||
return kept
|
||||
|
||||
|
||||
def record_snapshot(state_dir: str, vm_name: str, snapshot_url: str, max_total_gb: float) -> tuple[list, list]:
|
||||
def record_snapshot(state_dir: str, vm_name: str, snapshot_url: str, max_total_gb: float,
|
||||
events: list | None = None) -> tuple[list, list]:
|
||||
"""快照成功后登记;file:// 目标记实际大小。登记后若
|
||||
总量超上限,按最旧优先删登记外的旧快照文件(总量硬
|
||||
帽,删动作由调用方审计——本函数返回被删的旧条目)。
|
||||
返回(全部条目,被总量帽删掉的旧条目)。"""
|
||||
返回(全部条目,被总量帽删掉的旧条目)。
|
||||
|
||||
feat-chv-p1fix(P1-06):单条快照本身大于总量帽时,
|
||||
在任何落盘/删旧动作之前直接拒绝
|
||||
(snapshot_quota_exceeded),不悄悄保留超大快照——
|
||||
旧循环 `while len > 1` 在单条超帽时直接停、超大快照
|
||||
被保留。大小未知(size 为 None,如远端目标或文件
|
||||
缺失)时无法判定,不拒绝,沿用旧口径。
|
||||
feat-chv-p1fix(P1-08):总量帽删文件动作记 events
|
||||
(("deleted"/"delete_failed"/"pruned_no_file", entry,
|
||||
error)),供调用方审计;删失败时登记条目仍移除
|
||||
(旧语义不变),但失败本身不再静默。
|
||||
"""
|
||||
import time
|
||||
path = None
|
||||
size = None
|
||||
@@ -235,12 +269,29 @@ def record_snapshot(state_dir: str, vm_name: str, snapshot_url: str, max_total_g
|
||||
size = os.path.getsize(path)
|
||||
except OSError:
|
||||
size = None
|
||||
if max_total_gb is not None and size is not None \
|
||||
and size > max_total_gb * 1024 ** 3:
|
||||
raise ChvFailure(
|
||||
"snapshot_quota_exceeded",
|
||||
"单条快照 %d 字节已超过总量上限 %.1f GB(VM %s),"
|
||||
"拒绝记录" % (size, max_total_gb, vm_name))
|
||||
entries = _load_snapshots(state_dir, vm_name)
|
||||
entries.append({"url": snapshot_url, "path": path,
|
||||
"size": size, "created_at": time.time()})
|
||||
_save_snapshots(state_dir, vm_name, entries)
|
||||
removed = _tighten_snapshot_cap(entries, max_total_gb, events)
|
||||
if removed:
|
||||
_save_snapshots(state_dir, vm_name, entries)
|
||||
return entries, removed
|
||||
|
||||
|
||||
def _tighten_snapshot_cap(entries, max_total_gb, events):
|
||||
"""总量帽收紧(record_snapshot 提取,守 50 行门禁):
|
||||
总量超帽时按最旧优先删登记外的旧快照文件,返回被删
|
||||
的旧条目;删动作记 events(见 record_snapshot)。"""
|
||||
removed = []
|
||||
if max_total_gb is not None:
|
||||
if max_total_gb is None:
|
||||
return removed
|
||||
cap = max_total_gb * 1024 ** 3
|
||||
total = sum(e.get("size") or 0 for e in entries)
|
||||
while len(entries) > 1 and total > cap:
|
||||
@@ -248,10 +299,13 @@ def record_snapshot(state_dir: str, vm_name: str, snapshot_url: str, max_total_g
|
||||
if oldest.get("path") and os.path.exists(oldest["path"]):
|
||||
try:
|
||||
os.remove(oldest["path"])
|
||||
except OSError:
|
||||
pass
|
||||
if events is not None:
|
||||
events.append(("deleted", oldest, None))
|
||||
except OSError as exc:
|
||||
if events is not None:
|
||||
events.append(("delete_failed", oldest, exc))
|
||||
elif events is not None:
|
||||
events.append(("pruned_no_file", oldest, None))
|
||||
total -= oldest.get("size") or 0
|
||||
removed.append(oldest)
|
||||
if removed:
|
||||
_save_snapshots(state_dir, vm_name, entries)
|
||||
return entries, removed
|
||||
return removed
|
||||
+187
-41
@@ -55,6 +55,7 @@ from cloud.chv.hardening import (
|
||||
validate_net_entries,
|
||||
validate_snapshot_url,
|
||||
)
|
||||
from cloud.chv.locking import StateLock
|
||||
from cloud.chv.ssh_transport import build_ssh_command as _build_ssh_cmd
|
||||
from cloud.chv.ssh_transport import known_hosts_path as _known_hosts_path
|
||||
from cloud.chv.ssh_transport import (
|
||||
@@ -299,6 +300,7 @@ class CloudHypervisorExecutor(Executor):
|
||||
sleeper: "Callable[..., object]"
|
||||
audit_callback: object
|
||||
audit_failure_count: int
|
||||
_state_lock: StateLock
|
||||
_vms: dict
|
||||
_guest_env_delivered: dict
|
||||
|
||||
@@ -365,6 +367,11 @@ class CloudHypervisorExecutor(Executor):
|
||||
# 名 source 客内文件,未投递的密钥名一律拒绝(防止
|
||||
# “wake 没投递、执行时却按名 source 一个旧文件”)。
|
||||
self._guest_env_delivered = {}
|
||||
# feat-chv-p1fix(P1-05):容量/net/磁盘 check→act
|
||||
# 与 purge/cleanup 的跨进程锁(flock,见 locking.py)。
|
||||
# 只在 wake/restore/cleanup 时加锁,不在构造时建锁
|
||||
# 文件(构造本身不应有 state_dir 写副作用)。
|
||||
self._state_lock = StateLock(self.state_dir)
|
||||
self.api_client = api_client if api_client is not None \
|
||||
else _unix_socket_api
|
||||
self.sleeper = sleeper if sleeper is not None else time.sleep
|
||||
@@ -465,16 +472,23 @@ class CloudHypervisorExecutor(Executor):
|
||||
# P2-6:配置校验时机统一到 preflight。CPU 非整数、内存
|
||||
# 字符串非法过去只在 build_vm_config 才炸,现在 preflight
|
||||
# 一次报出(kind=config_invalid)。
|
||||
# feat-chv-p1fix(P1-07):构造时校验拦不住“属性事后
|
||||
# 改写/注入”的非法值,preflight 这里再验 >0——CPU
|
||||
# 0/负、内存 0、任务超时 0/负一律干净 preflight 失败,
|
||||
# 不进 VmConfig、不等 VMM 侧才炸。
|
||||
try:
|
||||
int(self.cpus)
|
||||
if int(self.cpus) <= 0:
|
||||
raise ValueError("CHV_CPU")
|
||||
except (TypeError, ValueError):
|
||||
missing.append(("config_invalid",
|
||||
"CHV_CPU 必须是整数:%r" % (self.cpus,)))
|
||||
"CHV_CPU 必须是大于 0 的整数:%r"
|
||||
% (self.cpus,)))
|
||||
try:
|
||||
_parse_memory_bytes(self.memory)
|
||||
if _parse_memory_bytes(self.memory) <= 0:
|
||||
raise ValueError("CHV_MEM")
|
||||
except ValueError:
|
||||
missing.append(("config_invalid",
|
||||
"CHV_MEM 格式非法:%r" % (self.memory,)))
|
||||
"CHV_MEM 格式非法或为 0:%r" % (self.memory,)))
|
||||
self._preflight_numeric_checks(missing)
|
||||
def _preflight_numeric_checks(self, missing: list) -> None:
|
||||
"""preflight 数值面:端口/配额/超时/快照统一校验,纯移动。
|
||||
@@ -513,6 +527,7 @@ class CloudHypervisorExecutor(Executor):
|
||||
except (TypeError, ValueError):
|
||||
missing.append(("config_invalid",
|
||||
"CHV_*_TIMEOUT/INTERVAL 必须是正数"))
|
||||
self._preflight_task_timeout_check(missing)
|
||||
try:
|
||||
if int(getattr(self, "warm_pool_size", 0)) < 0:
|
||||
raise ValueError("warm_pool_size")
|
||||
@@ -520,6 +535,19 @@ class CloudHypervisorExecutor(Executor):
|
||||
missing.append(("config_invalid",
|
||||
"CHV_WARM_POOL_SIZE 必须是非负整数"))
|
||||
self._preflight_snapshot_checks(missing)
|
||||
def _preflight_task_timeout_check(self, missing: list) -> None:
|
||||
"""preflight 任务超时面(feat-chv-p1fix,P1-07 提取:
|
||||
保持 _preflight_numeric_checks 在 50 行硬门禁内)。
|
||||
None = 不限(旧语义);显式 0/负数即 preflight 失败。"""
|
||||
try:
|
||||
_task_timeout = getattr(self, "task_timeout", None)
|
||||
if _task_timeout is not None \
|
||||
and float(_task_timeout) <= 0:
|
||||
raise ValueError("task_timeout")
|
||||
except (TypeError, ValueError):
|
||||
missing.append(("config_invalid",
|
||||
"CHV_TASK_TIMEOUT 必须是大于 0 的数字"))
|
||||
|
||||
def _preflight_snapshot_checks(self, missing: list) -> None:
|
||||
"""preflight 快照/host-key 面:统一校验,纯移动。"""
|
||||
try:
|
||||
@@ -815,13 +843,15 @@ class CloudHypervisorExecutor(Executor):
|
||||
public_text: str) -> None:
|
||||
"""写预置 known_hosts(to10 提取)。主机部分跟 SSH
|
||||
目标走(客机 IP 时不再写死回环,否则钉死校验永远
|
||||
对不上);解析失败时回环仅作钉死写入,不放宽校验。"""
|
||||
对不上)。feat-chv-p1fix(P1-04 同款):解析失败即
|
||||
抛 ChvFailure,绝不回落回环写钉死——回环钉死会让
|
||||
SSH 校验钉在一个“解析失败后猜出来的目标”上;
|
||||
wake 路径在 provision 之前已在 _wake_prechecks 里
|
||||
严格解析过,此处失败只发生在直接调用时,抛错是
|
||||
正确语义。"""
|
||||
from cloud.chv.hostkeys import known_hosts_line
|
||||
from cloud.chv.ssh_transport import write_known_hosts
|
||||
try:
|
||||
ssh_host = self.resolve_ssh_host()
|
||||
except ChvFailure:
|
||||
ssh_host = "127.0.0.1"
|
||||
write_known_hosts(
|
||||
paths["known_hosts"],
|
||||
[known_hosts_line(self.ssh_port, public_text,
|
||||
@@ -921,12 +951,14 @@ class CloudHypervisorExecutor(Executor):
|
||||
即 CHV + 网关密钥现在要求 provisioned 模式(已是
|
||||
默认),keyscan 模式下的网关任务 fail-closed,不再
|
||||
把密钥值拼进 argv 将就跑。"""
|
||||
try:
|
||||
# feat-chv-p1fix(P1-04):删除旧回环回落——构造失败
|
||||
# 即抛 ssh_target_undeclared,调用方无法再把“显式
|
||||
# 回环”与“解析失败后回落”混为一谈。显式回环仍可用:
|
||||
# 设 CHV_SSH_HOST=127.0.0.1,或显式宣称
|
||||
# CHV_SSH_FORWARD=1;wake/wait 路径本来就严格解析,
|
||||
# 本改动只收紧公开构造方法,不放宽任何严格路径,
|
||||
# 也没有新增 dry-run 例外参数(干跑同样不许猜目标)。
|
||||
ssh_host = self.resolve_ssh_host()
|
||||
except ChvFailure:
|
||||
# 纯命令构造(旧测试/干跑)不炸:回环缺省仅作
|
||||
# 构造兼容,wake/wait 路径仍严格解析。
|
||||
ssh_host = "127.0.0.1"
|
||||
secrets, plain = split_secret_env(extra_env)
|
||||
guest_env_file = None
|
||||
if secrets:
|
||||
@@ -1065,6 +1097,13 @@ class CloudHypervisorExecutor(Executor):
|
||||
names.discard(exclude)
|
||||
return len(names)
|
||||
|
||||
def _state_lock_ctx(self) -> StateLock:
|
||||
"""当前 state_dir 的 StateLock(state_dir 若被事后改写,
|
||||
换用新路径的锁实例,避免锁在旧目录上)。"""
|
||||
if self._state_lock.state_dir != self.state_dir:
|
||||
self._state_lock = StateLock(self.state_dir)
|
||||
return self._state_lock
|
||||
|
||||
def _check_capacity(self, exclude=None):
|
||||
"""宿主容量门:持久化计数(见 _persistent_vm_count)
|
||||
达到 max_vms 即明确拒绝(kind=host_capacity_exceeded,
|
||||
@@ -1442,7 +1481,18 @@ class CloudHypervisorExecutor(Executor):
|
||||
密钥子集在 provisioned 模式下写进 seed 客内文件,
|
||||
keyscan 模式下有密钥子集即 fail-closed;起 VM 前的
|
||||
preflight/容量/配额失败也发 vm_failed 审计(L3);
|
||||
net 隔离在起 VMM 前校验(H7),成功后登记 net.json。"""
|
||||
net 隔离在起 VMM 前校验(H7),成功后登记 net.json。
|
||||
feat-chv-p1fix(P1-05):全程持 state_dir flock——
|
||||
容量/net/磁盘的 check→act 只有在“查完立即占位、
|
||||
占位前不放行”时才成立;锁提前释放会在 socket 尚
|
||||
未绑定前留出计数窗口,仍可超限。串行化代价只落在
|
||||
wake/restore/cleanup 之间,已运行 VM 的任务执行
|
||||
(_run_guest)不持此锁。"""
|
||||
with self._state_lock_ctx():
|
||||
return self._wake_vm_locked(task, extra_env)
|
||||
|
||||
def _wake_vm_locked(self, task: dict, extra_env: dict | None = None) -> dict:
|
||||
"""持 state 锁的 wake 本体(见 wake_vm)。"""
|
||||
if not isinstance(task, dict) or task.get("id") is None:
|
||||
raise ValueError("任务缺少 id")
|
||||
name = self.vm_name(task)
|
||||
@@ -1499,19 +1549,48 @@ class CloudHypervisorExecutor(Executor):
|
||||
feat-chv-secfix(L2/M3):ch-remote 之前先过 URL
|
||||
白名单(file:// 须在 state_dir/CHV_SNAPSHOT_DIR 内,
|
||||
远端须显式放行)与数量/总量/保留期上限;成功后登
|
||||
记并在总量超帽时按最旧优先剪枝(删动作审计)。"""
|
||||
记并在总量超帽时按最旧优先剪枝(删动作审计)。
|
||||
feat-chv-p1fix(P1-08):保留期剪枝与总量帽剪枝的
|
||||
删文件成功/失败均发审计;失败路径统一发
|
||||
vm_snapshot_failed 审计再抛(此前失败路径无任何
|
||||
快照失败审计,只有 wake 前置失败才有 vm_failed)。
|
||||
本体拆到 _snapshot_create 以守 50 行硬门禁。"""
|
||||
if not snapshot_url:
|
||||
raise ValueError("snapshot_url 不能为空")
|
||||
task = task_or_name if isinstance(task_or_name, dict) \
|
||||
else {"id": task_or_name}
|
||||
vm_name = self.vm_name(task_or_name)
|
||||
try:
|
||||
self._snapshot_create(task_or_name, task, vm_name,
|
||||
snapshot_url)
|
||||
except Exception as exc:
|
||||
kind = getattr(exc, "kind", None) or type(exc).__name__
|
||||
self._emit("vm_snapshot_failed", task,
|
||||
"vm=%s snapshot=%s kind=%s error=%s"
|
||||
% (vm_name, snapshot_url, kind, exc), kind)
|
||||
raise
|
||||
self._emit("vm_snapshot", task,
|
||||
"vm=%s snapshot=%s"
|
||||
% (self.vm_name(task_or_name), snapshot_url))
|
||||
return snapshot_url
|
||||
|
||||
def _snapshot_create(self, task_or_name, task, vm_name,
|
||||
snapshot_url) -> None:
|
||||
"""快照创建段:白名单→上限/保留期剪枝→ch-remote→
|
||||
落盘校验→登记/总量剪枝(见 snapshot_vm)。"""
|
||||
validate_snapshot_url(
|
||||
snapshot_url, self.state_dir,
|
||||
snapshot_dir=getattr(self, "snapshot_dir", None),
|
||||
allow_remote=getattr(self, "allow_remote_snapshot", False))
|
||||
vm_name = self.vm_name(task_or_name)
|
||||
allow_remote=getattr(
|
||||
self, "allow_remote_snapshot", False))
|
||||
prune_events: list = []
|
||||
check_snapshot_limits(
|
||||
self.state_dir, vm_name,
|
||||
getattr(self, "snapshot_max_count", 10),
|
||||
getattr(self, "snapshot_max_total_gb", 20.0),
|
||||
getattr(self, "snapshot_retention_days", 30.0))
|
||||
getattr(self, "snapshot_retention_days", 30.0),
|
||||
events=prune_events)
|
||||
self._emit_prune_events(task, vm_name, prune_events, "保留期")
|
||||
self._ch_remote(task_or_name, "snapshot",
|
||||
"source_url=%s" % snapshot_url)
|
||||
# feat-chv-fix(P1-7):只查 returncode 不够。file://
|
||||
@@ -1526,19 +1605,39 @@ class CloudHypervisorExecutor(Executor):
|
||||
raise ChvFailure(
|
||||
"snapshot_not_persisted",
|
||||
"快照未落盘或为空:%s" % snapshot_url)
|
||||
task = task_or_name if isinstance(task_or_name, dict) \
|
||||
else {"id": task_or_name}
|
||||
_entries, _removed = record_snapshot(
|
||||
record_events: list = []
|
||||
record_snapshot(
|
||||
self.state_dir, vm_name, snapshot_url,
|
||||
getattr(self, "snapshot_max_total_gb", 20.0))
|
||||
for _old in _removed:
|
||||
getattr(self, "snapshot_max_total_gb", 20.0),
|
||||
events=record_events)
|
||||
self._emit_prune_events(task, vm_name, record_events,
|
||||
"总量超帽")
|
||||
|
||||
def _emit_prune_events(self, task, vm_name, events, reason) -> None:
|
||||
"""剪枝事件转审计(feat-chv-p1fix,P1-08):成功删
|
||||
文件与登记剪枝发 vm_snapshot_pruned,删失败发
|
||||
vm_snapshot_prune_failed;无文件可删的登记剪枝只
|
||||
发 pruned(删动作本身不存在,无失败可言)。"""
|
||||
deleted = 0
|
||||
failed = 0
|
||||
for status, entry, error in events:
|
||||
url = entry.get("url") if isinstance(entry, dict) else None
|
||||
if status == "delete_failed":
|
||||
failed += 1
|
||||
self._emit("vm_snapshot_prune_failed", task,
|
||||
"vm=%s pruned snapshot=%s reason=%s "
|
||||
"error=%s" % (vm_name, url, reason, error))
|
||||
else:
|
||||
deleted += 1
|
||||
self._emit("vm_snapshot_pruned", task,
|
||||
"vm=%s pruned snapshot=%s(总量超帽,最旧优先)"
|
||||
% (vm_name, _old.get("url")))
|
||||
self._emit("vm_snapshot", task,
|
||||
"vm=%s snapshot=%s"
|
||||
% (self.vm_name(task_or_name), snapshot_url))
|
||||
return snapshot_url
|
||||
"vm=%s pruned snapshot=%s reason=%s "
|
||||
"deleted=%s" % (
|
||||
vm_name, url, reason,
|
||||
status == "deleted"))
|
||||
if failed:
|
||||
_LOGGER.warning("chv_snapshot_prune_failed vm=%s "
|
||||
"reason=%s failed=%d", vm_name, reason,
|
||||
failed)
|
||||
|
||||
def _validate_restore(self, task, snapshot_url):
|
||||
"""restore 前置校验:返回归一化后的 snapshot_url。
|
||||
@@ -1588,7 +1687,14 @@ class CloudHypervisorExecutor(Executor):
|
||||
resume → 等 SSH 就绪。恢复后默认 paused,不 resume
|
||||
就不算醒。任一步失败都关 VM 再抛;成功返回
|
||||
{vm, api_socket, state: "running", ready: True,
|
||||
mode: "restored"}。"""
|
||||
mode: "restored"}。feat-chv-p1fix(P1-05):与 wake
|
||||
同款,全程持 state 锁——restore 同样查容量、建 VM,
|
||||
不持锁会与并发 wake 互相超限。"""
|
||||
with self._state_lock_ctx():
|
||||
return self._restore_vm_locked(task, snapshot_url)
|
||||
|
||||
def _restore_vm_locked(self, task: dict, snapshot_url: str) -> dict:
|
||||
"""持 state 锁的 restore 本体(见 restore_vm)。"""
|
||||
snapshot_url = self._validate_restore(task, snapshot_url)
|
||||
name = self.vm_name(task)
|
||||
self._check_capacity(exclude=name)
|
||||
@@ -1781,15 +1887,22 @@ class CloudHypervisorExecutor(Executor):
|
||||
def _clean_one_vm_secrets(self, entry, vm_dir):
|
||||
"""单 VM 残留清理:私钥一律删(live VM 私钥在 seed 盘
|
||||
建好后即删,残留只可能是崩溃遗留);非 live VM 的
|
||||
seed.iso/seed 目录同理删。返回 (删私钥, 删 seed)。"""
|
||||
seed.iso/seed 目录同理删。返回 (删私钥, 删 seed,
|
||||
删失败)。
|
||||
|
||||
feat-chv-p1fix(P1-03):删失败不再静默 pass——
|
||||
第三元组就是失败标志,调用方据此发审计、写脏标记,
|
||||
删失败的 VM 绝不进 cleaned(调用方可区分成功与失败)。
|
||||
"""
|
||||
removed_priv = False
|
||||
failed = False
|
||||
priv = os.path.join(vm_dir, "ssh_host_ed25519_key")
|
||||
if os.path.exists(priv):
|
||||
try:
|
||||
os.remove(priv)
|
||||
removed_priv = True
|
||||
except Exception:
|
||||
pass
|
||||
failed = True
|
||||
removed_seed = False
|
||||
if not self._is_live_vm(entry):
|
||||
for _suffix in ("seed.iso", "seed"):
|
||||
@@ -1802,13 +1915,18 @@ class CloudHypervisorExecutor(Executor):
|
||||
os.remove(_p)
|
||||
removed_seed = True
|
||||
except Exception:
|
||||
pass
|
||||
return removed_priv, removed_seed
|
||||
failed = True
|
||||
return removed_priv, removed_seed, failed
|
||||
|
||||
def cleanup_stale_secrets(self) -> list:
|
||||
"""崩溃残留扫描(H5):私钥一律删、非 live VM 的 seed
|
||||
同理删并审计;返回已清理 VM 名。worker 启动与
|
||||
cleanup_stale_vms 时调它,不依赖 socket 是否存在。"""
|
||||
cleanup_stale_vms 时调它,不依赖 socket 是否存在。
|
||||
feat-chv-p1fix(P1-03):删失败的 VM 不进 cleaned,
|
||||
另发 host_secret_residue_delete_failed 审计并写脏
|
||||
标记(下次 provision_host_key 见标记即拒起,见
|
||||
vm_dirty_refused),直至人工清理——最敏感物残留
|
||||
不再“重试前不可观测”。"""
|
||||
cleaned: list = []
|
||||
try:
|
||||
entries = os.listdir(self.state_dir)
|
||||
@@ -1820,9 +1938,18 @@ class CloudHypervisorExecutor(Executor):
|
||||
vm_dir = os.path.join(self.state_dir, entry)
|
||||
if not os.path.isdir(vm_dir):
|
||||
continue
|
||||
removed_priv, removed_seed = \
|
||||
removed_priv, removed_seed, failed = \
|
||||
self._clean_one_vm_secrets(entry, vm_dir)
|
||||
if removed_priv or removed_seed:
|
||||
if failed:
|
||||
self._emit("host_secret_residue_delete_failed",
|
||||
{"id": entry},
|
||||
"vm=%s private_key_removed=%s "
|
||||
"seed_removed=%s" % (
|
||||
entry, removed_priv, removed_seed))
|
||||
self._write_dirty_marker(entry, True)
|
||||
# 删失败时即使部分删掉了,也不进 cleaned:残留仍在,
|
||||
# “已清理”名单里绝不出现未清干净的 VM。
|
||||
if not failed and (removed_priv or removed_seed):
|
||||
cleaned.append(entry)
|
||||
self._emit("host_secret_residue_cleaned",
|
||||
{"id": entry},
|
||||
@@ -1865,7 +1992,9 @@ class CloudHypervisorExecutor(Executor):
|
||||
def _stop_dead_socket_vms(self, entries):
|
||||
"""socket 扫描:登记进程仍活的跳过,其余走 stop_vm。
|
||||
|
||||
返回已清理 VM 名;非 chv- 前缀 socket 一律跳过。"""
|
||||
返回已清理 VM 名;非 chv- 前缀 socket 一律跳过。
|
||||
feat-chv-p1fix(P1-05):登记进程的读持 _vms_lock,
|
||||
不再无锁读 dict(与 purge/注册并发会读到中间态)。"""
|
||||
cleaned = []
|
||||
for entry in entries:
|
||||
if not entry.endswith(".sock"):
|
||||
@@ -1873,6 +2002,7 @@ class CloudHypervisorExecutor(Executor):
|
||||
name = entry[:-len(".sock")]
|
||||
if not name.startswith("chv-"):
|
||||
continue
|
||||
with self._vms_lock:
|
||||
proc = self._vms.get(name)
|
||||
if proc is not None:
|
||||
try:
|
||||
@@ -1887,20 +2017,36 @@ class CloudHypervisorExecutor(Executor):
|
||||
|
||||
def _purge_dead_registry(self):
|
||||
"""登记进程已死但 socket 已被删的登记项顺手清掉
|
||||
(没有 socket 可删,不计入清理数)。"""
|
||||
for name in list(self._vms.keys()):
|
||||
(没有 socket 可删,不计入清理数)。
|
||||
feat-chv-p1fix(P1-05):先在锁内快照键列表,再
|
||||
逐项在锁内读/删——旧“无锁 pop 与持锁遍历并发”
|
||||
会致 dictionary changed size during iteration,
|
||||
且 pop 本身无锁。"""
|
||||
with self._vms_lock:
|
||||
names = list(self._vms.keys())
|
||||
for name in names:
|
||||
with self._vms_lock:
|
||||
proc = self._vms.get(name)
|
||||
try:
|
||||
dead = proc is not None and proc.poll() is not None
|
||||
except Exception:
|
||||
dead = True
|
||||
if dead:
|
||||
with self._vms_lock:
|
||||
self._vms.pop(name, None)
|
||||
|
||||
def cleanup_stale_vms(self) -> list:
|
||||
"""清理残留 VM:先跑残留盘 GC 与残留密钥扫描,再走
|
||||
socket 扫描与登记清理。只清 socket 残留与已死进程
|
||||
对应的 VM;state_dir 不存在返回 [],绝不抛异常。"""
|
||||
对应的 VM;state_dir 不存在返回 [],绝不抛异常。
|
||||
feat-chv-p1fix(P1-05):全程持 state 锁,与 wake/
|
||||
restore 互斥——清理删盘/删 socket 时不许另一路
|
||||
wake 同时为同名 VM 供盘、建 VM。"""
|
||||
with self._state_lock_ctx():
|
||||
return self._cleanup_stale_vms_locked()
|
||||
|
||||
def _cleanup_stale_vms_locked(self) -> list:
|
||||
"""持 state 锁的 cleanup 本体(见 cleanup_stale_vms)。"""
|
||||
try:
|
||||
self.cleanup_stale_secrets()
|
||||
except Exception:
|
||||
|
||||
@@ -159,8 +159,14 @@ class WarmPool:
|
||||
|
||||
对每个条目尽力 shutdown_vm,条目清掉并落盘,返回
|
||||
已清理的 VM 名列表。生产启动/关闭不走本方法,走
|
||||
cloud.chv.orphans.reconcile_orphans_for_executor()。"""
|
||||
cloud.chv.orphans.reconcile_orphans_for_executor()。
|
||||
feat-chv-p1fix(P1-01 同款语义):shutdown 失败的
|
||||
条目不进 cleaned、不从内存条目里 pop(下次 reconcile
|
||||
可重试),并记 failed 审计;文件落盘仍只写
|
||||
warm/assigned 态(见 P2-16,实验路径的文件语义
|
||||
本轨不改,内存态保留即重试依据)。"""
|
||||
cleaned = []
|
||||
failed = []
|
||||
for name, entry in list(self._entries.items()):
|
||||
if entry["state"] not in ("orphan", "assigned"):
|
||||
continue
|
||||
@@ -168,7 +174,8 @@ class WarmPool:
|
||||
try:
|
||||
self.executor.shutdown_vm(vm)
|
||||
except Exception:
|
||||
pass
|
||||
failed.append(vm)
|
||||
continue
|
||||
self._entries.pop(name, None)
|
||||
cleaned.append(vm)
|
||||
if cleaned:
|
||||
@@ -179,6 +186,14 @@ class WarmPool:
|
||||
"cleaned=%s" % ",".join(cleaned))
|
||||
except Exception:
|
||||
pass
|
||||
if failed:
|
||||
try:
|
||||
self.executor._emit(
|
||||
"warm_pool_reconcile_failed", {"id": None},
|
||||
"failed=%s cleaned=%s" % (
|
||||
",".join(failed), ",".join(cleaned)))
|
||||
except Exception:
|
||||
pass
|
||||
return cleaned
|
||||
|
||||
# -- 池操作 ----------------------------------------------------------
|
||||
|
||||
@@ -0,0 +1,107 @@
|
||||
"""CHV state_dir 跨进程锁(feat-chv-p1fix,P1-05)。
|
||||
|
||||
问题:容量 / net / 磁盘的 check→act 原来只靠进程内
|
||||
threading.Lock 护 `_vms` dict;多 worker 进程共用同一
|
||||
state_dir 时,两路 wake 可双双通过容量/磁盘/net 门并
|
||||
双双建 VM(check-then-act 竞态),`_purge_dead_registry`
|
||||
的无锁 pop 与持锁遍历并发还可致字典迭代异常。
|
||||
|
||||
解法:同一把锁护住 check→act 全程——文件锁(flock)
|
||||
+ 进程内可重入锁的组合:
|
||||
|
||||
- 文件锁:state_dir/chv-state.lock 上的 `LOCK_EX`。
|
||||
不同进程(多 worker)靠它互斥;同一进程内不同
|
||||
executor 实例(各有自己的 StateLock 对象)同样靠它
|
||||
互斥,因为 flock 按打开的文件描述符区分,两个实例
|
||||
各自 open → 第二个会阻塞,直到第一个释放。
|
||||
- 进程内可重入:同一 executor 实例内,wake 全程会
|
||||
嵌套调用同样需要锁的方法(如 wake → 失败时 stop →
|
||||
cleanup_stale_vms 等路径),因此 StateLock 自带
|
||||
threading.RLock + 深度计数:同一线程重入时不再重复
|
||||
flock,只在最外层加锁/释放,避免同一线程把自己锁死。
|
||||
注意:同一线程经“不同 StateLock 实例”嵌套加锁不在
|
||||
保证内——代码里不存在这条路径(wake/restore/cleanup/
|
||||
reconcile 各自只用本实例或本 registry 的锁,且
|
||||
reconcile 不在 wake 持锁期间被调用)。
|
||||
|
||||
持有范围(见 executor.py):wake_vm 全程(prechecks →
|
||||
prepare → seed → boot 含 SSH 就绪等待)、restore_vm 全程、
|
||||
cleanup_stale_vms 全程(含 gc/密钥扫描/socket 扫描/purge)、
|
||||
orphans.reconcile 全程。锁内串行化 wake 是有意的代价:
|
||||
容量门只有“查完立即占位、占位前不放行”才成立;把锁
|
||||
提前释放到 vm.create 之后会在 socket 尚未绑定前留出
|
||||
计数窗口,仍可超限。串行 wake 不影响已运行 VM 的任务
|
||||
执行(_run_guest 不持此锁)。
|
||||
|
||||
本模块只依赖标准库,是叶子模块;fcntl 只在 POSIX 上
|
||||
可用——本执行器只面向 Linux 宿主(KVM 路径本身即
|
||||
Linux 专属),非 POSIX 平台 import 即失败,不静默回落
|
||||
纯线程锁(回落会把多 worker 保证悄悄降格)。
|
||||
"""
|
||||
|
||||
import fcntl
|
||||
import os
|
||||
import threading
|
||||
|
||||
LOCK_FILENAME = "chv-state.lock"
|
||||
|
||||
|
||||
def lock_path_for_state_dir(state_dir: str) -> str:
|
||||
"""state_dir 对应的锁文件路径。"""
|
||||
return os.path.join(state_dir, LOCK_FILENAME)
|
||||
|
||||
|
||||
class StateLock:
|
||||
"""state_dir 跨进程 + 进程内可重入锁(上下文管理器)。
|
||||
|
||||
用法:`with executor_state_lock:`。acquire 时先拿进程内
|
||||
RLock,再在深度为 0 时 open 锁文件并 flock LOCK_EX
|
||||
(阻塞,直到其它进程/实例释放);release 时深度归零才
|
||||
解锁并关闭文件描述符。锁文件本身永不删除(删文件会
|
||||
破坏 flock 的互斥语义:两个进程可能各持一个已删文件
|
||||
的锁而互不感知)。
|
||||
"""
|
||||
|
||||
def __init__(self, state_dir: str) -> None:
|
||||
self.state_dir = state_dir
|
||||
self.lock_path = lock_path_for_state_dir(state_dir)
|
||||
self._rlock = threading.RLock()
|
||||
self._depth = 0
|
||||
self._fd: int | None = None
|
||||
|
||||
def acquire(self) -> None:
|
||||
self._rlock.acquire()
|
||||
try:
|
||||
if self._depth == 0:
|
||||
os.makedirs(self.state_dir, exist_ok=True)
|
||||
fd = os.open(self.lock_path,
|
||||
os.O_RDWR | os.O_CREAT, 0o600)
|
||||
try:
|
||||
fcntl.flock(fd, fcntl.LOCK_EX)
|
||||
except Exception:
|
||||
os.close(fd)
|
||||
raise
|
||||
self._fd = fd
|
||||
self._depth += 1
|
||||
except Exception:
|
||||
self._rlock.release()
|
||||
raise
|
||||
|
||||
def release(self) -> None:
|
||||
try:
|
||||
self._depth -= 1
|
||||
if self._depth == 0 and self._fd is not None:
|
||||
try:
|
||||
fcntl.flock(self._fd, fcntl.LOCK_UN)
|
||||
finally:
|
||||
os.close(self._fd)
|
||||
self._fd = None
|
||||
finally:
|
||||
self._rlock.release()
|
||||
|
||||
def __enter__(self) -> "StateLock":
|
||||
self.acquire()
|
||||
return self
|
||||
|
||||
def __exit__(self, *exc_info: object) -> None:
|
||||
self.release()
|
||||
+109
-24
@@ -17,12 +17,19 @@ Registry 语义与旧 WarmPool._load_registry 完全一致
|
||||
VM,绝不发给任务,只能清理。
|
||||
- 损坏 fail-safe:文件不是合法 JSON / 顶层不是 dict 时,
|
||||
整个文件 quarantine 成 <path>.corrupt,清理从空开始;
|
||||
损坏本身发审计(feat-chv-p1fix,P1-02),绝不静默
|
||||
返回空——损坏文件的孤儿 VM 此前只能靠 socket 扫描。
|
||||
单条目结构非法(无 vm、或 state 不在 warm/assigned)
|
||||
时只丢该条,不连坐其它条目。
|
||||
- 清理后落盘为空 dict(原子 tmp + rename);写失败只记
|
||||
日志警告,不炸 worker 启动/关闭。shutdown_vm 抛错时该
|
||||
条目仍计入已清理(旧 reconcile 语义:尽力清理、不因
|
||||
单条失败卡住整批),_emit 失败同样 best-effort。
|
||||
时只丢该条,不连坐其它条目;非法条目发审计并计数
|
||||
(feat-chv-p1fix,P1-02),落盘时不再写回。
|
||||
- 清理后落盘:全部成功时为空 dict;有失败项时只写回
|
||||
失败项的原始条目(原子 tmp + rename),下次 reconcile
|
||||
可重试,失败项绝不被“已清理”掩盖(feat-chv-p1fix,
|
||||
P1-01)。写失败只记日志警告,不炸 worker 启动/关闭。
|
||||
shutdown_vm 抛错的条目不进 cleaned、不触发“全清”,
|
||||
而是记 failed 列表并经 executor._emit 发审计;
|
||||
_emit 失败同样 best-effort。整个 reconcile 在
|
||||
state_dir 的 flock 下跑(见 locking.py),与 wake 互斥。
|
||||
"""
|
||||
|
||||
import json
|
||||
@@ -91,52 +98,130 @@ def _orphan_entries(data: dict) -> list[dict]:
|
||||
"""从原始 registry dict 里筛出可清理的上一世条目。
|
||||
|
||||
只留 state 为 warm/assigned 且有 vm 的条目;非法条目
|
||||
静默丢弃(不连坐)。返回条目为 {"vm", "task_id",
|
||||
不连坐其它条目(计数与审计见 reconcile 本体,不再
|
||||
静默丢弃)。返回条目为 {"vm", "task_id",
|
||||
"snapshot_url"} 的新 dict,不改原数据。"""
|
||||
entries: list[dict] = []
|
||||
for entry in data.values():
|
||||
for _key, entry in _orphan_entries_with_keys(data)[0]:
|
||||
entries.append(entry)
|
||||
return entries
|
||||
|
||||
|
||||
def _orphan_entries_with_keys(data: dict) -> tuple[list, list]:
|
||||
"""筛条目并保留原始 registry 键:返回(可清理条目,
|
||||
非法条目键)。
|
||||
|
||||
可清理条目为 (原始键, {"vm", "task_id",
|
||||
"snapshot_url"});非法条目只记键(键本身不含密钥,
|
||||
可进审计 detail),原始值不进审计。"""
|
||||
entries: list = []
|
||||
invalid_keys: list = []
|
||||
for key, entry in data.items():
|
||||
if not isinstance(entry, dict):
|
||||
invalid_keys.append(key)
|
||||
continue
|
||||
if not entry.get("vm"):
|
||||
invalid_keys.append(key)
|
||||
continue
|
||||
if entry.get("state") not in ("warm", "assigned"):
|
||||
invalid_keys.append(key)
|
||||
continue
|
||||
entries.append({
|
||||
entries.append((key, {
|
||||
"vm": entry["vm"],
|
||||
"task_id": entry.get("task_id"),
|
||||
"snapshot_url": entry.get("snapshot_url"),
|
||||
})
|
||||
return entries
|
||||
}))
|
||||
return entries, invalid_keys
|
||||
|
||||
|
||||
def _best_effort_emit(executor: object, event: str, detail: str) -> None:
|
||||
"""经 executor._emit 发审计(best-effort,兼容只接受
|
||||
(event, task, detail) 三参的假执行器;无 _emit 时跳过)。"""
|
||||
emit = getattr(executor, "_emit", None)
|
||||
if not callable(emit):
|
||||
return
|
||||
try:
|
||||
emit(event, {"id": None}, detail)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
def reconcile_orphans_for_executor(executor: object) -> list[str]:
|
||||
"""生产路径唯一的残留清理入口:返回已清理的 VM 名列表。
|
||||
|
||||
只做清理:读 registry → 每个上一世条目尽力
|
||||
shutdown_vm → 落盘为空 → best-effort 审计事件。
|
||||
shutdown_vm → 落盘 → best-effort 审计事件。
|
||||
绝不构造 WarmPool、绝不调 prewarm/acquire/release,
|
||||
绝不把任何条目发给任务。无 registry 路径时返回 []。"""
|
||||
绝不把任何条目发给任务。无 registry 路径时返回 []。
|
||||
|
||||
feat-chv-p1fix(P1-01/P1-02/P1-05)语义:
|
||||
- shutdown 失败的条目不进 cleaned,记 failed 并审计,
|
||||
落盘时只写回失败项原始条目(可重试),绝不清库;
|
||||
- registry 损坏时 quarantine 后仍发审计,不静默返回空;
|
||||
- 非法条目发审计并计数,落盘时丢弃(已审计,不再
|
||||
每次重启重复审计同一批非法条目);
|
||||
- 全程持 state_dir flock,与 wake/restore/cleanup 互斥。
|
||||
返回值仍只含 cleaned(调用方兼容不变);失败与
|
||||
损坏/非法的可见性走 executor._emit(worker 接线后
|
||||
即任务 timeline 审计),不靠返回值传递。
|
||||
"""
|
||||
path = registry_path_for_executor(executor)
|
||||
if not path:
|
||||
return []
|
||||
data, _corrupt = _read_registry(path)
|
||||
if data is None:
|
||||
if not os.path.exists(path):
|
||||
return []
|
||||
entries = _orphan_entries(data)
|
||||
state_dir = getattr(executor, "state_dir", None)
|
||||
if state_dir:
|
||||
from cloud.chv.locking import StateLock
|
||||
lock = StateLock(state_dir)
|
||||
else: # pragma: no cover - state_dir 缺失时 path 已为 None
|
||||
from contextlib import nullcontext
|
||||
lock = nullcontext() # type: ignore[assignment]
|
||||
with lock:
|
||||
return _reconcile_locked(executor, path)
|
||||
|
||||
|
||||
def _reconcile_locked(executor: object, path: str) -> list[str]:
|
||||
"""持锁的 reconcile 本体(见 reconcile_orphans_for_executor)。"""
|
||||
data, corrupt = _read_registry(path)
|
||||
if data is None:
|
||||
if corrupt:
|
||||
_best_effort_emit(
|
||||
executor, "warm_pool_registry_corrupt",
|
||||
"path=%s registry corrupt, quarantined" % path)
|
||||
return []
|
||||
entries, invalid_keys = _orphan_entries_with_keys(data)
|
||||
if invalid_keys:
|
||||
_best_effort_emit(
|
||||
executor, "warm_pool_registry_invalid_entries",
|
||||
"invalid=%d keys=%s"
|
||||
% (len(invalid_keys), ",".join(str(k) for k in invalid_keys)))
|
||||
cleaned: list[str] = []
|
||||
for entry in entries:
|
||||
failed: list[str] = []
|
||||
failed_errors: list[str] = []
|
||||
failed_keys: list[str] = []
|
||||
for key, entry in entries:
|
||||
vm = entry["vm"]
|
||||
try:
|
||||
executor.shutdown_vm(vm) # type: ignore[attr-defined]
|
||||
except Exception:
|
||||
pass
|
||||
except Exception as exc:
|
||||
failed.append(vm)
|
||||
failed_keys.append(key)
|
||||
failed_errors.append("%s:%s" % (vm, type(exc).__name__))
|
||||
continue
|
||||
cleaned.append(vm)
|
||||
if cleaned or failed or invalid_keys:
|
||||
remaining = {key: data[key] for key in failed_keys
|
||||
if key in data}
|
||||
_write_registry(path, remaining)
|
||||
if cleaned:
|
||||
_write_registry(path, {})
|
||||
try:
|
||||
executor._emit( # type: ignore[attr-defined]
|
||||
"warm_pool_reconcile", {"id": None},
|
||||
_best_effort_emit(
|
||||
executor, "warm_pool_reconcile",
|
||||
"cleaned=%s" % ",".join(cleaned))
|
||||
except Exception:
|
||||
pass
|
||||
if failed:
|
||||
_best_effort_emit(
|
||||
executor, "warm_pool_reconcile_failed",
|
||||
"failed=%s cleaned=%s errors=%s"
|
||||
% (",".join(failed), ",".join(cleaned),
|
||||
";".join(failed_errors)))
|
||||
return cleaned
|
||||
@@ -156,6 +156,8 @@ class BreakingCleanupTest(unittest.TestCase):
|
||||
|
||||
class OrphansTest(unittest.TestCase):
|
||||
def test_valid_warm_and_assigned_cleaned_invalid_dropped(self):
|
||||
# feat-chv-p1fix(P1-02):非法条目不再静默丢弃,
|
||||
# 先发 invalid 审计并计数,再发 cleaned 审计。
|
||||
with tempfile.TemporaryDirectory() as d:
|
||||
path = _write_registry(d, {
|
||||
"a": {"state": "warm", "vm": "chv-a",
|
||||
@@ -171,19 +173,26 @@ class OrphansTest(unittest.TestCase):
|
||||
reconcile_orphans_for_executor(ex),
|
||||
["chv-a", "chv-b"])
|
||||
self.assertEqual(ex.shutdowns, ["chv-a", "chv-b"])
|
||||
self.assertEqual(ex.emits, ["warm_pool_reconcile"])
|
||||
self.assertEqual(ex.emits, [
|
||||
"warm_pool_registry_invalid_entries",
|
||||
"warm_pool_reconcile"])
|
||||
with open(path) as fh:
|
||||
self.assertEqual(json.load(fh), {})
|
||||
|
||||
def test_invalid_only_returns_empty_and_leaves_file(self):
|
||||
def test_invalid_only_returns_empty_and_drops_audited_entries(self):
|
||||
# feat-chv-p1fix(P1-02):仅非法条目时返回 [] 且
|
||||
# 不 shutdown;非法条目已审计计数,落盘时丢弃,
|
||||
# 不再每次重启重复审计同一批非法条目。
|
||||
with tempfile.TemporaryDirectory() as d:
|
||||
path = _write_registry(d, {
|
||||
"c": {"state": "warm", "task_id": "c"}})
|
||||
ex = FakeProdExecutor(d)
|
||||
self.assertEqual(reconcile_orphans_for_executor(ex), [])
|
||||
self.assertEqual(ex.shutdowns, [])
|
||||
self.assertEqual(
|
||||
ex.emits, ["warm_pool_registry_invalid_entries"])
|
||||
with open(path) as fh:
|
||||
self.assertIn("task_id", fh.read())
|
||||
self.assertEqual(json.load(fh), {})
|
||||
|
||||
def test_corrupt_json_quarantined(self):
|
||||
with tempfile.TemporaryDirectory() as d:
|
||||
@@ -212,9 +221,12 @@ class OrphansTest(unittest.TestCase):
|
||||
self.assertEqual(
|
||||
reconcile_orphans_for_executor(NoDir()), [])
|
||||
|
||||
def test_shutdown_and_emit_failures_do_not_break_cleanup(self):
|
||||
def test_shutdown_failure_not_cleaned_and_retained(self):
|
||||
# feat-chv-p1fix(P1-01):shutdown 失败的条目不进
|
||||
# cleaned、不触发清库,落盘保留原始条目可重试;
|
||||
# _emit 失败仍 best-effort,不炸清理流程。
|
||||
with tempfile.TemporaryDirectory() as d:
|
||||
_write_registry(d, {
|
||||
path = _write_registry(d, {
|
||||
"a": {"state": "warm", "vm": "chv-a"}})
|
||||
ex = FakeProdExecutor(d)
|
||||
|
||||
@@ -225,7 +237,11 @@ class OrphansTest(unittest.TestCase):
|
||||
ex._emit = lambda *a, **k: (_ for _ in ()).throw(
|
||||
RuntimeError("audit down"))
|
||||
self.assertEqual(
|
||||
reconcile_orphans_for_executor(ex), ["chv-a"])
|
||||
reconcile_orphans_for_executor(ex), [])
|
||||
with open(path) as fh:
|
||||
self.assertEqual(
|
||||
json.load(fh),
|
||||
{"a": {"state": "warm", "vm": "chv-a"}})
|
||||
|
||||
def test_experimental_pool_registry_roundtrip_unchanged(self):
|
||||
# 实验池自身的 registry 语义(载入即 orphan、reconcile
|
||||
|
||||
@@ -889,25 +889,31 @@ class DefensiveBranchTest(CloseupCase):
|
||||
ex.runner = keyscan_runner
|
||||
self.assertFalse(ex.capture_host_key(self._task()))
|
||||
|
||||
def test_build_ssh_command_fallback_and_secret_not_delivered(self):
|
||||
def test_build_ssh_command_undeclared_raises_and_secret_not_delivered(self):
|
||||
# feat-chv-p1fix(P1-04):删除回环回落——目标未声明
|
||||
# 时构造即抛 ssh_target_undeclared,不再悄悄连回环。
|
||||
ex = self.make_exec(env_extra={"CHV_SSH_FORWARD": "0"})
|
||||
cmd = ex.build_ssh_command(self._task(), ["true"])
|
||||
self.assertIn("127.0.0.1", " ".join(cmd))
|
||||
with self.assertRaises(ChvFailure) as ctx:
|
||||
ex.build_ssh_command(self._task(), ["true"])
|
||||
self.assertEqual(ctx.exception.kind, "ssh_target_undeclared")
|
||||
ex = self.make_exec()
|
||||
with self.assertRaises(ChvFailure) as ctx:
|
||||
ex.build_ssh_command(self._task(), ["true"],
|
||||
extra_env={"SUB2API_API_KEY": "v"})
|
||||
self.assertEqual(ctx.exception.kind, "secret_env_not_delivered")
|
||||
|
||||
def test_write_provisioned_known_hosts_fallback(self):
|
||||
def test_write_provisioned_known_hosts_undeclared_raises(self):
|
||||
# feat-chv-p1fix(P1-04 同款):预置 known_hosts 解析
|
||||
# 失败即抛,不再回落回环写钉死。
|
||||
ex = self.make_exec(env_extra={"CHV_SSH_FORWARD": "0",
|
||||
"CHV_HOST_KEY_MODE": "provisioned"})
|
||||
vm_dir = os.path.join(self.state_dir, "chv-abc123")
|
||||
os.makedirs(vm_dir, exist_ok=True)
|
||||
paths = {"known_hosts": os.path.join(vm_dir, "known_hosts")}
|
||||
with self.assertRaises(ChvFailure) as ctx:
|
||||
ex._write_provisioned_known_hosts(paths, "ssh-ed25519 AAAA")
|
||||
with open(paths["known_hosts"]) as fh:
|
||||
self.assertIn("127.0.0.1", fh.read())
|
||||
self.assertEqual(ctx.exception.kind, "ssh_target_undeclared")
|
||||
self.assertFalse(os.path.exists(paths["known_hosts"]))
|
||||
|
||||
def test_provision_host_key_dirty_refused(self):
|
||||
ex = self.make_exec()
|
||||
@@ -1083,22 +1089,26 @@ class DefensiveBranchTest(CloseupCase):
|
||||
self.assertFalse(ex._vm_is_running("chv-bad"))
|
||||
|
||||
def test_clean_one_vm_secrets_remove_failures(self):
|
||||
# feat-chv-p1fix(P1-03):返回 3 元组,删失败以
|
||||
# failed=True 区分,不再静默 pass。
|
||||
ex = self.make_exec()
|
||||
vm_dir = os.path.join(self.state_dir, "chv-x")
|
||||
os.makedirs(os.path.join(vm_dir, "ssh_host_ed25519_key"),
|
||||
exist_ok=True)
|
||||
removed_priv, removed_seed = ex._clean_one_vm_secrets(
|
||||
removed_priv, removed_seed, failed = ex._clean_one_vm_secrets(
|
||||
"chv-x", vm_dir)
|
||||
self.assertFalse(removed_priv)
|
||||
self.assertTrue(failed)
|
||||
vm2 = os.path.join(self.state_dir, "chv-y")
|
||||
os.makedirs(os.path.join(vm2, "seed"), exist_ok=True)
|
||||
import shutil
|
||||
real_rmtree = shutil.rmtree
|
||||
with mock.patch("shutil.rmtree",
|
||||
side_effect=OSError("rmtree boom")):
|
||||
removed_priv, removed_seed = ex._clean_one_vm_secrets(
|
||||
removed_priv, removed_seed, failed = ex._clean_one_vm_secrets(
|
||||
"chv-y", vm2)
|
||||
self.assertFalse(removed_seed)
|
||||
self.assertTrue(failed)
|
||||
self.assertTrue(real_rmtree is not None)
|
||||
|
||||
def test_cleanup_stale_vms_subcall_failures_and_poll_branches(self):
|
||||
|
||||
@@ -559,14 +559,17 @@ class ExperimentalPoolCover100Test(unittest.TestCase):
|
||||
self.assertIn("warm-1", pool._entries)
|
||||
self.assertNotIn("old-1", pool._entries)
|
||||
|
||||
def test_reconcile_shutdown_failure_still_cleans(self):
|
||||
def test_reconcile_shutdown_failure_not_cleaned(self):
|
||||
# feat-chv-p1fix(P1-01 同款语义,实验池):shutdown
|
||||
# 失败的条目不进 cleaned、不从内存条目里 pop,
|
||||
# 下次 reconcile 可重试。
|
||||
stub = _PoolStub()
|
||||
stub.shutdown_raises = True
|
||||
pool = WarmPool(stub, registry_path=None)
|
||||
pool._entries["old-1"] = {"state": "orphan", "vm": "vm-old",
|
||||
"task": {"id": "old-1"}}
|
||||
self.assertEqual(pool.reconcile_orphans(), ["vm-old"])
|
||||
self.assertEqual(pool._entries, {})
|
||||
self.assertEqual(pool.reconcile_orphans(), [])
|
||||
self.assertIn("old-1", pool._entries)
|
||||
|
||||
def test_reconcile_emit_failure_is_best_effort(self):
|
||||
stub = _PoolStub()
|
||||
|
||||
@@ -0,0 +1,730 @@
|
||||
"""CHV 生产级审核 P1 修复轨(feat-chv-p1fix,堆叠于 #270 之上)测试。
|
||||
|
||||
逐项对应审核报告 §5 的 P1-01~P1-10(P1-10 是文档项,
|
||||
无代码测试;文档数字见 docs/chv-production-readiness.md
|
||||
§15 与 docs/chv-architecture.md 头范围):
|
||||
- P1-01:生产 reconcile 失败项不进 cleaned、不清库、
|
||||
落盘保留失败项原始条目、可重试;实验池同款语义。
|
||||
- P1-02:registry 损坏与非法条目发审计并计数,损坏不
|
||||
静默返回空。
|
||||
- P1-03:残留私钥删失败发审计、写脏标记,调用方可区分
|
||||
成功与失败(failed VM 不进 cleaned)。
|
||||
- P1-04:build_ssh_command 删除回环回落,构造失败即抛;
|
||||
显式回环(CHV_SSH_HOST / CHV_SSH_FORWARD=1)仍可用。
|
||||
- P1-05:state_dir flock(locking.StateLock)——线程内
|
||||
互斥、跨进程互斥、wake 全程串行化、purge 快照遍历。
|
||||
- P1-06:单条快照超总量帽时拒绝记录,且不删旧快照、
|
||||
不落盘新条目。
|
||||
- P1-07:CHV_CPU / CHV_MEM / CHV_TASK_TIMEOUT 的 >0
|
||||
校验(config 构造层 + preflight 事后改写兜底)。
|
||||
- P1-08:快照保留期剪枝删失败发审计;snapshot_vm 失败
|
||||
路径发 vm_snapshot_failed 审计。
|
||||
- P1-09:4 处三元 else 分支——str 形态 pause/resume/
|
||||
snapshot + 非 dict spill task。
|
||||
|
||||
全部用标准库与 mock runner/launcher/api_client,绝不启动
|
||||
真实 VM(本机无 /dev/kvm,真机验证仍是未验证项)。
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import threading
|
||||
import time
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
from unittest import mock
|
||||
|
||||
try:
|
||||
from cloud.chv import (
|
||||
ChvFailure,
|
||||
CloudHypervisorExecutor,
|
||||
reconcile_orphans_for_executor,
|
||||
)
|
||||
from cloud.chv.config import build_chv_config
|
||||
from cloud.chv.disk import (
|
||||
prune_snapshots,
|
||||
record_snapshot,
|
||||
)
|
||||
from cloud.chv.locking import StateLock, lock_path_for_state_dir
|
||||
except ImportError: # discover -s cloud/tests 时把项目根目录补进 sys.path
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[2]))
|
||||
from cloud.chv import (
|
||||
ChvFailure,
|
||||
CloudHypervisorExecutor,
|
||||
reconcile_orphans_for_executor,
|
||||
)
|
||||
from cloud.chv.config import build_chv_config
|
||||
from cloud.chv.disk import (
|
||||
prune_snapshots,
|
||||
record_snapshot,
|
||||
)
|
||||
from cloud.chv.locking import StateLock, lock_path_for_state_dir
|
||||
|
||||
|
||||
class FakeResult:
|
||||
def __init__(self, returncode=0, stdout="", stderr=""):
|
||||
self.returncode = returncode
|
||||
self.stdout = stdout
|
||||
self.stderr = stderr
|
||||
|
||||
|
||||
class FakeProc:
|
||||
def __init__(self, alive=True):
|
||||
self._alive = alive
|
||||
|
||||
def poll(self):
|
||||
return None if self._alive else 1
|
||||
|
||||
def terminate(self):
|
||||
self._alive = False
|
||||
|
||||
def wait(self, timeout=None):
|
||||
return 0
|
||||
|
||||
def kill(self):
|
||||
self._alive = False
|
||||
|
||||
|
||||
def _write(path, content=b"x"):
|
||||
with open(path, "wb") as fh:
|
||||
fh.write(content)
|
||||
os.chmod(path, 0o755)
|
||||
return path
|
||||
|
||||
|
||||
class P1FixCase(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.tmp = tempfile.TemporaryDirectory()
|
||||
self.dir = self.tmp.name
|
||||
self.binary = _write(os.path.join(self.dir, "cloud-hypervisor"))
|
||||
self.ch_remote = _write(os.path.join(self.dir, "ch-remote"))
|
||||
self.kernel = _write(os.path.join(self.dir, "vmlinux"))
|
||||
self.rootfs = _write(os.path.join(self.dir, "disk.raw"),
|
||||
b"R" * 100)
|
||||
self.ssh_key = _write(os.path.join(self.dir, "id_chv"))
|
||||
self.kvm = _write(os.path.join(self.dir, "kvm"))
|
||||
self.cpuinfo = os.path.join(self.dir, "cpuinfo")
|
||||
with open(self.cpuinfo, "w") as fh:
|
||||
fh.write("flags : fpu vmx sse\n")
|
||||
self.state_dir = os.path.join(self.dir, "state")
|
||||
self.calls = []
|
||||
self.audit = []
|
||||
|
||||
def tearDown(self):
|
||||
self.tmp.cleanup()
|
||||
|
||||
def make_exec(self, runner=None, api_client=None, launcher=None,
|
||||
env_extra=None, **overrides):
|
||||
env = {"CHV_EGRESS_PROXY": "http://10.0.0.1:8888",
|
||||
"CHV_SSH_FORWARD": "1",
|
||||
"CHV_HOST_KEY_MODE": "keyscan"}
|
||||
if env_extra:
|
||||
env.update(env_extra)
|
||||
if runner is None:
|
||||
runner = self._default_runner
|
||||
if api_client is None:
|
||||
api_client = self._default_api
|
||||
if launcher is None:
|
||||
launcher = self._default_launcher
|
||||
kwargs = dict(
|
||||
binary=self.binary, ch_remote=self.ch_remote,
|
||||
kernel=self.kernel, rootfs=self.rootfs,
|
||||
ssh_key=self.ssh_key, kvm_path=self.kvm,
|
||||
cpuinfo_path=self.cpuinfo, state_dir=self.state_dir,
|
||||
runner=runner, launcher=launcher, api_client=api_client,
|
||||
sleeper=lambda s: None, env=env)
|
||||
kwargs.update(overrides)
|
||||
ex = CloudHypervisorExecutor(**kwargs)
|
||||
ex.audit_callback = lambda *a: self.audit.append(a)
|
||||
return ex
|
||||
|
||||
def _default_runner(self, cmd, timeout=None):
|
||||
self.calls.append(list(cmd))
|
||||
return FakeResult(0, "", "")
|
||||
|
||||
def _default_api(self, socket_path, method, path, payload=None):
|
||||
return (200, {})
|
||||
|
||||
def _default_launcher(self, cmd):
|
||||
return FakeProc(alive=True)
|
||||
|
||||
def _task(self, task_id="abc123"):
|
||||
return {"id": task_id, "prompt": "x", "project_id": "proj1"}
|
||||
|
||||
def _audit_events(self):
|
||||
return [a[0] for a in self.audit]
|
||||
|
||||
|
||||
# -- P1-01 / P1-02:生产 reconcile --------------------------------------
|
||||
|
||||
class FakeProdExecutor:
|
||||
def __init__(self, state_dir, fail_vms=()):
|
||||
self.state_dir = state_dir
|
||||
self.fail_vms = set(fail_vms)
|
||||
self.shutdowns = []
|
||||
self.emits = []
|
||||
|
||||
def shutdown_vm(self, vm):
|
||||
self.shutdowns.append(vm)
|
||||
if vm in self.fail_vms:
|
||||
raise RuntimeError("shutdown down")
|
||||
|
||||
def _emit(self, event, task, detail):
|
||||
self.emits.append(event)
|
||||
|
||||
|
||||
def _write_registry(state_dir, data):
|
||||
path = os.path.join(state_dir, "warm-pool.json")
|
||||
os.makedirs(state_dir, exist_ok=True)
|
||||
if isinstance(data, str):
|
||||
with open(path, "w") as fh:
|
||||
fh.write(data)
|
||||
else:
|
||||
with open(path, "w") as fh:
|
||||
json.dump(data, fh)
|
||||
return path
|
||||
|
||||
|
||||
class ReconcileP1FixTest(unittest.TestCase):
|
||||
def test_failure_not_cleaned_registry_retained_and_retryable(self):
|
||||
with tempfile.TemporaryDirectory() as d:
|
||||
path = _write_registry(d, {
|
||||
"a": {"state": "warm", "vm": "chv-good"},
|
||||
"b": {"state": "warm", "vm": "chv-bad"},
|
||||
})
|
||||
ex = FakeProdExecutor(d, fail_vms={"chv-bad"})
|
||||
self.assertEqual(reconcile_orphans_for_executor(ex),
|
||||
["chv-good"])
|
||||
self.assertIn("warm_pool_reconcile_failed", ex.emits)
|
||||
with open(path) as fh:
|
||||
remaining = json.load(fh)
|
||||
self.assertEqual(
|
||||
remaining,
|
||||
{"b": {"state": "warm", "vm": "chv-bad"}})
|
||||
# 修复后重试:失败项可被清理,库最终为空。
|
||||
ex.fail_vms = set()
|
||||
self.assertEqual(reconcile_orphans_for_executor(ex),
|
||||
["chv-bad"])
|
||||
with open(path) as fh:
|
||||
self.assertEqual(json.load(fh), {})
|
||||
|
||||
def test_corrupt_registry_audited_not_silent(self):
|
||||
with tempfile.TemporaryDirectory() as d:
|
||||
path = _write_registry(d, "{not json")
|
||||
ex = FakeProdExecutor(d)
|
||||
self.assertEqual(reconcile_orphans_for_executor(ex), [])
|
||||
self.assertIn("warm_pool_registry_corrupt", ex.emits)
|
||||
self.assertTrue(os.path.exists(path + ".corrupt"))
|
||||
|
||||
def test_invalid_entries_audited_and_counted(self):
|
||||
with tempfile.TemporaryDirectory() as d:
|
||||
path = _write_registry(d, {
|
||||
"a": {"state": "warm", "vm": "chv-a"},
|
||||
"c": {"state": "warm"},
|
||||
"e": {"state": "running", "vm": "chv-e"},
|
||||
})
|
||||
ex = FakeProdExecutor(d)
|
||||
self.assertEqual(reconcile_orphans_for_executor(ex),
|
||||
["chv-a"])
|
||||
self.assertIn("warm_pool_registry_invalid_entries",
|
||||
ex.emits)
|
||||
with open(path) as fh:
|
||||
self.assertEqual(json.load(fh), {})
|
||||
|
||||
|
||||
# -- P1-03:残留私钥删失败 ----------------------------------------------
|
||||
|
||||
class StaleSecretsP1FixTest(P1FixCase):
|
||||
def test_delete_failure_not_cleaned_audited_and_dirty_marked(self):
|
||||
from cloud.chv.hostkeys import dirty_marker_path
|
||||
ex = self.make_exec()
|
||||
vm_dir = os.path.join(self.state_dir, "chv-bad")
|
||||
os.makedirs(vm_dir, exist_ok=True)
|
||||
priv = os.path.join(vm_dir, "ssh_host_ed25519_key")
|
||||
with open(priv, "w") as fh:
|
||||
fh.write("PRIVATE")
|
||||
real_remove = os.remove
|
||||
|
||||
def remove(path):
|
||||
if path == priv:
|
||||
raise OSError("删不掉")
|
||||
return real_remove(path)
|
||||
|
||||
with mock.patch.object(os, "remove", side_effect=remove):
|
||||
cleaned = ex.cleanup_stale_secrets()
|
||||
self.assertNotIn("chv-bad", cleaned)
|
||||
self.assertIn("host_secret_residue_delete_failed",
|
||||
self._audit_events())
|
||||
self.assertTrue(os.path.exists(
|
||||
dirty_marker_path(self.state_dir, "chv-bad")))
|
||||
self.assertTrue(os.path.exists(priv))
|
||||
|
||||
def test_delete_success_cleaned(self):
|
||||
ex = self.make_exec()
|
||||
vm_dir = os.path.join(self.state_dir, "chv-ok")
|
||||
os.makedirs(vm_dir, exist_ok=True)
|
||||
with open(os.path.join(vm_dir, "ssh_host_ed25519_key"),
|
||||
"w") as fh:
|
||||
fh.write("PRIVATE")
|
||||
cleaned = ex.cleanup_stale_secrets()
|
||||
self.assertIn("chv-ok", cleaned)
|
||||
self.assertIn("host_secret_residue_cleaned",
|
||||
self._audit_events())
|
||||
|
||||
|
||||
# -- P1-04:SSH 回落删除 -------------------------------------------------
|
||||
|
||||
class SshFallbackP1FixTest(P1FixCase):
|
||||
def test_undeclared_build_raises_not_loopback(self):
|
||||
ex = self.make_exec(env_extra={"CHV_SSH_FORWARD": "0"})
|
||||
with self.assertRaises(ChvFailure) as ctx:
|
||||
ex.build_ssh_command(self._task(), ["true"])
|
||||
self.assertEqual(ctx.exception.kind, "ssh_target_undeclared")
|
||||
|
||||
def test_explicit_loopback_still_allowed(self):
|
||||
ex = self.make_exec(env_extra={"CHV_SSH_HOST": "127.0.0.1"})
|
||||
cmd = ex.build_ssh_command(self._task(), ["true"])
|
||||
self.assertIn("root@127.0.0.1", " ".join(cmd))
|
||||
ex2 = self.make_exec(env_extra={"CHV_SSH_FORWARD": "1"})
|
||||
cmd2 = ex2.build_ssh_command(self._task(), ["true"])
|
||||
self.assertIn("root@127.0.0.1", " ".join(cmd2))
|
||||
|
||||
|
||||
# -- P1-05:flock ---------------------------------------------------------
|
||||
|
||||
class StateLockP1FixTest(unittest.TestCase):
|
||||
def test_thread_mutual_exclusion(self):
|
||||
with tempfile.TemporaryDirectory() as d:
|
||||
state_dir = os.path.join(d, "state")
|
||||
lock_a = StateLock(state_dir)
|
||||
lock_b = StateLock(state_dir)
|
||||
acquired = []
|
||||
lock_a.acquire()
|
||||
try:
|
||||
def _take():
|
||||
# RLock 须由加锁线程释放,故获取与释放同线程。
|
||||
lock_b.acquire()
|
||||
acquired.append(True)
|
||||
lock_b.release()
|
||||
|
||||
t = threading.Thread(target=_take)
|
||||
t.start()
|
||||
time.sleep(0.3)
|
||||
self.assertEqual(acquired, [])
|
||||
self.assertTrue(t.is_alive())
|
||||
finally:
|
||||
lock_a.release()
|
||||
t.join(timeout=5)
|
||||
self.assertEqual(acquired, [True])
|
||||
self.assertTrue(os.path.exists(
|
||||
lock_path_for_state_dir(state_dir)))
|
||||
|
||||
def test_cross_process_mutual_exclusion(self):
|
||||
with tempfile.TemporaryDirectory() as d:
|
||||
state_dir = os.path.join(d, "state")
|
||||
os.makedirs(state_dir, exist_ok=True)
|
||||
root = str(Path(__file__).resolve().parents[2])
|
||||
ready = os.path.join(state_dir, "child-locked")
|
||||
script = (
|
||||
"import sys, time; sys.path.insert(0, %r); "
|
||||
"from cloud.chv.locking import StateLock; "
|
||||
"lock = StateLock(%r); lock.acquire(); "
|
||||
"open(%r, 'w').write('locked'); "
|
||||
"time.sleep(10)" % (root, state_dir, ready))
|
||||
proc = subprocess.Popen(
|
||||
[sys.executable, "-c", script],
|
||||
stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
|
||||
try:
|
||||
deadline = time.time() + 10
|
||||
while not os.path.exists(ready) and time.time() < deadline:
|
||||
time.sleep(0.05)
|
||||
self.assertTrue(os.path.exists(ready))
|
||||
self.assertIsNone(proc.poll())
|
||||
lock = StateLock(state_dir)
|
||||
acquired = []
|
||||
|
||||
def _take():
|
||||
lock.acquire()
|
||||
acquired.append(True)
|
||||
lock.release()
|
||||
|
||||
t = threading.Thread(target=_take)
|
||||
t.start()
|
||||
time.sleep(0.5)
|
||||
# 子进程持锁期间,本进程的加锁必须阻塞。
|
||||
self.assertEqual(acquired, [])
|
||||
proc.terminate()
|
||||
proc.wait(timeout=5)
|
||||
t.join(timeout=5)
|
||||
self.assertEqual(acquired, [True])
|
||||
finally:
|
||||
if proc.poll() is None:
|
||||
proc.kill()
|
||||
proc.wait(timeout=5)
|
||||
|
||||
def test_purge_snapshot_traversal_under_concurrent_mutation(self):
|
||||
# 旧无锁 pop + 持锁遍历并发可致字典迭代异常;新实现
|
||||
# 快照键列表后逐项持锁删,并发变更加锁路径下不得抛错。
|
||||
with tempfile.TemporaryDirectory():
|
||||
case = P1FixCase()
|
||||
case.setUp()
|
||||
try:
|
||||
ex = case.make_exec()
|
||||
ex.state_dir = case.state_dir
|
||||
stop = threading.Event()
|
||||
|
||||
def mutate():
|
||||
i = 0
|
||||
while not stop.is_set():
|
||||
name = "chv-m%d" % (i % 5)
|
||||
with ex._vms_lock:
|
||||
ex._vms[name] = FakeProc(alive=False)
|
||||
i += 1
|
||||
|
||||
t = threading.Thread(target=mutate)
|
||||
t.start()
|
||||
try:
|
||||
for _ in range(50):
|
||||
ex._purge_dead_registry()
|
||||
finally:
|
||||
stop.set()
|
||||
t.join(timeout=5)
|
||||
finally:
|
||||
case.tearDown()
|
||||
|
||||
|
||||
class WakeSerializationP1FixTest(P1FixCase):
|
||||
def test_concurrent_wakes_serialized_by_state_lock(self):
|
||||
# 两路并发 wake:每路 api 往返睡 0.2 秒;串行则总耗时
|
||||
# 至少 0.35 秒,并行则约 0.2 秒。阈值留足余量防抖动。
|
||||
def sleeping_api(socket_path, method, path, payload=None):
|
||||
if path == "/vmm.ping":
|
||||
time.sleep(0.2)
|
||||
return (200, {})
|
||||
|
||||
ex1 = self.make_exec(api_client=sleeping_api)
|
||||
ex2 = self.make_exec(api_client=sleeping_api)
|
||||
results = []
|
||||
start = time.time()
|
||||
|
||||
def wake(ex, task_id):
|
||||
try:
|
||||
results.append(ex.wake_vm({"id": task_id})["state"])
|
||||
except Exception as exc: # noqa: BLE001 - 失败即测试失败
|
||||
results.append("error:%s" % exc)
|
||||
|
||||
t1 = threading.Thread(target=wake, args=(ex1, "s1"))
|
||||
t2 = threading.Thread(target=wake, args=(ex2, "s2"))
|
||||
t1.start()
|
||||
t2.start()
|
||||
t1.join(timeout=30)
|
||||
t2.join(timeout=30)
|
||||
elapsed = time.time() - start
|
||||
self.assertEqual(sorted(results), ["running", "running"])
|
||||
self.assertGreaterEqual(elapsed, 0.35)
|
||||
self.assertTrue(os.path.exists(
|
||||
lock_path_for_state_dir(self.state_dir)))
|
||||
|
||||
|
||||
# -- P1-06:单条快照超帽 ---------------------------------------------------
|
||||
|
||||
class SingleSnapshotCapP1FixTest(unittest.TestCase):
|
||||
def test_single_oversize_snapshot_rejected_without_side_effects(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
old = os.path.join(tmp, "vm1", "old.snap")
|
||||
os.makedirs(os.path.join(tmp, "vm1"), exist_ok=True)
|
||||
with open(old, "wb") as fh:
|
||||
fh.write(b"O" * 100)
|
||||
from cloud.chv.disk import _save_snapshots
|
||||
_save_snapshots(tmp, "vm1", [{
|
||||
"url": "file://" + old, "path": old, "size": 100,
|
||||
"created_at": 0}])
|
||||
new = os.path.join(tmp, "vm1", "new.snap")
|
||||
with open(new, "wb") as fh:
|
||||
fh.write(b"N" * 1000)
|
||||
# 总量帽 500 字节:新单条 1000 字节本身已超帽。
|
||||
with self.assertRaises(ChvFailure) as ctx:
|
||||
record_snapshot(tmp, "vm1", "file://" + new,
|
||||
500 / (1024 ** 3))
|
||||
self.assertEqual(ctx.exception.kind,
|
||||
"snapshot_quota_exceeded")
|
||||
# 拒绝无副作用:旧快照文件仍在、登记仍只有旧条目。
|
||||
self.assertTrue(os.path.exists(old))
|
||||
from cloud.chv.disk import _load_snapshots
|
||||
entries = _load_snapshots(tmp, "vm1")
|
||||
self.assertEqual(len(entries), 1)
|
||||
self.assertEqual(entries[0]["path"], old)
|
||||
|
||||
|
||||
# -- P1-07:>0 校验 ---------------------------------------------------------
|
||||
|
||||
class PositiveConfigP1FixTest(P1FixCase):
|
||||
def test_config_layer_rejects_non_positive(self):
|
||||
base_env = {}
|
||||
for env_extra, field in (
|
||||
({"CHV_CPU": "0"}, "cpu"),
|
||||
({"CHV_CPU": "-2"}, "cpu"),
|
||||
({"CHV_MEM": "0"}, "mem"),
|
||||
({"CHV_MEM": "0G"}, "mem"),
|
||||
({"CHV_TASK_TIMEOUT": "0"}, "timeout"),
|
||||
({"CHV_TASK_TIMEOUT": "-5"}, "timeout")):
|
||||
env = dict(base_env)
|
||||
env.update(env_extra)
|
||||
with self.assertRaises(ValueError, msg=field):
|
||||
build_chv_config({}, env)
|
||||
# 合法值与 None(不限)不受影响。
|
||||
cfg = build_chv_config({}, {"CHV_TASK_TIMEOUT": "30"})
|
||||
self.assertEqual(cfg.task_timeout, 30.0)
|
||||
cfg = build_chv_config({}, {})
|
||||
self.assertIsNone(cfg.task_timeout)
|
||||
|
||||
def test_preflight_rejects_tampered_attributes(self):
|
||||
for attr, value in (("cpus", "0"), ("cpus", "-1"),
|
||||
("memory", "0"), ("task_timeout", 0),
|
||||
("task_timeout", -3)):
|
||||
ex = self.make_exec()
|
||||
setattr(ex, attr, value)
|
||||
with self.assertRaises(ChvFailure) as ctx:
|
||||
ex.preflight()
|
||||
self.assertEqual(ctx.exception.kind, "config_invalid")
|
||||
|
||||
|
||||
# -- P1-08:剪枝审计与快照失败审计 -----------------------------------------
|
||||
|
||||
class SnapshotAuditP1FixTest(P1FixCase):
|
||||
def test_retention_prune_delete_failure_audited(self):
|
||||
ex = self.make_exec()
|
||||
vm_dir = os.path.join(self.state_dir, "chv-abc123")
|
||||
os.makedirs(vm_dir, exist_ok=True)
|
||||
old = os.path.join(self.state_dir, "old.snap")
|
||||
with open(old, "wb") as fh:
|
||||
fh.write(b"O" * 10)
|
||||
with open(os.path.join(vm_dir, "snapshots.json"), "w") as fh:
|
||||
json.dump([{"url": "file://" + old, "path": old,
|
||||
"size": 10, "created_at": 0}], fh)
|
||||
new = os.path.join(self.state_dir, "new.snap")
|
||||
with open(new, "wb") as fh:
|
||||
fh.write(b"N" * 10)
|
||||
real_remove = os.remove
|
||||
|
||||
def remove(path):
|
||||
if path == old:
|
||||
raise OSError("删不掉")
|
||||
return real_remove(path)
|
||||
|
||||
with mock.patch.object(os, "remove", side_effect=remove):
|
||||
out = ex.snapshot_vm(self._task(), "file://" + new)
|
||||
self.assertEqual(out, "file://" + new)
|
||||
self.assertIn("vm_snapshot_prune_failed",
|
||||
self._audit_events())
|
||||
self.assertTrue(os.path.exists(old))
|
||||
|
||||
def test_snapshot_failure_audited(self):
|
||||
def failing_runner(cmd, timeout=None):
|
||||
if cmd and cmd[0] == "ch-remote":
|
||||
raise RuntimeError("ch-remote 挂了")
|
||||
return FakeResult(0, "", "")
|
||||
|
||||
ex = self.make_exec(runner=failing_runner)
|
||||
# 目标须在 state_dir 内才过白名单(ch-remote 失败在后)。
|
||||
target = "file://" + os.path.join(self.state_dir, "x.snap")
|
||||
with self.assertRaises(RuntimeError):
|
||||
ex.snapshot_vm(self._task(), target)
|
||||
self.assertIn("vm_snapshot_failed", self._audit_events())
|
||||
|
||||
|
||||
# -- P1-09:4 处三元 else 分支 ----------------------------------------------
|
||||
|
||||
class BranchCoverP1FixTest(P1FixCase):
|
||||
def test_pause_resume_snapshot_str_forms(self):
|
||||
ex = self.make_exec()
|
||||
os.makedirs(self.state_dir, exist_ok=True)
|
||||
self.assertTrue(ex.pause_vm("chv-abc"))
|
||||
self.assertTrue(ex.resume_vm("chv-abc"))
|
||||
snap = os.path.join(self.state_dir, "snap.raw")
|
||||
with open(snap, "wb") as fh:
|
||||
fh.write(b"S" * 10)
|
||||
out = ex.snapshot_vm("chv-abc", "file://" + snap)
|
||||
self.assertEqual(out, "file://" + snap)
|
||||
self.assertIn("vm_pause", self._audit_events())
|
||||
self.assertIn("vm_resume", self._audit_events())
|
||||
self.assertIn("vm_snapshot", self._audit_events())
|
||||
|
||||
def test_spill_audit_non_dict_task(self):
|
||||
ex = self.make_exec()
|
||||
os.makedirs(self.state_dir, exist_ok=True)
|
||||
ex._spill_audit("ev", "plain-task-id", "detail", None)
|
||||
path = os.path.join(self.state_dir, "chv-audit-spill.jsonl")
|
||||
with open(path) as fh:
|
||||
row = json.loads(fh.readline())
|
||||
self.assertEqual(row["task_id"], "plain-task-id")
|
||||
self.assertEqual(row["event"], "ev")
|
||||
|
||||
|
||||
# -- 覆盖补齐:新分支的行覆盖(诚实口径,不凑数改代码) --------
|
||||
|
||||
class CoverageCompletionP1FixTest(P1FixCase):
|
||||
def test_prune_events_no_file_and_no_delete(self):
|
||||
from cloud.chv.disk import _save_snapshots
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
_save_snapshots(tmp, "vm1", [
|
||||
{"url": "file:///gone.snap", "path": "/gone.snap",
|
||||
"size": 5, "created_at": 0},
|
||||
{"url": "https://example.invalid/x", "path": None,
|
||||
"size": 5, "created_at": 0}])
|
||||
events = []
|
||||
kept, pruned = prune_snapshots(
|
||||
tmp, "vm1", retention_days=30, now=10 ** 9,
|
||||
events=events)
|
||||
self.assertEqual(kept, [])
|
||||
self.assertEqual(len(pruned), 2)
|
||||
kinds = [e[0] for e in events]
|
||||
self.assertEqual(kinds, ["pruned_no_file", "pruned_no_file"])
|
||||
events2 = []
|
||||
_save_snapshots(tmp, "vm1", [
|
||||
{"url": "file:///gone2.snap", "path": "/gone2.snap",
|
||||
"size": 5, "created_at": 0}])
|
||||
kept2, _ = prune_snapshots(
|
||||
tmp, "vm1", retention_days=30, now=10 ** 9,
|
||||
delete_files=False, events=events2)
|
||||
self.assertEqual(kept2, [])
|
||||
self.assertEqual([e[0] for e in events2],
|
||||
["pruned_no_file"])
|
||||
|
||||
def test_record_without_cap_and_tighten_delete_failure_event(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
entries, removed = record_snapshot(
|
||||
tmp, "vm1", "https://example.invalid/new", None)
|
||||
self.assertEqual(len(entries), 1)
|
||||
self.assertEqual(removed, [])
|
||||
old = os.path.join(tmp, "vm1", "old.snap")
|
||||
os.makedirs(os.path.join(tmp, "vm1"), exist_ok=True)
|
||||
with open(old, "wb") as fh:
|
||||
fh.write(b"O" * 100)
|
||||
from cloud.chv.disk import _save_snapshots
|
||||
_save_snapshots(tmp, "vm1", [
|
||||
{"url": "file://" + old, "path": old, "size": 100,
|
||||
"created_at": 0}])
|
||||
real_remove = os.remove
|
||||
|
||||
def remove(path):
|
||||
if path == old:
|
||||
raise OSError("删不掉")
|
||||
return real_remove(path)
|
||||
|
||||
events = []
|
||||
with mock.patch.object(os, "remove", side_effect=remove):
|
||||
entries2, removed2 = record_snapshot(
|
||||
tmp, "vm1", "https://example.invalid/new2",
|
||||
0.0, events=events)
|
||||
self.assertEqual(len(removed2), 1)
|
||||
self.assertEqual([e[0] for e in events], ["delete_failed"])
|
||||
self.assertTrue(os.path.exists(old))
|
||||
|
||||
def test_tighten_no_file_entry_event(self):
|
||||
from cloud.chv.disk import _save_snapshots
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
_save_snapshots(tmp, "vm1", [
|
||||
{"url": "https://example.invalid/old", "path": None,
|
||||
"size": 100, "created_at": 0}])
|
||||
events = []
|
||||
entries, removed = record_snapshot(
|
||||
tmp, "vm1", "https://example.invalid/new",
|
||||
0.0, events=events)
|
||||
self.assertEqual(len(removed), 1)
|
||||
self.assertEqual([e[0] for e in events],
|
||||
["pruned_no_file"])
|
||||
|
||||
def test_state_lock_ctx_refreshes_on_state_dir_change(self):
|
||||
ex = self.make_exec()
|
||||
old_lock = ex._state_lock
|
||||
new_dir = os.path.join(self.dir, "state2")
|
||||
ex.state_dir = new_dir
|
||||
self.assertIsNot(ex._state_lock_ctx(), old_lock)
|
||||
self.assertEqual(ex._state_lock.state_dir, new_dir)
|
||||
|
||||
def test_gc_and_secrets_listdir_failure_returns_empty(self):
|
||||
ex = self.make_exec()
|
||||
ex.state_dir = os.path.join(self.dir, "missing-dir")
|
||||
self.assertEqual(ex.gc_stale_disks(),
|
||||
{"deleted": [], "failed": []})
|
||||
self.assertEqual(ex.cleanup_stale_secrets(), [])
|
||||
|
||||
def test_cleanup_listdir_failure_returns_empty(self):
|
||||
ex = self.make_exec()
|
||||
os.makedirs(self.state_dir, exist_ok=True)
|
||||
real_listdir = os.listdir
|
||||
calls = {"n": 0}
|
||||
|
||||
def listdir(path):
|
||||
calls["n"] += 1
|
||||
if calls["n"] >= 3:
|
||||
raise OSError("listdir 挂了")
|
||||
return real_listdir(path)
|
||||
|
||||
with mock.patch.object(os, "listdir", side_effect=listdir):
|
||||
self.assertEqual(ex.cleanup_stale_vms(), [])
|
||||
|
||||
def test_orphan_entries_compat_wrapper_and_no_emit_executor(self):
|
||||
from cloud.chv.orphans import _orphan_entries
|
||||
data = {"a": {"state": "warm", "vm": "chv-a",
|
||||
"task_id": "a", "snapshot_url": None},
|
||||
"b": {"state": "running", "vm": "chv-b"}}
|
||||
self.assertEqual(_orphan_entries(data),
|
||||
[{"vm": "chv-a", "task_id": "a",
|
||||
"snapshot_url": None}])
|
||||
|
||||
class NoEmit:
|
||||
def __init__(self, state_dir):
|
||||
self.state_dir = state_dir
|
||||
self.shutdowns = []
|
||||
|
||||
def shutdown_vm(self, vm):
|
||||
self.shutdowns.append(vm)
|
||||
|
||||
with tempfile.TemporaryDirectory() as d:
|
||||
_write_registry(d, {
|
||||
"a": {"state": "warm", "vm": "chv-a"}})
|
||||
ex = NoEmit(d)
|
||||
self.assertEqual(reconcile_orphans_for_executor(ex),
|
||||
["chv-a"])
|
||||
|
||||
def test_state_lock_flock_failure_closes_and_releases(self):
|
||||
import fcntl
|
||||
with tempfile.TemporaryDirectory() as d:
|
||||
state_dir = os.path.join(d, "state")
|
||||
lock = StateLock(state_dir)
|
||||
with mock.patch.object(fcntl, "flock",
|
||||
side_effect=OSError("flock 挂了")):
|
||||
with self.assertRaises(OSError):
|
||||
lock.acquire()
|
||||
self.assertIsNone(lock._fd)
|
||||
# 失败后锁仍可用(RLock 已释放、fd 已关闭)。
|
||||
lock.acquire()
|
||||
lock.release()
|
||||
|
||||
def test_pool_reconcile_failed_emit_failure_swallowed(self):
|
||||
from cloud.chv.experimental_pool import WarmPool
|
||||
stub = _PoolStubForP1Fix()
|
||||
pool = WarmPool(stub, registry_path=None)
|
||||
pool._entries["old-1"] = {"state": "orphan", "vm": "vm-old",
|
||||
"task": {"id": "old-1"}}
|
||||
self.assertEqual(pool.reconcile_orphans(), [])
|
||||
self.assertIn("old-1", pool._entries)
|
||||
|
||||
|
||||
class _PoolStubForP1Fix:
|
||||
"""实验池失败审计吞错测试用假执行器。"""
|
||||
|
||||
def shutdown_vm(self, task_or_name):
|
||||
raise RuntimeError("shutdown 失败")
|
||||
|
||||
def _emit(self, event, task, detail="", kind=None):
|
||||
raise RuntimeError("audit 失败")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -1,6 +1,6 @@
|
||||
# Cloud Hypervisor 执行器架构决策记录(ADR)
|
||||
|
||||
范围:cloud/chv/ 执行器链(#216 → #221 → #230 → #232 → #234 → #239 → #246 → #252 → #254 → 本轨 feat-chv-arch-maint-fe)。
|
||||
范围:cloud/chv/ 执行器链(#216 → #221 → #230 → #232 → #234 → #239 → #246 → #252 → #254 → feat-chv-arch-maint-fe → feat-chv-closeup → feat-chv-cover100 → 本轨 feat-chv-p1fix)。
|
||||
本文件只记录已落代码的决策,不记录未验证为已验证;真机、压测、动态安全
|
||||
仍见 docs/chv-production-readiness.md §11。
|
||||
|
||||
@@ -323,3 +323,35 @@
|
||||
缺 268 → 242);未覆盖大头仍是 executor 深层生命周期
|
||||
分支(需假 VMM 全链路)与 disk 配额分支,不靠删测试
|
||||
或改分母凑数。
|
||||
|
||||
## ADR-17 P1 修复轨语义收紧(feat-chv-p1fix,生产级审核 §5)
|
||||
|
||||
本轨只修审核报告 §5 的 P1 十项(P2 不动),决策均为
|
||||
“失败可见、猜测收紧、竞态加锁”,不放宽任何既有安全语义:
|
||||
|
||||
- 失败不算成功:孤儿清理失败项不进 cleaned、不清库、
|
||||
落盘保留失败项(P1-01);残留私钥删失败不进 cleaned、
|
||||
发审计并写脏标记(P1-03);单条超帽快照拒绝记录而不
|
||||
是悄悄保留(P1-06);快照失败与剪枝删失败均发审计
|
||||
(P1-08)。“尽力清理、不因单条失败卡住整批”的旧语义
|
||||
被修正为“尽力清理、失败项留痕可重试”——整批仍不卡,
|
||||
但失败不再被成功名单掩盖。
|
||||
- 损坏与非法不静默:registry 损坏与非法条目均发审计并
|
||||
计数(P1-02);非法条目落盘时丢弃(已审计,不再每次
|
||||
重启重复审计)。
|
||||
- 不猜目标:SSH 命令构造删除回环回落,失败即抛
|
||||
(P1-04);显式回环(CHV_SSH_HOST / CHV_SSH_FORWARD=1)
|
||||
仍可用,wake/wait 严格路径不受影响。
|
||||
- 竞态用文件锁:容量/net/磁盘 check→act 持
|
||||
`state_dir/chv-state.lock` 的 flock 全程(P1-05),
|
||||
拒绝“只加进程内锁就声称多 worker 已解决”的写法;
|
||||
串行 wake 是有意的代价,已运行 VM 的任务执行不持锁。
|
||||
- 非正配置在入口拦:CPU/内存/任务超时非正在构造层与
|
||||
preflight 层双重校验(P1-07);None 任务超时 = 不限
|
||||
的旧语义保留。
|
||||
- 覆盖口径诚实:行覆盖与分支覆盖分开报(P1-09);
|
||||
`__init__.py` 计入 coverage.py 分母(4 行);长函数
|
||||
50 行硬门禁靠提取维持,新增 helper 均有命名职责
|
||||
(`_snapshot_create`、`_tighten_snapshot_cap`、
|
||||
`_preflight_task_timeout_check`、`_emit_prune_events`、
|
||||
`_state_lock_ctx`),不是为凑行数而拆。
|
||||
@@ -580,6 +580,9 @@
|
||||
|
||||
### 14.1 本轨量化结果(实测,非上一轨回执转述)
|
||||
|
||||
(本节数字为 closeup 轨快照,历史数字保留不改;
|
||||
P1 修复轨后的最新数字见 §15,§14.1 不再是最新口径。)
|
||||
|
||||
- 后端:232 项 → 317 项(新增 `cloud/tests/test_chv_closeup.py`
|
||||
85 项),全过。命令(仓库根目录,cwd 须为根目录——
|
||||
在 `cloud/` 下跑会撞 `cloud/queue.py` 遮蔽 stdlib queue,
|
||||
@@ -761,6 +764,10 @@
|
||||
|
||||
### 14.7 本轨逐项评分(目标 10,无一项硬写 10)
|
||||
|
||||
(本节评分为 closeup 轨快照;P1 修复轨后的评分更新见
|
||||
§15.7,其中“80 行未覆”“317 项/36 项”扣分理由已过期,
|
||||
不再作为当前扣分依据。)
|
||||
|
||||
- 代码质量 9.8 → 9.8,未达 10:executor/worker_adapter
|
||||
覆盖到 100%,但 chv 仍有 80 行未覆(非 executor
|
||||
模块),且默认口径 mypy 余 2 错在作用域外。差:代码内。
|
||||
@@ -785,3 +792,194 @@
|
||||
UNSTABLE、未合并、未部署、真机 0 项。差:外部为主。
|
||||
- 动态安全 2.0 → 2.0,未达 10:本轨无任何动态验证,
|
||||
V-7 未跑。差:外部(KVM 宿主 + 审计结论)。
|
||||
## 15. P1 修复轨 feat-chv-p1fix(堆叠于 #270 之上)
|
||||
|
||||
本节是工程生产级审核(`~/workspace/chv-prod-audit/REPORT.md`,
|
||||
审核对象 #270,提交 `8c8d62f`)§5 的 P1-01~P1-10 修复轨。
|
||||
本轨只到 PR OPEN,不合并、不部署、不做合并试验(P0-01
|
||||
仍未做,见 §15.6);P2-01~P2-21 一律不动。
|
||||
|
||||
### 15.1 本轨量化结果(实测)
|
||||
|
||||
- 后端:362 项 → 389 项(新增 `cloud/tests/test_chv_p1fix.py`
|
||||
27 项),全过。命令(仓库根目录):
|
||||
`python3 -m unittest cloud.tests.test_chv_arch95
|
||||
cloud.tests.test_chv_arch95plus cloud.tests.test_chv_arch_maint_fe
|
||||
cloud.tests.test_chv_fix cloud.tests.test_chv_gaps
|
||||
cloud.tests.test_chv_optimize cloud.tests.test_chv_secfix
|
||||
cloud.tests.test_chv_to10 cloud.tests.test_cloud_hypervisor
|
||||
cloud.tests.test_chv_closeup cloud.tests.test_chv_cover100
|
||||
cloud.tests.test_chv_p1fix`
|
||||
- 覆盖率(同命令 + `coverage run --source=cloud/chv`,再
|
||||
`--append test_cloud_hypervisor`,`.coverage` 写 /tmp):
|
||||
行覆盖 100%(2097 行,缺 0);分支覆盖 99%
|
||||
(714 分支,28 处部分分支未全走——行 100% 不等于
|
||||
分支 100%,PR 描述按此口径写,不写裸“100%”)。
|
||||
`cloud/chv/__init__.py` 计入分母:4 行,100%(仅
|
||||
docstring + 3 个 re-export,coverage.py 口径;审核报告
|
||||
的 trace 口径把它排除在外,两口径差在此说明,不混用)。
|
||||
审核报告 P1-09 点名的 4 处三元 else 分支(str 形态
|
||||
pause/resume/snapshot + 非 dict spill task)已由
|
||||
`test_chv_p1fix.py` 的 BranchCover 测试覆盖。
|
||||
- mypy:silent 口径 `cloud/chv/` 0 错(不变);
|
||||
默认口径 `cloud/marker_tools.py` 仍 2 错(作用域外,
|
||||
本轨未改该文件)。
|
||||
- ruff:`--select F,E9,W,I` 口径 `cloud/chv/` +
|
||||
本轨新测试 0 错(不变;默认全规则基线不声称通过)。
|
||||
- 前端:Vitest 37 项不变(`chv.test.ts` 23 +
|
||||
`chv.closeup.test.ts` 14),本轨未改前端代码;三轮
|
||||
连跑 37/37/37,无抖动回归。
|
||||
- 全量:`python3 -m unittest discover -s cloud/tests`
|
||||
(仓库根目录)1819 项全过(skipped=8),392 秒;
|
||||
上一轨引用口径 1792 项的差额来自本轨新增 27 项。
|
||||
- 长函数:AST 扫描 `cloud/chv/`,>50 行函数 0 个
|
||||
(本轨新增的 snapshot_vm / record_snapshot /
|
||||
_preflight_numeric_checks 均经提取守住 50 行门禁,
|
||||
见 ADR-17)。
|
||||
- §13.1 的 main 头 `aebc6238` 与前端产物哈希
|
||||
`269820d1…f389` 仍是 to10 轨快照,本轨不更新它们:
|
||||
合并前必须按 §13.1 逐项重跑重核,绝不把旧快照写成
|
||||
新数字(审核报告 P1-10 的要求原样保留)。
|
||||
|
||||
### 15.2 P1 逐项修法(证据:文件与测试名)
|
||||
|
||||
- P1-01(orphans.py + experimental_pool.py):失败项不进
|
||||
cleaned、不触发清库;生产 reconcile 落盘只写回失败项
|
||||
原始条目并审计 `warm_pool_reconcile_failed`,下次可重试;
|
||||
实验池失败条目留在内存条目里(文件落盘语义属 P2-16,
|
||||
本轨不改)。同步改旧测试:`test_chv_arch_maint_fe.py`
|
||||
的 shutdown 失败用例与 `test_chv_cover100.py` 的
|
||||
`test_reconcile_shutdown_failure_still_cleans`(已改名
|
||||
`test_reconcile_shutdown_failure_not_cleaned`)——代码
|
||||
与测试同改,不存在“只改一侧”。
|
||||
新测试:`ReconcileP1FixTest.test_failure_not_cleaned_registry_retained_and_retryable`。
|
||||
- P1-02(orphans.py):registry 损坏发
|
||||
`warm_pool_registry_corrupt` 审计(quarantine 后仍审计,
|
||||
不静默返回空);非法条目发
|
||||
`warm_pool_registry_invalid_entries` 审计并计数,落盘
|
||||
时丢弃已审计条目(不再每次重启重复审计)。
|
||||
新测试:`test_corrupt_registry_audited_not_silent`、
|
||||
`test_invalid_entries_audited_and_counted`。
|
||||
- P1-03(executor.py `_clean_one_vm_secrets` /
|
||||
`cleanup_stale_secrets`):返回值由 2 元组改为
|
||||
(removed_priv, removed_seed, failed),删失败发
|
||||
`host_secret_residue_delete_failed` 审计并写脏标记
|
||||
(下次 provision 见标记即 `vm_dirty_refused` 拒起),
|
||||
删失败的 VM 不进 cleaned(即使部分删掉也不算已清理)。
|
||||
旧测试 `test_clean_one_vm_secrets_remove_failures` 已
|
||||
同步改 3 元组解包与 failed 断言。
|
||||
- P1-04(executor.py):删除 `build_ssh_command` 的回环
|
||||
回落(构造失败即抛 `ssh_target_undeclared`),不设
|
||||
dry-run 例外参数——干跑同样不许猜目标;同款回落
|
||||
`_write_provisioned_known_hosts` 一并删除(wake 路径
|
||||
在 provision 前已在 `_wake_prechecks` 里严格解析,
|
||||
不受影响;wait 路径的 `capture_host_key` 本来就
|
||||
解析失败返回 False,同样严格)。显式回环仍可用:
|
||||
`CHV_SSH_HOST=127.0.0.1` 或 `CHV_SSH_FORWARD=1`。
|
||||
旧测试 `test_build_ssh_command_fallback_and_secret_not_delivered`
|
||||
与 `test_write_provisioned_known_hosts_fallback` 已
|
||||
同步改为抛错断言。
|
||||
- P1-05(新增 `cloud/chv/locking.py` + executor.py):
|
||||
选文件锁方案(非单 worker 声明):`StateLock` =
|
||||
`flock`(`state_dir/chv-state.lock`,永不删除)+
|
||||
进程内 RLock 可重入(同一实例嵌套不自锁;多实例/
|
||||
多进程靠 flock 互斥)。持有范围:wake_vm 全程、
|
||||
restore_vm 全程、cleanup_stale_vms 全程、orphans
|
||||
reconcile 全程;`_purge_dead_registry` 改为锁内快照
|
||||
键列表、逐项锁内读/删,`_stop_dead_socket_vms` 的
|
||||
登记读持锁。串行 wake 是有意的代价(容量门只有
|
||||
“查完立即占位”才成立,提前释放会在 socket 未绑定
|
||||
前留出计数窗口);已运行 VM 的任务执行不持此锁。
|
||||
新测试:`StateLockP1FixTest`(线程互斥、跨进程互斥、
|
||||
purge 并发变更加锁)、`WakeSerializationP1FixTest`
|
||||
(两路并发 wake 实测串行化)。
|
||||
- P1-06(disk.py `record_snapshot`):单条快照本身大于
|
||||
总量帽时,在任何落盘/删旧动作之前直接拒绝
|
||||
(`snapshot_quota_exceeded`),不悄悄保留超大快照;
|
||||
大小未知(size 为 None,如远端目标)时无法判定,
|
||||
沿用旧口径。新测试:
|
||||
`SingleSnapshotCapP1FixTest`(拒绝无副作用:旧文件
|
||||
与旧登记均在)。
|
||||
- P1-07(config.py + executor.py preflight):构造层
|
||||
拒绝能解析出的 CPU/内存非正值与显式任务超时非正值
|
||||
(`CHV_CPU`/`CHV_MEM`/`CHV_TASK_TIMEOUT`);preflight
|
||||
再验属性事后改写(CPU 0/负、内存 0、任务超时 0/负
|
||||
一律 `config_invalid` 干净失败,不进 VmConfig)。
|
||||
`None` 任务超时 = 不限(旧语义保留);解析不出的
|
||||
格式仍留给 preflight(构造时机分工不变)。
|
||||
新测试:`PositiveConfigP1FixTest`。
|
||||
- P1-08(disk.py + executor.py `snapshot_vm`):剪枝
|
||||
事件外发(`prune_snapshots` / `check_snapshot_limits` /
|
||||
`record_snapshot` 均接受 `events` 列表:deleted /
|
||||
delete_failed / pruned_no_file);`snapshot_vm` 据此发
|
||||
`vm_snapshot_pruned` / `vm_snapshot_prune_failed`
|
||||
审计,失败路径(白名单/上限/ch-remote/落盘校验/
|
||||
登记拒绝)统一发 `vm_snapshot_failed` 审计再抛。
|
||||
新测试:`SnapshotAuditP1FixTest`。
|
||||
- P1-09:见 §15.1(行 100% / 分支 99% 分开写;
|
||||
`__init__.py` 计入分母说明;4 分支测试在
|
||||
`BranchCoverP1FixTest`)。
|
||||
- P1-10:本节即同步结果;§14.1/§14.7 已加“已被 §15
|
||||
取代”指针(历史数字保留不改);架构文档头范围与
|
||||
ADR-17 见 `docs/chv-architecture.md`;§13.1 快照数
|
||||
字按要求保留、合并前重跑(见 §15.1 末)。
|
||||
|
||||
### 15.3 安全语义未放宽清单
|
||||
|
||||
egress 默认 fail-closed、key seed 投递(provisioned 才
|
||||
投递、keyscan 下密钥 fail-closed)、磁盘删除(删失败
|
||||
审计 + 脏标记)、默认 provisioned、tap/mac/ip 校验、
|
||||
StrictHostKeyChecking 钉死、网关 loud fail 守卫——
|
||||
本轨一律未改动,只收紧(回落删除、失败可见性、
|
||||
check→act 加锁)。前端无改动,不存在绕后端白名单
|
||||
的新路径。
|
||||
|
||||
### 15.4 新增与改动文件
|
||||
|
||||
- 新增:`cloud/chv/locking.py`(StateLock)、
|
||||
`cloud/tests/test_chv_p1fix.py`(27 项)。
|
||||
- 改动:`cloud/chv/orphans.py`、`cloud/chv/disk.py`、
|
||||
`cloud/chv/executor.py`、`cloud/chv/experimental_pool.py`、
|
||||
`cloud/chv/config.py`;旧测试同步改:
|
||||
`cloud/tests/test_chv_arch_maint_fe.py`、
|
||||
`cloud/tests/test_chv_closeup.py`、
|
||||
`cloud/tests/test_chv_cover100.py`;
|
||||
文档:本节 + `docs/chv-architecture.md`(头范围 +
|
||||
ADR-17)。
|
||||
|
||||
### 15.5 逐项完成情况
|
||||
|
||||
P1-01~P1-10 全部修完(每项均有新测试或同步改的旧
|
||||
测试,见 §15.2;无“修不动”的项)。未修完的部分:
|
||||
无。但“修完”只等于代码层完成,不等于生产就绪,
|
||||
见 §15.6。
|
||||
|
||||
### 15.6 仍未验证项(原样保留,不冒充)
|
||||
|
||||
- 合并试验 P0-01 仍未做(本轨禁令):隔离演练区以
|
||||
当前远端 HEAD 为目标的三方合并 + 全量测试未跑,
|
||||
直接合入仍可能以链侧文件覆盖 main 新增功能。
|
||||
- CI 仍 UNSTABLE(账单秒级失败,本轨未改 CI 配置)。
|
||||
- KVM 宿主 provision、客机镜像真实构建与发布未做;
|
||||
真机验证 V-1~V-9 全部未跑。
|
||||
- 前端 live 点击验证未做;渲染测试未做。
|
||||
- 旧安全审计 No-Go 未复审、未推翻。
|
||||
|
||||
### 15.7 本轨逐项评分更新(目标 10,无一项硬写 10)
|
||||
|
||||
- 代码质量 9.8 → 9.8,未达 10:P1 语义缺陷已修,但
|
||||
分支覆盖 99% 未满、长函数门禁靠提取维持(边界脆弱)。
|
||||
差:代码内(小)。
|
||||
- 测试 9.8 → 9.9,未达 10:后端 389 项、前端 37 项
|
||||
全过,行覆盖 100% 恢复且口径诚实分开写;但真机
|
||||
用例 0 项已跑、分支 28 处部分。差:外部(真机)为主。
|
||||
- 文档 9.9 → 9.9,未达 10:§15 与 ADR-17 已同步,但
|
||||
§13.1 仍是旧快照、真机证据表仍空。差:外部。
|
||||
- 架构 9.7 → 9.7,未达 10:新增 flock 锁是并发语义
|
||||
收紧而非架构重构,ADR 取舍不变。差:外部。
|
||||
- 前端 9.6 → 9.6,未达 10:本轨未改前端。差:代码内
|
||||
(有意不做)+ 外部(live)。
|
||||
- 生产就绪 5.5 → 5.5,未达 10:P0-01 未做、CI 未绿、
|
||||
未合并、未部署、真机 0 项。差:外部为主。
|
||||
- 动态安全 2.0 → 2.0,未达 10:本轨无任何动态验证。
|
||||
差:外部。
|
||||
Reference in new issue
Block a user