fix(chv): 生产级审核 P1-01～P1-10 修复（堆叠于 #270 之上）
This commit is contained in:
muse-agent committed
1 parent
8c8d62f70d
commit
971e054938
12 files changed
+1545
-120
No files matched your search
+110
-25
@@ -17,12 +17,19 @@ Registry 语义与旧 WarmPool._load_registry 完全一致
|
||||
VM，绝不发给任务，只能清理。
|
||||
- 损坏 fail-safe：文件不是合法 JSON / 顶层不是 dict 时，
|
||||
整个文件 quarantine 成 <path>.corrupt，清理从空开始；
|
||||
损坏本身发审计（feat-chv-p1fix，P1-02），绝不静默
|
||||
返回空——损坏文件的孤儿 VM 此前只能靠 socket 扫描。
|
||||
单条目结构非法（无 vm、或 state 不在 warm/assigned）
|
||||
时只丢该条，不连坐其它条目。
|
||||
- 清理后落盘为空 dict（原子 tmp + rename）；写失败只记
|
||||
日志警告，不炸 worker 启动/关闭。shutdown_vm 抛错时该
|
||||
条目仍计入已清理（旧 reconcile 语义：尽力清理、不因
|
||||
单条失败卡住整批），_emit 失败同样 best-effort。
|
||||
时只丢该条，不连坐其它条目；非法条目发审计并计数
|
||||
（feat-chv-p1fix，P1-02），落盘时不再写回。
|
||||
- 清理后落盘：全部成功时为空 dict；有失败项时只写回
|
||||
失败项的原始条目（原子 tmp + rename），下次 reconcile
|
||||
可重试，失败项绝不被“已清理”掩盖（feat-chv-p1fix，
|
||||
P1-01）。写失败只记日志警告，不炸 worker 启动/关闭。
|
||||
shutdown_vm 抛错的条目不进 cleaned、不触发“全清”，
|
||||
而是记 failed 列表并经 executor._emit 发审计；
|
||||
_emit 失败同样 best-effort。整个 reconcile 在
|
||||
state_dir 的 flock 下跑（见 locking.py），与 wake 互斥。
|
||||
"""
|
||||
|
||||
import json
|
||||
@@ -91,52 +98,130 @@ def _orphan_entries(data: dict) -> list[dict]:
|
||||
"""从原始 registry dict 里筛出可清理的上一世条目。
|
||||
|
||||
只留 state 为 warm/assigned 且有 vm 的条目；非法条目
|
||||
静默丢弃（不连坐）。返回条目为 {"vm", "task_id",
|
||||
不连坐其它条目（计数与审计见 reconcile 本体，不再
|
||||
静默丢弃）。返回条目为 {"vm", "task_id",
|
||||
"snapshot_url"} 的新 dict，不改原数据。"""
|
||||
entries: list[dict] = []
|
||||
for entry in data.values():
|
||||
for _key, entry in _orphan_entries_with_keys(data)[0]:
|
||||
entries.append(entry)
|
||||
return entries
|
||||
|
||||
|
||||
def _orphan_entries_with_keys(data: dict) -> tuple[list, list]:
|
||||
"""筛条目并保留原始 registry 键：返回（可清理条目，
|
||||
非法条目键）。
|
||||
|
||||
可清理条目为 (原始键, {"vm", "task_id",
|
||||
"snapshot_url"})；非法条目只记键（键本身不含密钥，
|
||||
可进审计 detail），原始值不进审计。"""
|
||||
entries: list = []
|
||||
invalid_keys: list = []
|
||||
for key, entry in data.items():
|
||||
if not isinstance(entry, dict):
|
||||
invalid_keys.append(key)
|
||||
continue
|
||||
if not entry.get("vm"):
|
||||
invalid_keys.append(key)
|
||||
continue
|
||||
if entry.get("state") not in ("warm", "assigned"):
|
||||
invalid_keys.append(key)
|
||||
continue
|
||||
entries.append({
|
||||
entries.append((key, {
|
||||
"vm": entry["vm"],
|
||||
"task_id": entry.get("task_id"),
|
||||
"snapshot_url": entry.get("snapshot_url"),
|
||||
})
|
||||
return entries
|
||||
}))
|
||||
return entries, invalid_keys
|
||||
|
||||
|
||||
def _best_effort_emit(executor: object, event: str, detail: str) -> None:
|
||||
"""经 executor._emit 发审计（best-effort，兼容只接受
|
||||
(event, task, detail) 三参的假执行器；无 _emit 时跳过）。"""
|
||||
emit = getattr(executor, "_emit", None)
|
||||
if not callable(emit):
|
||||
return
|
||||
try:
|
||||
emit(event, {"id": None}, detail)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
def reconcile_orphans_for_executor(executor: object) -> list[str]:
|
||||
"""生产路径唯一的残留清理入口：返回已清理的 VM 名列表。
|
||||
|
||||
只做清理：读 registry → 每个上一世条目尽力
|
||||
shutdown_vm → 落盘为空 → best-effort 审计事件。
|
||||
shutdown_vm → 落盘 → best-effort 审计事件。
|
||||
绝不构造 WarmPool、绝不调 prewarm/acquire/release，
|
||||
绝不把任何条目发给任务。无 registry 路径时返回 []。"""
|
||||
绝不把任何条目发给任务。无 registry 路径时返回 []。
|
||||
|
||||
feat-chv-p1fix（P1-01/P1-02/P1-05）语义：
|
||||
- shutdown 失败的条目不进 cleaned，记 failed 并审计，
|
||||
落盘时只写回失败项原始条目（可重试），绝不清库；
|
||||
- registry 损坏时 quarantine 后仍发审计，不静默返回空；
|
||||
- 非法条目发审计并计数，落盘时丢弃（已审计，不再
|
||||
每次重启重复审计同一批非法条目）；
|
||||
- 全程持 state_dir flock，与 wake/restore/cleanup 互斥。
|
||||
返回值仍只含 cleaned（调用方兼容不变）；失败与
|
||||
损坏/非法的可见性走 executor._emit（worker 接线后
|
||||
即任务 timeline 审计），不靠返回值传递。
|
||||
"""
|
||||
path = registry_path_for_executor(executor)
|
||||
if not path:
|
||||
return []
|
||||
data, _corrupt = _read_registry(path)
|
||||
if data is None:
|
||||
if not os.path.exists(path):
|
||||
return []
|
||||
entries = _orphan_entries(data)
|
||||
state_dir = getattr(executor, "state_dir", None)
|
||||
if state_dir:
|
||||
from cloud.chv.locking import StateLock
|
||||
lock = StateLock(state_dir)
|
||||
else: # pragma: no cover - state_dir 缺失时 path 已为 None
|
||||
from contextlib import nullcontext
|
||||
lock = nullcontext() # type: ignore[assignment]
|
||||
with lock:
|
||||
return _reconcile_locked(executor, path)
|
||||
|
||||
|
||||
def _reconcile_locked(executor: object, path: str) -> list[str]:
|
||||
"""持锁的 reconcile 本体（见 reconcile_orphans_for_executor）。"""
|
||||
data, corrupt = _read_registry(path)
|
||||
if data is None:
|
||||
if corrupt:
|
||||
_best_effort_emit(
|
||||
executor, "warm_pool_registry_corrupt",
|
||||
"path=%s registry corrupt, quarantined" % path)
|
||||
return []
|
||||
entries, invalid_keys = _orphan_entries_with_keys(data)
|
||||
if invalid_keys:
|
||||
_best_effort_emit(
|
||||
executor, "warm_pool_registry_invalid_entries",
|
||||
"invalid=%d keys=%s"
|
||||
% (len(invalid_keys), ",".join(str(k) for k in invalid_keys)))
|
||||
cleaned: list[str] = []
|
||||
for entry in entries:
|
||||
failed: list[str] = []
|
||||
failed_errors: list[str] = []
|
||||
failed_keys: list[str] = []
|
||||
for key, entry in entries:
|
||||
vm = entry["vm"]
|
||||
try:
|
||||
executor.shutdown_vm(vm) # type: ignore[attr-defined]
|
||||
except Exception:
|
||||
pass
|
||||
except Exception as exc:
|
||||
failed.append(vm)
|
||||
failed_keys.append(key)
|
||||
failed_errors.append("%s:%s" % (vm, type(exc).__name__))
|
||||
continue
|
||||
cleaned.append(vm)
|
||||
if cleaned or failed or invalid_keys:
|
||||
remaining = {key: data[key] for key in failed_keys
|
||||
if key in data}
|
||||
_write_registry(path, remaining)
|
||||
if cleaned:
|
||||
_write_registry(path, {})
|
||||
try:
|
||||
executor._emit( # type: ignore[attr-defined]
|
||||
"warm_pool_reconcile", {"id": None},
|
||||
"cleaned=%s" % ",".join(cleaned))
|
||||
except Exception:
|
||||
pass
|
||||
_best_effort_emit(
|
||||
executor, "warm_pool_reconcile",
|
||||
"cleaned=%s" % ",".join(cleaned))
|
||||
if failed:
|
||||
_best_effort_emit(
|
||||
executor, "warm_pool_reconcile_failed",
|
||||
"failed=%s cleaned=%s errors=%s"
|
||||
% (",".join(failed), ",".join(cleaned),
|
||||
";".join(failed_errors)))
|
||||
return cleaned
|
||||
Reference in new issue
Block a user