feat: CHV gap fill — worker breakpoint wiring, pool reconcile, per-VM host keys, egress validation, guest build script
Stack: #216 -> #221 -> this. Worker breakpoint branch now calls any executor providing goal_resume_breakpoint_task (Local keeps its original path, gateway tasks still loud-fail); WarmPool reconcile_orphans is called on CHV worker startup and close with audit-on-failure; per-VM SSH host keys are provisioned before VMM start via cloud-init NoCloud seed (keyscan kept only as weak fallback, CHV_HOST_KEY_MODE=provisioned|keyscan); CHV_NET_JSON is validated in preflight and explicit direct egress is audited; chv-guest/build.sh is now real steps with --dry-run and fails fast without /dev/kvm. Tests: new test_chv_gaps.py (18), cloud full 1514 OK (skipped=4). No real VM verification: no KVM host.
This commit is contained in:
1 parent
fde7bf4c3e
commit
22b83f118a
8 files changed
+922
-59
No files matched your search
+86
-1
@@ -120,6 +120,7 @@ Docker 下 repos_dir 必须放在 extra-mounts 已挂载的目录下,
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import inspect
|
||||
import json
|
||||
import os
|
||||
from datetime import datetime, timezone
|
||||
@@ -332,6 +333,24 @@ def _merge_env(base, override):
|
||||
return merged
|
||||
|
||||
|
||||
def _call_goal_resume_breakpoint(executor, task, model_provider=None):
|
||||
"""通用断点调用:执行器提供 goal_resume_breakpoint_task 时调用它。
|
||||
|
||||
Local 仍走原路径(调用方直接传 model_provider,不经过这里,
|
||||
以免改生产语义);这里只用于非 Local 执行器:签名接受
|
||||
model_provider 时传它(CHV/Docker 均接受),否则只传 task
|
||||
(旧假执行器单参)。网关 key 绝不经这里传值:worker 侧
|
||||
gateway_env 只按名注入,见 _run_attempt 的网关守卫。"""
|
||||
method = getattr(executor, "goal_resume_breakpoint_task")
|
||||
try:
|
||||
params = inspect.signature(method).parameters
|
||||
except (TypeError, ValueError):
|
||||
return method(task)
|
||||
if "model_provider" in params:
|
||||
return method(task, model_provider=model_provider)
|
||||
return method(task)
|
||||
|
||||
|
||||
def _resolve_model(model=None, config_path=None):
|
||||
"""模型名:显式参数 > host 配置文件里的 model > host 默认值。"""
|
||||
if model:
|
||||
@@ -525,6 +544,19 @@ class Worker:
|
||||
self.executor.audit_callback = _chv_audit
|
||||
except Exception:
|
||||
pass
|
||||
# CHV 补缺口轨:WarmPool 显式接线。池构造本身不自动
|
||||
# 清理(单活池假设,见 pool.py),worker 启动是显式
|
||||
# 入口,在这里建池并调 reconcile_orphans();关闭时
|
||||
# close() 再调一次。调用失败不炸 worker 构造/关闭,
|
||||
# 但要进审计(_audit 本身 best-effort)。
|
||||
self._warm_pool = None
|
||||
if isinstance(self.executor, CloudHypervisorExecutor):
|
||||
try:
|
||||
from cloud.chv.pool import WarmPool as _WarmPool
|
||||
self._warm_pool = _WarmPool(self.executor)
|
||||
self._reconcile_warm_pool("startup")
|
||||
except Exception:
|
||||
pass
|
||||
self._cost_store = cost_store
|
||||
self._cancel_requested = set()
|
||||
# 真取消用:process_factory 可注入(单测用它返回一个假长跑
|
||||
@@ -759,8 +791,49 @@ class Worker:
|
||||
self._cost_store = CostStore(self.db_path)
|
||||
return self._cost_store
|
||||
|
||||
def _reconcile_warm_pool(self, when):
|
||||
"""显式调 WarmPool.reconcile_orphans(),返回已清理 VM 名。
|
||||
|
||||
仅 CHV worker 且池已建时有效;池未建时若执行器是 CHV
|
||||
则现建现调(供测试与手动清理用)。调用失败不炸主流程,
|
||||
但进审计(失败本身也审计,不静默丢弃)。orphan 条目
|
||||
绝不发给任务,reconcile 只做清理。"""
|
||||
pool = getattr(self, "_warm_pool", None)
|
||||
if pool is None:
|
||||
executor = getattr(self, "executor", None)
|
||||
if not isinstance(executor, CloudHypervisorExecutor):
|
||||
return []
|
||||
try:
|
||||
from cloud.chv.pool import WarmPool as _WarmPool
|
||||
pool = _WarmPool(executor)
|
||||
self._warm_pool = pool
|
||||
except Exception:
|
||||
return []
|
||||
try:
|
||||
cleaned = pool.reconcile_orphans() or []
|
||||
except Exception as exc:
|
||||
try:
|
||||
self._audit(None, "chv_warm_pool_reconcile_failed",
|
||||
"when=%s error=%s" % (when, exc))
|
||||
except Exception:
|
||||
pass
|
||||
return []
|
||||
if cleaned:
|
||||
try:
|
||||
self._audit(None, "chv_warm_pool_reconcile",
|
||||
"when=%s cleaned=%s" % (when, ",".join(cleaned)))
|
||||
except Exception:
|
||||
pass
|
||||
return cleaned
|
||||
|
||||
def close(self):
|
||||
"""关闭惰性建的成本存储连接(注入的 cost_store 由调用方管)。"""
|
||||
"""关闭惰性建的成本存储连接(注入的 cost_store 由调用方管)。
|
||||
CHV worker 关闭时先显式调一次 warm-pool reconcile,
|
||||
失败不炸关闭流程。"""
|
||||
try:
|
||||
self._reconcile_warm_pool("shutdown")
|
||||
except Exception:
|
||||
pass
|
||||
if self._cost_store is not None:
|
||||
try:
|
||||
self._cost_store._conn.close()
|
||||
@@ -4712,6 +4785,18 @@ class Worker:
|
||||
self.executor, LocalExecutor):
|
||||
result = self.executor.goal_resume_breakpoint_task(
|
||||
task, model_provider=gateway_provider)
|
||||
elif task.get("_goal_resume_breakpoint") and hasattr(
|
||||
self.executor,
|
||||
"goal_resume_breakpoint_task"):
|
||||
# CHV 补缺口轨:任何提供断点方法的执行器
|
||||
# 都可走断点(CHV 的 goal_resume_breakpoint_task
|
||||
# 已接线;Docker 走上面 is_docker 分支,不到
|
||||
# 这里)。网关任务在进入这里之前已被
|
||||
# “网关不支持该执行器”守卫 loud fail,
|
||||
# 故这里不放宽网关规则。
|
||||
result = _call_goal_resume_breakpoint(
|
||||
self.executor, task,
|
||||
model_provider=gateway_provider)
|
||||
elif task.get("_goal_resume_breakpoint"):
|
||||
# 假执行器/CloudExecutor 没有断点方法时
|
||||
# 绝不回落成 goal(会撞已存在)或
|
||||
|
||||
Reference in new issue
Block a user