197 lines
8.1 KiB
Python
197 lines
8.1 KiB
Python
"""CHV worker 适配模块（to10 轨，见 ADR-13）。
|
||
|
||
worker.py 里散落的 CHV 强相关接线抽到这里，worker 只留
|
||
薄包装（方法名、签名、调用点不变，旧测试继续走 worker
|
||
方法）。本模块只依赖标准库 + cloud.chv 叶子模块 +
|
||
cloud.logging_config，不 import cloud.worker，绝无循环。
|
||
|
||
抽入本模块的（行为与错误文案逐字搬运）：
|
||
- call_goal_resume_breakpoint：通用断点调用（原 worker
|
||
顶层函数 _call_goal_resume_breakpoint 的本体）。
|
||
- make_chv_audit_callback：VM 生命周期事件进审计的回调
|
||
（原 Worker.__init__ 里的闭包本体）。
|
||
- reconcile_warm_pool：生产残留清理 facade 调用 + 审计
|
||
（原 Worker._reconcile_warm_pool 的本体）。
|
||
- cancel_watchdog_loop / quota_watchdog_loop：同步路径
|
||
两条看门狗循环（原 Worker._chv_*_watchdog 的本体，
|
||
self 依赖全部改成回调参数）。
|
||
- watchdog_vm_name：看门狗启动时的 VM 名派生（原
|
||
_run_attempt 里的 try/except 块）。
|
||
|
||
明确不抽的（见 ADR-13 诚实清单，worker 通用流程）：
|
||
cancel() 里的 cancel_vm 分支、cleanup_stale_containers
|
||
的 CHV 分支、网关守卫、_executor_name 的 CHV 分支、
|
||
_run_attempt 的看门狗线程启动、VM 死亡写回——它们都
|
||
嵌在 Docker/Local 共用的通用流程里，硬抽只会把通用
|
||
流程撕成两半，行为不变的收益为零。
|
||
"""
|
||
|
||
import inspect
|
||
import threading
|
||
from collections.abc import Callable
|
||
|
||
from cloud.chv.executor import CloudHypervisorExecutor
|
||
from cloud.logging_config import log_event
|
||
|
||
|
||
def is_chv_executor(executor: object) -> bool:
|
||
"""是否为 CHV 执行器（worker 各处 isinstance 的同款判断）。"""
|
||
return isinstance(executor, CloudHypervisorExecutor)
|
||
|
||
|
||
def call_goal_resume_breakpoint(executor: object, task: dict,
|
||
model_provider: str | None = None,
|
||
extra_env: dict | None = None) -> object:
|
||
"""通用断点调用：执行器提供 goal_resume_breakpoint_task
|
||
时调用它。
|
||
|
||
Local 仍走原路径（调用方直接传 model_provider，不经过
|
||
这里，以免改生产语义）；这里只用于非 Local 执行器：
|
||
签名接受 model_provider 时传它（CHV/Docker 均接受），
|
||
否则只传 task（旧假执行器单参）。feat-chv-secfix（H2）：
|
||
CHV 的 gateway_env 不再经 SSH env 前缀注入（密钥值进
|
||
argv 的旧路径已废除），改走 provisioned seed 盘客内文件
|
||
投递；CHV 没有 Local 那样的进程 env 通道，seed 文件是
|
||
唯一通道，keyscan 模式下密钥投递 fail-closed。"""
|
||
method = getattr(executor, "goal_resume_breakpoint_task")
|
||
try:
|
||
params = inspect.signature(method).parameters
|
||
except (TypeError, ValueError):
|
||
return method(task)
|
||
kwargs: dict = {}
|
||
if "model_provider" in params:
|
||
kwargs["model_provider"] = model_provider
|
||
if "extra_env" in params and extra_env is not None:
|
||
kwargs["extra_env"] = extra_env
|
||
return method(task, **kwargs)
|
||
|
||
|
||
def make_chv_audit_callback(audit_fn: Callable) -> Callable:
|
||
"""VM 生命周期事件进审计的回调（原 __init__ 闭包本体）。
|
||
|
||
audit_fn 即 Worker._audit（best-effort，失败不影响任务）；
|
||
时间线展示 VM 事件时读审计表即可，不另起一套 VM 事件表；
|
||
成本/配额仍走 worker 现有 quota 语义，本回调绝不记账。"""
|
||
|
||
def _chv_audit(event, task, detail, kind=None):
|
||
try:
|
||
project_id = task.get("project_id") \
|
||
if isinstance(task, dict) else None
|
||
audit_fn(project_id, "chv_vm_" + event, detail or "")
|
||
except Exception:
|
||
pass
|
||
|
||
return _chv_audit
|
||
|
||
|
||
def reconcile_warm_pool(executor: object, audit_fn: Callable,
|
||
when: str) -> list:
|
||
"""显式调生产残留清理 facade，返回已清理 VM 名。
|
||
|
||
仅 CHV worker 有效（其它执行器返回 []）。实现是
|
||
cloud.chv.orphans.reconcile_orphans_for_executor()：
|
||
直接读 registry 文件做清理，不构造 WarmPool、不碰
|
||
实验接口。调用失败不炸主流程，但进审计（失败本身
|
||
也审计，不静默丢弃）。orphan 条目绝不发给任务，
|
||
reconcile 只做清理。"""
|
||
if not isinstance(executor, CloudHypervisorExecutor):
|
||
return []
|
||
try:
|
||
from cloud.chv.orphans import reconcile_orphans_for_executor as _reconcile
|
||
cleaned = _reconcile(executor) or []
|
||
except Exception as exc:
|
||
try:
|
||
audit_fn(None, "chv_warm_pool_reconcile_failed",
|
||
"when=%s error=%s" % (when, exc))
|
||
except Exception:
|
||
pass
|
||
return []
|
||
if cleaned:
|
||
try:
|
||
audit_fn(None, "chv_warm_pool_reconcile",
|
||
"when=%s cleaned=%s" % (when, ",".join(cleaned)))
|
||
except Exception:
|
||
pass
|
||
return cleaned
|
||
|
||
|
||
def cancel_watchdog_loop(task_id: str, vm_name: str | None,
|
||
stop_event: threading.Event,
|
||
cancel_requested: set,
|
||
get_task: Callable,
|
||
cancel_vm: Callable | None,
|
||
logger: object) -> None:
|
||
"""CHV 同步路径的取消看门狗（feat-chv-fix，P1-2，Docker
|
||
取消看门狗同款模式）：同步 SSH 阻塞时 worker 的 cancel()
|
||
只能改库、清 _running 项，阻塞中的 run 本身不会醒；
|
||
本线程每 1 秒查库/取消标志，命中时调 cancel_vm 真关
|
||
VM，然后退出。查库失败时当没取消，不误杀。"""
|
||
while not stop_event.wait(1.0):
|
||
try:
|
||
if task_id in cancel_requested:
|
||
cancelled = True
|
||
else:
|
||
db_task = get_task(task_id)
|
||
cancelled = (db_task is not None and
|
||
db_task.get("status") == "interrupted")
|
||
except Exception:
|
||
cancelled = False
|
||
if not cancelled:
|
||
continue
|
||
try:
|
||
if cancel_vm is not None and vm_name:
|
||
cancel_vm(vm_name)
|
||
log_event(logger, "worker_chv_killed",
|
||
task_id=task_id,
|
||
message="cancel watchdog killed VM %s "
|
||
"(task interrupted/cancelled)"
|
||
% vm_name)
|
||
except Exception:
|
||
pass
|
||
return
|
||
|
||
|
||
def quota_watchdog_loop(task_id: str, vm_name: str | None,
|
||
stop_event: threading.Event,
|
||
quota_check: Callable | None,
|
||
mark_quota_hit: Callable,
|
||
cancel_vm: Callable | None,
|
||
logger: object,
|
||
interval: float = 30.0) -> None:
|
||
"""CHV 同步路径的配额看门狗（feat-chv-fix，P1-2，Docker
|
||
配额看门狗同款）：命中时记 _quota_hit_tasks 并调
|
||
cancel_vm 真关 VM；_run_attempt 在同步返回后把结果
|
||
改写成 quota_exceeded，走统一配额中断收尾。"""
|
||
while not stop_event.wait(interval):
|
||
try:
|
||
over = bool(quota_check()) if quota_check else False
|
||
except Exception:
|
||
over = False
|
||
if not over:
|
||
continue
|
||
mark_quota_hit(task_id)
|
||
try:
|
||
if cancel_vm is not None and vm_name:
|
||
cancel_vm(vm_name)
|
||
log_event(logger, "worker_chv_killed",
|
||
task_id=task_id,
|
||
message="quota watchdog killed VM %s "
|
||
"(quota exceeded)" % vm_name)
|
||
except Exception:
|
||
pass
|
||
return
|
||
|
||
|
||
def watchdog_vm_name(executor: object, task: dict) -> str | None:
|
||
"""看门狗启动时的 VM 名派生（原 _run_attempt 块）：
|
||
executor.vm_name(task) 抛错时回 None（不建看门狗），
|
||
绝不炸主派发流程。executor 无 vm_name 时等同抛错
|
||
（原代码里 AttributeError 同样被 except 吞掉回 None）。"""
|
||
get_vm_name = getattr(executor, "vm_name", None)
|
||
if not callable(get_vm_name):
|
||
return None
|
||
try:
|
||
return get_vm_name(task)
|
||
except Exception:
|
||
return None
|