"""CHV worker 适配模块(to10 轨,见 ADR-13)。 worker.py 里散落的 CHV 强相关接线抽到这里,worker 只留 薄包装(方法名、签名、调用点不变,旧测试继续走 worker 方法)。本模块只依赖标准库 + cloud.chv 叶子模块 + cloud.logging_config,不 import cloud.worker,绝无循环。 抽入本模块的(行为与错误文案逐字搬运): - call_goal_resume_breakpoint:通用断点调用(原 worker 顶层函数 _call_goal_resume_breakpoint 的本体)。 - make_chv_audit_callback:VM 生命周期事件进审计的回调 (原 Worker.__init__ 里的闭包本体)。 - reconcile_warm_pool:生产残留清理 facade 调用 + 审计 (原 Worker._reconcile_warm_pool 的本体)。 - cancel_watchdog_loop / quota_watchdog_loop:同步路径 两条看门狗循环(原 Worker._chv_*_watchdog 的本体, self 依赖全部改成回调参数)。 - watchdog_vm_name:看门狗启动时的 VM 名派生(原 _run_attempt 里的 try/except 块)。 明确不抽的(见 ADR-13 诚实清单,worker 通用流程): cancel() 里的 cancel_vm 分支、cleanup_stale_containers 的 CHV 分支、网关守卫、_executor_name 的 CHV 分支、 _run_attempt 的看门狗线程启动、VM 死亡写回——它们都 嵌在 Docker/Local 共用的通用流程里,硬抽只会把通用 流程撕成两半,行为不变的收益为零。 """ import inspect import threading from collections.abc import Callable from cloud.chv.executor import CloudHypervisorExecutor from cloud.logging_config import log_event def is_chv_executor(executor: object) -> bool: """是否为 CHV 执行器(worker 各处 isinstance 的同款判断)。""" return isinstance(executor, CloudHypervisorExecutor) def call_goal_resume_breakpoint(executor: object, task: dict, model_provider: str | None = None, extra_env: dict | None = None) -> object: """通用断点调用:执行器提供 goal_resume_breakpoint_task 时调用它。 Local 仍走原路径(调用方直接传 model_provider,不经过 这里,以免改生产语义);这里只用于非 Local 执行器: 签名接受 model_provider 时传它(CHV/Docker 均接受), 否则只传 task(旧假执行器单参)。feat-chv-secfix(H2): CHV 的 gateway_env 不再经 SSH env 前缀注入(密钥值进 argv 的旧路径已废除),改走 provisioned seed 盘客内文件 投递;CHV 没有 Local 那样的进程 env 通道,seed 文件是 唯一通道,keyscan 模式下密钥投递 fail-closed。""" method = getattr(executor, "goal_resume_breakpoint_task") try: params = inspect.signature(method).parameters except (TypeError, ValueError): return method(task) kwargs: dict = {} if "model_provider" in params: kwargs["model_provider"] = model_provider if "extra_env" in params and extra_env is not None: kwargs["extra_env"] = extra_env return method(task, **kwargs) def make_chv_audit_callback(audit_fn: Callable) -> Callable: """VM 生命周期事件进审计的回调(原 __init__ 闭包本体)。 audit_fn 即 Worker._audit(best-effort,失败不影响任务); 时间线展示 VM 事件时读审计表即可,不另起一套 VM 事件表; 成本/配额仍走 worker 现有 quota 语义,本回调绝不记账。""" def _chv_audit(event, task, detail, kind=None): try: project_id = task.get("project_id") \ if isinstance(task, dict) else None audit_fn(project_id, "chv_vm_" + event, detail or "") except Exception: pass return _chv_audit def reconcile_warm_pool(executor: object, audit_fn: Callable, when: str) -> list: """显式调生产残留清理 facade,返回已清理 VM 名。 仅 CHV worker 有效(其它执行器返回 [])。实现是 cloud.chv.orphans.reconcile_orphans_for_executor(): 直接读 registry 文件做清理,不构造 WarmPool、不碰 实验接口。调用失败不炸主流程,但进审计(失败本身 也审计,不静默丢弃)。orphan 条目绝不发给任务, reconcile 只做清理。""" if not isinstance(executor, CloudHypervisorExecutor): return [] try: from cloud.chv.orphans import reconcile_orphans_for_executor as _reconcile cleaned = _reconcile(executor) or [] except Exception as exc: try: audit_fn(None, "chv_warm_pool_reconcile_failed", "when=%s error=%s" % (when, exc)) except Exception: pass return [] if cleaned: try: audit_fn(None, "chv_warm_pool_reconcile", "when=%s cleaned=%s" % (when, ",".join(cleaned))) except Exception: pass return cleaned def cancel_watchdog_loop(task_id: str, vm_name: str | None, stop_event: threading.Event, cancel_requested: set, get_task: Callable, cancel_vm: Callable | None, logger: object) -> None: """CHV 同步路径的取消看门狗(feat-chv-fix,P1-2,Docker 取消看门狗同款模式):同步 SSH 阻塞时 worker 的 cancel() 只能改库、清 _running 项,阻塞中的 run 本身不会醒; 本线程每 1 秒查库/取消标志,命中时调 cancel_vm 真关 VM,然后退出。查库失败时当没取消,不误杀。""" while not stop_event.wait(1.0): try: if task_id in cancel_requested: cancelled = True else: db_task = get_task(task_id) cancelled = (db_task is not None and db_task.get("status") == "interrupted") except Exception: cancelled = False if not cancelled: continue try: if cancel_vm is not None and vm_name: cancel_vm(vm_name) log_event(logger, "worker_chv_killed", task_id=task_id, message="cancel watchdog killed VM %s " "(task interrupted/cancelled)" % vm_name) except Exception: pass return def quota_watchdog_loop(task_id: str, vm_name: str | None, stop_event: threading.Event, quota_check: Callable | None, mark_quota_hit: Callable, cancel_vm: Callable | None, logger: object, interval: float = 30.0) -> None: """CHV 同步路径的配额看门狗(feat-chv-fix,P1-2,Docker 配额看门狗同款):命中时记 _quota_hit_tasks 并调 cancel_vm 真关 VM;_run_attempt 在同步返回后把结果 改写成 quota_exceeded,走统一配额中断收尾。""" while not stop_event.wait(interval): try: over = bool(quota_check()) if quota_check else False except Exception: over = False if not over: continue mark_quota_hit(task_id) try: if cancel_vm is not None and vm_name: cancel_vm(vm_name) log_event(logger, "worker_chv_killed", task_id=task_id, message="quota watchdog killed VM %s " "(quota exceeded)" % vm_name) except Exception: pass return def watchdog_vm_name(executor: object, task: dict) -> str | None: """看门狗启动时的 VM 名派生(原 _run_attempt 块): executor.vm_name(task) 抛错时回 None(不建看门狗), 绝不炸主派发流程。executor 无 vm_name 时等同抛错 (原代码里 AttributeError 同样被 except 吞掉回 None)。""" get_vm_name = getattr(executor, "vm_name", None) if not callable(get_vm_name): return None try: return get_vm_name(task) except Exception: return None