feat-chv-to10:往 10 分方向提升(类型/lint 清零、worker 适配抽取、审计落盘、lockfile 生成脚本、前端四态与统一文案、就绪清单 §13)
This commit is contained in:
1 parent
df8c64bd64
commit
2f3ff8b3ec
22 files changed
+1400
-282
No files matched your search
+37
-117
@@ -120,7 +120,6 @@ Docker 下 repos_dir 必须放在 extra-mounts 已挂载的目录下,
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import inspect
|
||||
import json
|
||||
import os
|
||||
from datetime import datetime, timezone
|
||||
@@ -198,6 +197,9 @@ from cloud.queue import budget_exceeded, ready_tasks
|
||||
from cloud.repos import RepoError, checkout_dir, resolve_repo
|
||||
from cloud.sandbox import CloudExecutor, CloudHypervisorExecutor, \
|
||||
DockerExecutor, LocalExecutor
|
||||
# to10:CHV 强相关接线抽到 cloud.chv.worker_adapter,见 ADR-13;
|
||||
# worker 只留薄包装(方法名/签名/调用点不变)。
|
||||
from cloud.chv import worker_adapter as _chv_worker_adapter
|
||||
from cloud.uploads import upload_root as _upload_root
|
||||
from cloud.sandbox import _default_runner as _sandbox_default_runner
|
||||
from cloud.caps import effective_capacity, resolve_capacity
|
||||
@@ -335,27 +337,13 @@ def _merge_env(base, override):
|
||||
|
||||
def _call_goal_resume_breakpoint(executor, task, model_provider=None,
|
||||
extra_env=None):
|
||||
"""通用断点调用:执行器提供 goal_resume_breakpoint_task 时调用它。
|
||||
|
||||
Local 仍走原路径(调用方直接传 model_provider,不经过这里,
|
||||
以免改生产语义);这里只用于非 Local 执行器:签名接受
|
||||
model_provider 时传它(CHV/Docker 均接受),否则只传 task
|
||||
(旧假执行器单参)。feat-chv-secfix(H2):CHV 的
|
||||
gateway_env 不再经 SSH env 前缀注入(密钥值进 argv 的
|
||||
旧路径已废除),改走 provisioned seed 盘客内文件投递;
|
||||
CHV 没有 Local 那样的进程 env 通道,seed 文件是唯一通道,
|
||||
keyscan 模式下密钥投递 fail-closed。"""
|
||||
method = getattr(executor, "goal_resume_breakpoint_task")
|
||||
try:
|
||||
params = inspect.signature(method).parameters
|
||||
except (TypeError, ValueError):
|
||||
return method(task)
|
||||
kwargs = {}
|
||||
if "model_provider" in params:
|
||||
kwargs["model_provider"] = model_provider
|
||||
if "extra_env" in params and extra_env is not None:
|
||||
kwargs["extra_env"] = extra_env
|
||||
return method(task, **kwargs)
|
||||
"""通用断点调用(to10:本体已移到
|
||||
cloud.chv.worker_adapter.call_goal_resume_breakpoint,
|
||||
此处只留薄包装——旧测试从 cloud.worker import 它,名
|
||||
字与签名不变,行为逐字不变)。"""
|
||||
return _chv_worker_adapter.call_goal_resume_breakpoint(
|
||||
executor, task, model_provider=model_provider,
|
||||
extra_env=extra_env)
|
||||
|
||||
|
||||
def _resolve_model(model=None, config_path=None):
|
||||
@@ -539,16 +527,12 @@ class Worker:
|
||||
# 读审计表即可,不另起一套 VM 事件表;成本/配额仍走
|
||||
# worker 现有 quota 语义,本回调绝不记账。
|
||||
if isinstance(self.executor, CloudHypervisorExecutor):
|
||||
def _chv_audit(event, task, detail, kind=None):
|
||||
try:
|
||||
project_id = task.get("project_id") \
|
||||
if isinstance(task, dict) else None
|
||||
self._audit(project_id, "chv_vm_" + event,
|
||||
detail or "")
|
||||
except Exception:
|
||||
pass
|
||||
# to10:回调本体已移到 worker_adapter.make_chv_audit_
|
||||
# callback(原 __init__ 闭包逐字搬运),此处只挂接。
|
||||
try:
|
||||
self.executor.audit_callback = _chv_audit
|
||||
self.executor.audit_callback = \
|
||||
_chv_worker_adapter.make_chv_audit_callback(
|
||||
self._audit)
|
||||
except Exception:
|
||||
pass
|
||||
# CHV 残留清理:仅孤儿清理接线(feat-chv-arch95 定案,
|
||||
@@ -797,36 +781,11 @@ class Worker:
|
||||
return self._cost_store
|
||||
|
||||
def _reconcile_warm_pool(self, when):
|
||||
"""显式调生产残留清理 facade,返回已清理 VM 名。
|
||||
|
||||
仅 CHV worker 有效(其它执行器返回 [])。实现是
|
||||
cloud.chv.orphans.reconcile_orphans_for_executor():
|
||||
直接读 registry 文件做清理,不构造 WarmPool、不碰
|
||||
实验接口(方法内 import,失败时按下面分支审计后
|
||||
返回 [])。调用失败不炸主流程,但进审计(失败本身
|
||||
也审计,不静默丢弃)。orphan 条目绝不发给任务,
|
||||
reconcile 只做清理。"""
|
||||
executor = getattr(self, "executor", None)
|
||||
if not isinstance(executor, CloudHypervisorExecutor):
|
||||
return []
|
||||
try:
|
||||
from cloud.chv.orphans import \
|
||||
reconcile_orphans_for_executor as _reconcile
|
||||
cleaned = _reconcile(executor) or []
|
||||
except Exception as exc:
|
||||
try:
|
||||
self._audit(None, "chv_warm_pool_reconcile_failed",
|
||||
"when=%s error=%s" % (when, exc))
|
||||
except Exception:
|
||||
pass
|
||||
return []
|
||||
if cleaned:
|
||||
try:
|
||||
self._audit(None, "chv_warm_pool_reconcile",
|
||||
"when=%s cleaned=%s" % (when, ",".join(cleaned)))
|
||||
except Exception:
|
||||
pass
|
||||
return cleaned
|
||||
"""显式调生产残留清理 facade,返回已清理 VM 名
|
||||
(to10:本体已移到 worker_adapter.reconcile_warm_
|
||||
pool,此处只留薄包装,旧测试调用点不变)。"""
|
||||
return _chv_worker_adapter.reconcile_warm_pool(
|
||||
getattr(self, "executor", None), self._audit, when)
|
||||
|
||||
def close(self):
|
||||
"""关闭惰性建的成本存储连接(注入的 cost_store 由调用方管)。
|
||||
@@ -2479,61 +2438,22 @@ class Worker:
|
||||
return
|
||||
|
||||
def _chv_cancel_watchdog(self, task_id, vm_name, stop_event):
|
||||
"""CHV 同步路径的取消看门狗(feat-chv-fix,P1-2,Docker
|
||||
取消看门狗同款模式):同步 SSH 阻塞时 worker 的 cancel()
|
||||
只能改库、清 _running 项,阻塞中的 run 本身不会醒;
|
||||
本线程每 1 秒查库/取消标志,命中时调 cancel_vm 真关
|
||||
VM,然后退出。查库失败时当没取消,不误杀。"""
|
||||
while not stop_event.wait(1.0):
|
||||
try:
|
||||
if task_id in self._cancel_requested:
|
||||
cancelled = True
|
||||
else:
|
||||
db_task = self.get_task(task_id)
|
||||
cancelled = (db_task is not None and
|
||||
db_task.get("status") == "interrupted")
|
||||
except Exception:
|
||||
cancelled = False
|
||||
if not cancelled:
|
||||
continue
|
||||
try:
|
||||
cancel = getattr(self.executor, "cancel_vm", None)
|
||||
if cancel is not None and vm_name:
|
||||
cancel(vm_name)
|
||||
log_event(self.logger, "worker_chv_killed",
|
||||
task_id=task_id,
|
||||
message="cancel watchdog killed VM %s "
|
||||
"(task interrupted/cancelled)"
|
||||
% vm_name)
|
||||
except Exception:
|
||||
pass
|
||||
return
|
||||
"""CHV 同步路径的取消看门狗(to10:循环本体已移到
|
||||
worker_adapter.cancel_watchdog_loop,此处只传回调)。"""
|
||||
_chv_worker_adapter.cancel_watchdog_loop(
|
||||
task_id, vm_name, stop_event, self._cancel_requested,
|
||||
self.get_task, getattr(self.executor, "cancel_vm", None),
|
||||
self.logger)
|
||||
|
||||
def _chv_quota_watchdog(self, task_id, vm_name, stop_event,
|
||||
quota_check, interval=30.0):
|
||||
"""CHV 同步路径的配额看门狗(feat-chv-fix,P1-2,Docker
|
||||
配额看门狗同款):命中时记 _quota_hit_tasks 并调
|
||||
cancel_vm 真关 VM;_run_attempt 在同步返回后把结果
|
||||
改写成 quota_exceeded,走统一配额中断收尾。"""
|
||||
while not stop_event.wait(interval):
|
||||
try:
|
||||
over = bool(quota_check()) if quota_check else False
|
||||
except Exception:
|
||||
over = False
|
||||
if not over:
|
||||
continue
|
||||
self._mark_quota_hit(task_id)
|
||||
try:
|
||||
cancel = getattr(self.executor, "cancel_vm", None)
|
||||
if cancel is not None and vm_name:
|
||||
cancel(vm_name)
|
||||
log_event(self.logger, "worker_chv_killed",
|
||||
task_id=task_id,
|
||||
message="quota watchdog killed VM %s "
|
||||
"(quota exceeded)" % vm_name)
|
||||
except Exception:
|
||||
pass
|
||||
return
|
||||
"""CHV 同步路径的配额看门狗(to10:循环本体已移到
|
||||
worker_adapter.quota_watchdog_loop,此处只传回调)。"""
|
||||
_chv_worker_adapter.quota_watchdog_loop(
|
||||
task_id, vm_name, stop_event, quota_check,
|
||||
self._mark_quota_hit,
|
||||
getattr(self.executor, "cancel_vm", None),
|
||||
self.logger, interval=interval)
|
||||
|
||||
def cleanup_stale_containers(self):
|
||||
"""启动清理兜底:清掉上一轮残留的 codex- 容器。
|
||||
@@ -4717,10 +4637,10 @@ class Worker:
|
||||
# process 已轮询 quota_check)。
|
||||
if isinstance(self.executor, CloudHypervisorExecutor) \
|
||||
and hasattr(self.executor, "cancel_vm"):
|
||||
try:
|
||||
_chv_vm_name = self.executor.vm_name(task)
|
||||
except Exception:
|
||||
_chv_vm_name = None
|
||||
# to10:VM 名派生已移到 worker_adapter.watchdog_vm_
|
||||
# name(原 try/except 块逐字搬运)。
|
||||
_chv_vm_name = _chv_worker_adapter.watchdog_vm_name(
|
||||
self.executor, task)
|
||||
if _chv_vm_name:
|
||||
_watchdog_stop = threading.Event()
|
||||
_watchdog_thread = threading.Thread(
|
||||
|
||||
Reference in new issue
Block a user