feat: CHV 审核修复(P0+P1,堆叠于 #230)

P0:seed 磁盘 CIDATA ISO 挂进 VmConfig 第二块只读盘;
restore provisioned 下复用钉死 known_hosts、缺失 fail-fast
绝不 keyscan;preflight 拦截 CHV_DISK_MODE=reuse(显式
实验二次确认 + 审计放行);SSH 目标选客机 IP 方案(显式
host > net 静态 ip > 显式转发宣称回环,三者皆无 fail-fast)。
P1:网关 extra_env 经 SSH env 前缀注入、worker 守卫收窄到
Local/CHV;取消/清理/看门狗接 CHV 通用路径;WarmPool
acquire 内 resume;容量门持久化计数;磁盘配额总量+余量
与已存在磁盘校验重拷;API 按 Content-Length 验完整性;
快照 file:// 验落盘再 shutdown;execute 死端口归类
vm_not_running。
P2 全部排除未做(用户范围收窄令,留待以后一轨)。
测试:原 84 + 新增 29 = 113 项,mock,无真 VM。
This commit is contained in:
host committed 2026-10-31 01:54:13 +00:00
1 parent 22b83f118a
commit 9771ed9e8c
13 files changed
+1231 -128

No files matched your search

+138 -19
View File
@@ -333,22 +333,28 @@ def _merge_env(base, override):
return merged
def _call_goal_resume_breakpoint(executor, task, model_provider=None):
def _call_goal_resume_breakpoint(executor, task, model_provider=None,
extra_env=None):
"""通用断点调用:执行器提供 goal_resume_breakpoint_task 时调用它。
Local 仍走原路径(调用方直接传 model_provider,不经过这里,
以免改生产语义);这里只用于非 Local 执行器:签名接受
model_provider 时传它(CHV/Docker 均接受),否则只传 task
(旧假执行器单参)。网关 key 绝不经这里传值:worker 侧
gateway_env 只按名注入,见 _run_attempt 的网关守卫。"""
(旧假执行器单参)。feat-chv-fix(P1-1):CHV 的
gateway_env 经此进入 SSH env 前缀注入客机环境(不进
镜像/代码/prompt);CHV 没有 Local 那样的进程 env 通道,
SSH env 前缀是唯一通道,值只出现在宿主侧 ssh argv。"""
method = getattr(executor, "goal_resume_breakpoint_task")
try:
params = inspect.signature(method).parameters
except (TypeError, ValueError):
return method(task)
kwargs = {}
if "model_provider" in params:
return method(task, model_provider=model_provider)
return method(task)
kwargs["model_provider"] = model_provider
if "extra_env" in params and extra_env is not None:
kwargs["extra_env"] = extra_env
return method(task, **kwargs)
def _resolve_model(model=None, config_path=None):
@@ -2368,6 +2374,12 @@ class Worker:
self.executor.kill_container(entry["container"])
except Exception:
pass
# feat-chv-fix(P1-2):CHV 取消真关 VM,不只改库状态。
if hasattr(self.executor, "cancel_vm"):
try:
self.executor.cancel_vm(task_id)
except Exception:
pass
self._running_clear(task_id)
conn = _db_connect(self.db_path)
try:
@@ -2470,22 +2482,87 @@ class Worker:
pass
return
def _chv_cancel_watchdog(self, task_id, vm_name, stop_event):
"""CHV 同步路径的取消看门狗(feat-chv-fix,P1-2,Docker
取消看门狗同款模式):同步 SSH 阻塞时 worker 的 cancel()
只能改库、清 _running 项,阻塞中的 run 本身不会醒;
本线程每 1 秒查库/取消标志,命中时调 cancel_vm 真关
VM,然后退出。查库失败时当没取消,不误杀。"""
while not stop_event.wait(1.0):
try:
if task_id in self._cancel_requested:
cancelled = True
else:
db_task = self.get_task(task_id)
cancelled = (db_task is not None and
db_task.get("status") == "interrupted")
except Exception:
cancelled = False
if not cancelled:
continue
try:
cancel = getattr(self.executor, "cancel_vm", None)
if cancel is not None and vm_name:
cancel(vm_name)
log_event(self.logger, "worker_chv_killed",
task_id=task_id,
message="cancel watchdog killed VM %s "
"(task interrupted/cancelled)"
% vm_name)
except Exception:
pass
return
def _chv_quota_watchdog(self, task_id, vm_name, stop_event,
quota_check, interval=30.0):
"""CHV 同步路径的配额看门狗(feat-chv-fix,P1-2,Docker
配额看门狗同款):命中时记 _quota_hit_tasks 并调
cancel_vm 真关 VM;_run_attempt 在同步返回后把结果
改写成 quota_exceeded,走统一配额中断收尾。"""
while not stop_event.wait(interval):
try:
over = bool(quota_check()) if quota_check else False
except Exception:
over = False
if not over:
continue
self._mark_quota_hit(task_id)
try:
cancel = getattr(self.executor, "cancel_vm", None)
if cancel is not None and vm_name:
cancel(vm_name)
log_event(self.logger, "worker_chv_killed",
task_id=task_id,
message="quota watchdog killed VM %s "
"(quota exceeded)" % vm_name)
except Exception:
pass
return
def cleanup_stale_containers(self):
"""启动清理兜底:清掉上一轮残留的 codex- 容器。
只在执行器有 cleanup_stale_containers 时做(Docker);
Docker 执行器清容器、CHV 执行器清残留 VM(feat-chv-fix);
列出或删除失败只记日志,绝不抛异常炸掉 worker。
返回已清理的容器名列表(非 Docker 执行器返回 [])。
返回已清理的容器/VM 名列表(两者皆无的执行器返回 [])。
"""
cleaned = []
cleanup = getattr(self.executor, "cleanup_stale_containers", None)
if cleanup is None:
return []
try:
cleaned = cleanup()
except Exception as exc:
log_event(self.logger, "worker_cleanup_failed",
message="cleanup stale containers failed: %s" % exc)
return []
if cleanup is not None:
try:
cleaned = list(cleanup() or [])
except Exception as exc:
log_event(self.logger, "worker_cleanup_failed",
message="cleanup stale containers failed: %s" % exc)
# feat-chv-fix(P1-2):CHV 残留 VM 走通用清理路径,
# 不再只靠 worker 启动/关闭的 WarmPool reconcile 兜底。
chv_cleanup = getattr(self.executor, "cleanup_stale_vms", None)
if chv_cleanup is not None:
try:
cleaned = cleaned + list(chv_cleanup() or [])
except Exception as exc:
log_event(self.logger, "worker_cleanup_failed",
message="cleanup stale vms failed: %s" % exc)
if cleaned:
log_event(self.logger, "worker_cleanup",
message="cleaned stale containers: %s"
@@ -4639,6 +4716,29 @@ class Worker:
_watchdog_stop, quota_check),
daemon=True)
_quota_watchdog_thread.start()
# feat-chv-fix(P1-2):CHV 同步路径同款看门狗(VM 名
# 由任务 id 派生)。进程模式不重复起(_execute_via_
# process 已轮询 quota_check)。
if isinstance(self.executor, CloudHypervisorExecutor) \
and hasattr(self.executor, "cancel_vm"):
try:
_chv_vm_name = self.executor.vm_name(task)
except Exception:
_chv_vm_name = None
if _chv_vm_name:
_watchdog_stop = threading.Event()
_watchdog_thread = threading.Thread(
target=self._chv_cancel_watchdog,
args=(task_id, _chv_vm_name, _watchdog_stop),
daemon=True)
_watchdog_thread.start()
if quota_check is not None and not self._use_popen():
_quota_watchdog_thread = threading.Thread(
target=self._chv_quota_watchdog,
args=(task_id, _chv_vm_name,
_watchdog_stop, quota_check),
daemon=True)
_quota_watchdog_thread.start()
# 启动前分段日志(2026-10-29 任务 A):认领后 61 秒中断,
# 但阻塞的 docker run 中途没有任何标记、容器 --rm 后输出
# 也没了,事后完全无法定位卡在哪一段。派发开始/结束进
@@ -4774,8 +4874,12 @@ class Worker:
# Docker 路径上面已有 is_docker 守卫,不走这里。
# 网关任务在非 Local 执行器上无法注入 key 与
# provider,必须 loud fail,不静默跑错网关。
# feat-chv-fix(P1-1):守卫收窄——CHV 已补
# extra_env SSH 注入路径,不再被“非 Local 即
# 拦”;无注入能力的执行器仍 loud fail。
if gateway_provider and not isinstance(
self.executor, LocalExecutor):
self.executor,
(LocalExecutor, CloudHypervisorExecutor)):
raise RuntimeError(
"sub2api 网关不支持该执行器:%s"
% type(self.executor).__name__)
@@ -4796,7 +4900,8 @@ class Worker:
# 故这里不放宽网关规则。
result = _call_goal_resume_breakpoint(
self.executor, task,
model_provider=gateway_provider)
model_provider=gateway_provider,
extra_env=gateway_env)
elif task.get("_goal_resume_breakpoint"):
# 假执行器/CloudExecutor 没有断点方法时
# 绝不回落成 goal(会撞已存在)或
@@ -4825,6 +4930,11 @@ class Worker:
task, seed_dir=seed_host_dir,
extra_env=gateway_env,
model_provider=gateway_provider)
elif isinstance(
self.executor, CloudHypervisorExecutor):
result = self.executor.goal_task(
task, extra_env=gateway_env,
model_provider=gateway_provider)
else:
result = self.executor.goal_task(task)
else:
@@ -4833,6 +4943,11 @@ class Worker:
task, seed_dir=seed_host_dir,
extra_env=gateway_env,
model_provider=gateway_provider)
elif isinstance(
self.executor, CloudHypervisorExecutor):
result = self.executor.dispatch_task(
task, extra_env=gateway_env,
model_provider=gateway_provider)
else:
result = self.executor.dispatch_task(task)
combined = (result.get("stdout", "") or "") + \
@@ -4856,14 +4971,18 @@ class Worker:
quota_check)
else:
if gateway_provider and not isinstance(
self.executor, LocalExecutor):
self.executor,
(LocalExecutor,
CloudHypervisorExecutor)):
raise RuntimeError(
"sub2api 网关不支持该执行器:%s"
% type(self.executor).__name__)
result = self.executor.run_task(
task, extra_env=gateway_env,
model_provider=gateway_provider) \
if isinstance(self.executor, LocalExecutor) \
if isinstance(self.executor,
(LocalExecutor,
CloudHypervisorExecutor)) \
else self.executor.run_task(task)
except Exception as exc: # runner 异常也如实写回 failed
result = {"executor": "local", "task_id": task_id,