feat: CHV 审核修复(P0+P1,堆叠于 #230)
P0:seed 磁盘 CIDATA ISO 挂进 VmConfig 第二块只读盘; restore provisioned 下复用钉死 known_hosts、缺失 fail-fast 绝不 keyscan;preflight 拦截 CHV_DISK_MODE=reuse(显式 实验二次确认 + 审计放行);SSH 目标选客机 IP 方案(显式 host > net 静态 ip > 显式转发宣称回环,三者皆无 fail-fast)。 P1:网关 extra_env 经 SSH env 前缀注入、worker 守卫收窄到 Local/CHV;取消/清理/看门狗接 CHV 通用路径;WarmPool acquire 内 resume;容量门持久化计数;磁盘配额总量+余量 与已存在磁盘校验重拷;API 按 Content-Length 验完整性; 快照 file:// 验落盘再 shutdown;execute 死端口归类 vm_not_running。 P2 全部排除未做(用户范围收窄令,留待以后一轨)。 测试:原 84 + 新增 29 = 113 项,mock,无真 VM。
This commit is contained in:
1 parent
22b83f118a
commit
9771ed9e8c
13 files changed
+1231
-128
No files matched your search
+138
-19
@@ -333,22 +333,28 @@ def _merge_env(base, override):
|
||||
return merged
|
||||
|
||||
|
||||
def _call_goal_resume_breakpoint(executor, task, model_provider=None):
|
||||
def _call_goal_resume_breakpoint(executor, task, model_provider=None,
|
||||
extra_env=None):
|
||||
"""通用断点调用:执行器提供 goal_resume_breakpoint_task 时调用它。
|
||||
|
||||
Local 仍走原路径(调用方直接传 model_provider,不经过这里,
|
||||
以免改生产语义);这里只用于非 Local 执行器:签名接受
|
||||
model_provider 时传它(CHV/Docker 均接受),否则只传 task
|
||||
(旧假执行器单参)。网关 key 绝不经这里传值:worker 侧
|
||||
gateway_env 只按名注入,见 _run_attempt 的网关守卫。"""
|
||||
(旧假执行器单参)。feat-chv-fix(P1-1):CHV 的
|
||||
gateway_env 经此进入 SSH env 前缀注入客机环境(不进
|
||||
镜像/代码/prompt);CHV 没有 Local 那样的进程 env 通道,
|
||||
SSH env 前缀是唯一通道,值只出现在宿主侧 ssh argv。"""
|
||||
method = getattr(executor, "goal_resume_breakpoint_task")
|
||||
try:
|
||||
params = inspect.signature(method).parameters
|
||||
except (TypeError, ValueError):
|
||||
return method(task)
|
||||
kwargs = {}
|
||||
if "model_provider" in params:
|
||||
return method(task, model_provider=model_provider)
|
||||
return method(task)
|
||||
kwargs["model_provider"] = model_provider
|
||||
if "extra_env" in params and extra_env is not None:
|
||||
kwargs["extra_env"] = extra_env
|
||||
return method(task, **kwargs)
|
||||
|
||||
|
||||
def _resolve_model(model=None, config_path=None):
|
||||
@@ -2368,6 +2374,12 @@ class Worker:
|
||||
self.executor.kill_container(entry["container"])
|
||||
except Exception:
|
||||
pass
|
||||
# feat-chv-fix(P1-2):CHV 取消真关 VM,不只改库状态。
|
||||
if hasattr(self.executor, "cancel_vm"):
|
||||
try:
|
||||
self.executor.cancel_vm(task_id)
|
||||
except Exception:
|
||||
pass
|
||||
self._running_clear(task_id)
|
||||
conn = _db_connect(self.db_path)
|
||||
try:
|
||||
@@ -2470,22 +2482,87 @@ class Worker:
|
||||
pass
|
||||
return
|
||||
|
||||
def _chv_cancel_watchdog(self, task_id, vm_name, stop_event):
|
||||
"""CHV 同步路径的取消看门狗(feat-chv-fix,P1-2,Docker
|
||||
取消看门狗同款模式):同步 SSH 阻塞时 worker 的 cancel()
|
||||
只能改库、清 _running 项,阻塞中的 run 本身不会醒;
|
||||
本线程每 1 秒查库/取消标志,命中时调 cancel_vm 真关
|
||||
VM,然后退出。查库失败时当没取消,不误杀。"""
|
||||
while not stop_event.wait(1.0):
|
||||
try:
|
||||
if task_id in self._cancel_requested:
|
||||
cancelled = True
|
||||
else:
|
||||
db_task = self.get_task(task_id)
|
||||
cancelled = (db_task is not None and
|
||||
db_task.get("status") == "interrupted")
|
||||
except Exception:
|
||||
cancelled = False
|
||||
if not cancelled:
|
||||
continue
|
||||
try:
|
||||
cancel = getattr(self.executor, "cancel_vm", None)
|
||||
if cancel is not None and vm_name:
|
||||
cancel(vm_name)
|
||||
log_event(self.logger, "worker_chv_killed",
|
||||
task_id=task_id,
|
||||
message="cancel watchdog killed VM %s "
|
||||
"(task interrupted/cancelled)"
|
||||
% vm_name)
|
||||
except Exception:
|
||||
pass
|
||||
return
|
||||
|
||||
def _chv_quota_watchdog(self, task_id, vm_name, stop_event,
|
||||
quota_check, interval=30.0):
|
||||
"""CHV 同步路径的配额看门狗(feat-chv-fix,P1-2,Docker
|
||||
配额看门狗同款):命中时记 _quota_hit_tasks 并调
|
||||
cancel_vm 真关 VM;_run_attempt 在同步返回后把结果
|
||||
改写成 quota_exceeded,走统一配额中断收尾。"""
|
||||
while not stop_event.wait(interval):
|
||||
try:
|
||||
over = bool(quota_check()) if quota_check else False
|
||||
except Exception:
|
||||
over = False
|
||||
if not over:
|
||||
continue
|
||||
self._mark_quota_hit(task_id)
|
||||
try:
|
||||
cancel = getattr(self.executor, "cancel_vm", None)
|
||||
if cancel is not None and vm_name:
|
||||
cancel(vm_name)
|
||||
log_event(self.logger, "worker_chv_killed",
|
||||
task_id=task_id,
|
||||
message="quota watchdog killed VM %s "
|
||||
"(quota exceeded)" % vm_name)
|
||||
except Exception:
|
||||
pass
|
||||
return
|
||||
|
||||
def cleanup_stale_containers(self):
|
||||
"""启动清理兜底:清掉上一轮残留的 codex- 容器。
|
||||
|
||||
只在执行器有 cleanup_stale_containers 时做(Docker);
|
||||
Docker 执行器清容器、CHV 执行器清残留 VM(feat-chv-fix);
|
||||
列出或删除失败只记日志,绝不抛异常炸掉 worker。
|
||||
返回已清理的容器名列表(非 Docker 执行器返回 [])。
|
||||
返回已清理的容器/VM 名列表(两者皆无的执行器返回 [])。
|
||||
"""
|
||||
cleaned = []
|
||||
cleanup = getattr(self.executor, "cleanup_stale_containers", None)
|
||||
if cleanup is None:
|
||||
return []
|
||||
try:
|
||||
cleaned = cleanup()
|
||||
except Exception as exc:
|
||||
log_event(self.logger, "worker_cleanup_failed",
|
||||
message="cleanup stale containers failed: %s" % exc)
|
||||
return []
|
||||
if cleanup is not None:
|
||||
try:
|
||||
cleaned = list(cleanup() or [])
|
||||
except Exception as exc:
|
||||
log_event(self.logger, "worker_cleanup_failed",
|
||||
message="cleanup stale containers failed: %s" % exc)
|
||||
# feat-chv-fix(P1-2):CHV 残留 VM 走通用清理路径,
|
||||
# 不再只靠 worker 启动/关闭的 WarmPool reconcile 兜底。
|
||||
chv_cleanup = getattr(self.executor, "cleanup_stale_vms", None)
|
||||
if chv_cleanup is not None:
|
||||
try:
|
||||
cleaned = cleaned + list(chv_cleanup() or [])
|
||||
except Exception as exc:
|
||||
log_event(self.logger, "worker_cleanup_failed",
|
||||
message="cleanup stale vms failed: %s" % exc)
|
||||
if cleaned:
|
||||
log_event(self.logger, "worker_cleanup",
|
||||
message="cleaned stale containers: %s"
|
||||
@@ -4639,6 +4716,29 @@ class Worker:
|
||||
_watchdog_stop, quota_check),
|
||||
daemon=True)
|
||||
_quota_watchdog_thread.start()
|
||||
# feat-chv-fix(P1-2):CHV 同步路径同款看门狗(VM 名
|
||||
# 由任务 id 派生)。进程模式不重复起(_execute_via_
|
||||
# process 已轮询 quota_check)。
|
||||
if isinstance(self.executor, CloudHypervisorExecutor) \
|
||||
and hasattr(self.executor, "cancel_vm"):
|
||||
try:
|
||||
_chv_vm_name = self.executor.vm_name(task)
|
||||
except Exception:
|
||||
_chv_vm_name = None
|
||||
if _chv_vm_name:
|
||||
_watchdog_stop = threading.Event()
|
||||
_watchdog_thread = threading.Thread(
|
||||
target=self._chv_cancel_watchdog,
|
||||
args=(task_id, _chv_vm_name, _watchdog_stop),
|
||||
daemon=True)
|
||||
_watchdog_thread.start()
|
||||
if quota_check is not None and not self._use_popen():
|
||||
_quota_watchdog_thread = threading.Thread(
|
||||
target=self._chv_quota_watchdog,
|
||||
args=(task_id, _chv_vm_name,
|
||||
_watchdog_stop, quota_check),
|
||||
daemon=True)
|
||||
_quota_watchdog_thread.start()
|
||||
# 启动前分段日志(2026-10-29 任务 A):认领后 61 秒中断,
|
||||
# 但阻塞的 docker run 中途没有任何标记、容器 --rm 后输出
|
||||
# 也没了,事后完全无法定位卡在哪一段。派发开始/结束进
|
||||
@@ -4774,8 +4874,12 @@ class Worker:
|
||||
# Docker 路径上面已有 is_docker 守卫,不走这里。
|
||||
# 网关任务在非 Local 执行器上无法注入 key 与
|
||||
# provider,必须 loud fail,不静默跑错网关。
|
||||
# feat-chv-fix(P1-1):守卫收窄——CHV 已补
|
||||
# extra_env SSH 注入路径,不再被“非 Local 即
|
||||
# 拦”;无注入能力的执行器仍 loud fail。
|
||||
if gateway_provider and not isinstance(
|
||||
self.executor, LocalExecutor):
|
||||
self.executor,
|
||||
(LocalExecutor, CloudHypervisorExecutor)):
|
||||
raise RuntimeError(
|
||||
"sub2api 网关不支持该执行器:%s"
|
||||
% type(self.executor).__name__)
|
||||
@@ -4796,7 +4900,8 @@ class Worker:
|
||||
# 故这里不放宽网关规则。
|
||||
result = _call_goal_resume_breakpoint(
|
||||
self.executor, task,
|
||||
model_provider=gateway_provider)
|
||||
model_provider=gateway_provider,
|
||||
extra_env=gateway_env)
|
||||
elif task.get("_goal_resume_breakpoint"):
|
||||
# 假执行器/CloudExecutor 没有断点方法时
|
||||
# 绝不回落成 goal(会撞已存在)或
|
||||
@@ -4825,6 +4930,11 @@ class Worker:
|
||||
task, seed_dir=seed_host_dir,
|
||||
extra_env=gateway_env,
|
||||
model_provider=gateway_provider)
|
||||
elif isinstance(
|
||||
self.executor, CloudHypervisorExecutor):
|
||||
result = self.executor.goal_task(
|
||||
task, extra_env=gateway_env,
|
||||
model_provider=gateway_provider)
|
||||
else:
|
||||
result = self.executor.goal_task(task)
|
||||
else:
|
||||
@@ -4833,6 +4943,11 @@ class Worker:
|
||||
task, seed_dir=seed_host_dir,
|
||||
extra_env=gateway_env,
|
||||
model_provider=gateway_provider)
|
||||
elif isinstance(
|
||||
self.executor, CloudHypervisorExecutor):
|
||||
result = self.executor.dispatch_task(
|
||||
task, extra_env=gateway_env,
|
||||
model_provider=gateway_provider)
|
||||
else:
|
||||
result = self.executor.dispatch_task(task)
|
||||
combined = (result.get("stdout", "") or "") + \
|
||||
@@ -4856,14 +4971,18 @@ class Worker:
|
||||
quota_check)
|
||||
else:
|
||||
if gateway_provider and not isinstance(
|
||||
self.executor, LocalExecutor):
|
||||
self.executor,
|
||||
(LocalExecutor,
|
||||
CloudHypervisorExecutor)):
|
||||
raise RuntimeError(
|
||||
"sub2api 网关不支持该执行器:%s"
|
||||
% type(self.executor).__name__)
|
||||
result = self.executor.run_task(
|
||||
task, extra_env=gateway_env,
|
||||
model_provider=gateway_provider) \
|
||||
if isinstance(self.executor, LocalExecutor) \
|
||||
if isinstance(self.executor,
|
||||
(LocalExecutor,
|
||||
CloudHypervisorExecutor)) \
|
||||
else self.executor.run_task(task)
|
||||
except Exception as exc: # runner 异常也如实写回 failed
|
||||
result = {"executor": "local", "task_id": task_id,
|
||||
|
||||
Reference in new issue
Block a user