Real model fallback retry (opt-in, at most once)
Worker previously only recorded a model_fallback event on a fallbackable failure and failed the task; ModelRouter.next_after was never used to actually run the task on another model. With --fallback-models (default empty = off, old behaviour unchanged), a fallbackable failure (404/429/5xx/timeout; 401/400 and cancellations never fall back) now re-runs the same task, same worktree/branch, on the first fallback model via host.py's top-level --model flag (no host.py signature change). The second attempt's result is final: actual_model is the fallback model, fallback_count=1, and a double failure keeps both reasons in error_text. LocalExecutor/DockerExecutor pass task model through to host.py; CloudExecutor includes model only when set.
This commit is contained in:
host committed
1 parent
f1d42d04f3
commit
7ce603b610
4 files changed
+473
-51
No files matched your search
+40
-13
@@ -67,16 +67,24 @@ class LocalExecutor(Executor):
|
||||
"""
|
||||
|
||||
def __init__(self, host_py_path, config_path=None, runner=None,
|
||||
repo=None):
|
||||
repo=None, model=None):
|
||||
self.host_py_path = host_py_path
|
||||
self.config_path = config_path
|
||||
self.runner = runner if runner is not None else _default_runner
|
||||
self.repo = repo
|
||||
# 模型：host.py 的 --model 是顶层全局参数（dispatch/run 共用），
|
||||
# 不传时 host.py 用配置文件里的 model；传了就覆盖配置。
|
||||
# worker 的 fallback 靠它把换模型后的第二次尝试真正发到新模型上，
|
||||
# 不改 host.py 的 dispatch/run 签名。
|
||||
self.model = model
|
||||
|
||||
def _base_command(self):
|
||||
def _base_command(self, model=None):
|
||||
cmd = [sys.executable, self.host_py_path]
|
||||
if self.config_path is not None:
|
||||
cmd += ["--config", self.config_path]
|
||||
effective = model or self.model
|
||||
if effective:
|
||||
cmd += ["--model", effective]
|
||||
return cmd
|
||||
|
||||
def _command(self, task_id):
|
||||
@@ -136,17 +144,20 @@ class LocalExecutor(Executor):
|
||||
def dispatch_task(self, task) -> dict:
|
||||
"""调 host.py dispatch：建 host 任务并跑一轮 turn。
|
||||
|
||||
命令：host.py [--config CFG] dispatch --name <host 名>
|
||||
命令：host.py [--config CFG] [--model M] dispatch --name <host 名>
|
||||
--task <prompt> [--repo <repo>]。repo 取任务的 repo 字段，
|
||||
没有则取构造时传的 repo，都没有则不传（此时 host.py 用其
|
||||
配置文件里的 repo，配置里也没有时 host.py 自己报错，
|
||||
worker 把该报错原样写回 failed，不编造成功）。
|
||||
worker 把该报错原样写回 failed，不编造成功）。模型取任务的
|
||||
model 字段，没有则取构造时传的 model，都没有则不传 --model，
|
||||
host.py 用配置文件里的 model（fallback 时 worker 会把任务的
|
||||
model 换成 fallback 模型，第二次 dispatch/run 就跑在新模型上）。
|
||||
"""
|
||||
if not isinstance(task, dict) or task.get("id") is None:
|
||||
raise ValueError("任务缺少 id")
|
||||
if not task.get("prompt"):
|
||||
raise ValueError("任务缺少 prompt")
|
||||
cmd = self._base_command()
|
||||
cmd = self._base_command(model=task.get("model"))
|
||||
cmd += ["dispatch", "--name", self._host_name(task),
|
||||
"--task", task["prompt"]]
|
||||
repo = task.get("repo") or self.repo
|
||||
@@ -158,15 +169,16 @@ class LocalExecutor(Executor):
|
||||
def run_task(self, task) -> dict:
|
||||
"""调 host.py run：在已有 host 任务上再跑一轮 turn。
|
||||
|
||||
命令：host.py [--config CFG] run --task <host 名>
|
||||
命令：host.py [--config CFG] [--model M] run --task <host 名>
|
||||
--prompt <prompt>。worker 只在 dispatch 报“任务已存在”时
|
||||
用它续跑，不用它做首次派发。
|
||||
用它续跑，不用它做首次派发。模型与 dispatch_task 同规则：
|
||||
任务的 model 字段优先，fallback 续跑时就是新模型。
|
||||
"""
|
||||
if not isinstance(task, dict) or task.get("id") is None:
|
||||
raise ValueError("任务缺少 id")
|
||||
if not task.get("prompt"):
|
||||
raise ValueError("任务缺少 prompt")
|
||||
cmd = self._base_command()
|
||||
cmd = self._base_command(model=task.get("model"))
|
||||
cmd += ["run", "--task", self._host_name(task),
|
||||
"--prompt", task["prompt"]]
|
||||
return self._result_from_runner(
|
||||
@@ -184,7 +196,7 @@ class LocalExecutor(Executor):
|
||||
raise ValueError("任务缺少 id")
|
||||
if not task.get("prompt"):
|
||||
raise ValueError("任务缺少 prompt")
|
||||
cmd = self._base_command()
|
||||
cmd = self._base_command(model=task.get("model"))
|
||||
if kind == "run":
|
||||
cmd += ["run", "--task", self._host_name(task),
|
||||
"--prompt", task["prompt"]]
|
||||
@@ -251,12 +263,17 @@ class DockerExecutor(Executor):
|
||||
def __init__(self, host_py_path=None, config_path=None, runner=None,
|
||||
image="codex-app-server:latest", cpus="2", memory="2g",
|
||||
user="1000:1000", worktrees_root=None, env=None,
|
||||
env_key="XCM_API_KEY", timeout=None, extra_mounts=None):
|
||||
env_key="XCM_API_KEY", timeout=None, extra_mounts=None,
|
||||
model=None):
|
||||
if host_py_path is None:
|
||||
here = os.path.dirname(os.path.abspath(__file__))
|
||||
host_py_path = os.path.join(os.path.dirname(here), "host.py")
|
||||
self.host_py_path = host_py_path
|
||||
self.config_path = config_path
|
||||
# 模型与 LocalExecutor 同规则：任务的 model 字段优先，没有则
|
||||
# 取构造时传的 model，都没有则不传 --model（容器内 host.py
|
||||
# 用配置文件里的 model）。fallback 时 worker 换任务的 model。
|
||||
self.model = model
|
||||
self.runner = runner if runner is not None else _docker_default_runner
|
||||
self.image = image
|
||||
self.cpus = str(cpus)
|
||||
@@ -381,7 +398,7 @@ class DockerExecutor(Executor):
|
||||
cmd += list(inner_cmd)
|
||||
return cmd
|
||||
|
||||
def _inner_host_command(self, *args):
|
||||
def _inner_host_command(self, *args, model=None):
|
||||
"""容器内 host.py 命令：python3 /app/host.py [args...]。
|
||||
|
||||
config_path 若已给，容器内用 /app/<basename> 指代它——
|
||||
@@ -389,11 +406,16 @@ class DockerExecutor(Executor):
|
||||
本就放在项目目录）；不在代码目录时不静默猜路径，
|
||||
由 _inner_host_command 的调用方在传 config 前自行确认。
|
||||
这里只做最简单的同名映射，不展开宿主绝对路径进容器。
|
||||
--model 是 host.py 的顶层全局参数，必须放在子命令之前，
|
||||
所以 model 在这里插到 --config 之后、子命令之前。
|
||||
"""
|
||||
cmd = ["python3", "/app/host.py"]
|
||||
if self.config_path is not None:
|
||||
cmd += ["--config",
|
||||
"/app/" + os.path.basename(self.config_path)]
|
||||
effective = model or self.model
|
||||
if effective:
|
||||
cmd += ["--model", effective]
|
||||
cmd += list(args)
|
||||
return cmd
|
||||
|
||||
@@ -515,7 +537,7 @@ class DockerExecutor(Executor):
|
||||
raise ValueError("任务缺少 prompt")
|
||||
inner = self._inner_host_command(
|
||||
"dispatch", "--name", self._host_name(task),
|
||||
"--task", task["prompt"])
|
||||
"--task", task["prompt"], model=task.get("model"))
|
||||
repo = task.get("repo")
|
||||
if repo:
|
||||
inner += ["--repo", repo]
|
||||
@@ -529,7 +551,7 @@ class DockerExecutor(Executor):
|
||||
raise ValueError("任务缺少 prompt")
|
||||
inner = self._inner_host_command(
|
||||
"run", "--task", self._host_name(task),
|
||||
"--prompt", task["prompt"])
|
||||
"--prompt", task["prompt"], model=task.get("model"))
|
||||
return self._run_in_container(task, inner)
|
||||
|
||||
def execute(self, task) -> dict:
|
||||
@@ -660,6 +682,11 @@ class CloudExecutor(Executor):
|
||||
"prompt": task.get("prompt") or "",
|
||||
"repo_url": task.get("repo"),
|
||||
}
|
||||
# 任务带 model 时（worker 每次都设，fallback 时是新模型）
|
||||
# 才发 model 字段：runner 若认就按它跑，不认则忽略；无模型
|
||||
# 的任务请求体与以前完全一致，不含任何密钥。
|
||||
if task.get("model"):
|
||||
payload["model"] = task["model"]
|
||||
result = self._post(payload)
|
||||
if isinstance(result, dict):
|
||||
status = result.get("status")
|
||||
|
||||
Reference in new issue
Block a user