From 5827716a3a4166cb36634f3975b7618ffab6e099 Mon Sep 17 00:00:00 2001 From: host Date: Mon, 26 Oct 2026 06:07:07 +0800 Subject: [PATCH] Fallback: extract HTTP code from failed-turn JSON; Docker codex_home points at real relay home 1. Worker _fallback_token correction: real host.py never returns status='404'; a failed turn is status='failed' with the real error (e.g. '404 model not found') in text/error. The old code returned host_status verbatim, so 'failed' blocked every real fallback and the old event-only branch did too. Now: if host_status itself is a fallbackable code it is used; otherwise the code is extracted from turn JSON text/error, stdout, stderr or runner error, and 'failed' alone never counts as a signal. Tests: failed-turn JSON with 404 in text falls back and succeeds; with 401 it never falls back. 2. Docker config codex_home: /var/lib/codex-app-server/codex-home, mounted at the same path into the container, is the only home that holds the real relay config (xcmapi provider, base_url, env_key). Pointing codex_home at a fresh empty dir made codex POST to api.openai.com with no auth header and loop on connect timeouts until host.py's 600s turn timeout. The /app/work codex-home from PR #17 was necessary to get past initialize but insufficient for a real turn; this replaces it. --- cloud/tests/test_fallback_retry.py | 38 ++++++++++++++++++++++++++ cloud/worker.py | 44 ++++++++++++++++++------------ host-config-docker.json | 2 +- 3 files changed, 66 insertions(+), 18 deletions(-) diff --git a/cloud/tests/test_fallback_retry.py b/cloud/tests/test_fallback_retry.py index 48b7eb9..d012dff 100644 --- a/cloud/tests/test_fallback_retry.py +++ b/cloud/tests/test_fallback_retry.py @@ -346,6 +346,44 @@ class FallbackRetryTestCase(unittest.TestCase): self.assertIn("缺少 --repo", row[0]) self.assertEqual(result["events"], []) + def test_failed_turn_json_with_404_in_text_fallback_succeeds(self): + # 真实回合失败形态：host.py JSON 的 status 只是 "failed"， + # 真正的 404 藏在 text 里；必须从 text 里提取出来 fallback。 + def fake(cmd): + if _model_of(cmd) == "gpt-6-sol": + return _host_json("A", "completed", "done by fallback") + return _host_json( + "A", "failed", + "turn failed: 404 model not found: gpt-9.9-nonexistent") + + _insert_task(self.db_path, "A") + worker = self._worker( + runner=self._recording(fake), + model="gpt-6.1-sol", + fallback_models=["gpt-6-sol"], + max_retries=0) + result = worker.run_one() + self.assertEqual(result["status"], "completed") + self.assertEqual(result["actual_model"], "gpt-6-sol") + self.assertEqual(len(self.calls), 2) + self.assertEqual(result["events"][0]["reason"], "404") + + def test_failed_turn_json_with_401_in_text_never_fallbacks(self): + # 回合 JSON 里是 401 时绝不 fallback，只调一次。 + def fake(cmd): + return _host_json("A", "failed", "turn failed: 401 unauthorized") + + _insert_task(self.db_path, "A") + worker = self._worker( + runner=self._recording(fake), + model="gpt-6.1-sol", + fallback_models=["gpt-6-sol"], + max_retries=0) + result = worker.run_one() + self.assertEqual(result["status"], "failed") + self.assertEqual(len(self.calls), 1) + self.assertEqual(result["events"], []) + if __name__ == "__main__": unittest.main() diff --git a/cloud/worker.py b/cloud/worker.py index 56a00ff..0ee79ea 100644 --- a/cloud/worker.py +++ b/cloud/worker.py @@ -917,26 +917,32 @@ class Worker: return stdout.strip() or "host execution failed" def _fallback_token(self, result, error): - """一次失败尝试的可 fallback 信号（host_status 为空时的兜底）。 + """一次失败尝试的可 fallback 信号。 - 坏模型最常见的真实失败不在 turn 里，而在 thread/start： - host.py 的 create_thread 抛 RuntimeError、进程非 0 退出、 - 不打印 JSON，于是执行器结果的 host_status 为 None，旧逻辑 - 因此永远不 fallback（主机实测：坏模型任务直接 failed、 - 无 worker_fallback 审计）。此时从 stdout/stderr/runner 异常 - 文本里找显式 HTTP 状态码（404/429/5xx）或“model not found” - 类文本，前者按码判断，后者按 404 计；找不到显式信号时 - 返回 None，绝不凭“host execution failed”这类通用文本猜测， - 否则任何执行器故障都会被误当成模型故障换模型烧额度。 + host_status 的真实含义是 host.py 回合状态（completed / + failed / timeout / interrupted），不是 HTTP 状态码—— + 真实世界里 host.py 从不返回 status=\"404\"，只有单测里的 + 假 runner 才会这么写。所以这里不能把 host_status 原样 + 当 token：host_status 本身已是可 fallback 码（404/429/ + 5xx/timeout，含历史假 runner 形态）时直接返回；否则 + （最常见：host_status=\"failed\"）从回合 JSON 的 text/error + 字段、stdout、stderr、runner 异常里找显式 HTTP 状态码 + 或“model not found”类文本（按 404 计），找到且可 + fallback 才返回；找不到显式信号时返回 None，绝不凭 + “host execution failed”这类通用文本猜测，否则任何执行 + 器故障都会被误当成模型故障换模型烧额度。thread/start + 崩溃（host.py 非 0 退出、不打印 JSON）时 host_status + 为 None，同样走这条提取路径。 """ host_status = (result or {}).get("host_status") - if host_status is not None: + if host_status is not None and is_fallbackable(host_status): return host_status - combined = " ".join([ + combined_parts = [ (result or {}).get("stdout", "") or "", (result or {}).get("stderr", "") or "", error or "", - ]) + ] + combined = " ".join(combined_parts) if not combined: return None lowered = combined.lower() @@ -1138,16 +1144,20 @@ class Worker: if retry_count: error = "[retries=%d] %s" % (retry_count, error) # 未配 fallback_models 时的老行为：只记事件、不重试。 - if self.router is not None and host_status is not None \ - and is_fallbackable(host_status): - nxt = self.router.next_after(self.model, host_status) + # 注意用提取出的 token（真实回合失败时 host_status + # 只是 "failed"，真正的 404 藏在回合 JSON 里）， + # 不能直接用原始 host_status 判断。 + _token = self._fallback_token(result, error) + if self.router is not None and _token is not None \ + and is_fallbackable(_token): + nxt = self.router.next_after(self.model, _token) if nxt is not None: record = dict(task) record["events"] = events # record_fallback 本身会 append 到 events， # 这里不要再 append 一次，否则事件会重复。 record_fallback( - record, self.model, nxt, str(host_status)) + record, self.model, nxt, str(_token)) fallback_count = 1 self._audit(task["project_id"], "worker_fallback", "%s -> %s" % (self.model, nxt)) diff --git a/host-config-docker.json b/host-config-docker.json index 03d011d..579393d 100644 --- a/host-config-docker.json +++ b/host-config-docker.json @@ -1 +1 @@ -{"_note": "Docker 执行器专用配置（容器内路径，绝不是宿主路径）：state 必须用相对路径 work/host-state.json——相对配置文件所在目录 /app 解析，得到 /app/work/host-state.json，经额外挂载落到宿主 /opt/codex-app-server/work/host-state.json，属主是 codexsvc（uid 1003，与容器 --user 1003:1004 一致），容器可写。绝不能用默认的 /app/host-state.json（/app 是只读挂载，锁一开就报 Read-only file system），也绝不能用 /work/host-state.json（/work 挂载的宿主目录由 docker 自动创建、属主 root，容器用户写不进去，同样 1 秒内失败）。此文件曾因 deploy 清理被删过（不在仓库里），导致 Docker 派发静默全灭，所以它必须进仓库、随部署一起发。sandbox 用 danger-full-access：主机内核禁止不可信 user namespace，Docker + 内层 bwrap 双重沙箱不可行，生产隔离层就是 Docker 本身。API key 绝不放这里，只走环境变量。 默认仓库是宿主测试仓库：没有 repo_url 的项目走它；有 repo_url 的项目 worker 会把解析出的检出经 --repo 传进去，覆盖它。 codex_home 必须在 /app/work 下：容器内默认 HOME 是 /root，uid 1003 写不进 /root/.codex，app-server 连 initialize 都回不来（实测 60 秒超时）；/app/work 属主是 codexsvc，可写。", "approval": "deny", "approval_policy": "never", "effort": "low", "model": "gpt-6.1-sol", "sandbox": "danger-full-access", "state": "work/host-state.json", "repo": "/var/lib/codex-app-server/test-repo", "codex_home": "/app/work/codex-home"} \ No newline at end of file +{"_note": "Docker 执行器专用配置（容器内路径，绝不是宿主路径）：state 必须用相对路径 work/host-state.json——相对配置文件所在目录 /app 解析，得到 /app/work/host-state.json，经额外挂载落到宿主 /opt/codex-app-server/work/host-state.json，属主是 codexsvc（uid 1003，与容器 --user 1003:1004 一致），容器可写。绝不能用默认的 /app/host-state.json（/app 是只读挂载，锁一开就报 Read-only file system），也绝不能用 /work/host-state.json（/work 挂载的宿主目录由 docker 自动创建、属主 root，容器用户写不进去，同样 1 秒内失败）。此文件曾因 deploy 清理被删过（不在仓库里），导致 Docker 派发静默全灭，所以它必须进仓库、随部署一起发。sandbox 用 danger-full-access：主机内核禁止不可信 user namespace，Docker + 内层 bwrap 双重沙箱不可行，生产隔离层就是 Docker 本身。API key 绝不放这里，只走环境变量。 默认仓库是宿主测试仓库：没有 repo_url 的项目走它；有 repo_url 的项目 worker 会把解析出的检出经 --repo 传进去，覆盖它。 codex_home 必须指到 /var/lib/codex-app-server/codex-home（经额外挂载同路径进容器）：那里才有真正的 relay 配置（model_provider xcmapi、base_url、env_key），容器 codex 靠它才知道走 relay；指到空目录会让 codex 直接去 api.openai.com 且无 auth，turn 在连接超时里循环重试直到 host.py 600 秒超时。/var/lib 挂载属主是 codexsvc（uid 1003，与容器 --user 一致），可写。", "approval": "deny", "approval_policy": "never", "effort": "low", "model": "gpt-6.1-sol", "sandbox": "danger-full-access", "state": "work/host-state.json", "repo": "/var/lib/codex-app-server/test-repo", "codex_home": "/var/lib/codex-app-server/codex-home"} \ No newline at end of file