Fallback: extract HTTP code from failed-turn JSON; Docker codex_home points at real relay home
1. Worker _fallback_token correction: real host.py never returns status='404'; a failed turn is status='failed' with the real error (e.g. '404 model not found') in text/error. The old code returned host_status verbatim, so 'failed' blocked every real fallback and the old event-only branch did too. Now: if host_status itself is a fallbackable code it is used; otherwise the code is extracted from turn JSON text/error, stdout, stderr or runner error, and 'failed' alone never counts as a signal. Tests: failed-turn JSON with 404 in text falls back and succeeds; with 401 it never falls back. 2. Docker config codex_home: /var/lib/codex-app-server/codex-home, mounted at the same path into the container, is the only home that holds the real relay config (xcmapi provider, base_url, env_key). Pointing codex_home at a fresh empty dir made codex POST to api.openai.com with no auth header and loop on connect timeouts until host.py's 600s turn timeout. The /app/work codex-home from PR #17 was necessary to get past initialize but insufficient for a real turn; this replaces it.
This commit is contained in:
host committed
1 parent
edc187bb70
commit
5827716a3a
3 files changed
+66
-18
No files matched your search
@@ -346,6 +346,44 @@ class FallbackRetryTestCase(unittest.TestCase):
|
||||
self.assertIn("缺少 --repo", row[0])
|
||||
self.assertEqual(result["events"], [])
|
||||
|
||||
def test_failed_turn_json_with_404_in_text_fallback_succeeds(self):
|
||||
# 真实回合失败形态：host.py JSON 的 status 只是 "failed"，
|
||||
# 真正的 404 藏在 text 里；必须从 text 里提取出来 fallback。
|
||||
def fake(cmd):
|
||||
if _model_of(cmd) == "gpt-6-sol":
|
||||
return _host_json("A", "completed", "done by fallback")
|
||||
return _host_json(
|
||||
"A", "failed",
|
||||
"turn failed: 404 model not found: gpt-9.9-nonexistent")
|
||||
|
||||
_insert_task(self.db_path, "A")
|
||||
worker = self._worker(
|
||||
runner=self._recording(fake),
|
||||
model="gpt-6.1-sol",
|
||||
fallback_models=["gpt-6-sol"],
|
||||
max_retries=0)
|
||||
result = worker.run_one()
|
||||
self.assertEqual(result["status"], "completed")
|
||||
self.assertEqual(result["actual_model"], "gpt-6-sol")
|
||||
self.assertEqual(len(self.calls), 2)
|
||||
self.assertEqual(result["events"][0]["reason"], "404")
|
||||
|
||||
def test_failed_turn_json_with_401_in_text_never_fallbacks(self):
|
||||
# 回合 JSON 里是 401 时绝不 fallback，只调一次。
|
||||
def fake(cmd):
|
||||
return _host_json("A", "failed", "turn failed: 401 unauthorized")
|
||||
|
||||
_insert_task(self.db_path, "A")
|
||||
worker = self._worker(
|
||||
runner=self._recording(fake),
|
||||
model="gpt-6.1-sol",
|
||||
fallback_models=["gpt-6-sol"],
|
||||
max_retries=0)
|
||||
result = worker.run_one()
|
||||
self.assertEqual(result["status"], "failed")
|
||||
self.assertEqual(len(self.calls), 1)
|
||||
self.assertEqual(result["events"], [])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
+27
-17
@@ -917,26 +917,32 @@ class Worker:
|
||||
return stdout.strip() or "host execution failed"
|
||||
|
||||
def _fallback_token(self, result, error):
|
||||
"""一次失败尝试的可 fallback 信号（host_status 为空时的兜底）。
|
||||
"""一次失败尝试的可 fallback 信号。
|
||||
|
||||
坏模型最常见的真实失败不在 turn 里，而在 thread/start：
|
||||
host.py 的 create_thread 抛 RuntimeError、进程非 0 退出、
|
||||
不打印 JSON，于是执行器结果的 host_status 为 None，旧逻辑
|
||||
因此永远不 fallback（主机实测：坏模型任务直接 failed、
|
||||
无 worker_fallback 审计）。此时从 stdout/stderr/runner 异常
|
||||
文本里找显式 HTTP 状态码（404/429/5xx）或“model not found”
|
||||
类文本，前者按码判断，后者按 404 计；找不到显式信号时
|
||||
返回 None，绝不凭“host execution failed”这类通用文本猜测，
|
||||
否则任何执行器故障都会被误当成模型故障换模型烧额度。
|
||||
host_status 的真实含义是 host.py 回合状态（completed /
|
||||
failed / timeout / interrupted），不是 HTTP 状态码——
|
||||
真实世界里 host.py 从不返回 status=\"404\"，只有单测里的
|
||||
假 runner 才会这么写。所以这里不能把 host_status 原样
|
||||
当 token：host_status 本身已是可 fallback 码（404/429/
|
||||
5xx/timeout，含历史假 runner 形态）时直接返回；否则
|
||||
（最常见：host_status=\"failed\"）从回合 JSON 的 text/error
|
||||
字段、stdout、stderr、runner 异常里找显式 HTTP 状态码
|
||||
或“model not found”类文本（按 404 计），找到且可
|
||||
fallback 才返回；找不到显式信号时返回 None，绝不凭
|
||||
“host execution failed”这类通用文本猜测，否则任何执行
|
||||
器故障都会被误当成模型故障换模型烧额度。thread/start
|
||||
崩溃（host.py 非 0 退出、不打印 JSON）时 host_status
|
||||
为 None，同样走这条提取路径。
|
||||
"""
|
||||
host_status = (result or {}).get("host_status")
|
||||
if host_status is not None:
|
||||
if host_status is not None and is_fallbackable(host_status):
|
||||
return host_status
|
||||
combined = " ".join([
|
||||
combined_parts = [
|
||||
(result or {}).get("stdout", "") or "",
|
||||
(result or {}).get("stderr", "") or "",
|
||||
error or "",
|
||||
])
|
||||
]
|
||||
combined = " ".join(combined_parts)
|
||||
if not combined:
|
||||
return None
|
||||
lowered = combined.lower()
|
||||
@@ -1138,16 +1144,20 @@ class Worker:
|
||||
if retry_count:
|
||||
error = "[retries=%d] %s" % (retry_count, error)
|
||||
# 未配 fallback_models 时的老行为：只记事件、不重试。
|
||||
if self.router is not None and host_status is not None \
|
||||
and is_fallbackable(host_status):
|
||||
nxt = self.router.next_after(self.model, host_status)
|
||||
# 注意用提取出的 token（真实回合失败时 host_status
|
||||
# 只是 "failed"，真正的 404 藏在回合 JSON 里），
|
||||
# 不能直接用原始 host_status 判断。
|
||||
_token = self._fallback_token(result, error)
|
||||
if self.router is not None and _token is not None \
|
||||
and is_fallbackable(_token):
|
||||
nxt = self.router.next_after(self.model, _token)
|
||||
if nxt is not None:
|
||||
record = dict(task)
|
||||
record["events"] = events
|
||||
# record_fallback 本身会 append 到 events，
|
||||
# 这里不要再 append 一次，否则事件会重复。
|
||||
record_fallback(
|
||||
record, self.model, nxt, str(host_status))
|
||||
record, self.model, nxt, str(_token))
|
||||
fallback_count = 1
|
||||
self._audit(task["project_id"], "worker_fallback",
|
||||
"%s -> %s" % (self.model, nxt))
|
||||
|
||||
@@ -1 +1 @@
|
||||
{"_note": "Docker 执行器专用配置（容器内路径，绝不是宿主路径）：state 必须用相对路径 work/host-state.json——相对配置文件所在目录 /app 解析，得到 /app/work/host-state.json，经额外挂载落到宿主 /opt/codex-app-server/work/host-state.json，属主是 codexsvc（uid 1003，与容器 --user 1003:1004 一致），容器可写。绝不能用默认的 /app/host-state.json（/app 是只读挂载，锁一开就报 Read-only file system），也绝不能用 /work/host-state.json（/work 挂载的宿主目录由 docker 自动创建、属主 root，容器用户写不进去，同样 1 秒内失败）。此文件曾因 deploy 清理被删过（不在仓库里），导致 Docker 派发静默全灭，所以它必须进仓库、随部署一起发。sandbox 用 danger-full-access：主机内核禁止不可信 user namespace，Docker + 内层 bwrap 双重沙箱不可行，生产隔离层就是 Docker 本身。API key 绝不放这里，只走环境变量。 默认仓库是宿主测试仓库：没有 repo_url 的项目走它；有 repo_url 的项目 worker 会把解析出的检出经 --repo 传进去，覆盖它。 codex_home 必须在 /app/work 下：容器内默认 HOME 是 /root，uid 1003 写不进 /root/.codex，app-server 连 initialize 都回不来（实测 60 秒超时）；/app/work 属主是 codexsvc，可写。", "approval": "deny", "approval_policy": "never", "effort": "low", "model": "gpt-6.1-sol", "sandbox": "danger-full-access", "state": "work/host-state.json", "repo": "/var/lib/codex-app-server/test-repo", "codex_home": "/app/work/codex-home"}
|
||||
{"_note": "Docker 执行器专用配置（容器内路径，绝不是宿主路径）：state 必须用相对路径 work/host-state.json——相对配置文件所在目录 /app 解析，得到 /app/work/host-state.json，经额外挂载落到宿主 /opt/codex-app-server/work/host-state.json，属主是 codexsvc（uid 1003，与容器 --user 1003:1004 一致），容器可写。绝不能用默认的 /app/host-state.json（/app 是只读挂载，锁一开就报 Read-only file system），也绝不能用 /work/host-state.json（/work 挂载的宿主目录由 docker 自动创建、属主 root，容器用户写不进去，同样 1 秒内失败）。此文件曾因 deploy 清理被删过（不在仓库里），导致 Docker 派发静默全灭，所以它必须进仓库、随部署一起发。sandbox 用 danger-full-access：主机内核禁止不可信 user namespace，Docker + 内层 bwrap 双重沙箱不可行，生产隔离层就是 Docker 本身。API key 绝不放这里，只走环境变量。 默认仓库是宿主测试仓库：没有 repo_url 的项目走它；有 repo_url 的项目 worker 会把解析出的检出经 --repo 传进去，覆盖它。 codex_home 必须指到 /var/lib/codex-app-server/codex-home（经额外挂载同路径进容器）：那里才有真正的 relay 配置（model_provider xcmapi、base_url、env_key），容器 codex 靠它才知道走 relay；指到空目录会让 codex 直接去 api.openai.com 且无 auth，turn 在连接超时里循环重试直到 host.py 600 秒超时。/var/lib 挂载属主是 codexsvc（uid 1003，与容器 --user 一致），可写。", "approval": "deny", "approval_policy": "never", "effort": "low", "model": "gpt-6.1-sol", "sandbox": "danger-full-access", "state": "work/host-state.json", "repo": "/var/lib/codex-app-server/test-repo", "codex_home": "/var/lib/codex-app-server/codex-home"}
|
||||
Reference in new issue
Block a user