Surface failed-turn reason: host.py recovers it from thread_history, dispatch prints it

Real bad-model turn: relay returns 404, codex retries sampling
5 times, turn fails, but app-server's turn/completed event
carries status=failed with NO error and NO text, so worker
never saw any fallbackable signal. The 404 actually lives in
codex's thread_history_1.sqlite (thread_turns.error_json).
host.py Server.run_turn now best-effort recovers that message
(read-only query by thread_id+turn_id, any failure -> None,
never breaks the turn itself), and dispatch CLI prints the
error field (run CLI already printed the full result). Worker
_attempt_error prefers the error field when text is empty, so
a double failure keeps the real reasons, not just status codes.
Tests: host TurnErrorFromHistoryTest (recovery + missing/corrupt
DB -> None), full host + cloud suites pass.
This commit is contained in:
host committed 2026-10-26 06:13:27 +08:00
1 parent 6a857e06bd
commit c114178463
3 files changed
+105 -3

No files matched your search

+47
View File
@@ -248,6 +248,53 @@ class RetryAndBudgetTest(unittest.TestCase):
self.assertIn("t1 failed", Path(notify_log).read_text())
class TurnErrorFromHistoryTest(unittest.TestCase):
def _server_with_history(self, tmp, error_json):
import sqlite3
db = os.path.join(tmp, "thread_history_1.sqlite")
conn = sqlite3.connect(db)
conn.execute(
"CREATE TABLE thread_turns(thread_id TEXT, turn_id TEXT,"
" error_json TEXT)")
if error_json is not None:
conn.execute(
"INSERT INTO thread_turns VALUES('th1', 'tu1', ?)",
(error_json,))
conn.commit()
conn.close()
srv = object.__new__(host.Server)
srv.codex_home = tmp
return srv
def test_failed_turn_recovers_404_from_history(self):
# 事件里 status=failed 但 error 为空时,从 thread_history
# 库里按 thread_id + turn_id 捞回 404 原因(主机实测回归:
# 坏模型回合的 404 只在库里,事件里没有,导致 worker
# 永远看不到可 fallback 信号)。
import json as _json
with tempfile.TemporaryDirectory() as tmp:
err = _json.dumps({
"message": "unexpected status 404 Not Found: Model "
"\"gpt-9.9-nonexistent\" is not available",
"codexErrorInfo": {"httpConnectionFailed":
{"httpStatusCode": 404}}})
srv = self._server_with_history(tmp, err)
got = srv._turn_error_from_history("th1", "tu1")
self.assertIn("404", got)
self.assertIn("gpt-9.9-nonexistent", got)
def test_missing_history_returns_none_without_crashing(self):
# 库不存在、行不存在、JSON 损坏都只返回 None,不抛错。
with tempfile.TemporaryDirectory() as tmp:
srv = self._server_with_history(tmp, None)
self.assertIsNone(srv._turn_error_from_history("th1", "tu1"))
self.assertIsNone(srv._turn_error_from_history("th1", None))
self.assertIsNone(srv._turn_error_from_history(None, "tu1"))
with tempfile.TemporaryDirectory() as tmp:
srv = self._server_with_history(tmp, "not-json{")
self.assertIsNone(srv._turn_error_from_history("th1", "tu1"))
class StateVersionTest(unittest.TestCase):
def test_version_increments_and_conflict_logged(self):
with tempfile.TemporaryDirectory() as tmp: