Real model fallback retry (opt-in, at most once)
Worker previously only recorded a model_fallback event on a fallbackable failure and failed the task; ModelRouter.next_after was never used to actually run the task on another model. With --fallback-models (default empty = off, old behaviour unchanged), a fallbackable failure (404/429/5xx/timeout; 401/400 and cancellations never fall back) now re-runs the same task, same worktree/branch, on the first fallback model via host.py's top-level --model flag (no host.py signature change). The second attempt's result is final: actual_model is the fallback model, fallback_count=1, and a double failure keeps both reasons in error_text. LocalExecutor/DockerExecutor pass task model through to host.py; CloudExecutor includes model only when set.
This commit is contained in:
host committed
1 parent
f1d42d04f3
commit
7ce603b610
4 files changed
+473
-51
No files matched your search
@@ -0,0 +1,266 @@
|
||||
"""真实 fallback 换模型重试单测。
|
||||
|
||||
只用注入的假 runner（假 CLI），不跑真实模型回合、不联网、
|
||||
不登录任何服务；env 映射也是注入的普通 dict，里面的值只是
|
||||
占位用的假名，绝不是真实密钥。
|
||||
|
||||
假 runner 的约定：host.py 风格的 JSON 成功时 returncode=0，
|
||||
失败且可 fallback 时同样 returncode=0 但 JSON 的 status 字段
|
||||
是 "404"/"500"/"timeout"（_result_from_runner 据此把结果标
|
||||
failed 并把 host_status 置成该字段，这正是 worker 判定
|
||||
fallback 的依据）；returncode 非 0 时 host_status 为空。
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
import sqlite3
|
||||
import sys
|
||||
import tempfile
|
||||
import time
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
|
||||
try:
|
||||
from cloud.cost import CostStore
|
||||
from cloud.models import ModelRouter
|
||||
from cloud.server import init_db
|
||||
from cloud.worker import Worker
|
||||
except ImportError: # discover -s cloud/tests 时把项目根目录补进 sys.path
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[2]))
|
||||
from cloud.cost import CostStore
|
||||
from cloud.models import ModelRouter
|
||||
from cloud.server import init_db
|
||||
from cloud.worker import Worker
|
||||
|
||||
|
||||
def _insert_task(db_path, task_id, prompt="do work", status="queued",
|
||||
project_id="p1"):
|
||||
now = time.time()
|
||||
conn = sqlite3.connect(db_path)
|
||||
try:
|
||||
conn.execute(
|
||||
"INSERT INTO tasks(id, project_id, prompt, status,"
|
||||
" idempotency_key, depends_on, priority, created_by,"
|
||||
" created_at, updated_at, last_heartbeat)"
|
||||
" VALUES(?, ?, ?, ?, NULL, '[]', 0, 'tester', ?, ?, NULL)",
|
||||
(task_id, project_id, prompt, status, now, now),
|
||||
)
|
||||
conn.commit()
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
def _status(db_path, task_id):
|
||||
conn = sqlite3.connect(db_path)
|
||||
try:
|
||||
row = conn.execute(
|
||||
"SELECT status FROM tasks WHERE id = ?",
|
||||
(task_id,)).fetchone()
|
||||
finally:
|
||||
conn.close()
|
||||
return row[0] if row else None
|
||||
|
||||
|
||||
def _execution(db_path, task_id):
|
||||
conn = sqlite3.connect(db_path)
|
||||
try:
|
||||
row = conn.execute(
|
||||
"SELECT error_text, actual_model FROM tasks WHERE id = ?",
|
||||
(task_id,)).fetchone()
|
||||
cost = conn.execute(
|
||||
"SELECT model, fallback_count FROM cost_records"
|
||||
" WHERE task_id = ?",
|
||||
(task_id,)).fetchone()
|
||||
# 测试库里只有一个项目/任务，审计按项目全取：
|
||||
# worker_fallback 的 detail 是“模型 -> 模型”，不含任务 id，
|
||||
# 按 detail LIKE 任务 id 查会漏掉它。
|
||||
audits = [r[0] for r in conn.execute(
|
||||
"SELECT action FROM audit_log WHERE project_id = ?",
|
||||
("p1",)).fetchall()]
|
||||
finally:
|
||||
conn.close()
|
||||
return row, cost, audits
|
||||
|
||||
|
||||
def _host_json(name, status, text):
|
||||
return SimpleNamespace(
|
||||
returncode=0,
|
||||
stdout=json.dumps({"task": name, "status": status, "text": text}),
|
||||
stderr="")
|
||||
|
||||
|
||||
def _model_of(cmd):
|
||||
"""假 CLI 命令里的 --model 值（没有时返回 None，即配置模型）。"""
|
||||
if "--model" in cmd:
|
||||
return cmd[cmd.index("--model") + 1]
|
||||
return None
|
||||
|
||||
|
||||
class FallbackRetryTestCase(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.tmp = tempfile.TemporaryDirectory()
|
||||
self.db_path = os.path.join(self.tmp.name, "control.db")
|
||||
init_db(self.db_path)
|
||||
self.env = {"XCM_API_KEY": "test-placeholder"}
|
||||
self.calls = []
|
||||
|
||||
def tearDown(self):
|
||||
self.tmp.cleanup()
|
||||
|
||||
def _worker(self, runner=None, **kwargs):
|
||||
kwargs.setdefault("env", self.env)
|
||||
kwargs.setdefault("retry_backoff_seconds", 0)
|
||||
return Worker(self.db_path, "/fake/host.py",
|
||||
runner=runner, **kwargs)
|
||||
|
||||
def _recording(self, inner):
|
||||
def fake(cmd):
|
||||
self.calls.append(list(cmd))
|
||||
return inner(cmd)
|
||||
return fake
|
||||
|
||||
def _models_seen(self):
|
||||
"""每次调用里实际传给 host.py 的模型（None 表示配置模型）。"""
|
||||
seen = []
|
||||
for cmd in self.calls:
|
||||
if "dispatch" in cmd or "run" in cmd:
|
||||
seen.append(_model_of(cmd))
|
||||
return seen
|
||||
|
||||
def test_primary_404_fallback_succeeds(self):
|
||||
# 主模型 404，配了 fallback 列表：真的换模型跑第二次且成功。
|
||||
def fake(cmd):
|
||||
if _model_of(cmd) == "gpt-6-sol":
|
||||
return _host_json("A", "completed", "done by fallback")
|
||||
return _host_json("A", "404", "primary 404 boom")
|
||||
|
||||
_insert_task(self.db_path, "A")
|
||||
worker = self._worker(
|
||||
runner=self._recording(fake),
|
||||
model="gpt-6.1-sol",
|
||||
fallback_models=["gpt-6-sol"],
|
||||
max_retries=0)
|
||||
result = worker.run_one()
|
||||
self.assertEqual(result["status"], "completed")
|
||||
self.assertEqual(result["actual_model"], "gpt-6-sol")
|
||||
self.assertEqual(len(self.calls), 2)
|
||||
self.assertEqual(self._models_seen(),
|
||||
["gpt-6.1-sol", "gpt-6-sol"])
|
||||
self.assertEqual(len(result["events"]), 1)
|
||||
event = result["events"][0]
|
||||
self.assertEqual(event["type"], "model_fallback")
|
||||
self.assertEqual(event["from_model"], "gpt-6.1-sol")
|
||||
self.assertEqual(event["to_model"], "gpt-6-sol")
|
||||
self.assertEqual(_status(self.db_path, "A"), "completed")
|
||||
row, cost, audits = _execution(self.db_path, "A")
|
||||
self.assertEqual(row[1], "gpt-6-sol")
|
||||
self.assertEqual(cost[0], "gpt-6-sol")
|
||||
self.assertEqual(cost[1], 1)
|
||||
self.assertIn("worker_fallback", audits)
|
||||
|
||||
def test_primary_401_never_fallbacks(self):
|
||||
# 401 绝不 fallback：只调一次，直接 failed，无 fallback 事件。
|
||||
def fake(cmd):
|
||||
return _host_json("A", "401", "bad credentials")
|
||||
|
||||
_insert_task(self.db_path, "A")
|
||||
worker = self._worker(
|
||||
runner=self._recording(fake),
|
||||
model="gpt-6.1-sol",
|
||||
fallback_models=["gpt-6-sol"],
|
||||
max_retries=0)
|
||||
result = worker.run_one()
|
||||
self.assertEqual(result["status"], "failed")
|
||||
self.assertEqual(len(self.calls), 1)
|
||||
self.assertEqual(result["events"], [])
|
||||
row, cost, audits = _execution(self.db_path, "A")
|
||||
self.assertEqual(cost[1], 0)
|
||||
self.assertNotIn("worker_fallback", audits)
|
||||
|
||||
def test_no_fallback_list_keeps_old_event_only_behavior(self):
|
||||
# 未配 fallback 列表：老行为——只记事件、直接 failed、调一次。
|
||||
def fake(cmd):
|
||||
return _host_json("A", "404", "primary 404 boom")
|
||||
|
||||
_insert_task(self.db_path, "A")
|
||||
router = ModelRouter(["gpt-6.1-sol", "gpt-6-sol"])
|
||||
worker = self._worker(
|
||||
runner=self._recording(fake),
|
||||
model="gpt-6.1-sol",
|
||||
router=router,
|
||||
max_retries=0)
|
||||
result = worker.run_one()
|
||||
self.assertEqual(result["status"], "failed")
|
||||
self.assertEqual(len(self.calls), 1)
|
||||
self.assertEqual(len(result["events"]), 1)
|
||||
self.assertEqual(result["events"][0]["type"], "model_fallback")
|
||||
self.assertEqual(result["actual_model"], "gpt-6.1-sol")
|
||||
row, cost, audits = _execution(self.db_path, "A")
|
||||
self.assertEqual(cost[1], 1)
|
||||
self.assertIn("worker_fallback", audits)
|
||||
|
||||
def test_both_attempts_fail_error_contains_both_reasons(self):
|
||||
# 第二次也失败：error_text 同时含两次原因，fallback_count=1。
|
||||
def fake(cmd):
|
||||
if _model_of(cmd) == "gpt-6-sol":
|
||||
return _host_json("A", "500", "fallback 500 boom")
|
||||
return _host_json("A", "404", "primary 404 boom")
|
||||
|
||||
_insert_task(self.db_path, "A")
|
||||
worker = self._worker(
|
||||
runner=self._recording(fake),
|
||||
model="gpt-6.1-sol",
|
||||
fallback_models=["gpt-6-sol"],
|
||||
max_retries=0)
|
||||
result = worker.run_one()
|
||||
self.assertEqual(result["status"], "failed")
|
||||
self.assertIn("primary 404 boom", result["error"])
|
||||
self.assertIn("fallback 500 boom", result["error"])
|
||||
self.assertIn("gpt-6.1-sol", result["error"])
|
||||
self.assertIn("gpt-6-sol", result["error"])
|
||||
self.assertEqual(len(self.calls), 2)
|
||||
row, cost, audits = _execution(self.db_path, "A")
|
||||
self.assertEqual(cost[1], 1)
|
||||
self.assertEqual(row[1], "gpt-6-sol")
|
||||
|
||||
def test_fallback_at_most_once_even_with_two_models(self):
|
||||
# 配两个 fallback 模型也只试第一个：总共调两次，绝不试第三个。
|
||||
def fake(cmd):
|
||||
return _host_json("A", "429", "always rate limited")
|
||||
|
||||
_insert_task(self.db_path, "A")
|
||||
worker = self._worker(
|
||||
runner=self._recording(fake),
|
||||
model="gpt-6.1-sol",
|
||||
fallback_models=["gpt-6-sol", "gpt-5.6-sol"],
|
||||
max_retries=0)
|
||||
result = worker.run_one()
|
||||
self.assertEqual(result["status"], "failed")
|
||||
self.assertEqual(len(self.calls), 2)
|
||||
self.assertEqual(self._models_seen(),
|
||||
["gpt-6.1-sol", "gpt-6-sol"])
|
||||
self.assertNotIn("gpt-5.6-sol", self._models_seen())
|
||||
|
||||
def test_timeout_first_attempt_fallback_succeeds(self):
|
||||
# 超时也算可 fallback：第一次 timeout，换模型后成功。
|
||||
def fake(cmd):
|
||||
if _model_of(cmd) == "gpt-6-sol":
|
||||
return _host_json("A", "completed", "done after timeout")
|
||||
return _host_json("A", "timeout", "")
|
||||
|
||||
_insert_task(self.db_path, "A")
|
||||
worker = self._worker(
|
||||
runner=self._recording(fake),
|
||||
model="gpt-6.1-sol",
|
||||
fallback_models=["gpt-6-sol"],
|
||||
max_retries=0)
|
||||
result = worker.run_one()
|
||||
self.assertEqual(result["status"], "completed")
|
||||
self.assertEqual(result["actual_model"], "gpt-6-sol")
|
||||
self.assertEqual(len(self.calls), 2)
|
||||
self.assertEqual(result["events"][0]["reason"], "timeout")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in new issue
Block a user