goal 异常退出记 interrupted,可断点续跑

长 goal 实弹(2026-10-28):宿主机内存挤兑致回合挂起 59 分钟,恢复时 goal 进程异常退出、无结论行,云端旧语义(returncode 非 0 一律 failed)把可续跑的断点判成终局。

修法:host.py run_goal_loop 回合异常统一记 interrupted 并打印 goal interrupted after rounds 标记行;cloud/sandbox.py 四处 goal 解析同步新语义(标记或无结论非 0 退出 -> interrupted,有结论的非 0 退出仍 failed);worker 终局审计名与状态对齐。单测:sandbox 新测 5 个、host 新测 1 个、更新旧测 1 个;三套件 cloud 1017、host 61、deploy 29 全绿。只本地提交,未推未部署。
This commit is contained in:
host committed 2026-10-28 15:18:26 +08:00
1 parent d129f7bf8c
commit 94137c4a29
6 files changed
+287 -29

No files matched your search

+46 -5
View File
@@ -345,6 +345,42 @@ class RetryAndBudgetTest(unittest.TestCase):
self.assertIn("budget-exceeded",
Path(notify_log).read_text())
def test_goal_turn_exception_is_interrupted_not_failed(self):
# 2026-10-28 长 goal 实弹:回合抛错必须记 interrupted +
# 打印标记行(云端据此可断点续跑),绝不能向上传播致
# 进程非 0 退出、无结论行(旧语义云端判 terminal failed)。
import contextlib
import io
class FakeServer:
def run_turn(self, *args, **kwargs):
raise RuntimeError("relay boom")
with tempfile.TemporaryDirectory() as tmp:
state_path = os.path.join(tmp, "host-state.json")
state = {"tasks": {}, "automations": [], "usage": {}}
task = {"name": "g", "thread_id": "x", "cwd": tmp,
"sandbox": None, "max_retries": 0, "retry_count": 0,
"turns": []}
state["tasks"]["g"] = task
buf = io.StringIO()
with contextlib.redirect_stdout(buf):
result = host.run_goal_loop(
FakeServer(), state, state_path,
task, "make it", "true", 3, "m",
"low", None, timeout=600)
self.assertTrue(result["interrupted"])
self.assertFalse(result["ok"])
self.assertEqual(task["status"], "interrupted")
self.assertEqual(task["goal_rounds_done"], 0)
self.assertIn("relay boom",
task["goal_interrupted_reason"])
self.assertIn("goal interrupted after rounds",
buf.getvalue())
saved = json.loads(Path(state_path).read_text())
self.assertEqual(saved["tasks"]["g"]["status"],
"interrupted")
class TurnErrorFromHistoryTest(unittest.TestCase):
def _server_with_history(self, tmp, error_json):
@@ -934,16 +970,21 @@ class GoalLoopLongTaskTest(unittest.TestCase):
state = {"tasks": {}, "automations": []}
task = self._task(tmp)
state["tasks"]["g"] = task
with self.assertRaises(RuntimeError):
host.run_goal_loop(FakeServer(), state, state_path,
task, "make it", "true", 3, "m", "low",
None, cfg={}, timeout=600)
# 2026-10-28 起:回合抛错不再向上传播(旧断言),而
# 是记 interrupted + 标记行(云端可断点续跑);首轮
# 前落盘的字段仍必须都在。
result = host.run_goal_loop(FakeServer(), state, state_path,
task, "make it", "true", 3, "m",
"low", None, cfg={}, timeout=600)
self.assertTrue(result["interrupted"])
saved = host.load_state(state_path)
saved_task = saved["tasks"]["g"]
self.assertEqual(saved_task["goal"], "make it")
self.assertEqual(saved_task["verify_command"], "true")
self.assertEqual(saved_task["goal_rounds_done"], 0)
self.assertEqual(saved_task["status"], "goal-running")
self.assertEqual(saved_task["status"], "interrupted")
self.assertIn("killed during turn",
saved_task["goal_interrupted_reason"])
def test_goal_budget_stop_is_interrupted_not_unmet(self):
class FakeServer: