Merge pull request #109 from openqj/feat-goal-fail-resume

goal 异常退出记 interrupted，可断点续跑

CHANGELOG.md

@@ -3,6 +3,37 @@
 语义化版本见 `VERSION`（当前 0.1.0）。以下按日期倒序，只列已交付
 的大项，每项附实测证据；未建成的东西在“诚实边界”里明说，不夸大。
 
+## 2026-10-28：长 goal 任务异常退出可断点续跑（不再 terminal failed）
+
+- 实弹事故：11:49 认领的 8 轮 goal 长任务跑约 60 分钟后终局
+  failed、无 goal 结论；host-state 仍是 goal-running、
+  goal_rounds_done=0、turns 为空——第 1 轮 turn 从 11:53 起
+  再无进展，600 秒回合超时未触发（同期探针脚本与 autoscaler
+  一并静默，12:48:48 全局 OOM 杀掉另一租户的 supermemory-ser
+  后系统恢复，12:48:56 任务判 failed）。根因链：宿主机内存
+  挤兑致全机停滞 + 恢复时 goal 进程异常退出、无结论行，云端
+  旧语义（returncode 非 0 一律 failed）把可续跑的断点判成终局。
+- 修法：host.py `run_goal_loop` 回合异常（turn 抛错、验收起
+  不来、检查点落盘失败）不再向上传播，统一记 interrupted、
+  打印 "goal interrupted after rounds: <原因>" 标记行、exit 0；
+  cloud/sandbox.py 四处 goal 结论解析（local goal、docker
+  goal、docker 两处 resume）同步新语义：interrupted 标记 →
+  interrupted；returncode 非 0 但无结论 → interrupted（断点在
+  盘上，保住 /goal-resume 活路）；有 met/unmet 结论的非 0 退出
+  仍 failed，rc==0 无结论仍 failed；worker 终局审计名与状态
+  对齐（interrupted 记 worker_interrupted，不再误记 failed）。
+- 单测：sandbox 新测 5 个（local 无结论非 0 → interrupted、
+  非 0 带 unmet 结论仍 failed、interrupted 标记、docker goal 与
+  docker 两处 resume 无结论非 0 → interrupted），host 新测 1 个
+  （turn 抛错 → interrupted + 标记行 + 落盘原因），并更新 1 个
+  旧测（首轮前落盘断言：旧“异常上传播 + 状态 goal-running”
+  改为新语义）。
+  诚实边界：导致恢复时退出的确切异常已不可考（任务行与执行
+  结果已被探针清理删掉，stderr 无存档）；本次只修“异常退出
+  的终局语义”，不修宿主机内存挤兑本身（另一租户容器，不动）；
+  本批只本地提交，未部署、未做生产实弹复验，部署与复验另等
+  用户点头。
+
 ## 2026-10-26：收尾（Docker 沙箱实测 + 云端合并状态）
 
 - Docker 沙箱执行器接通并在外部主机上真实跑通：镜像

cloud/sandbox.py

@@ -112,6 +112,22 @@ def _parse_goal_output(stdout):
     return None, None
 
 
+def _goal_interrupted_reason(stdout):
+    """goal interrupted 标记行的原因，无标记返回 None。
+
+    host.py run_goal_loop 在回合异常时打印
+    "goal interrupted after rounds: <原因>" 并记 interrupted；
+    云端据此把任务标 interrupted（可 /goal-resume 断点续跑），
+    而不是 terminal 的 failed——异常退出时断点（goal_rounds_done、
+    线程、worktree）都在盘上，判 failed 会把可续跑的活路断掉。
+    """
+    for line in (stdout or "").splitlines():
+        if "goal interrupted after rounds" in line:
+            reason = line.split("goal interrupted after rounds", 1)[1]
+            return reason.strip().lstrip(":").strip()[:500]
+    return None
+
+
 def _goal_stdout_tail(stdout):
     """goal 结果存库的 stdout：结论在末尾，截尾不截头。
 
@@ -512,16 +528,20 @@ class LocalExecutor(Executor):
         """把 host.py goal 的 runner 返回值解析成 goal 结果 dict。
 
         解析用完整 stdout（结论在末尾），存库的 stdout 才截尾；
-        returncode 非 0 一律 failed；为 0 时无结论也判 failed，
-        绝不把“没打印结论”当成达标。host 任务已存在时（同任务
-        /retry 撞名，goal 没有 run 续跑语义）错误写明“goal 失败
-        请用继续而非重试”（P1：重试在已存在 host-state 上必失败，
-        断点续跑才是正确路径，见 worker 的回落逻辑）。
+        returncode 非 0 时：已打印 met/unmet 结论的仍判 failed，
+        无结论的异常退出判 interrupted（断点在盘上，可续跑，
+        绝不能一律 failed）；host 打印了 interrupted 标记时也判
+        interrupted。为 0 时无结论仍判 failed，绝不把“没打印
+        结论”当成达标。host 任务已存在时（同任务 /retry 撞名，
+        goal 没有 run 续跑语义）错误写明“goal 失败请用继续而
+        非重试”（P1：重试在已存在 host-state 上必失败，断点续
+        跑才是正确路径，见 worker 的回落逻辑）。
         """
         returncode = getattr(result, "returncode", 1)
         stdout_full = getattr(result, "stdout", "") or ""
         stderr = getattr(result, "stderr", "") or ""
         met, goal_error = _parse_goal_output(stdout_full)
+        interrupted_reason = _goal_interrupted_reason(stdout_full)
         combined = stdout_full + stderr
         if "已存在" in combined:
             met, goal_error = None, (
@@ -530,12 +550,23 @@ class LocalExecutor(Executor):
                 "尝试断点续跑，断点续跑仍失败时请用继续（不要"
                 "重排同一任务）")
             status = "failed"
+        elif interrupted_reason is not None:
+            # host 侧已记 interrupted：异常中断不是失败，云端跟
+            # 着记 interrupted，任务可 /goal-resume 断点续跑。
+            status = "interrupted"
+            goal_error = (interrupted_reason
+                          or "host goal 异常中断，断点可续跑")
         elif returncode != 0:
-            status = "failed"
             if met is None:
+                # 无结论的异常退出（进程被杀、连接断后恢复时抛错
+                # 等）：断点在盘上，记 interrupted 保住续跑活路，
+                # 绝不能一律 failed——2026-10-28 长 goal 实弹教训。
+                status = "interrupted"
                 goal_error = ((stderr.strip()[-500:] if stderr.strip()
                                else stdout_full.strip()[-500:])
-                              or "host goal failed")
+                              or "host goal 异常退出（无结论），断点可续跑")
+            else:
+                status = "failed"
         elif met is True:
             status = "completed"
         elif met is False:
@@ -1223,6 +1254,7 @@ class DockerExecutor(Executor):
         stdout_full = getattr(result, "stdout", "") or ""
         stderr = getattr(result, "stderr", "") or ""
         met, goal_error = _parse_goal_output(stdout_full)
+        interrupted_reason = _goal_interrupted_reason(stdout_full)
         combined = stdout_full + stderr
         if "已存在" in combined:
             met, goal_error = None, (
@@ -1231,12 +1263,23 @@ class DockerExecutor(Executor):
                 "尝试断点续跑，断点续跑仍失败时请用继续（不要"
                 "重排同一任务）")
             status = "failed"
+        elif interrupted_reason is not None:
+            # host 侧已记 interrupted：异常中断不是失败，云端跟
+            # 着记 interrupted，任务可 /goal-resume 断点续跑。
+            status = "interrupted"
+            goal_error = (interrupted_reason
+                          or "host goal 异常中断，断点可续跑")
         elif returncode != 0:
-            status = "failed"
             if met is None:
+                # 无结论的异常退出（进程被杀、连接断后恢复时抛错
+                # 等）：断点在盘上，记 interrupted 保住续跑活路，
+                # 绝不能一律 failed——2026-10-28 长 goal 实弹教训。
+                status = "interrupted"
                 goal_error = ((stderr.strip()[-500:] if stderr.strip()
                                else stdout_full.strip()[-500:])
-                              or "host goal failed")
+                              or "host goal 异常退出（无结论），断点可续跑")
+            else:
+                status = "failed"
         elif met is True:
             status = "completed"
         elif met is False:
@@ -1269,7 +1312,9 @@ class DockerExecutor(Executor):
         目标才传 goal_text（带 --goal-text 与 --max-rounds，默认 0）。
         解析与 goal_task 同款（完整 stdout 结论、存库截尾），但
         没有“已存在”分支：goal-resume 本来就是操作已有 host 任务，
-        不存在 dispatch 撞名问题；returncode 非 0 或无结论一律 failed。
+        不存在 dispatch 撞名问题；returncode 非 0 时：有结论仍
+        failed，无结论判 interrupted（断点可续跑）；rc==0 无结论
+        仍 failed。
         """
         if not isinstance(task, dict) or task.get("id") is None:
             raise ValueError("任务缺少 id")
@@ -1315,12 +1360,24 @@ class DockerExecutor(Executor):
         stdout_full = getattr(result, "stdout", "") or ""
         stderr = getattr(result, "stderr", "") or ""
         met, goal_error = _parse_goal_output(stdout_full)
-        if returncode != 0:
-            status = "failed"
+        interrupted_reason = _goal_interrupted_reason(stdout_full)
+        if interrupted_reason is not None:
+            # host 侧已记 interrupted：异常中断不是失败，云端跟
+            # 着记 interrupted，任务可 /goal-resume 断点续跑。
+            status = "interrupted"
+            goal_error = (interrupted_reason
+                          or "host goal-resume 异常中断，断点可续跑")
+        elif returncode != 0:
             if met is None:
+                # 无结论的异常退出：断点在盘上，记 interrupted
+                # 保住续跑活路，绝不能一律 failed。
+                status = "interrupted"
                 goal_error = ((stderr.strip()[-500:] if stderr.strip()
                                else stdout_full.strip()[-500:])
-                              or "host goal-resume failed")
+                              or "host goal-resume 异常退出（无结论），"
+                                 "断点可续跑")
+            else:
+                status = "failed"
         elif met is True:
             status = "completed"
         elif met is False:
@@ -1386,12 +1443,24 @@ class DockerExecutor(Executor):
         stdout_full = getattr(result, "stdout", "") or ""
         stderr = getattr(result, "stderr", "") or ""
         met, goal_error = _parse_goal_output(stdout_full)
-        if returncode != 0:
-            status = "failed"
+        interrupted_reason = _goal_interrupted_reason(stdout_full)
+        if interrupted_reason is not None:
+            # host 侧已记 interrupted：异常中断不是失败，云端跟
+            # 着记 interrupted，任务可 /goal-resume 断点续跑。
+            status = "interrupted"
+            goal_error = (interrupted_reason
+                          or "host goal-resume 异常中断，断点可续跑")
+        elif returncode != 0:
             if met is None:
+                # 无结论的异常退出：断点在盘上，记 interrupted
+                # 保住续跑活路，绝不能一律 failed。
+                status = "interrupted"
                 goal_error = ((stderr.strip()[-500:] if stderr.strip()
                                else stdout_full.strip()[-500:])
-                              or "host goal-resume failed")
+                              or "host goal-resume 异常退出（无结论），"
+                                 "断点可续跑")
+            else:
+                status = "failed"
         elif met is True:
             status = "completed"
         elif met is False:

cloud/tests/test_queue_sandbox.py

@@ -309,7 +309,10 @@ class TestGoalExecutors(unittest.TestCase):
         self.assertIsNone(result["goal_met"])
         self.assertIn("无结论", result["goal_error"])
 
-    def test_local_goal_task_nonzero_is_failed(self):
+    def test_local_goal_task_nonzero_without_conclusion_is_interrupted(self):
+        # 2026-10-28 长 goal 实弹：无结论的异常退出（进程被杀、
+        # 连接断后恢复时抛错）必须记 interrupted 保住断点续跑，
+        # 绝不能一律 failed（旧语义直接断了 60 分钟工作的活路）。
         def fake(cmd):
             return SimpleNamespace(returncode=1, stdout="", stderr="boom")
 
@@ -317,9 +320,77 @@ class TestGoalExecutors(unittest.TestCase):
                                runner=fake).goal_task(
             {"id": "t1", "prompt": "do it",
              "verify_command": "true"})
-        self.assertEqual(result["status"], "failed")
+        self.assertEqual(result["status"], "interrupted")
+        self.assertIsNone(result["goal_met"])
         self.assertIn("boom", result["goal_error"])
 
+    def test_local_goal_task_nonzero_with_unmet_stays_failed(self):
+        def fake(cmd):
+            return SimpleNamespace(
+                returncode=1,
+                stdout="round 1: turn=completed verify_rc=1 "
+                       "verify_out=file missing\n"
+                       "goal unmet after rounds\n",
+                stderr="")
+
+        result = LocalExecutor("/fake/host.py",
+                               runner=fake).goal_task(
+            {"id": "t1", "prompt": "do it",
+             "verify_command": "true"})
+        self.assertEqual(result["status"], "failed")
+        self.assertIs(result["goal_met"], False)
+
+    def test_local_goal_task_interrupted_marker(self):
+        def fake(cmd):
+            return SimpleNamespace(
+                returncode=0,
+                stdout="round 1: turn=interrupted verify_rc=- "
+                       "verify_out=\n"
+                       "goal interrupted after rounds: relay boom\n",
+                stderr="")
+
+        result = LocalExecutor("/fake/host.py",
+                               runner=fake).goal_task(
+            {"id": "t1", "prompt": "do it",
+             "verify_command": "true"})
+        self.assertEqual(result["status"], "interrupted")
+        self.assertIn("relay boom", result["goal_error"])
+
+    def test_docker_goal_task_nonzero_without_conclusion_is_interrupted(self):
+        def fake_runner(cmd, timeout=None):
+            return SimpleNamespace(returncode=1, stdout="",
+                                   stderr="boom")
+
+        with mock.patch("sandbox.shutil.which",
+                        return_value="/usr/bin/docker"):
+            executor = DockerExecutor(
+                "/fake/host.py", runner=fake_runner,
+                env={"XCM_API_KEY": "placeholder-not-a-real-key"})
+            result = executor.goal_task(
+                {"id": "t1", "prompt": "do it",
+                 "verify_command": "test -f x"})
+        self.assertEqual(result["status"], "interrupted")
+        self.assertIn("boom", result["goal_error"])
+
+    def test_docker_goal_resume_nonzero_without_conclusion_is_interrupted(self):
+        # 覆盖 docker 侧两处 resume 解析（goal_resume_task 与
+        # goal_resume_breakpoint_task 共用 shape B 链路，两个都测）。
+        def fake_runner(cmd, timeout=None):
+            return SimpleNamespace(returncode=1, stdout="",
+                                   stderr="boom")
+
+        with mock.patch("sandbox.shutil.which",
+                        return_value="/usr/bin/docker"):
+            executor = DockerExecutor(
+                "/fake/host.py", runner=fake_runner,
+                env={"XCM_API_KEY": "placeholder-not-a-real-key"})
+            for method in ("goal_resume_task",
+                           "goal_resume_breakpoint_task"):
+                result = getattr(executor, method)(
+                    {"id": "t1", "prompt": "do it"})
+                self.assertEqual(result["status"], "interrupted")
+                self.assertIn("boom", result["goal_error"])
+
     def test_local_goal_task_already_exists_names_retry(self):
         def fake(cmd):
             return SimpleNamespace(

cloud/worker.py

@@ -3770,13 +3770,19 @@ class Worker:
             cost_id = self._record_usage_cost(
                 task, actual_model, usage_total, stdout, fallback_count)
         if final_written:
+            # 审计名须与终态一致：interrupted 任务记 worker_failed
+            # 会把“可续跑的中断”误读成失败，误导后续排查。
             self._audit(task["project_id"],
                         "worker_quota_stop" if quota_stop else
                         ("worker_completed" if final == "completed"
+                         else "worker_interrupted"
+                         if final == "interrupted"
                          else "worker_failed"), task_id)
             log_event(self.logger,
                       "worker_quota_stop" if quota_stop else
                       ("worker_completed" if final == "completed"
+                       else "worker_interrupted"
+                       if final == "interrupted"
                        else "worker_failed"),
                       task_id=task_id, project_id=task["project_id"],
                       status=final)

host.py

@@ -1896,6 +1896,13 @@ def run_goal_loop(server, state, state_path, task, goal_text, verify,
     不能动这三行；两种结局都 exit 0（main 不抛异常即 0）。
     预算 stop 是第三种结局：打印 "goal stopped after rounds: ..."
     且任务记 interrupted（明天预算重置后可续跑），不冒充 unmet。
+    回合异常是第四种结局：turn 或验收抛错时不向上传播（否则
+    进程非 0 退出、无结论行，云端旧语义会判 terminal failed，
+    断点续跑活路就断了），而是记 interrupted、打印
+    "goal interrupted after rounds: <原因>"，exit 0，任务可
+    /goal-resume 断点续跑（2026-10-28 长 goal 实弹：宿主机
+    内存挤兑致回合挂起 59 分钟、恢复时异常退出，旧语义直接
+    判 failed，60 分钟工作无续跑口）。
     goal-unmet 穷尽轮次后不允许 goal-resume（按既有规则须新建
     任务），goal-resume 只接受 goal-running / interrupted / running。
     """
@@ -1912,6 +1919,28 @@ def run_goal_loop(server, state, state_path, task, goal_text, verify,
     ok = False
     budget_stopped = False
     goal_usage = None
+
+    def _interrupt_loop(exc):
+        # 回合异常的统一出口：不断言、不向上传播，记 interrupted
+        # 并打印标记行，云端据此标 interrupted（可断点续跑）。
+        # 再落盘一次已尽力（二次异常直接吞掉，标记行照常打印；
+        # 即使落盘也失败，云端无结论 + 非 0 退出时仍有
+        # interrupted 兜底，正常路径下本函数返回后 main exit 0）。
+        task["status"] = "interrupted"
+        task["goal_last_turn_status"] = "interrupted"
+        task["goal_interrupted_reason"] = str(exc).replace(
+            "\n", " ")[:200]
+        try:
+            save_state(state_path, state)
+        except Exception:
+            pass
+        print("goal interrupted after rounds: %s"
+              % str(exc).replace("\n", " ")[:500])
+        return {"ok": False, "budget_stopped": False,
+                "interrupted": True,
+                "rounds_done": task["goal_rounds_done"],
+                "last_verify": last_verify, "usage": goal_usage}
+
     # max_rounds=0 是不限轮次：while 循环只在模型宣告、预算耗尽
     # 或取消时结束；非 0 时按旧语义穷尽轮次即 unmet。
     round_no = int(start_round) - 1
@@ -1962,11 +1991,14 @@ def run_goal_loop(server, state, state_path, task, goal_text, verify,
                       f"目标：{goal_text}\n"
                       f"验收命令：{verify}\n"
                       f"上一轮验收输出：\n{last_verify}")
-        result = run_task_turn(server, state, task, prompt,
-                               model, effort, sandbox, timeout=timeout,
-                               interrupt_file=_interrupt_request_path(
-                                   state_path, task["name"])
-                               if task.get("name") else None)
+        try:
+            result = run_task_turn(server, state, task, prompt,
+                                   model, effort, sandbox, timeout=timeout,
+                                   interrupt_file=_interrupt_request_path(
+                                       state_path, task["name"])
+                                   if task.get("name") else None)
+        except Exception as exc:
+            return _interrupt_loop(exc)
         goal_usage = _add_usage(goal_usage, result.get("usage"))
         last_turn_status = result["status"]
         last_turn_text = result.get("text", "") or ""
@@ -1997,6 +2029,10 @@ def run_goal_loop(server, state, state_path, task, goal_text, verify,
                     else ""
                 v_out = (f"verify timeout after {verify_timeout}s: "
                          f"{(out + err).strip()[:900]}")
+            except Exception as exc:
+                # 验收命令本身起不来（OSError 等）不是验收失败，
+                # 同样走中断出口，不让异常杀掉整个循环。
+                return _interrupt_loop(exc)
         last_verify = v_out
         task["goal_rounds_done"] = round_no
         task["goal_last_verify"] = last_verify
@@ -2004,7 +2040,10 @@ def run_goal_loop(server, state, state_path, task, goal_text, verify,
         # 检查点：非终局轮次先记 goal-running，终局轮次由下面
         # 统一改写；这样被杀后 state 里能看出循环没跑完。
         task["status"] = "goal-running"
-        save_state(state_path, state)
+        try:
+            save_state(state_path, state)
+        except Exception as exc:
+            return _interrupt_loop(exc)
         print(f"round {round_no}: turn={result['status']} "
               f"verify_rc={v_rc if v_rc is not None else '-'} "
               f"verify_out={last_verify[:300]}")
@@ -2044,6 +2083,7 @@ def run_goal_loop(server, state, state_path, task, goal_text, verify,
               f"total={goal_usage['total_tokens']} "
               f"completions={goal_usage['completions']}")
     return {"ok": ok, "budget_stopped": budget_stopped,
+            "interrupted": False,
             "rounds_done": task["goal_rounds_done"],
             "last_verify": last_verify, "usage": goal_usage}
 

tests/test_host.py

@@ -345,6 +345,42 @@ class RetryAndBudgetTest(unittest.TestCase):
             self.assertIn("budget-exceeded",
                           Path(notify_log).read_text())
 
+    def test_goal_turn_exception_is_interrupted_not_failed(self):
+        # 2026-10-28 长 goal 实弹：回合抛错必须记 interrupted +
+        # 打印标记行（云端据此可断点续跑），绝不能向上传播致
+        # 进程非 0 退出、无结论行（旧语义云端判 terminal failed）。
+        import contextlib
+        import io
+
+        class FakeServer:
+            def run_turn(self, *args, **kwargs):
+                raise RuntimeError("relay boom")
+
+        with tempfile.TemporaryDirectory() as tmp:
+            state_path = os.path.join(tmp, "host-state.json")
+            state = {"tasks": {}, "automations": [], "usage": {}}
+            task = {"name": "g", "thread_id": "x", "cwd": tmp,
+                    "sandbox": None, "max_retries": 0, "retry_count": 0,
+                    "turns": []}
+            state["tasks"]["g"] = task
+            buf = io.StringIO()
+            with contextlib.redirect_stdout(buf):
+                result = host.run_goal_loop(
+                    FakeServer(), state, state_path,
+                    task, "make it", "true", 3, "m",
+                    "low", None, timeout=600)
+            self.assertTrue(result["interrupted"])
+            self.assertFalse(result["ok"])
+            self.assertEqual(task["status"], "interrupted")
+            self.assertEqual(task["goal_rounds_done"], 0)
+            self.assertIn("relay boom",
+                          task["goal_interrupted_reason"])
+            self.assertIn("goal interrupted after rounds",
+                          buf.getvalue())
+            saved = json.loads(Path(state_path).read_text())
+            self.assertEqual(saved["tasks"]["g"]["status"],
+                             "interrupted")
+
 
 class TurnErrorFromHistoryTest(unittest.TestCase):
     def _server_with_history(self, tmp, error_json):
@@ -934,16 +970,21 @@ class GoalLoopLongTaskTest(unittest.TestCase):
             state = {"tasks": {}, "automations": []}
             task = self._task(tmp)
             state["tasks"]["g"] = task
-            with self.assertRaises(RuntimeError):
-                host.run_goal_loop(FakeServer(), state, state_path,
-                                   task, "make it", "true", 3, "m", "low",
-                                   None, cfg={}, timeout=600)
+            # 2026-10-28 起：回合抛错不再向上传播（旧断言），而
+            # 是记 interrupted + 标记行（云端可断点续跑）；首轮
+            # 前落盘的字段仍必须都在。
+            result = host.run_goal_loop(FakeServer(), state, state_path,
+                                        task, "make it", "true", 3, "m",
+                                        "low", None, cfg={}, timeout=600)
+            self.assertTrue(result["interrupted"])
             saved = host.load_state(state_path)
             saved_task = saved["tasks"]["g"]
             self.assertEqual(saved_task["goal"], "make it")
             self.assertEqual(saved_task["verify_command"], "true")
             self.assertEqual(saved_task["goal_rounds_done"], 0)
-            self.assertEqual(saved_task["status"], "goal-running")
+            self.assertEqual(saved_task["status"], "interrupted")
+            self.assertIn("killed during turn",
+                          saved_task["goal_interrupted_reason"])
 
     def test_goal_budget_stop_is_interrupted_not_unmet(self):
         class FakeServer: