goal 异常退出记 interrupted，可断点续跑

长 goal 实弹（2026-10-28）：宿主机内存挤兑致回合挂起 59 分钟，恢复时 goal 进程异常退出、无结论行，云端旧语义（returncode 非 0 一律 failed）把可续跑的断点判成终局。修法：host.py run_goal_loop 回合异常统一记 interrupted 并打印 goal interrupted after rounds 标记行；cloud/sandbox.py 四处 goal 解析同步新语义（标记或无结论非 0 退出 -> interrupted，有结论的非 0 退出仍 failed）；worker 终局审计名与状态对齐。单测：sandbox 新测 5 个、host 新测 1 个、更新旧测 1 个；三套件 cloud 1017、host 61、deploy 29 全绿。只本地提交，未推未部署。
2026-10-28 15:18:26 +08:00
1 parent d129f7bf8c
commit 94137c4a29
@@ -112,6 +112,22 @@ def _parse_goal_output(stdout):
    return None, None


+def _goal_interrupted_reason(stdout):
+    """goal interrupted 标记行的原因，无标记返回 None。
+
+    host.py run_goal_loop 在回合异常时打印
+    "goal interrupted after rounds: <原因>" 并记 interrupted；
+    云端据此把任务标 interrupted（可 /goal-resume 断点续跑），
+    而不是 terminal 的 failed——异常退出时断点（goal_rounds_done、
+    线程、worktree）都在盘上，判 failed 会把可续跑的活路断掉。
+    """
+    for line in (stdout or "").splitlines():
+        if "goal interrupted after rounds" in line:
+            reason = line.split("goal interrupted after rounds", 1)[1]
+            return reason.strip().lstrip(":").strip()[:500]
+    return None
+
+
 def _goal_stdout_tail(stdout):
    """goal 结果存库的 stdout：结论在末尾，截尾不截头。

@@ -512,16 +528,20 @@ class LocalExecutor(Executor):
        """把 host.py goal 的 runner 返回值解析成 goal 结果 dict。

        解析用完整 stdout（结论在末尾），存库的 stdout 才截尾；
-        returncode 非 0 一律 failed；为 0 时无结论也判 failed，
-        绝不把“没打印结论”当成达标。host 任务已存在时（同任务
-        /retry 撞名，goal 没有 run 续跑语义）错误写明“goal 失败
-        请用继续而非重试”（P1：重试在已存在 host-state 上必失败，
-        断点续跑才是正确路径，见 worker 的回落逻辑）。
+        returncode 非 0 时：已打印 met/unmet 结论的仍判 failed，
+        无结论的异常退出判 interrupted（断点在盘上，可续跑，
+        绝不能一律 failed）；host 打印了 interrupted 标记时也判
+        interrupted。为 0 时无结论仍判 failed，绝不把“没打印
+        结论”当成达标。host 任务已存在时（同任务 /retry 撞名，
+        goal 没有 run 续跑语义）错误写明“goal 失败请用继续而
+        非重试”（P1：重试在已存在 host-state 上必失败，断点续
+        跑才是正确路径，见 worker 的回落逻辑）。
        """
        returncode = getattr(result, "returncode", 1)
        stdout_full = getattr(result, "stdout", "") or ""
        stderr = getattr(result, "stderr", "") or ""
        met, goal_error = _parse_goal_output(stdout_full)
+        interrupted_reason = _goal_interrupted_reason(stdout_full)
        combined = stdout_full + stderr
        if "已存在" in combined:
            met, goal_error = None, (
@@ -530,12 +550,23 @@ class LocalExecutor(Executor):
                "尝试断点续跑，断点续跑仍失败时请用继续（不要"
                "重排同一任务）")
            status = "failed"
+        elif interrupted_reason is not None:
+            # host 侧已记 interrupted：异常中断不是失败，云端跟
+            # 着记 interrupted，任务可 /goal-resume 断点续跑。
+            status = "interrupted"
+            goal_error = (interrupted_reason
+                          or "host goal 异常中断，断点可续跑")
        elif returncode != 0:
-            status = "failed"
            if met is None:
+                # 无结论的异常退出（进程被杀、连接断后恢复时抛错
+                # 等）：断点在盘上，记 interrupted 保住续跑活路，
+                # 绝不能一律 failed——2026-10-28 长 goal 实弹教训。
+                status = "interrupted"
                goal_error = ((stderr.strip()[-500:] if stderr.strip()
                               else stdout_full.strip()[-500:])
-                              or "host goal failed")
+                              or "host goal 异常退出（无结论），断点可续跑")
+            else:
+                status = "failed"
        elif met is True:
            status = "completed"
        elif met is False:
@@ -1223,6 +1254,7 @@ class DockerExecutor(Executor):
        stdout_full = getattr(result, "stdout", "") or ""
        stderr = getattr(result, "stderr", "") or ""
        met, goal_error = _parse_goal_output(stdout_full)
+        interrupted_reason = _goal_interrupted_reason(stdout_full)
        combined = stdout_full + stderr
        if "已存在" in combined:
            met, goal_error = None, (
@@ -1231,12 +1263,23 @@ class DockerExecutor(Executor):
                "尝试断点续跑，断点续跑仍失败时请用继续（不要"
                "重排同一任务）")
            status = "failed"
+        elif interrupted_reason is not None:
+            # host 侧已记 interrupted：异常中断不是失败，云端跟
+            # 着记 interrupted，任务可 /goal-resume 断点续跑。
+            status = "interrupted"
+            goal_error = (interrupted_reason
+                          or "host goal 异常中断，断点可续跑")
        elif returncode != 0:
-            status = "failed"
            if met is None:
+                # 无结论的异常退出（进程被杀、连接断后恢复时抛错
+                # 等）：断点在盘上，记 interrupted 保住续跑活路，
+                # 绝不能一律 failed——2026-10-28 长 goal 实弹教训。
+                status = "interrupted"
                goal_error = ((stderr.strip()[-500:] if stderr.strip()
                               else stdout_full.strip()[-500:])
-                              or "host goal failed")
+                              or "host goal 异常退出（无结论），断点可续跑")
+            else:
+                status = "failed"
        elif met is True:
            status = "completed"
        elif met is False:
@@ -1269,7 +1312,9 @@ class DockerExecutor(Executor):
        目标才传 goal_text（带 --goal-text 与 --max-rounds，默认 0）。
        解析与 goal_task 同款（完整 stdout 结论、存库截尾），但
        没有“已存在”分支：goal-resume 本来就是操作已有 host 任务，
-        不存在 dispatch 撞名问题；returncode 非 0 或无结论一律 failed。
+        不存在 dispatch 撞名问题；returncode 非 0 时：有结论仍
+        failed，无结论判 interrupted（断点可续跑）；rc==0 无结论
+        仍 failed。
        """
        if not isinstance(task, dict) or task.get("id") is None:
            raise ValueError("任务缺少 id")
@@ -1315,12 +1360,24 @@ class DockerExecutor(Executor):
        stdout_full = getattr(result, "stdout", "") or ""
        stderr = getattr(result, "stderr", "") or ""
        met, goal_error = _parse_goal_output(stdout_full)
-        if returncode != 0:
-            status = "failed"
+        interrupted_reason = _goal_interrupted_reason(stdout_full)
+        if interrupted_reason is not None:
+            # host 侧已记 interrupted：异常中断不是失败，云端跟
+            # 着记 interrupted，任务可 /goal-resume 断点续跑。
+            status = "interrupted"
+            goal_error = (interrupted_reason
+                          or "host goal-resume 异常中断，断点可续跑")
+        elif returncode != 0:
            if met is None:
+                # 无结论的异常退出：断点在盘上，记 interrupted
+                # 保住续跑活路，绝不能一律 failed。
+                status = "interrupted"
                goal_error = ((stderr.strip()[-500:] if stderr.strip()
                               else stdout_full.strip()[-500:])
-                              or "host goal-resume failed")
+                              or "host goal-resume 异常退出（无结论），"
+                                 "断点可续跑")
+            else:
+                status = "failed"
        elif met is True:
            status = "completed"
        elif met is False:
@@ -1386,12 +1443,24 @@ class DockerExecutor(Executor):
        stdout_full = getattr(result, "stdout", "") or ""
        stderr = getattr(result, "stderr", "") or ""
        met, goal_error = _parse_goal_output(stdout_full)
-        if returncode != 0:
-            status = "failed"
+        interrupted_reason = _goal_interrupted_reason(stdout_full)
+        if interrupted_reason is not None:
+            # host 侧已记 interrupted：异常中断不是失败，云端跟
+            # 着记 interrupted，任务可 /goal-resume 断点续跑。
+            status = "interrupted"
+            goal_error = (interrupted_reason
+                          or "host goal-resume 异常中断，断点可续跑")
+        elif returncode != 0:
            if met is None:
+                # 无结论的异常退出：断点在盘上，记 interrupted
+                # 保住续跑活路，绝不能一律 failed。
+                status = "interrupted"
                goal_error = ((stderr.strip()[-500:] if stderr.strip()
                               else stdout_full.strip()[-500:])
-                              or "host goal-resume failed")
+                              or "host goal-resume 异常退出（无结论），"
+                                 "断点可续跑")
+            else:
+                status = "failed"
        elif met is True:
            status = "completed"
        elif met is False:
@@ -309,7 +309,10 @@ class TestGoalExecutors(unittest.TestCase):
        self.assertIsNone(result["goal_met"])
        self.assertIn("无结论", result["goal_error"])

-    def test_local_goal_task_nonzero_is_failed(self):
+    def test_local_goal_task_nonzero_without_conclusion_is_interrupted(self):
+        # 2026-10-28 长 goal 实弹：无结论的异常退出（进程被杀、
+        # 连接断后恢复时抛错）必须记 interrupted 保住断点续跑，
+        # 绝不能一律 failed（旧语义直接断了 60 分钟工作的活路）。
        def fake(cmd):
            return SimpleNamespace(returncode=1, stdout="", stderr="boom")

@@ -317,9 +320,77 @@ class TestGoalExecutors(unittest.TestCase):
                               runner=fake).goal_task(
            {"id": "t1", "prompt": "do it",
             "verify_command": "true"})
-        self.assertEqual(result["status"], "failed")
+        self.assertEqual(result["status"], "interrupted")
+        self.assertIsNone(result["goal_met"])
        self.assertIn("boom", result["goal_error"])

+    def test_local_goal_task_nonzero_with_unmet_stays_failed(self):
+        def fake(cmd):
+            return SimpleNamespace(
+                returncode=1,
+                stdout="round 1: turn=completed verify_rc=1 "
+                       "verify_out=file missing\n"
+                       "goal unmet after rounds\n",
+                stderr="")
+
+        result = LocalExecutor("/fake/host.py",
+                               runner=fake).goal_task(
+            {"id": "t1", "prompt": "do it",
+             "verify_command": "true"})
+        self.assertEqual(result["status"], "failed")
+        self.assertIs(result["goal_met"], False)
+
+    def test_local_goal_task_interrupted_marker(self):
+        def fake(cmd):
+            return SimpleNamespace(
+                returncode=0,
+                stdout="round 1: turn=interrupted verify_rc=- "
+                       "verify_out=\n"
+                       "goal interrupted after rounds: relay boom\n",
+                stderr="")
+
+        result = LocalExecutor("/fake/host.py",
+                               runner=fake).goal_task(
+            {"id": "t1", "prompt": "do it",
+             "verify_command": "true"})
+        self.assertEqual(result["status"], "interrupted")
+        self.assertIn("relay boom", result["goal_error"])
+
+    def test_docker_goal_task_nonzero_without_conclusion_is_interrupted(self):
+        def fake_runner(cmd, timeout=None):
+            return SimpleNamespace(returncode=1, stdout="",
+                                   stderr="boom")
+
+        with mock.patch("sandbox.shutil.which",
+                        return_value="/usr/bin/docker"):
+            executor = DockerExecutor(
+                "/fake/host.py", runner=fake_runner,
+                env={"XCM_API_KEY": "placeholder-not-a-real-key"})
+            result = executor.goal_task(
+                {"id": "t1", "prompt": "do it",
+                 "verify_command": "test -f x"})
+        self.assertEqual(result["status"], "interrupted")
+        self.assertIn("boom", result["goal_error"])
+
+    def test_docker_goal_resume_nonzero_without_conclusion_is_interrupted(self):
+        # 覆盖 docker 侧两处 resume 解析（goal_resume_task 与
+        # goal_resume_breakpoint_task 共用 shape B 链路，两个都测）。
+        def fake_runner(cmd, timeout=None):
+            return SimpleNamespace(returncode=1, stdout="",
+                                   stderr="boom")
+
+        with mock.patch("sandbox.shutil.which",
+                        return_value="/usr/bin/docker"):
+            executor = DockerExecutor(
+                "/fake/host.py", runner=fake_runner,
+                env={"XCM_API_KEY": "placeholder-not-a-real-key"})
+            for method in ("goal_resume_task",
+                           "goal_resume_breakpoint_task"):
+                result = getattr(executor, method)(
+                    {"id": "t1", "prompt": "do it"})
+                self.assertEqual(result["status"], "interrupted")
+                self.assertIn("boom", result["goal_error"])
+
    def test_local_goal_task_already_exists_names_retry(self):
        def fake(cmd):
            return SimpleNamespace(
@@ -3770,13 +3770,19 @@ class Worker:
            cost_id = self._record_usage_cost(
                task, actual_model, usage_total, stdout, fallback_count)
        if final_written:
+            # 审计名须与终态一致：interrupted 任务记 worker_failed
+            # 会把“可续跑的中断”误读成失败，误导后续排查。
            self._audit(task["project_id"],
                        "worker_quota_stop" if quota_stop else
                        ("worker_completed" if final == "completed"
+                         else "worker_interrupted"
+                         if final == "interrupted"
                         else "worker_failed"), task_id)
            log_event(self.logger,
                      "worker_quota_stop" if quota_stop else
                      ("worker_completed" if final == "completed"
+                       else "worker_interrupted"
+                       if final == "interrupted"
                       else "worker_failed"),
                      task_id=task_id, project_id=task["project_id"],
                      status=final)