diff --git a/CHANGELOG.md b/CHANGELOG.md index b338848..9397237 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,37 @@ 语义化版本见 `VERSION`(当前 0.1.0)。以下按日期倒序,只列已交付 的大项,每项附实测证据;未建成的东西在“诚实边界”里明说,不夸大。 +## 2026-10-28:长 goal 任务异常退出可断点续跑(不再 terminal failed) + +- 实弹事故:11:49 认领的 8 轮 goal 长任务跑约 60 分钟后终局 + failed、无 goal 结论;host-state 仍是 goal-running、 + goal_rounds_done=0、turns 为空——第 1 轮 turn 从 11:53 起 + 再无进展,600 秒回合超时未触发(同期探针脚本与 autoscaler + 一并静默,12:48:48 全局 OOM 杀掉另一租户的 supermemory-ser + 后系统恢复,12:48:56 任务判 failed)。根因链:宿主机内存 + 挤兑致全机停滞 + 恢复时 goal 进程异常退出、无结论行,云端 + 旧语义(returncode 非 0 一律 failed)把可续跑的断点判成终局。 +- 修法:host.py `run_goal_loop` 回合异常(turn 抛错、验收起 + 不来、检查点落盘失败)不再向上传播,统一记 interrupted、 + 打印 "goal interrupted after rounds: <原因>" 标记行、exit 0; + cloud/sandbox.py 四处 goal 结论解析(local goal、docker + goal、docker 两处 resume)同步新语义:interrupted 标记 → + interrupted;returncode 非 0 但无结论 → interrupted(断点在 + 盘上,保住 /goal-resume 活路);有 met/unmet 结论的非 0 退出 + 仍 failed,rc==0 无结论仍 failed;worker 终局审计名与状态 + 对齐(interrupted 记 worker_interrupted,不再误记 failed)。 +- 单测:sandbox 新测 5 个(local 无结论非 0 → interrupted、 + 非 0 带 unmet 结论仍 failed、interrupted 标记、docker goal 与 + docker 两处 resume 无结论非 0 → interrupted),host 新测 1 个 + (turn 抛错 → interrupted + 标记行 + 落盘原因),并更新 1 个 + 旧测(首轮前落盘断言:旧“异常上传播 + 状态 goal-running” + 改为新语义)。 + 诚实边界:导致恢复时退出的确切异常已不可考(任务行与执行 + 结果已被探针清理删掉,stderr 无存档);本次只修“异常退出 + 的终局语义”,不修宿主机内存挤兑本身(另一租户容器,不动); + 本批只本地提交,未部署、未做生产实弹复验,部署与复验另等 + 用户点头。 + ## 2026-10-26:收尾(Docker 沙箱实测 + 云端合并状态) - Docker 沙箱执行器接通并在外部主机上真实跑通:镜像 diff --git a/cloud/sandbox.py b/cloud/sandbox.py index f5f54f1..7a5a060 100644 --- a/cloud/sandbox.py +++ b/cloud/sandbox.py @@ -112,6 +112,22 @@ def _parse_goal_output(stdout): return None, None +def _goal_interrupted_reason(stdout): + """goal interrupted 标记行的原因,无标记返回 None。 + + host.py run_goal_loop 在回合异常时打印 + "goal interrupted after rounds: <原因>" 并记 interrupted; + 云端据此把任务标 interrupted(可 /goal-resume 断点续跑), + 而不是 terminal 的 failed——异常退出时断点(goal_rounds_done、 + 线程、worktree)都在盘上,判 failed 会把可续跑的活路断掉。 + """ + for line in (stdout or "").splitlines(): + if "goal interrupted after rounds" in line: + reason = line.split("goal interrupted after rounds", 1)[1] + return reason.strip().lstrip(":").strip()[:500] + return None + + def _goal_stdout_tail(stdout): """goal 结果存库的 stdout:结论在末尾,截尾不截头。 @@ -512,16 +528,20 @@ class LocalExecutor(Executor): """把 host.py goal 的 runner 返回值解析成 goal 结果 dict。 解析用完整 stdout(结论在末尾),存库的 stdout 才截尾; - returncode 非 0 一律 failed;为 0 时无结论也判 failed, - 绝不把“没打印结论”当成达标。host 任务已存在时(同任务 - /retry 撞名,goal 没有 run 续跑语义)错误写明“goal 失败 - 请用继续而非重试”(P1:重试在已存在 host-state 上必失败, - 断点续跑才是正确路径,见 worker 的回落逻辑)。 + returncode 非 0 时:已打印 met/unmet 结论的仍判 failed, + 无结论的异常退出判 interrupted(断点在盘上,可续跑, + 绝不能一律 failed);host 打印了 interrupted 标记时也判 + interrupted。为 0 时无结论仍判 failed,绝不把“没打印 + 结论”当成达标。host 任务已存在时(同任务 /retry 撞名, + goal 没有 run 续跑语义)错误写明“goal 失败请用继续而 + 非重试”(P1:重试在已存在 host-state 上必失败,断点续 + 跑才是正确路径,见 worker 的回落逻辑)。 """ returncode = getattr(result, "returncode", 1) stdout_full = getattr(result, "stdout", "") or "" stderr = getattr(result, "stderr", "") or "" met, goal_error = _parse_goal_output(stdout_full) + interrupted_reason = _goal_interrupted_reason(stdout_full) combined = stdout_full + stderr if "已存在" in combined: met, goal_error = None, ( @@ -530,12 +550,23 @@ class LocalExecutor(Executor): "尝试断点续跑,断点续跑仍失败时请用继续(不要" "重排同一任务)") status = "failed" + elif interrupted_reason is not None: + # host 侧已记 interrupted:异常中断不是失败,云端跟 + # 着记 interrupted,任务可 /goal-resume 断点续跑。 + status = "interrupted" + goal_error = (interrupted_reason + or "host goal 异常中断,断点可续跑") elif returncode != 0: - status = "failed" if met is None: + # 无结论的异常退出(进程被杀、连接断后恢复时抛错 + # 等):断点在盘上,记 interrupted 保住续跑活路, + # 绝不能一律 failed——2026-10-28 长 goal 实弹教训。 + status = "interrupted" goal_error = ((stderr.strip()[-500:] if stderr.strip() else stdout_full.strip()[-500:]) - or "host goal failed") + or "host goal 异常退出(无结论),断点可续跑") + else: + status = "failed" elif met is True: status = "completed" elif met is False: @@ -1223,6 +1254,7 @@ class DockerExecutor(Executor): stdout_full = getattr(result, "stdout", "") or "" stderr = getattr(result, "stderr", "") or "" met, goal_error = _parse_goal_output(stdout_full) + interrupted_reason = _goal_interrupted_reason(stdout_full) combined = stdout_full + stderr if "已存在" in combined: met, goal_error = None, ( @@ -1231,12 +1263,23 @@ class DockerExecutor(Executor): "尝试断点续跑,断点续跑仍失败时请用继续(不要" "重排同一任务)") status = "failed" + elif interrupted_reason is not None: + # host 侧已记 interrupted:异常中断不是失败,云端跟 + # 着记 interrupted,任务可 /goal-resume 断点续跑。 + status = "interrupted" + goal_error = (interrupted_reason + or "host goal 异常中断,断点可续跑") elif returncode != 0: - status = "failed" if met is None: + # 无结论的异常退出(进程被杀、连接断后恢复时抛错 + # 等):断点在盘上,记 interrupted 保住续跑活路, + # 绝不能一律 failed——2026-10-28 长 goal 实弹教训。 + status = "interrupted" goal_error = ((stderr.strip()[-500:] if stderr.strip() else stdout_full.strip()[-500:]) - or "host goal failed") + or "host goal 异常退出(无结论),断点可续跑") + else: + status = "failed" elif met is True: status = "completed" elif met is False: @@ -1269,7 +1312,9 @@ class DockerExecutor(Executor): 目标才传 goal_text(带 --goal-text 与 --max-rounds,默认 0)。 解析与 goal_task 同款(完整 stdout 结论、存库截尾),但 没有“已存在”分支:goal-resume 本来就是操作已有 host 任务, - 不存在 dispatch 撞名问题;returncode 非 0 或无结论一律 failed。 + 不存在 dispatch 撞名问题;returncode 非 0 时:有结论仍 + failed,无结论判 interrupted(断点可续跑);rc==0 无结论 + 仍 failed。 """ if not isinstance(task, dict) or task.get("id") is None: raise ValueError("任务缺少 id") @@ -1315,12 +1360,24 @@ class DockerExecutor(Executor): stdout_full = getattr(result, "stdout", "") or "" stderr = getattr(result, "stderr", "") or "" met, goal_error = _parse_goal_output(stdout_full) - if returncode != 0: - status = "failed" + interrupted_reason = _goal_interrupted_reason(stdout_full) + if interrupted_reason is not None: + # host 侧已记 interrupted:异常中断不是失败,云端跟 + # 着记 interrupted,任务可 /goal-resume 断点续跑。 + status = "interrupted" + goal_error = (interrupted_reason + or "host goal-resume 异常中断,断点可续跑") + elif returncode != 0: if met is None: + # 无结论的异常退出:断点在盘上,记 interrupted + # 保住续跑活路,绝不能一律 failed。 + status = "interrupted" goal_error = ((stderr.strip()[-500:] if stderr.strip() else stdout_full.strip()[-500:]) - or "host goal-resume failed") + or "host goal-resume 异常退出(无结论)," + "断点可续跑") + else: + status = "failed" elif met is True: status = "completed" elif met is False: @@ -1386,12 +1443,24 @@ class DockerExecutor(Executor): stdout_full = getattr(result, "stdout", "") or "" stderr = getattr(result, "stderr", "") or "" met, goal_error = _parse_goal_output(stdout_full) - if returncode != 0: - status = "failed" + interrupted_reason = _goal_interrupted_reason(stdout_full) + if interrupted_reason is not None: + # host 侧已记 interrupted:异常中断不是失败,云端跟 + # 着记 interrupted,任务可 /goal-resume 断点续跑。 + status = "interrupted" + goal_error = (interrupted_reason + or "host goal-resume 异常中断,断点可续跑") + elif returncode != 0: if met is None: + # 无结论的异常退出:断点在盘上,记 interrupted + # 保住续跑活路,绝不能一律 failed。 + status = "interrupted" goal_error = ((stderr.strip()[-500:] if stderr.strip() else stdout_full.strip()[-500:]) - or "host goal-resume failed") + or "host goal-resume 异常退出(无结论)," + "断点可续跑") + else: + status = "failed" elif met is True: status = "completed" elif met is False: diff --git a/cloud/tests/test_queue_sandbox.py b/cloud/tests/test_queue_sandbox.py index 42607d2..cfcdaf0 100644 --- a/cloud/tests/test_queue_sandbox.py +++ b/cloud/tests/test_queue_sandbox.py @@ -309,7 +309,10 @@ class TestGoalExecutors(unittest.TestCase): self.assertIsNone(result["goal_met"]) self.assertIn("无结论", result["goal_error"]) - def test_local_goal_task_nonzero_is_failed(self): + def test_local_goal_task_nonzero_without_conclusion_is_interrupted(self): + # 2026-10-28 长 goal 实弹:无结论的异常退出(进程被杀、 + # 连接断后恢复时抛错)必须记 interrupted 保住断点续跑, + # 绝不能一律 failed(旧语义直接断了 60 分钟工作的活路)。 def fake(cmd): return SimpleNamespace(returncode=1, stdout="", stderr="boom") @@ -317,9 +320,77 @@ class TestGoalExecutors(unittest.TestCase): runner=fake).goal_task( {"id": "t1", "prompt": "do it", "verify_command": "true"}) - self.assertEqual(result["status"], "failed") + self.assertEqual(result["status"], "interrupted") + self.assertIsNone(result["goal_met"]) self.assertIn("boom", result["goal_error"]) + def test_local_goal_task_nonzero_with_unmet_stays_failed(self): + def fake(cmd): + return SimpleNamespace( + returncode=1, + stdout="round 1: turn=completed verify_rc=1 " + "verify_out=file missing\n" + "goal unmet after rounds\n", + stderr="") + + result = LocalExecutor("/fake/host.py", + runner=fake).goal_task( + {"id": "t1", "prompt": "do it", + "verify_command": "true"}) + self.assertEqual(result["status"], "failed") + self.assertIs(result["goal_met"], False) + + def test_local_goal_task_interrupted_marker(self): + def fake(cmd): + return SimpleNamespace( + returncode=0, + stdout="round 1: turn=interrupted verify_rc=- " + "verify_out=\n" + "goal interrupted after rounds: relay boom\n", + stderr="") + + result = LocalExecutor("/fake/host.py", + runner=fake).goal_task( + {"id": "t1", "prompt": "do it", + "verify_command": "true"}) + self.assertEqual(result["status"], "interrupted") + self.assertIn("relay boom", result["goal_error"]) + + def test_docker_goal_task_nonzero_without_conclusion_is_interrupted(self): + def fake_runner(cmd, timeout=None): + return SimpleNamespace(returncode=1, stdout="", + stderr="boom") + + with mock.patch("sandbox.shutil.which", + return_value="/usr/bin/docker"): + executor = DockerExecutor( + "/fake/host.py", runner=fake_runner, + env={"XCM_API_KEY": "placeholder-not-a-real-key"}) + result = executor.goal_task( + {"id": "t1", "prompt": "do it", + "verify_command": "test -f x"}) + self.assertEqual(result["status"], "interrupted") + self.assertIn("boom", result["goal_error"]) + + def test_docker_goal_resume_nonzero_without_conclusion_is_interrupted(self): + # 覆盖 docker 侧两处 resume 解析(goal_resume_task 与 + # goal_resume_breakpoint_task 共用 shape B 链路,两个都测)。 + def fake_runner(cmd, timeout=None): + return SimpleNamespace(returncode=1, stdout="", + stderr="boom") + + with mock.patch("sandbox.shutil.which", + return_value="/usr/bin/docker"): + executor = DockerExecutor( + "/fake/host.py", runner=fake_runner, + env={"XCM_API_KEY": "placeholder-not-a-real-key"}) + for method in ("goal_resume_task", + "goal_resume_breakpoint_task"): + result = getattr(executor, method)( + {"id": "t1", "prompt": "do it"}) + self.assertEqual(result["status"], "interrupted") + self.assertIn("boom", result["goal_error"]) + def test_local_goal_task_already_exists_names_retry(self): def fake(cmd): return SimpleNamespace( diff --git a/cloud/worker.py b/cloud/worker.py index b3dfaa9..0226e42 100644 --- a/cloud/worker.py +++ b/cloud/worker.py @@ -3770,13 +3770,19 @@ class Worker: cost_id = self._record_usage_cost( task, actual_model, usage_total, stdout, fallback_count) if final_written: + # 审计名须与终态一致:interrupted 任务记 worker_failed + # 会把“可续跑的中断”误读成失败,误导后续排查。 self._audit(task["project_id"], "worker_quota_stop" if quota_stop else ("worker_completed" if final == "completed" + else "worker_interrupted" + if final == "interrupted" else "worker_failed"), task_id) log_event(self.logger, "worker_quota_stop" if quota_stop else ("worker_completed" if final == "completed" + else "worker_interrupted" + if final == "interrupted" else "worker_failed"), task_id=task_id, project_id=task["project_id"], status=final) diff --git a/host.py b/host.py index 6a1eed1..2b4582a 100644 --- a/host.py +++ b/host.py @@ -1896,6 +1896,13 @@ def run_goal_loop(server, state, state_path, task, goal_text, verify, 不能动这三行;两种结局都 exit 0(main 不抛异常即 0)。 预算 stop 是第三种结局:打印 "goal stopped after rounds: ..." 且任务记 interrupted(明天预算重置后可续跑),不冒充 unmet。 + 回合异常是第四种结局:turn 或验收抛错时不向上传播(否则 + 进程非 0 退出、无结论行,云端旧语义会判 terminal failed, + 断点续跑活路就断了),而是记 interrupted、打印 + "goal interrupted after rounds: <原因>",exit 0,任务可 + /goal-resume 断点续跑(2026-10-28 长 goal 实弹:宿主机 + 内存挤兑致回合挂起 59 分钟、恢复时异常退出,旧语义直接 + 判 failed,60 分钟工作无续跑口)。 goal-unmet 穷尽轮次后不允许 goal-resume(按既有规则须新建 任务),goal-resume 只接受 goal-running / interrupted / running。 """ @@ -1912,6 +1919,28 @@ def run_goal_loop(server, state, state_path, task, goal_text, verify, ok = False budget_stopped = False goal_usage = None + + def _interrupt_loop(exc): + # 回合异常的统一出口:不断言、不向上传播,记 interrupted + # 并打印标记行,云端据此标 interrupted(可断点续跑)。 + # 再落盘一次已尽力(二次异常直接吞掉,标记行照常打印; + # 即使落盘也失败,云端无结论 + 非 0 退出时仍有 + # interrupted 兜底,正常路径下本函数返回后 main exit 0)。 + task["status"] = "interrupted" + task["goal_last_turn_status"] = "interrupted" + task["goal_interrupted_reason"] = str(exc).replace( + "\n", " ")[:200] + try: + save_state(state_path, state) + except Exception: + pass + print("goal interrupted after rounds: %s" + % str(exc).replace("\n", " ")[:500]) + return {"ok": False, "budget_stopped": False, + "interrupted": True, + "rounds_done": task["goal_rounds_done"], + "last_verify": last_verify, "usage": goal_usage} + # max_rounds=0 是不限轮次:while 循环只在模型宣告、预算耗尽 # 或取消时结束;非 0 时按旧语义穷尽轮次即 unmet。 round_no = int(start_round) - 1 @@ -1962,11 +1991,14 @@ def run_goal_loop(server, state, state_path, task, goal_text, verify, f"目标:{goal_text}\n" f"验收命令:{verify}\n" f"上一轮验收输出:\n{last_verify}") - result = run_task_turn(server, state, task, prompt, - model, effort, sandbox, timeout=timeout, - interrupt_file=_interrupt_request_path( - state_path, task["name"]) - if task.get("name") else None) + try: + result = run_task_turn(server, state, task, prompt, + model, effort, sandbox, timeout=timeout, + interrupt_file=_interrupt_request_path( + state_path, task["name"]) + if task.get("name") else None) + except Exception as exc: + return _interrupt_loop(exc) goal_usage = _add_usage(goal_usage, result.get("usage")) last_turn_status = result["status"] last_turn_text = result.get("text", "") or "" @@ -1997,6 +2029,10 @@ def run_goal_loop(server, state, state_path, task, goal_text, verify, else "" v_out = (f"verify timeout after {verify_timeout}s: " f"{(out + err).strip()[:900]}") + except Exception as exc: + # 验收命令本身起不来(OSError 等)不是验收失败, + # 同样走中断出口,不让异常杀掉整个循环。 + return _interrupt_loop(exc) last_verify = v_out task["goal_rounds_done"] = round_no task["goal_last_verify"] = last_verify @@ -2004,7 +2040,10 @@ def run_goal_loop(server, state, state_path, task, goal_text, verify, # 检查点:非终局轮次先记 goal-running,终局轮次由下面 # 统一改写;这样被杀后 state 里能看出循环没跑完。 task["status"] = "goal-running" - save_state(state_path, state) + try: + save_state(state_path, state) + except Exception as exc: + return _interrupt_loop(exc) print(f"round {round_no}: turn={result['status']} " f"verify_rc={v_rc if v_rc is not None else '-'} " f"verify_out={last_verify[:300]}") @@ -2044,6 +2083,7 @@ def run_goal_loop(server, state, state_path, task, goal_text, verify, f"total={goal_usage['total_tokens']} " f"completions={goal_usage['completions']}") return {"ok": ok, "budget_stopped": budget_stopped, + "interrupted": False, "rounds_done": task["goal_rounds_done"], "last_verify": last_verify, "usage": goal_usage} diff --git a/tests/test_host.py b/tests/test_host.py index 688e9e2..fecd64e 100644 --- a/tests/test_host.py +++ b/tests/test_host.py @@ -345,6 +345,42 @@ class RetryAndBudgetTest(unittest.TestCase): self.assertIn("budget-exceeded", Path(notify_log).read_text()) + def test_goal_turn_exception_is_interrupted_not_failed(self): + # 2026-10-28 长 goal 实弹:回合抛错必须记 interrupted + + # 打印标记行(云端据此可断点续跑),绝不能向上传播致 + # 进程非 0 退出、无结论行(旧语义云端判 terminal failed)。 + import contextlib + import io + + class FakeServer: + def run_turn(self, *args, **kwargs): + raise RuntimeError("relay boom") + + with tempfile.TemporaryDirectory() as tmp: + state_path = os.path.join(tmp, "host-state.json") + state = {"tasks": {}, "automations": [], "usage": {}} + task = {"name": "g", "thread_id": "x", "cwd": tmp, + "sandbox": None, "max_retries": 0, "retry_count": 0, + "turns": []} + state["tasks"]["g"] = task + buf = io.StringIO() + with contextlib.redirect_stdout(buf): + result = host.run_goal_loop( + FakeServer(), state, state_path, + task, "make it", "true", 3, "m", + "low", None, timeout=600) + self.assertTrue(result["interrupted"]) + self.assertFalse(result["ok"]) + self.assertEqual(task["status"], "interrupted") + self.assertEqual(task["goal_rounds_done"], 0) + self.assertIn("relay boom", + task["goal_interrupted_reason"]) + self.assertIn("goal interrupted after rounds", + buf.getvalue()) + saved = json.loads(Path(state_path).read_text()) + self.assertEqual(saved["tasks"]["g"]["status"], + "interrupted") + class TurnErrorFromHistoryTest(unittest.TestCase): def _server_with_history(self, tmp, error_json): @@ -934,16 +970,21 @@ class GoalLoopLongTaskTest(unittest.TestCase): state = {"tasks": {}, "automations": []} task = self._task(tmp) state["tasks"]["g"] = task - with self.assertRaises(RuntimeError): - host.run_goal_loop(FakeServer(), state, state_path, - task, "make it", "true", 3, "m", "low", - None, cfg={}, timeout=600) + # 2026-10-28 起:回合抛错不再向上传播(旧断言),而 + # 是记 interrupted + 标记行(云端可断点续跑);首轮 + # 前落盘的字段仍必须都在。 + result = host.run_goal_loop(FakeServer(), state, state_path, + task, "make it", "true", 3, "m", + "low", None, cfg={}, timeout=600) + self.assertTrue(result["interrupted"]) saved = host.load_state(state_path) saved_task = saved["tasks"]["g"] self.assertEqual(saved_task["goal"], "make it") self.assertEqual(saved_task["verify_command"], "true") self.assertEqual(saved_task["goal_rounds_done"], 0) - self.assertEqual(saved_task["status"], "goal-running") + self.assertEqual(saved_task["status"], "interrupted") + self.assertIn("killed during turn", + saved_task["goal_interrupted_reason"]) def test_goal_budget_stop_is_interrupted_not_unmet(self): class FakeServer: