goal 异常退出记 interrupted,可断点续跑
长 goal 实弹(2026-10-28):宿主机内存挤兑致回合挂起 59 分钟,恢复时 goal 进程异常退出、无结论行,云端旧语义(returncode 非 0 一律 failed)把可续跑的断点判成终局。 修法:host.py run_goal_loop 回合异常统一记 interrupted 并打印 goal interrupted after rounds 标记行;cloud/sandbox.py 四处 goal 解析同步新语义(标记或无结论非 0 退出 -> interrupted,有结论的非 0 退出仍 failed);worker 终局审计名与状态对齐。单测:sandbox 新测 5 个、host 新测 1 个、更新旧测 1 个;三套件 cloud 1017、host 61、deploy 29 全绿。只本地提交,未推未部署。
This commit is contained in:
1 parent
d129f7bf8c
commit
94137c4a29
6 files changed
+287
-29
No files matched your search
@@ -1896,6 +1896,13 @@ def run_goal_loop(server, state, state_path, task, goal_text, verify,
|
||||
不能动这三行;两种结局都 exit 0(main 不抛异常即 0)。
|
||||
预算 stop 是第三种结局:打印 "goal stopped after rounds: ..."
|
||||
且任务记 interrupted(明天预算重置后可续跑),不冒充 unmet。
|
||||
回合异常是第四种结局:turn 或验收抛错时不向上传播(否则
|
||||
进程非 0 退出、无结论行,云端旧语义会判 terminal failed,
|
||||
断点续跑活路就断了),而是记 interrupted、打印
|
||||
"goal interrupted after rounds: <原因>",exit 0,任务可
|
||||
/goal-resume 断点续跑(2026-10-28 长 goal 实弹:宿主机
|
||||
内存挤兑致回合挂起 59 分钟、恢复时异常退出,旧语义直接
|
||||
判 failed,60 分钟工作无续跑口)。
|
||||
goal-unmet 穷尽轮次后不允许 goal-resume(按既有规则须新建
|
||||
任务),goal-resume 只接受 goal-running / interrupted / running。
|
||||
"""
|
||||
@@ -1912,6 +1919,28 @@ def run_goal_loop(server, state, state_path, task, goal_text, verify,
|
||||
ok = False
|
||||
budget_stopped = False
|
||||
goal_usage = None
|
||||
|
||||
def _interrupt_loop(exc):
|
||||
# 回合异常的统一出口:不断言、不向上传播,记 interrupted
|
||||
# 并打印标记行,云端据此标 interrupted(可断点续跑)。
|
||||
# 再落盘一次已尽力(二次异常直接吞掉,标记行照常打印;
|
||||
# 即使落盘也失败,云端无结论 + 非 0 退出时仍有
|
||||
# interrupted 兜底,正常路径下本函数返回后 main exit 0)。
|
||||
task["status"] = "interrupted"
|
||||
task["goal_last_turn_status"] = "interrupted"
|
||||
task["goal_interrupted_reason"] = str(exc).replace(
|
||||
"\n", " ")[:200]
|
||||
try:
|
||||
save_state(state_path, state)
|
||||
except Exception:
|
||||
pass
|
||||
print("goal interrupted after rounds: %s"
|
||||
% str(exc).replace("\n", " ")[:500])
|
||||
return {"ok": False, "budget_stopped": False,
|
||||
"interrupted": True,
|
||||
"rounds_done": task["goal_rounds_done"],
|
||||
"last_verify": last_verify, "usage": goal_usage}
|
||||
|
||||
# max_rounds=0 是不限轮次:while 循环只在模型宣告、预算耗尽
|
||||
# 或取消时结束;非 0 时按旧语义穷尽轮次即 unmet。
|
||||
round_no = int(start_round) - 1
|
||||
@@ -1962,11 +1991,14 @@ def run_goal_loop(server, state, state_path, task, goal_text, verify,
|
||||
f"目标:{goal_text}\n"
|
||||
f"验收命令:{verify}\n"
|
||||
f"上一轮验收输出:\n{last_verify}")
|
||||
result = run_task_turn(server, state, task, prompt,
|
||||
model, effort, sandbox, timeout=timeout,
|
||||
interrupt_file=_interrupt_request_path(
|
||||
state_path, task["name"])
|
||||
if task.get("name") else None)
|
||||
try:
|
||||
result = run_task_turn(server, state, task, prompt,
|
||||
model, effort, sandbox, timeout=timeout,
|
||||
interrupt_file=_interrupt_request_path(
|
||||
state_path, task["name"])
|
||||
if task.get("name") else None)
|
||||
except Exception as exc:
|
||||
return _interrupt_loop(exc)
|
||||
goal_usage = _add_usage(goal_usage, result.get("usage"))
|
||||
last_turn_status = result["status"]
|
||||
last_turn_text = result.get("text", "") or ""
|
||||
@@ -1997,6 +2029,10 @@ def run_goal_loop(server, state, state_path, task, goal_text, verify,
|
||||
else ""
|
||||
v_out = (f"verify timeout after {verify_timeout}s: "
|
||||
f"{(out + err).strip()[:900]}")
|
||||
except Exception as exc:
|
||||
# 验收命令本身起不来(OSError 等)不是验收失败,
|
||||
# 同样走中断出口,不让异常杀掉整个循环。
|
||||
return _interrupt_loop(exc)
|
||||
last_verify = v_out
|
||||
task["goal_rounds_done"] = round_no
|
||||
task["goal_last_verify"] = last_verify
|
||||
@@ -2004,7 +2040,10 @@ def run_goal_loop(server, state, state_path, task, goal_text, verify,
|
||||
# 检查点:非终局轮次先记 goal-running,终局轮次由下面
|
||||
# 统一改写;这样被杀后 state 里能看出循环没跑完。
|
||||
task["status"] = "goal-running"
|
||||
save_state(state_path, state)
|
||||
try:
|
||||
save_state(state_path, state)
|
||||
except Exception as exc:
|
||||
return _interrupt_loop(exc)
|
||||
print(f"round {round_no}: turn={result['status']} "
|
||||
f"verify_rc={v_rc if v_rc is not None else '-'} "
|
||||
f"verify_out={last_verify[:300]}")
|
||||
@@ -2044,6 +2083,7 @@ def run_goal_loop(server, state, state_path, task, goal_text, verify,
|
||||
f"total={goal_usage['total_tokens']} "
|
||||
f"completions={goal_usage['completions']}")
|
||||
return {"ok": ok, "budget_stopped": budget_stopped,
|
||||
"interrupted": False,
|
||||
"rounds_done": task["goal_rounds_done"],
|
||||
"last_verify": last_verify, "usage": goal_usage}
|
||||
|
||||
|
||||
Reference in new issue
Block a user