goal 异常退出记 interrupted,可断点续跑

长 goal 实弹(2026-10-28):宿主机内存挤兑致回合挂起 59 分钟,恢复时 goal 进程异常退出、无结论行,云端旧语义(returncode 非 0 一律 failed)把可续跑的断点判成终局。

修法:host.py run_goal_loop 回合异常统一记 interrupted 并打印 goal interrupted after rounds 标记行;cloud/sandbox.py 四处 goal 解析同步新语义(标记或无结论非 0 退出 -> interrupted,有结论的非 0 退出仍 failed);worker 终局审计名与状态对齐。单测:sandbox 新测 5 个、host 新测 1 个、更新旧测 1 个;三套件 cloud 1017、host 61、deploy 29 全绿。只本地提交,未推未部署。
This commit is contained in:
host committed 2026-10-28 15:18:26 +08:00
1 parent d129f7bf8c
commit 94137c4a29
6 files changed
+287 -29

No files matched your search

+85 -16
View File
@@ -112,6 +112,22 @@ def _parse_goal_output(stdout):
return None, None
def _goal_interrupted_reason(stdout):
"""goal interrupted 标记行的原因,无标记返回 None。
host.py run_goal_loop 在回合异常时打印
"goal interrupted after rounds: <原因>" 并记 interrupted;
云端据此把任务标 interrupted(可 /goal-resume 断点续跑),
而不是 terminal 的 failed——异常退出时断点(goal_rounds_done、
线程、worktree)都在盘上,判 failed 会把可续跑的活路断掉。
"""
for line in (stdout or "").splitlines():
if "goal interrupted after rounds" in line:
reason = line.split("goal interrupted after rounds", 1)[1]
return reason.strip().lstrip(":").strip()[:500]
return None
def _goal_stdout_tail(stdout):
"""goal 结果存库的 stdout:结论在末尾,截尾不截头。
@@ -512,16 +528,20 @@ class LocalExecutor(Executor):
"""把 host.py goal 的 runner 返回值解析成 goal 结果 dict。
解析用完整 stdout(结论在末尾),存库的 stdout 才截尾;
returncode 非 0 一律 failed;为 0 时无结论也判 failed,
绝不把“没打印结论”当成达标。host 任务已存在时(同任务
/retry 撞名,goal 没有 run 续跑语义)错误写明“goal 失败
请用继续而非重试”(P1:重试在已存在 host-state 上必失败,
断点续跑才是正确路径,见 worker 的回落逻辑)。
returncode 非 0 时:已打印 met/unmet 结论的仍判 failed,
无结论的异常退出判 interrupted(断点在盘上,可续跑,
绝不能一律 failed);host 打印了 interrupted 标记时也判
interrupted。为 0 时无结论仍判 failed,绝不把“没打印
结论”当成达标。host 任务已存在时(同任务 /retry 撞名,
goal 没有 run 续跑语义)错误写明“goal 失败请用继续而
非重试”(P1:重试在已存在 host-state 上必失败,断点续
跑才是正确路径,见 worker 的回落逻辑)。
"""
returncode = getattr(result, "returncode", 1)
stdout_full = getattr(result, "stdout", "") or ""
stderr = getattr(result, "stderr", "") or ""
met, goal_error = _parse_goal_output(stdout_full)
interrupted_reason = _goal_interrupted_reason(stdout_full)
combined = stdout_full + stderr
if "已存在" in combined:
met, goal_error = None, (
@@ -530,12 +550,23 @@ class LocalExecutor(Executor):
"尝试断点续跑,断点续跑仍失败时请用继续(不要"
"重排同一任务)")
status = "failed"
elif interrupted_reason is not None:
# host 侧已记 interrupted:异常中断不是失败,云端跟
# 着记 interrupted,任务可 /goal-resume 断点续跑。
status = "interrupted"
goal_error = (interrupted_reason
or "host goal 异常中断,断点可续跑")
elif returncode != 0:
status = "failed"
if met is None:
# 无结论的异常退出(进程被杀、连接断后恢复时抛错
# 等):断点在盘上,记 interrupted 保住续跑活路,
# 绝不能一律 failed——2026-10-28 长 goal 实弹教训。
status = "interrupted"
goal_error = ((stderr.strip()[-500:] if stderr.strip()
else stdout_full.strip()[-500:])
or "host goal failed")
or "host goal 异常退出(无结论),断点可续跑")
else:
status = "failed"
elif met is True:
status = "completed"
elif met is False:
@@ -1223,6 +1254,7 @@ class DockerExecutor(Executor):
stdout_full = getattr(result, "stdout", "") or ""
stderr = getattr(result, "stderr", "") or ""
met, goal_error = _parse_goal_output(stdout_full)
interrupted_reason = _goal_interrupted_reason(stdout_full)
combined = stdout_full + stderr
if "已存在" in combined:
met, goal_error = None, (
@@ -1231,12 +1263,23 @@ class DockerExecutor(Executor):
"尝试断点续跑,断点续跑仍失败时请用继续(不要"
"重排同一任务)")
status = "failed"
elif interrupted_reason is not None:
# host 侧已记 interrupted:异常中断不是失败,云端跟
# 着记 interrupted,任务可 /goal-resume 断点续跑。
status = "interrupted"
goal_error = (interrupted_reason
or "host goal 异常中断,断点可续跑")
elif returncode != 0:
status = "failed"
if met is None:
# 无结论的异常退出(进程被杀、连接断后恢复时抛错
# 等):断点在盘上,记 interrupted 保住续跑活路,
# 绝不能一律 failed——2026-10-28 长 goal 实弹教训。
status = "interrupted"
goal_error = ((stderr.strip()[-500:] if stderr.strip()
else stdout_full.strip()[-500:])
or "host goal failed")
or "host goal 异常退出(无结论),断点可续跑")
else:
status = "failed"
elif met is True:
status = "completed"
elif met is False:
@@ -1269,7 +1312,9 @@ class DockerExecutor(Executor):
目标才传 goal_text(带 --goal-text 与 --max-rounds,默认 0)。
解析与 goal_task 同款(完整 stdout 结论、存库截尾),但
没有“已存在”分支:goal-resume 本来就是操作已有 host 任务,
不存在 dispatch 撞名问题;returncode 非 0 或无结论一律 failed。
不存在 dispatch 撞名问题;returncode 非 0 时:有结论仍
failed,无结论判 interrupted(断点可续跑);rc==0 无结论
仍 failed。
"""
if not isinstance(task, dict) or task.get("id") is None:
raise ValueError("任务缺少 id")
@@ -1315,12 +1360,24 @@ class DockerExecutor(Executor):
stdout_full = getattr(result, "stdout", "") or ""
stderr = getattr(result, "stderr", "") or ""
met, goal_error = _parse_goal_output(stdout_full)
if returncode != 0:
status = "failed"
interrupted_reason = _goal_interrupted_reason(stdout_full)
if interrupted_reason is not None:
# host 侧已记 interrupted:异常中断不是失败,云端跟
# 着记 interrupted,任务可 /goal-resume 断点续跑。
status = "interrupted"
goal_error = (interrupted_reason
or "host goal-resume 异常中断,断点可续跑")
elif returncode != 0:
if met is None:
# 无结论的异常退出:断点在盘上,记 interrupted
# 保住续跑活路,绝不能一律 failed。
status = "interrupted"
goal_error = ((stderr.strip()[-500:] if stderr.strip()
else stdout_full.strip()[-500:])
or "host goal-resume failed")
or "host goal-resume 异常退出(无结论),"
"断点可续跑")
else:
status = "failed"
elif met is True:
status = "completed"
elif met is False:
@@ -1386,12 +1443,24 @@ class DockerExecutor(Executor):
stdout_full = getattr(result, "stdout", "") or ""
stderr = getattr(result, "stderr", "") or ""
met, goal_error = _parse_goal_output(stdout_full)
if returncode != 0:
status = "failed"
interrupted_reason = _goal_interrupted_reason(stdout_full)
if interrupted_reason is not None:
# host 侧已记 interrupted:异常中断不是失败,云端跟
# 着记 interrupted,任务可 /goal-resume 断点续跑。
status = "interrupted"
goal_error = (interrupted_reason
or "host goal-resume 异常中断,断点可续跑")
elif returncode != 0:
if met is None:
# 无结论的异常退出:断点在盘上,记 interrupted
# 保住续跑活路,绝不能一律 failed。
status = "interrupted"
goal_error = ((stderr.strip()[-500:] if stderr.strip()
else stdout_full.strip()[-500:])
or "host goal-resume failed")
or "host goal-resume 异常退出(无结论),"
"断点可续跑")
else:
status = "failed"
elif met is True:
status = "completed"
elif met is False: