fix: 断点丢失时 goal-resume 拒绝成 failed,阻断自动恢复空转
2026-10-29 长任务实弹:Docker goal 断点续跑在 host-state 里找 不到任务时抛 KeyError,worker 把无结论异常退出判 interrupted, 自动恢复再排队、再 KeyError,12 秒一轮空转 49 次(人工暂停 拦下,第 50 次人工续跑又烧掉),文件 32 章原样未动。 修法:host.py goal-resume 先查任务在不在 state 里,不在就打印 拒绝行、正常退出;sandbox 三处解析(本地、Docker resume、 Docker 断点)见拒绝行即判 failed 终态,不进自动恢复循环。 单测:host 缺任务拒绝不炸 + 三处解析拒绝判 failed。
This commit is contained in:
1 parent
e48b756f96
commit
a5f2e4e091
4 files changed
+138
-3
No files matched your search
+31
-3
@@ -117,6 +117,19 @@ def _parse_goal_output(stdout):
|
||||
return None, None
|
||||
|
||||
|
||||
def _goal_resume_refusal(stdout):
|
||||
"""host.py goal-resume 的拒绝行(断点丢失时),无则返回 None。
|
||||
|
||||
断点丢失不是可续跑的中断:host-state 里没有任务,任何一次
|
||||
续跑都会原样失败;调用方必须判 failed 终态,绝不能判
|
||||
interrupted——否则 worker 自动恢复会 12 秒一轮空转到上限。
|
||||
"""
|
||||
for line in (stdout or "").splitlines():
|
||||
if line.startswith("goal-resume 拒绝:"):
|
||||
return line.strip()
|
||||
return None
|
||||
|
||||
|
||||
def _goal_interrupted_reason(stdout):
|
||||
"""goal interrupted 标记行的原因,无标记返回 None。
|
||||
|
||||
@@ -569,8 +582,13 @@ class LocalExecutor(Executor):
|
||||
stderr = getattr(result, "stderr", "") or ""
|
||||
met, goal_error = _parse_goal_output(stdout_full)
|
||||
interrupted_reason = _goal_interrupted_reason(stdout_full)
|
||||
refusal = _goal_resume_refusal(stdout_full)
|
||||
combined = stdout_full + stderr
|
||||
if "已存在" in combined:
|
||||
if refusal is not None and met is None:
|
||||
# 断点丢失拒绝:终态 failed,不进自动恢复(见 helper)。
|
||||
met, goal_error = None, refusal
|
||||
status = "failed"
|
||||
elif "已存在" in combined:
|
||||
met, goal_error = None, (
|
||||
"host 任务已存在:goal 失败请用继续而非重试——"
|
||||
"同任务重试会撞已存在 host-state,worker 会先"
|
||||
@@ -1388,7 +1406,12 @@ class DockerExecutor(Executor):
|
||||
stderr = getattr(result, "stderr", "") or ""
|
||||
met, goal_error = _parse_goal_output(stdout_full)
|
||||
interrupted_reason = _goal_interrupted_reason(stdout_full)
|
||||
if interrupted_reason is not None:
|
||||
refusal = _goal_resume_refusal(stdout_full)
|
||||
if refusal is not None and met is None:
|
||||
# 断点丢失拒绝:终态 failed,不进自动恢复(见 helper)。
|
||||
status = "failed"
|
||||
goal_error = refusal
|
||||
elif interrupted_reason is not None:
|
||||
# host 侧已记 interrupted:异常中断不是失败,云端跟
|
||||
# 着记 interrupted,任务可 /goal-resume 断点续跑。
|
||||
status = "interrupted"
|
||||
@@ -1484,7 +1507,12 @@ class DockerExecutor(Executor):
|
||||
stderr = getattr(result, "stderr", "") or ""
|
||||
met, goal_error = _parse_goal_output(stdout_full)
|
||||
interrupted_reason = _goal_interrupted_reason(stdout_full)
|
||||
if interrupted_reason is not None:
|
||||
refusal = _goal_resume_refusal(stdout_full)
|
||||
if refusal is not None and met is None:
|
||||
# 断点丢失拒绝:终态 failed,不进自动恢复(见 helper)。
|
||||
status = "failed"
|
||||
goal_error = refusal
|
||||
elif interrupted_reason is not None:
|
||||
# host 侧已记 interrupted:异常中断不是失败,云端跟
|
||||
# 着记 interrupted,任务可 /goal-resume 断点续跑。
|
||||
status = "interrupted"
|
||||
|
||||
@@ -391,6 +391,37 @@ class TestGoalExecutors(unittest.TestCase):
|
||||
self.assertEqual(result["status"], "interrupted")
|
||||
self.assertIn("boom", result["goal_error"])
|
||||
|
||||
def test_goal_resume_refusal_is_failed_not_interrupted(self):
|
||||
# 断点丢失拒绝(2026-10-29 实弹):host-state 里没有任务时
|
||||
# goal-resume 只打印拒绝行、正常退出;三处解析(本地、
|
||||
# Docker goal_resume_task、Docker 断点)都必须判 failed
|
||||
# 终态,绝不能判 interrupted——否则自动恢复 12 秒一轮
|
||||
# 空转到 50 次上限。
|
||||
refusal = ("goal-resume 拒绝:host-state 里没有该任务 t1,"
|
||||
"断点已丢失,不能续跑\n")
|
||||
fake = SimpleNamespace(returncode=0, stdout=refusal, stderr="")
|
||||
result = LocalExecutor("/fake/host.py",
|
||||
runner=lambda cmd: fake
|
||||
)._goal_result_from_runner("t1", fake)
|
||||
self.assertEqual(result["status"], "failed")
|
||||
self.assertIn("goal-resume 拒绝", result["goal_error"])
|
||||
|
||||
def fake_runner(cmd, timeout=None, env=None):
|
||||
return SimpleNamespace(returncode=0, stdout=refusal,
|
||||
stderr="")
|
||||
|
||||
with mock.patch("sandbox.shutil.which",
|
||||
return_value="/usr/bin/docker"):
|
||||
executor = DockerExecutor(
|
||||
"/fake/host.py", runner=fake_runner,
|
||||
env={"XCM_API_KEY": "placeholder-not-a-real-key"})
|
||||
for method in ("goal_resume_task",
|
||||
"goal_resume_breakpoint_task"):
|
||||
result = getattr(executor, method)(
|
||||
{"id": "t1", "prompt": "do it"})
|
||||
self.assertEqual(result["status"], "failed")
|
||||
self.assertIn("goal-resume 拒绝", result["goal_error"])
|
||||
|
||||
def test_local_goal_task_already_exists_names_retry(self):
|
||||
def fake(cmd):
|
||||
return SimpleNamespace(
|
||||
|
||||
Reference in new issue
Block a user