长任务 P0 三件:断点进库、错误分类+熔断、Run/Step 时间线

断点进库:task_checkpoints 表(PG 新库 + 老库幂等补表 + 迁移清单
对齐);worker 在 goal 任务执行期间每 30 秒把磁盘 host-state 断点
镜像进库,终局再补一次;断点续跑认领前,若 host-state 条目丢失
但库里有断点,先按库断点重建最小条目再跑,库里也没有断点时不
造假、走原有拒绝路径。

错误分类+熔断:sandbox 四处解析加永久错误分支(KeyError、配置
缺失等无结论异常退出判 failed 死信,不再判 interrupted;host 已
明言 interrupted 的不受影响,未知错误不分类);worker 自动恢复
加退避(10 秒起指数退避、封顶 300 秒)与无进展熔断(库断点轮次
连续 3 次不变即标 auto_resume_stalled 等人工)。

时间线:task_steps 表 + host 每轮写 host-steps 步骤文件 + worker
导入;GET /tasks/{id}/timeline 一次返回 steps 与 checkpoints;
认领、暂停、自动恢复、终局都记步骤。旧任务返回空数组,不编造。

单测:cloud/tests/test_p0_three.py 14 项(分类、熔断、镜像重建、
步骤导入去重、timeline 接口)。
This commit is contained in:
host committed 2026-10-29 15:13:01 +08:00
1 parent d56bd3a99e
commit 585242fc8f
7 files changed
+982 -3

No files matched your search

+86
View File
@@ -117,6 +117,57 @@ def _parse_goal_output(stdout):
return None, None
# P0 三件其二:goal 错误签名。permanent = 同代码、同断点、同配置
# 下重试必然再失败(断点丢失、配置缺失、旧语义误用),必须判
# failed 终态进死信,绝不能判 interrupted 进自动恢复;transient =
# 网络/网关抖动,重试有意义,走原有 interrupted 路径。未知错误
# 不分类(None),走原有逻辑,兜底靠 worker 的无进展熔断。
_GOAL_PERMANENT_MARKERS = (
"goal-resume 拒绝",
"KeyError",
"没有会话线程",
"没有 goal 记录",
"provider not found",
"Model provider",
"Missing environment variable",
"任务已存在",
)
_GOAL_TRANSIENT_MARKERS = (
"timed out",
"Timeout",
"429",
"502",
"503",
"RemoteDisconnected",
"IncompleteRead",
"Connection reset",
"connection refused",
)
def _classify_goal_error(stdout, stderr):
"""返回 (kind, snippet):kind 为 permanent/transient/None。
snippet 取命中标记所在行(截 500 字),供终态 error_text 用;
命中多个时 permanent 优先于 transient(永久错误里常混着
网络字样,反过来会把死信误判成可重试)。
"""
combined = (stdout or "") + "\n" + (stderr or "")
for marker in _GOAL_PERMANENT_MARKERS:
if marker in combined:
for line in combined.splitlines():
if marker in line:
return "permanent", line.strip()[:500]
return "permanent", marker
for marker in _GOAL_TRANSIENT_MARKERS:
if marker in combined:
for line in combined.splitlines():
if marker in line:
return "transient", line.strip()[:500]
return "transient", marker
return None, None
def _goal_resume_refusal(stdout):
"""host.py goal-resume 的拒绝行(断点丢失时),无则返回 None。
@@ -584,6 +635,8 @@ class LocalExecutor(Executor):
interrupted_reason = _goal_interrupted_reason(stdout_full)
refusal = _goal_resume_refusal(stdout_full)
combined = stdout_full + stderr
perm_kind, perm_snippet = _classify_goal_error(
stdout_full, stderr)
if refusal is not None and met is None:
# 断点丢失拒绝:终态 failed,不进自动恢复(见 helper)。
met, goal_error = None, refusal
@@ -601,6 +654,14 @@ class LocalExecutor(Executor):
status = "interrupted"
goal_error = (interrupted_reason
or "host goal 异常中断,断点可续跑")
elif (met is None and returncode != 0
and perm_kind == "permanent"):
# P0 三件其二:永久错误(断点 KeyError、配置缺失等)
# 的无结论异常退出——同代码同断点重试必然再失败,
# 判 failed 终态进死信,绝不判 interrupted 空转。
# host 已明言 interrupted 的在上一分支,不受影响。
status = "failed"
goal_error = perm_snippet or "goal 永久错误"
elif returncode != 0:
if met is None:
# 无结论的异常退出(进程被杀、连接断后恢复时抛错
@@ -1301,6 +1362,8 @@ class DockerExecutor(Executor):
met, goal_error = _parse_goal_output(stdout_full)
interrupted_reason = _goal_interrupted_reason(stdout_full)
combined = stdout_full + stderr
perm_kind, perm_snippet = _classify_goal_error(
stdout_full, stderr)
if "已存在" in combined:
met, goal_error = None, (
"host 任务已存在:goal 失败请用继续而非重试——"
@@ -1314,6 +1377,13 @@ class DockerExecutor(Executor):
status = "interrupted"
goal_error = (interrupted_reason
or "host goal 异常中断,断点可续跑")
elif (met is None and returncode != 0
and perm_kind == "permanent"):
# P0 三件其二:永久错误无结论异常退出 → failed 死信
# (同 local 侧注释;host 已明言 interrupted 的在上
# 一分支,不受影响)。
status = "failed"
goal_error = perm_snippet or "goal 永久错误"
elif returncode != 0:
if met is None:
# 无结论的异常退出(进程被杀、连接断后恢复时抛错
@@ -1407,6 +1477,8 @@ class DockerExecutor(Executor):
met, goal_error = _parse_goal_output(stdout_full)
interrupted_reason = _goal_interrupted_reason(stdout_full)
refusal = _goal_resume_refusal(stdout_full)
perm_kind, perm_snippet = _classify_goal_error(
stdout_full, stderr)
if refusal is not None and met is None:
# 断点丢失拒绝:终态 failed,不进自动恢复(见 helper)。
status = "failed"
@@ -1417,6 +1489,13 @@ class DockerExecutor(Executor):
status = "interrupted"
goal_error = (interrupted_reason
or "host goal-resume 异常中断,断点可续跑")
elif (met is None and returncode != 0
and perm_kind == "permanent"):
# P0 三件其二:续跑侧永久错误(典型如断点 KeyError)
# → failed 死信;任务 2 的 12 秒空转就是此分支缺失
# 造成的(当时无结论异常退出被判 interrupted)。
status = "failed"
goal_error = perm_snippet or "goal 永久错误"
elif returncode != 0:
if met is None:
# 无结论的异常退出:断点在盘上,记 interrupted
@@ -1508,6 +1587,8 @@ class DockerExecutor(Executor):
met, goal_error = _parse_goal_output(stdout_full)
interrupted_reason = _goal_interrupted_reason(stdout_full)
refusal = _goal_resume_refusal(stdout_full)
perm_kind, perm_snippet = _classify_goal_error(
stdout_full, stderr)
if refusal is not None and met is None:
# 断点丢失拒绝:终态 failed,不进自动恢复(见 helper)。
status = "failed"
@@ -1518,6 +1599,11 @@ class DockerExecutor(Executor):
status = "interrupted"
goal_error = (interrupted_reason
or "host goal-resume 异常中断,断点可续跑")
elif (met is None and returncode != 0
and perm_kind == "permanent"):
# P0 三件其二:续跑侧永久错误 → failed 死信(同上)。
status = "failed"
goal_error = perm_snippet or "goal 永久错误"
elif returncode != 0:
if met is None:
# 无结论的异常退出:断点在盘上,记 interrupted