长任务 P0 三件:断点进库、错误分类+熔断、Run/Step 时间线
断点进库:task_checkpoints 表(PG 新库 + 老库幂等补表 + 迁移清单
对齐);worker 在 goal 任务执行期间每 30 秒把磁盘 host-state 断点
镜像进库,终局再补一次;断点续跑认领前,若 host-state 条目丢失
但库里有断点,先按库断点重建最小条目再跑,库里也没有断点时不
造假、走原有拒绝路径。
错误分类+熔断:sandbox 四处解析加永久错误分支(KeyError、配置
缺失等无结论异常退出判 failed 死信,不再判 interrupted;host 已
明言 interrupted 的不受影响,未知错误不分类);worker 自动恢复
加退避(10 秒起指数退避、封顶 300 秒)与无进展熔断(库断点轮次
连续 3 次不变即标 auto_resume_stalled 等人工)。
时间线:task_steps 表 + host 每轮写 host-steps 步骤文件 + worker
导入;GET /tasks/{id}/timeline 一次返回 steps 与 checkpoints;
认领、暂停、自动恢复、终局都记步骤。旧任务返回空数组,不编造。
单测:cloud/tests/test_p0_three.py 14 项(分类、熔断、镜像重建、
步骤导入去重、timeline 接口)。
This commit is contained in:
1 parent
d56bd3a99e
commit
585242fc8f
7 files changed
+982
-3
No files matched your search
@@ -117,6 +117,57 @@ def _parse_goal_output(stdout):
|
||||
return None, None
|
||||
|
||||
|
||||
# P0 三件其二:goal 错误签名。permanent = 同代码、同断点、同配置
|
||||
# 下重试必然再失败(断点丢失、配置缺失、旧语义误用),必须判
|
||||
# failed 终态进死信,绝不能判 interrupted 进自动恢复;transient =
|
||||
# 网络/网关抖动,重试有意义,走原有 interrupted 路径。未知错误
|
||||
# 不分类(None),走原有逻辑,兜底靠 worker 的无进展熔断。
|
||||
_GOAL_PERMANENT_MARKERS = (
|
||||
"goal-resume 拒绝",
|
||||
"KeyError",
|
||||
"没有会话线程",
|
||||
"没有 goal 记录",
|
||||
"provider not found",
|
||||
"Model provider",
|
||||
"Missing environment variable",
|
||||
"任务已存在",
|
||||
)
|
||||
_GOAL_TRANSIENT_MARKERS = (
|
||||
"timed out",
|
||||
"Timeout",
|
||||
"429",
|
||||
"502",
|
||||
"503",
|
||||
"RemoteDisconnected",
|
||||
"IncompleteRead",
|
||||
"Connection reset",
|
||||
"connection refused",
|
||||
)
|
||||
|
||||
|
||||
def _classify_goal_error(stdout, stderr):
|
||||
"""返回 (kind, snippet):kind 为 permanent/transient/None。
|
||||
|
||||
snippet 取命中标记所在行(截 500 字),供终态 error_text 用;
|
||||
命中多个时 permanent 优先于 transient(永久错误里常混着
|
||||
网络字样,反过来会把死信误判成可重试)。
|
||||
"""
|
||||
combined = (stdout or "") + "\n" + (stderr or "")
|
||||
for marker in _GOAL_PERMANENT_MARKERS:
|
||||
if marker in combined:
|
||||
for line in combined.splitlines():
|
||||
if marker in line:
|
||||
return "permanent", line.strip()[:500]
|
||||
return "permanent", marker
|
||||
for marker in _GOAL_TRANSIENT_MARKERS:
|
||||
if marker in combined:
|
||||
for line in combined.splitlines():
|
||||
if marker in line:
|
||||
return "transient", line.strip()[:500]
|
||||
return "transient", marker
|
||||
return None, None
|
||||
|
||||
|
||||
def _goal_resume_refusal(stdout):
|
||||
"""host.py goal-resume 的拒绝行(断点丢失时),无则返回 None。
|
||||
|
||||
@@ -584,6 +635,8 @@ class LocalExecutor(Executor):
|
||||
interrupted_reason = _goal_interrupted_reason(stdout_full)
|
||||
refusal = _goal_resume_refusal(stdout_full)
|
||||
combined = stdout_full + stderr
|
||||
perm_kind, perm_snippet = _classify_goal_error(
|
||||
stdout_full, stderr)
|
||||
if refusal is not None and met is None:
|
||||
# 断点丢失拒绝:终态 failed,不进自动恢复(见 helper)。
|
||||
met, goal_error = None, refusal
|
||||
@@ -601,6 +654,14 @@ class LocalExecutor(Executor):
|
||||
status = "interrupted"
|
||||
goal_error = (interrupted_reason
|
||||
or "host goal 异常中断,断点可续跑")
|
||||
elif (met is None and returncode != 0
|
||||
and perm_kind == "permanent"):
|
||||
# P0 三件其二:永久错误(断点 KeyError、配置缺失等)
|
||||
# 的无结论异常退出——同代码同断点重试必然再失败,
|
||||
# 判 failed 终态进死信,绝不判 interrupted 空转。
|
||||
# host 已明言 interrupted 的在上一分支,不受影响。
|
||||
status = "failed"
|
||||
goal_error = perm_snippet or "goal 永久错误"
|
||||
elif returncode != 0:
|
||||
if met is None:
|
||||
# 无结论的异常退出(进程被杀、连接断后恢复时抛错
|
||||
@@ -1301,6 +1362,8 @@ class DockerExecutor(Executor):
|
||||
met, goal_error = _parse_goal_output(stdout_full)
|
||||
interrupted_reason = _goal_interrupted_reason(stdout_full)
|
||||
combined = stdout_full + stderr
|
||||
perm_kind, perm_snippet = _classify_goal_error(
|
||||
stdout_full, stderr)
|
||||
if "已存在" in combined:
|
||||
met, goal_error = None, (
|
||||
"host 任务已存在:goal 失败请用继续而非重试——"
|
||||
@@ -1314,6 +1377,13 @@ class DockerExecutor(Executor):
|
||||
status = "interrupted"
|
||||
goal_error = (interrupted_reason
|
||||
or "host goal 异常中断,断点可续跑")
|
||||
elif (met is None and returncode != 0
|
||||
and perm_kind == "permanent"):
|
||||
# P0 三件其二:永久错误无结论异常退出 → failed 死信
|
||||
# (同 local 侧注释;host 已明言 interrupted 的在上
|
||||
# 一分支,不受影响)。
|
||||
status = "failed"
|
||||
goal_error = perm_snippet or "goal 永久错误"
|
||||
elif returncode != 0:
|
||||
if met is None:
|
||||
# 无结论的异常退出(进程被杀、连接断后恢复时抛错
|
||||
@@ -1407,6 +1477,8 @@ class DockerExecutor(Executor):
|
||||
met, goal_error = _parse_goal_output(stdout_full)
|
||||
interrupted_reason = _goal_interrupted_reason(stdout_full)
|
||||
refusal = _goal_resume_refusal(stdout_full)
|
||||
perm_kind, perm_snippet = _classify_goal_error(
|
||||
stdout_full, stderr)
|
||||
if refusal is not None and met is None:
|
||||
# 断点丢失拒绝:终态 failed,不进自动恢复(见 helper)。
|
||||
status = "failed"
|
||||
@@ -1417,6 +1489,13 @@ class DockerExecutor(Executor):
|
||||
status = "interrupted"
|
||||
goal_error = (interrupted_reason
|
||||
or "host goal-resume 异常中断,断点可续跑")
|
||||
elif (met is None and returncode != 0
|
||||
and perm_kind == "permanent"):
|
||||
# P0 三件其二:续跑侧永久错误(典型如断点 KeyError)
|
||||
# → failed 死信;任务 2 的 12 秒空转就是此分支缺失
|
||||
# 造成的(当时无结论异常退出被判 interrupted)。
|
||||
status = "failed"
|
||||
goal_error = perm_snippet or "goal 永久错误"
|
||||
elif returncode != 0:
|
||||
if met is None:
|
||||
# 无结论的异常退出:断点在盘上,记 interrupted
|
||||
@@ -1508,6 +1587,8 @@ class DockerExecutor(Executor):
|
||||
met, goal_error = _parse_goal_output(stdout_full)
|
||||
interrupted_reason = _goal_interrupted_reason(stdout_full)
|
||||
refusal = _goal_resume_refusal(stdout_full)
|
||||
perm_kind, perm_snippet = _classify_goal_error(
|
||||
stdout_full, stderr)
|
||||
if refusal is not None and met is None:
|
||||
# 断点丢失拒绝:终态 failed,不进自动恢复(见 helper)。
|
||||
status = "failed"
|
||||
@@ -1518,6 +1599,11 @@ class DockerExecutor(Executor):
|
||||
status = "interrupted"
|
||||
goal_error = (interrupted_reason
|
||||
or "host goal-resume 异常中断,断点可续跑")
|
||||
elif (met is None and returncode != 0
|
||||
and perm_kind == "permanent"):
|
||||
# P0 三件其二:续跑侧永久错误 → failed 死信(同上)。
|
||||
status = "failed"
|
||||
goal_error = perm_snippet or "goal 永久错误"
|
||||
elif returncode != 0:
|
||||
if met is None:
|
||||
# 无结论的异常退出:断点在盘上,记 interrupted
|
||||
|
||||
Reference in new issue
Block a user