feat: 长任务中断自动恢复，仅用户暂停才暂停，不限时
- 新增 tasks 列 paused_by_user/interrupted_reason/auto_resume_count/no_time_limit（幂等迁移，server+worker 双侧）
- POST /tasks/{id}/pause：用户暂停标 interrupted + paused_by_user=1；/cancel 同款标记；人工 /retry、/goal-resume 清标记
- sweep 标 sweep、配额标 quota、协调员孤儿标 coordinator_orphan、worker 退出标 worker_shutdown
- worker.run_loop 每轮后 auto_resume_interrupted：只接系统中断的 goal 任务，CAS 回 queued + goal_resume_pending=1（G10 断点续跑），用户暂停/配额/孤儿/非 goal 不接，上限 50 次防无限烧额度
- no_time_limit=1 的任务 worker/Docker 级不限时（其它任务仍默认 1800 秒；host 单轮 600 秒仍是单轮护栏）
- 前置 cherry-pick：G10 断点续跑透传 model_provider（否则 Docker 断点续跑在 sub2api 下必死，自动恢复无法实证；本 PR supersedes #127，勿另行合并 #127）
This commit is contained in:
host committed
1 parent
a19b1567bb
commit
c9cfe22a36
3 files changed
+454
-11
No files matched your search
+142
-7
@@ -360,6 +360,12 @@ def _task_from_row(row):
|
||||
"resume_message": _col("resume_message"),
|
||||
# G10：goal 断点续跑标记，老行缺列时 0 = 普通排队。
|
||||
"goal_resume_pending": _col("goal_resume_pending") or 0,
|
||||
# 长任务自动恢复：用户暂停标记、中断原因、自动续跑
|
||||
# 次数、不限时标记，老行缺列时按默认（可恢复、不限时关）。
|
||||
"paused_by_user": _col("paused_by_user") or 0,
|
||||
"interrupted_reason": _col("interrupted_reason"),
|
||||
"auto_resume_count": _col("auto_resume_count") or 0,
|
||||
"no_time_limit": _col("no_time_limit") or 0,
|
||||
# 协调员：老行缺列时 coordinator_parts 为 []（= 普通任务），
|
||||
# parent_task_id/coordinator_part 缺列时为 None。
|
||||
"coordinator_parts": _coord_parse_parts(_col("coordinator_parts")),
|
||||
@@ -395,6 +401,7 @@ class Worker:
|
||||
outbox_backoff_seconds=60,
|
||||
outbox_max_attempts=3,
|
||||
quota_poll_interval_seconds=30,
|
||||
auto_resume_max=50,
|
||||
ignore_host_budget=True):
|
||||
self.db_path = str(db_path)
|
||||
self.host_py_path = host_py_path
|
||||
@@ -494,6 +501,11 @@ class Worker:
|
||||
self.task_timeout = task_timeout
|
||||
self.max_retries = max_retries
|
||||
self.retry_backoff_seconds = retry_backoff_seconds
|
||||
# 长任务自动恢复上限（次数，不是时间）：同一任务被
|
||||
# 系统中断后自动续跑超过该次数即停住、等人工处理，
|
||||
# 防确定性失败（每次续跑立刻又中断）无限烧额度；
|
||||
# 正常部署重启/偶发中断远达不到 50 次。
|
||||
self.auto_resume_max = auto_resume_max
|
||||
# 项目仓库检出根目录：CLI --repos-dir 可调（默认 ./repos）；
|
||||
# Docker 执行器下它必须放在 extra-mounts 已挂载的目录下，
|
||||
# 否则容器内 task["repo"] 路径不存在——见 cloud/README.md。
|
||||
@@ -813,6 +825,96 @@ class Worker:
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
def _mark_interrupted_reason(self, task_id, reason):
|
||||
"""给已是 interrupted 的任务补中断原因（不改状态本身）。
|
||||
|
||||
只写 reason，不碰 paused_by_user：调用方是系统路径
|
||||
（配额、sweep 已在 server 侧写、worker 退出、协调员
|
||||
孤儿），用户暂停的 paused 标记由 server /pause、
|
||||
/cancel 写，worker 侧绝不覆盖它。
|
||||
老库缺列时先迁移，迁移失败则静默跳过（不炸终局路径）。
|
||||
"""
|
||||
try:
|
||||
conn = _db_connect(self.db_path)
|
||||
try:
|
||||
self._ensure_result_columns(conn)
|
||||
conn.execute(
|
||||
"UPDATE tasks SET interrupted_reason = ?"
|
||||
" WHERE id = ? AND status = 'interrupted'",
|
||||
(reason, task_id),
|
||||
)
|
||||
conn.commit()
|
||||
finally:
|
||||
conn.close()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
def auto_resume_interrupted(self):
|
||||
"""长任务自动恢复：只接系统中断的 goal 任务。
|
||||
|
||||
只有用户暂停才暂停：paused_by_user=1（/pause、/cancel）
|
||||
的任务绝不自动动；配额中断（quota）与协调员孤儿
|
||||
（coordinator_orphan）也不接——前者要等配额恢复、
|
||||
后者要由父协调员收口，自动重跑只会烧钱或跑丢父子关系。
|
||||
非 goal 任务不接：它没有断点语义，重跑等于从头再来，
|
||||
可能重复副作用，留给人工 /retry 决定。
|
||||
接到的任务 CAS 回 queued 并置 goal_resume_pending=1，
|
||||
下一轮认领即走 G10 断点续跑（同一线程、同一 worktree、
|
||||
从 goal_rounds_done 下一轮起），同时清旧结果、清
|
||||
interrupted_reason、auto_resume_count+1；上限
|
||||
self.auto_resume_max 到顶的任务停住并记审计，等人工。
|
||||
返回自动续跑的任务 id 列表；任何异常只记日志，不炸
|
||||
run_loop（恢复是辅助路径，不能带走常驻 worker）。
|
||||
"""
|
||||
resumed = []
|
||||
try:
|
||||
conn = _db_connect(self.db_path)
|
||||
try:
|
||||
self._ensure_result_columns(conn)
|
||||
rows = conn.execute(
|
||||
"SELECT id, project_id FROM tasks"
|
||||
" WHERE status = 'interrupted'"
|
||||
" AND (paused_by_user IS NULL OR paused_by_user = 0)"
|
||||
" AND (interrupted_reason IS NULL OR"
|
||||
" interrupted_reason NOT IN"
|
||||
" ('quota', 'coordinator_orphan'))"
|
||||
" AND parent_task_id IS NULL"
|
||||
" AND (goal_mode = 1 OR verify_command IS NOT NULL"
|
||||
" OR goal IS NOT NULL)"
|
||||
" AND (auto_resume_count IS NULL OR"
|
||||
" auto_resume_count < ?)",
|
||||
(self.auto_resume_max,),
|
||||
).fetchall()
|
||||
now = time.time()
|
||||
for row in rows:
|
||||
task_id = row[0]
|
||||
project_id = row[1]
|
||||
cur = conn.execute(
|
||||
"UPDATE tasks SET status = 'queued',"
|
||||
" goal_resume_pending = 1,"
|
||||
" result_text = NULL, error_text = NULL,"
|
||||
" actual_model = NULL,"
|
||||
" interrupted_reason = NULL,"
|
||||
" auto_resume_count ="
|
||||
" COALESCE(auto_resume_count, 0) + 1,"
|
||||
" updated_at = ?"
|
||||
" WHERE id = ? AND status = 'interrupted'"
|
||||
" AND (paused_by_user IS NULL"
|
||||
" OR paused_by_user = 0)",
|
||||
(now, task_id),
|
||||
)
|
||||
if cur.rowcount == 1:
|
||||
resumed.append(task_id)
|
||||
self._audit(project_id, "worker_auto_resumed",
|
||||
task_id)
|
||||
conn.commit()
|
||||
finally:
|
||||
conn.close()
|
||||
except Exception as exc:
|
||||
log_event(self.logger, "worker_auto_resume_failed",
|
||||
message="auto resume failed: %s" % exc)
|
||||
return resumed
|
||||
|
||||
def _ensure_result_columns(self, conn):
|
||||
"""worker 侧的迁移兜底：与 server.migrate_tasks_table 同列。
|
||||
|
||||
@@ -851,7 +953,11 @@ class Worker:
|
||||
"coordinator_parts": "TEXT",
|
||||
"parent_task_id": "TEXT",
|
||||
"coordinator_part": "TEXT",
|
||||
"goal_resume_pending": "INTEGER DEFAULT 0"}.items():
|
||||
"goal_resume_pending": "INTEGER DEFAULT 0",
|
||||
"paused_by_user": "INTEGER DEFAULT 0",
|
||||
"interrupted_reason": "TEXT",
|
||||
"auto_resume_count": "INTEGER DEFAULT 0",
|
||||
"no_time_limit": "INTEGER DEFAULT 0"}.items():
|
||||
if name not in cols:
|
||||
try:
|
||||
conn.execute(
|
||||
@@ -2930,21 +3036,29 @@ class Worker:
|
||||
container=self.executor.container_name(task))
|
||||
except Exception:
|
||||
pass
|
||||
# 长任务不限时：任务带 no_time_limit=1 时，本次 worker
|
||||
# 级与 Docker 级超时一律按 None 算（不限时），不改全局
|
||||
# task_timeout（其它任务仍走默认 1800 秒）；host 单轮
|
||||
# turn_timeout（默认 600 秒）不在 worker 侧传参，仍是
|
||||
# 单轮护栏——不限的是任务总墙钟与总轮次，不是单轮无限。
|
||||
_no_time_limit = bool(task.get("no_time_limit"))
|
||||
_effective_task_timeout = None if _no_time_limit \
|
||||
else self.task_timeout
|
||||
# Docker 超时取较小值：两个超时都不应静默延长另一个。
|
||||
# 注意绝不能改共享的 executor.timeout（并发下两个任务会
|
||||
# 互相覆盖对方的超时）；超时只作为本次调用的参数传下去。
|
||||
effective_docker_timeout = None
|
||||
is_docker = isinstance(self.executor, DockerExecutor)
|
||||
if is_docker and self.task_timeout is not None:
|
||||
if is_docker and _effective_task_timeout is not None:
|
||||
if self.executor.timeout is None:
|
||||
effective_docker_timeout = self.task_timeout
|
||||
effective_docker_timeout = _effective_task_timeout
|
||||
else:
|
||||
try:
|
||||
effective_docker_timeout = min(
|
||||
float(self.task_timeout),
|
||||
float(_effective_task_timeout),
|
||||
float(self.executor.timeout))
|
||||
except (TypeError, ValueError):
|
||||
effective_docker_timeout = self.task_timeout
|
||||
effective_docker_timeout = _effective_task_timeout
|
||||
# P0 安全批后：Docker 同步路径起取消看门狗线程
|
||||
# （阻塞在 docker run 里时，库状态变 interrupted 就
|
||||
# 按容器名真杀；非 Docker 路径不启动，避免空线程）。
|
||||
@@ -2988,7 +3102,7 @@ class Worker:
|
||||
popen_kind = "dispatch"
|
||||
result = self._execute_via_process(
|
||||
task, kind=popen_kind,
|
||||
deadline_seconds=self.task_timeout,
|
||||
deadline_seconds=_effective_task_timeout,
|
||||
seed_host_dir=seed_host_dir,
|
||||
quota_check=quota_check)
|
||||
combined = (result.get("stdout", "") or "") + \
|
||||
@@ -2999,7 +3113,7 @@ class Worker:
|
||||
_ALREADY_EXISTS_MARKER in combined:
|
||||
result = self._execute_via_process(
|
||||
task, kind="run",
|
||||
deadline_seconds=self.task_timeout,
|
||||
deadline_seconds=_effective_task_timeout,
|
||||
quota_check=quota_check)
|
||||
else:
|
||||
if is_docker:
|
||||
@@ -3428,6 +3542,7 @@ class Worker:
|
||||
for ch in pending:
|
||||
cur = conn.execute(
|
||||
"UPDATE tasks SET status = 'interrupted',"
|
||||
" interrupted_reason = 'coordinator_orphan',"
|
||||
" updated_at = ? WHERE id = ? AND status NOT IN"
|
||||
" ('completed', 'failed', 'interrupted', 'merged',"
|
||||
" 'merge-failed')",
|
||||
@@ -3490,6 +3605,7 @@ class Worker:
|
||||
continue
|
||||
cur = conn.execute(
|
||||
"UPDATE tasks SET status = 'interrupted',"
|
||||
" interrupted_reason = 'coordinator_orphan',"
|
||||
" updated_at = ? WHERE id = ? AND status NOT IN"
|
||||
" ('completed', 'failed', 'interrupted', 'merged',"
|
||||
" 'merge-failed')",
|
||||
@@ -3830,6 +3946,9 @@ class Worker:
|
||||
quota_error = self._quota_exceeded(task, None, on_error="closed")
|
||||
if quota_error is not None:
|
||||
self._set_status(task_id, "interrupted")
|
||||
# 配额中断不自动恢复：要等配额恢复/调高，自动
|
||||
# 回 queued 只会让 autoscaler 按队列深度白拉 worker。
|
||||
self._mark_interrupted_reason(task_id, "quota")
|
||||
self._write_execution_result(
|
||||
task_id, error_text=quota_error, actual_model=self.model,
|
||||
started_at=started_at, finished_at=time.time(),
|
||||
@@ -4280,6 +4399,8 @@ class Worker:
|
||||
self._audit(task["project_id"], "worker_fallback",
|
||||
"%s -> %s" % (self.model, nxt))
|
||||
final_written = self._set_status_if_running(task_id, final)
|
||||
if quota_stop and final_written and final == "interrupted":
|
||||
self._mark_interrupted_reason(task_id, "quota")
|
||||
finished_at = time.time()
|
||||
# P0 安全批后：CAS 没改到行（任务已被取消/回收）时
|
||||
# 绝不写回结果，否则迟到的旧进程会覆盖新一轮的结果
|
||||
@@ -4764,6 +4885,13 @@ class Worker:
|
||||
except Exception as exc:
|
||||
log_event(self.logger, "worker_orphan_sweep_failed",
|
||||
message="orphan sweep failed: %s" % exc)
|
||||
# 长任务自动恢复：接在 sweep 之后——刚被 sweep
|
||||
# 标 interrupted 的系统中断 goal 任务，本轮即可
|
||||
# 回 queued、下一轮认领即断点续跑；用户暂停、
|
||||
# 配额、协调员孤儿一律不接（见方法 docstring）。
|
||||
# 放在 sweep 块外：sweep 关掉时，已由其它路径
|
||||
# （如 worker 退出）标 interrupted 的任务仍要恢复。
|
||||
self.auto_resume_interrupted()
|
||||
# 每轮结束写心跳：哪怕本轮一个任务都没派，worker 活着
|
||||
# 这件事本身也要被 server 看到（文件心跳 + 注册表心跳）。
|
||||
self._write_heartbeat()
|
||||
@@ -4810,6 +4938,8 @@ class Worker:
|
||||
pass
|
||||
self._running_clear(task_id)
|
||||
self._set_status(task_id, "interrupted")
|
||||
# worker 退出时的系统中断，可自动恢复。
|
||||
self._mark_interrupted_reason(task_id, "worker_shutdown")
|
||||
return all_results
|
||||
|
||||
|
||||
@@ -4970,6 +5100,10 @@ def main(argv=None):
|
||||
"取消与已记 fallback 的失败不重试）")
|
||||
parser.add_argument("--retry-backoff-seconds", type=float, default=5.0,
|
||||
help="重试前等待秒数（默认 5）")
|
||||
parser.add_argument("--auto-resume-max", type=int, default=50,
|
||||
help="长任务自动恢复上限（次数，非时间，默认 50）："
|
||||
"同一 goal 任务被系统中断后自动续跑超过该次数"
|
||||
"即停住等人工，防确定性失败无限烧额度")
|
||||
parser.add_argument("--loop", action="store_true",
|
||||
help="常驻循环，每 --interval 秒跑一轮")
|
||||
parser.add_argument("--interval", type=int, default=300)
|
||||
@@ -5002,6 +5136,7 @@ def main(argv=None):
|
||||
task_timeout=args.task_timeout,
|
||||
max_retries=args.max_retries,
|
||||
retry_backoff_seconds=args.retry_backoff_seconds,
|
||||
auto_resume_max=args.auto_resume_max,
|
||||
repos_dir=args.repos_dir,
|
||||
heartbeat_file=args.heartbeat_file,
|
||||
worker_id=args.worker_id,
|
||||
|
||||
Reference in new issue
Block a user