feat: 长任务无敌补齐三件:连续失败上限 + worker退出审计 + 配额恢复自动续跑
1. host goal 循环连续失败护栏:3 轮连续失败即中断出口, 任务记 interrupted、可断点续跑,不再无限空转 (2026-10-29 实弹:断点后 274 轮全失败)。 2. worker 退出杀残留路径补审计 worker_shutdown_interrupted, detail 为任务 id,与 sweep_interrupted 同口径。 3. 配额中断自动恢复:每轮现查配额,仍超限/查不到时留 在 interrupted 不重排队,下轮再查;现查为 None 时 (新月份/额度上调/无价上限重置)才 CAS 回 queued。 单测:配额仍超限不动、配额恢复续跑、连续 3 轮失败停转。
This commit is contained in:
1 parent
88fcfeebd3
commit
ae3aff8602
4 files changed
+100
-6
No files matched your search
+43
-5
@@ -853,9 +853,14 @@ class Worker:
|
||||
"""长任务自动恢复:只接系统中断的 goal 任务。
|
||||
|
||||
只有用户暂停才暂停:paused_by_user=1(/pause、/cancel)
|
||||
的任务绝不自动动;配额中断(quota)与协调员孤儿
|
||||
(coordinator_orphan)也不接——前者要等配额恢复、
|
||||
后者要由父协调员收口,自动重跑只会烧钱或跑丢父子关系。
|
||||
的任务绝不自动动;协调员孤儿(coordinator_orphan)
|
||||
不接——要由父协调员收口,自动重跑会跑丢父子关系。
|
||||
配额中断(quota)只在配额已恢复时接:每轮先跑
|
||||
_quota_exceeded 现查,仍超限就留在 interrupted 不
|
||||
重排队(不触发 autoscaler 拉 worker、不烧钱),等
|
||||
下一轮再查;查不到配额或仍超限都不接。恢复的判定
|
||||
是现查结果为 None(新月份、额度上调、无价上限重置
|
||||
都算恢复),不是猜时间。
|
||||
非 goal 任务不接:它没有断点语义,重跑等于从头再来,
|
||||
可能重复副作用,留给人工 /retry 决定。
|
||||
接到的任务 CAS 回 queued 并置 goal_resume_pending=1,
|
||||
@@ -872,12 +877,13 @@ class Worker:
|
||||
try:
|
||||
self._ensure_result_columns(conn)
|
||||
rows = conn.execute(
|
||||
"SELECT id, project_id FROM tasks"
|
||||
"SELECT id, project_id, created_by, parent_task_id,"
|
||||
" model, interrupted_reason FROM tasks"
|
||||
" WHERE status = 'interrupted'"
|
||||
" AND (paused_by_user IS NULL OR paused_by_user = 0)"
|
||||
" AND (interrupted_reason IS NULL OR"
|
||||
" interrupted_reason NOT IN"
|
||||
" ('quota', 'coordinator_orphan'))"
|
||||
" ('coordinator_orphan'))"
|
||||
" AND parent_task_id IS NULL"
|
||||
" AND (goal_mode = 1 OR verify_command IS NOT NULL"
|
||||
" OR goal IS NOT NULL)"
|
||||
@@ -889,6 +895,19 @@ class Worker:
|
||||
for row in rows:
|
||||
task_id = row[0]
|
||||
project_id = row[1]
|
||||
if row[5] == "quota":
|
||||
# 配额中断:现查配额,仍超限或查不到
|
||||
# 就留在 interrupted,下轮再查,不重排队。
|
||||
try:
|
||||
_qerr = self._quota_exceeded(
|
||||
{"created_by": row[2],
|
||||
"parent_task_id": row[3],
|
||||
"model": row[4]},
|
||||
None, on_error="closed")
|
||||
except Exception:
|
||||
_qerr = "quota_check_failed"
|
||||
if _qerr is not None:
|
||||
continue
|
||||
cur = conn.execute(
|
||||
"UPDATE tasks SET status = 'queued',"
|
||||
" goal_resume_pending = 1,"
|
||||
@@ -4940,6 +4959,25 @@ class Worker:
|
||||
self._set_status(task_id, "interrupted")
|
||||
# worker 退出时的系统中断,可自动恢复。
|
||||
self._mark_interrupted_reason(task_id, "worker_shutdown")
|
||||
# 审计:此前这条路径只改状态不记审计,中断只能靠
|
||||
# 重启时间 + 后续 auto_resumed 推断(2026-10-29
|
||||
# 实弹);记一条 worker_shutdown_interrupted,
|
||||
# detail 为任务 id,与 sweep_interrupted 同口径。
|
||||
try:
|
||||
_conn = _db_connect(self.db_path)
|
||||
try:
|
||||
_row = _conn.execute(
|
||||
"SELECT project_id FROM tasks WHERE id = ?",
|
||||
(task_id,)).fetchone()
|
||||
finally:
|
||||
_conn.close()
|
||||
_pid = _row[0] if _row is not None else ""
|
||||
if _pid is None:
|
||||
_pid = ""
|
||||
except Exception:
|
||||
_pid = ""
|
||||
self._audit(_pid, "worker_shutdown_interrupted",
|
||||
task_id)
|
||||
return all_results
|
||||
|
||||
|
||||
|
||||
Reference in new issue
Block a user