feat: 长任务无敌补齐三件:连续失败上限 + worker退出审计 + 配额恢复自动续跑

1. host goal 循环连续失败护栏:3 轮连续失败即中断出口,
   任务记 interrupted、可断点续跑,不再无限空转
   (2026-10-29 实弹:断点后 274 轮全失败)。
2. worker 退出杀残留路径补审计 worker_shutdown_interrupted,
   detail 为任务 id,与 sweep_interrupted 同口径。
3. 配额中断自动恢复:每轮现查配额,仍超限/查不到时留
   在 interrupted 不重排队,下轮再查;现查为 None 时
   (新月份/额度上调/无价上限重置)才 CAS 回 queued。

单测:配额仍超限不动、配额恢复续跑、连续 3 轮失败停转。
This commit is contained in:
host committed 2026-10-29 13:07:57 +08:00
1 parent 88fcfeebd3
commit ae3aff8602
4 files changed
+100 -6

No files matched your search

+43 -5
View File
@@ -853,9 +853,14 @@ class Worker:
"""长任务自动恢复:只接系统中断的 goal 任务。
只有用户暂停才暂停:paused_by_user=1(/pause、/cancel)
的任务绝不自动动;配额中断(quota)与协调员孤儿
(coordinator_orphan)也不接——前者要等配额恢复、
后者要由父协调员收口,自动重跑只会烧钱或跑丢父子关系。
的任务绝不自动动;协调员孤儿(coordinator_orphan)
不接——要由父协调员收口,自动重跑会跑丢父子关系。
配额中断(quota)只在配额已恢复时接:每轮先跑
_quota_exceeded 现查,仍超限就留在 interrupted 不
重排队(不触发 autoscaler 拉 worker、不烧钱),等
下一轮再查;查不到配额或仍超限都不接。恢复的判定
是现查结果为 None(新月份、额度上调、无价上限重置
都算恢复),不是猜时间。
非 goal 任务不接:它没有断点语义,重跑等于从头再来,
可能重复副作用,留给人工 /retry 决定。
接到的任务 CAS 回 queued 并置 goal_resume_pending=1,
@@ -872,12 +877,13 @@ class Worker:
try:
self._ensure_result_columns(conn)
rows = conn.execute(
"SELECT id, project_id FROM tasks"
"SELECT id, project_id, created_by, parent_task_id,"
" model, interrupted_reason FROM tasks"
" WHERE status = 'interrupted'"
" AND (paused_by_user IS NULL OR paused_by_user = 0)"
" AND (interrupted_reason IS NULL OR"
" interrupted_reason NOT IN"
" ('quota', 'coordinator_orphan'))"
" ('coordinator_orphan'))"
" AND parent_task_id IS NULL"
" AND (goal_mode = 1 OR verify_command IS NOT NULL"
" OR goal IS NOT NULL)"
@@ -889,6 +895,19 @@ class Worker:
for row in rows:
task_id = row[0]
project_id = row[1]
if row[5] == "quota":
# 配额中断:现查配额,仍超限或查不到
# 就留在 interrupted,下轮再查,不重排队。
try:
_qerr = self._quota_exceeded(
{"created_by": row[2],
"parent_task_id": row[3],
"model": row[4]},
None, on_error="closed")
except Exception:
_qerr = "quota_check_failed"
if _qerr is not None:
continue
cur = conn.execute(
"UPDATE tasks SET status = 'queued',"
" goal_resume_pending = 1,"
@@ -4940,6 +4959,25 @@ class Worker:
self._set_status(task_id, "interrupted")
# worker 退出时的系统中断,可自动恢复。
self._mark_interrupted_reason(task_id, "worker_shutdown")
# 审计:此前这条路径只改状态不记审计,中断只能靠
# 重启时间 + 后续 auto_resumed 推断(2026-10-29
# 实弹);记一条 worker_shutdown_interrupted,
# detail 为任务 id,与 sweep_interrupted 同口径。
try:
_conn = _db_connect(self.db_path)
try:
_row = _conn.execute(
"SELECT project_id FROM tasks WHERE id = ?",
(task_id,)).fetchone()
finally:
_conn.close()
_pid = _row[0] if _row is not None else ""
if _pid is None:
_pid = ""
except Exception:
_pid = ""
self._audit(_pid, "worker_shutdown_interrupted",
task_id)
return all_results