feat: 长任务无敌补齐三件:连续失败上限 + worker退出审计 + 配额恢复自动续跑
1. host goal 循环连续失败护栏:3 轮连续失败即中断出口, 任务记 interrupted、可断点续跑,不再无限空转 (2026-10-29 实弹:断点后 274 轮全失败)。 2. worker 退出杀残留路径补审计 worker_shutdown_interrupted, detail 为任务 id,与 sweep_interrupted 同口径。 3. 配额中断自动恢复:每轮现查配额,仍超限/查不到时留 在 interrupted 不重排队,下轮再查;现查为 None 时 (新月份/额度上调/无价上限重置)才 CAS 回 queued。 单测:配额仍超限不动、配额恢复续跑、连续 3 轮失败停转。
This commit is contained in:
1 parent
88fcfeebd3
commit
ae3aff8602
4 files changed
+100
-6
No files matched your search
@@ -4,7 +4,8 @@
|
||||
规则(见 worker.auto_resume_interrupted docstring):
|
||||
- 系统中断的 goal 任务 -> 回 queued + goal_resume_pending=1 + count+1
|
||||
- 用户暂停/取消(paused_by_user=1)-> 不动
|
||||
- 配额中断、协调员孤儿、非 goal 任务 -> 不动
|
||||
- 配额仍超限 -> 不动(不重排队);配额已恢复 -> 自动续跑
|
||||
- 协调员孤儿、非 goal 任务 -> 不动
|
||||
- 自动续跑次数到上限 -> 不动
|
||||
- no_time_limit 任务的 _run_attempt 不设 worker 级截止
|
||||
"""
|
||||
@@ -100,6 +101,9 @@ class LongtaskAutoResumeTestCase(unittest.TestCase):
|
||||
self.assertEqual(count, 0)
|
||||
|
||||
def test_quota_orphan_nongoal_do_not_resume(self):
|
||||
# 配额仍超限时不重排队(mock 现查超限)。
|
||||
self.worker._quota_exceeded = lambda task, usage, on_error="closed": (
|
||||
"quota_exceeded: 本月配额已用完")
|
||||
_insert_task(self.db_path, "T1", goal_mode=1, goal="g",
|
||||
interrupted_reason="quota", paused_by_user=0)
|
||||
_insert_task(self.db_path, "T2", goal_mode=1, goal="g",
|
||||
@@ -111,6 +115,18 @@ class LongtaskAutoResumeTestCase(unittest.TestCase):
|
||||
status, _pending, _c, _p, _r, _n = _row(self.db_path, tid)
|
||||
self.assertEqual(status, "interrupted")
|
||||
|
||||
def test_quota_restored_resumes(self):
|
||||
# 配额已恢复(现查为 None)时配额中断也自动续跑。
|
||||
self.worker._quota_exceeded = lambda task, usage, on_error="closed": None
|
||||
_insert_task(self.db_path, "T1", goal_mode=1, goal="g",
|
||||
interrupted_reason="quota", paused_by_user=0)
|
||||
self.assertEqual(self.worker.auto_resume_interrupted(), ["T1"])
|
||||
status, pending, count, _p, reason, _n = _row(self.db_path, "T1")
|
||||
self.assertEqual(status, "queued")
|
||||
self.assertEqual(pending, 1)
|
||||
self.assertEqual(count, 1)
|
||||
self.assertIsNone(reason)
|
||||
|
||||
def test_auto_resume_max_cap(self):
|
||||
_insert_task(self.db_path, "T1", goal_mode=1, goal="g",
|
||||
interrupted_reason="sweep", paused_by_user=0,
|
||||
|
||||
Reference in new issue
Block a user