feat: 长任务中断自动恢复，仅用户暂停才暂停，不限时
- 新增 tasks 列 paused_by_user/interrupted_reason/auto_resume_count/no_time_limit（幂等迁移，server+worker 双侧）
- POST /tasks/{id}/pause：用户暂停标 interrupted + paused_by_user=1；/cancel 同款标记；人工 /retry、/goal-resume 清标记
- sweep 标 sweep、配额标 quota、协调员孤儿标 coordinator_orphan、worker 退出标 worker_shutdown
- worker.run_loop 每轮后 auto_resume_interrupted：只接系统中断的 goal 任务，CAS 回 queued + goal_resume_pending=1（G10 断点续跑），用户暂停/配额/孤儿/非 goal 不接，上限 50 次防无限烧额度
- no_time_limit=1 的任务 worker/Docker 级不限时（其它任务仍默认 1800 秒；host 单轮 600 秒仍是单轮护栏）
- 前置 cherry-pick：G10 断点续跑透传 model_provider（否则 Docker 断点续跑在 sub2api 下必死，自动恢复无法实证；本 PR supersedes #127，勿另行合并 #127）
This commit is contained in:
host committed
1 parent
a19b1567bb
commit
c9cfe22a36
3 files changed
+454
-11
No files matched your search
+85
-4
@@ -1113,6 +1113,19 @@ def migrate_tasks_table(conn):
|
||||
# 话发一条新消息、开新循环”，本标记是“从
|
||||
# goal_rounds_done 的下一轮接着跑同一循环”。
|
||||
"goal_resume_pending": "INTEGER DEFAULT 0",
|
||||
# 长任务自动恢复：paused_by_user=1 只由用户暂停/
|
||||
# 取消（/pause、/cancel）写，系统中断（sweep、worker
|
||||
# 退出、配额除外）绝不写它；interrupted_reason 记
|
||||
# 中断来源（user_pause/user_cancel/sweep/quota/
|
||||
# coordinator_orphan/worker_shutdown），NULL = 老行或
|
||||
# 未分类系统中断，按可自动恢复处理；auto_resume_count
|
||||
# 记自动续跑次数（防确定性失败无限循环的上限门）；
|
||||
# no_time_limit=1 的任务 worker 级不限时（见 worker
|
||||
# _run_attempt，host 单轮 600 秒仍是单轮护栏）。
|
||||
"paused_by_user": "INTEGER DEFAULT 0",
|
||||
"interrupted_reason": "TEXT",
|
||||
"auto_resume_count": "INTEGER DEFAULT 0",
|
||||
"no_time_limit": "INTEGER DEFAULT 0",
|
||||
}
|
||||
for name, coltype in new_cols.items():
|
||||
if name not in cols:
|
||||
@@ -1635,6 +1648,10 @@ def _task_to_dict(row, attachments=None):
|
||||
"parent_task_id": _row_get(row, "parent_task_id"),
|
||||
"coordinator_part": _row_get(row, "coordinator_part"),
|
||||
"goal_resume_pending": _row_get(row, "goal_resume_pending") or 0,
|
||||
"paused_by_user": _row_get(row, "paused_by_user") or 0,
|
||||
"interrupted_reason": _row_get(row, "interrupted_reason"),
|
||||
"auto_resume_count": _row_get(row, "auto_resume_count") or 0,
|
||||
"no_time_limit": _row_get(row, "no_time_limit") or 0,
|
||||
}
|
||||
|
||||
|
||||
@@ -1659,6 +1676,8 @@ def sweep_stale_tasks(db_path, threshold_seconds=300):
|
||||
心跳取 last_heartbeat，无心跳则用 updated_at；早于
|
||||
now - threshold_seconds 即过期。审计记一条 "sweep_interrupted"
|
||||
（user_id 记 "system"，因为回收不是某个用户的操作）。
|
||||
这是系统中断：interrupted_reason='sweep'、paused_by_user=0，
|
||||
长任务自动恢复可接它（用户暂停绝不走这条路）。
|
||||
独立可单测，不依赖 HTTP。
|
||||
"""
|
||||
now = time.time()
|
||||
@@ -1674,7 +1693,9 @@ def sweep_stale_tasks(db_path, threshold_seconds=300):
|
||||
swept = []
|
||||
for row in rows:
|
||||
cur = conn.execute(
|
||||
"UPDATE tasks SET status = 'interrupted', updated_at = ?"
|
||||
"UPDATE tasks SET status = 'interrupted',"
|
||||
" interrupted_reason = 'sweep', paused_by_user = 0,"
|
||||
" updated_at = ?"
|
||||
" WHERE id = ? AND status = 'running'",
|
||||
(now, row["id"]),
|
||||
)
|
||||
@@ -2059,6 +2080,8 @@ class CloudControlPlaneHandler(BaseHTTPRequestHandler):
|
||||
self._task_agents(user_id, rest[: -len("/agents")])
|
||||
elif rest.endswith("/cancel") and method == "POST":
|
||||
self._cancel_task(user_id, rest[: -len("/cancel")])
|
||||
elif rest.endswith("/pause") and method == "POST":
|
||||
self._pause_task(user_id, rest[: -len("/pause")])
|
||||
elif rest.endswith("/merge") and method == "POST":
|
||||
self._merge_task(user_id, rest[: -len("/merge")])
|
||||
elif rest.endswith("/pr") and method == "POST":
|
||||
@@ -4213,6 +4236,13 @@ class CloudControlPlaneHandler(BaseHTTPRequestHandler):
|
||||
json.dumps(coord_parts) if coord_parts else None,
|
||||
None, None, child_model, child_effort),
|
||||
)
|
||||
# 长任务不限时：API 字段 no_time_limit=1 时写标记，
|
||||
# worker 认领后按不限时跑（单轮 host 600 秒仍在）；
|
||||
# 不传或非真值时保持默认 0，现有任务行为不变。
|
||||
if body.get("no_time_limit") in (True, 1):
|
||||
conn.execute(
|
||||
"UPDATE tasks SET no_time_limit = 1"
|
||||
" WHERE id = ?", (task_id,))
|
||||
for rel_path, size in bundle_files:
|
||||
conn.execute(
|
||||
"INSERT INTO task_attachments"
|
||||
@@ -6442,8 +6472,12 @@ class CloudControlPlaneHandler(BaseHTTPRequestHandler):
|
||||
now = time.time()
|
||||
conn = self._db()
|
||||
try:
|
||||
# 用户取消 = 用户暂停：自动恢复绝不接它，人工
|
||||
# /retry 或 /goal-resume 才能再跑（那两处会清标记）。
|
||||
conn.execute(
|
||||
"UPDATE tasks SET status = 'interrupted', updated_at = ?"
|
||||
"UPDATE tasks SET status = 'interrupted',"
|
||||
" paused_by_user = 1, interrupted_reason = 'user_cancel',"
|
||||
" updated_at = ?"
|
||||
" WHERE id = ?",
|
||||
(now, task_id),
|
||||
)
|
||||
@@ -6476,6 +6510,49 @@ class CloudControlPlaneHandler(BaseHTTPRequestHandler):
|
||||
store.audit(user_id, task["project_id"], "cancel_task", task_id)
|
||||
self._send_json(200, task)
|
||||
|
||||
def _pause_task(self, user_id, task_id):
|
||||
"""用户暂停：与 /cancel 同终态、不同语义标记。
|
||||
|
||||
只有用户暂停才暂停：本端点把任务标 interrupted 并置
|
||||
paused_by_user=1、interrupted_reason='user_pause'，worker
|
||||
自动恢复（auto_resume_interrupted）绝不接它；人工
|
||||
/retry 或 /goal-resume 才能再跑（那两处会清标记）。
|
||||
权限与 /cancel 同款（create_task），终态任务 400。
|
||||
"""
|
||||
row = self._fetch_task(task_id)
|
||||
if row is None:
|
||||
self._send_json(404, {"error": "task not found"})
|
||||
return
|
||||
task = _task_to_dict(row)
|
||||
with _auth_store(self.server.db_path) as store:
|
||||
if not store.check_permission(
|
||||
user_id, task["project_id"], "create_task"):
|
||||
self._send_json(403, {"error": "forbidden"})
|
||||
return
|
||||
if task["status"] in TERMINAL_STATUSES:
|
||||
self._send_json(400, {"error": "task already terminal"})
|
||||
return
|
||||
now = time.time()
|
||||
conn = self._db()
|
||||
try:
|
||||
conn.execute(
|
||||
"UPDATE tasks SET status = 'interrupted',"
|
||||
" paused_by_user = 1, interrupted_reason = 'user_pause',"
|
||||
" updated_at = ? WHERE id = ?",
|
||||
(now, task_id),
|
||||
)
|
||||
conn.commit()
|
||||
task = _task_to_dict(
|
||||
conn.execute(
|
||||
"SELECT * FROM tasks WHERE id = ?", (task_id,)
|
||||
).fetchone()
|
||||
)
|
||||
finally:
|
||||
conn.close()
|
||||
with _auth_store(self.server.db_path) as store:
|
||||
store.audit(user_id, task["project_id"], "pause_task", task_id)
|
||||
self._send_json(200, task)
|
||||
|
||||
def _merge_task(self, user_id, task_id):
|
||||
"""把云端任务标记为已合并：只补状态，不执行合并本身。
|
||||
|
||||
@@ -6924,7 +7001,9 @@ class CloudControlPlaneHandler(BaseHTTPRequestHandler):
|
||||
cur = conn.execute(
|
||||
"UPDATE tasks SET status = 'queued',"
|
||||
" goal_resume_pending = 1, result_text = NULL,"
|
||||
" error_text = NULL, actual_model = NULL, updated_at = ?"
|
||||
" error_text = NULL, actual_model = NULL,"
|
||||
" paused_by_user = 0, interrupted_reason = NULL,"
|
||||
" updated_at = ?"
|
||||
" WHERE id = ? AND status = 'interrupted'",
|
||||
(now, task_id),
|
||||
)
|
||||
@@ -6993,7 +7072,9 @@ class CloudControlPlaneHandler(BaseHTTPRequestHandler):
|
||||
try:
|
||||
cur = conn.execute(
|
||||
"UPDATE tasks SET status = 'queued', result_text = NULL,"
|
||||
" error_text = NULL, actual_model = NULL, updated_at = ?"
|
||||
" error_text = NULL, actual_model = NULL,"
|
||||
" paused_by_user = 0, interrupted_reason = NULL,"
|
||||
" updated_at = ?"
|
||||
" WHERE id = ? AND status IN"
|
||||
" ('failed', 'interrupted', 'merge-failed')",
|
||||
(now, task_id),
|
||||
|
||||
Reference in new issue
Block a user