长任务 P0 三件:断点进库、错误分类+熔断、Run/Step 时间线
断点进库:task_checkpoints 表(PG 新库 + 老库幂等补表 + 迁移清单
对齐);worker 在 goal 任务执行期间每 30 秒把磁盘 host-state 断点
镜像进库,终局再补一次;断点续跑认领前,若 host-state 条目丢失
但库里有断点,先按库断点重建最小条目再跑,库里也没有断点时不
造假、走原有拒绝路径。
错误分类+熔断:sandbox 四处解析加永久错误分支(KeyError、配置
缺失等无结论异常退出判 failed 死信,不再判 interrupted;host 已
明言 interrupted 的不受影响,未知错误不分类);worker 自动恢复
加退避(10 秒起指数退避、封顶 300 秒)与无进展熔断(库断点轮次
连续 3 次不变即标 auto_resume_stalled 等人工)。
时间线:task_steps 表 + host 每轮写 host-steps 步骤文件 + worker
导入;GET /tasks/{id}/timeline 一次返回 steps 与 checkpoints;
认领、暂停、自动恢复、终局都记步骤。旧任务返回空数组,不编造。
单测:cloud/tests/test_p0_three.py 14 项(分类、熔断、镜像重建、
步骤导入去重、timeline 接口)。
This commit is contained in:
1 parent
d56bd3a99e
commit
585242fc8f
7 files changed
+982
-3
No files matched your search
+111
@@ -1126,6 +1126,19 @@ def migrate_tasks_table(conn):
|
||||
"interrupted_reason": "TEXT",
|
||||
"auto_resume_count": "INTEGER DEFAULT 0",
|
||||
"no_time_limit": "INTEGER DEFAULT 0",
|
||||
# P0 三件:自动恢复退避与无进展熔断(见 worker
|
||||
# auto_resume_interrupted):resume_after 是下次允许
|
||||
# 重排队的时间戳(指数退避);auto_resume_stall_count
|
||||
# 记连续无进展重排队次数(到 3 即停、标
|
||||
# auto_resume_stalled 等人工);checkpoint_round_at_last_
|
||||
# resume 记上次重排队时库中断点轮次(无进展判定的
|
||||
# 基线,NULL = 从未记过);last_error_signature 记
|
||||
# 最近一次终局的错误签名(permanent/transient/None,
|
||||
# 供排障,不作恢复判定——判定在 sandbox 终态里)。
|
||||
"resume_after": "DOUBLE PRECISION DEFAULT 0",
|
||||
"auto_resume_stall_count": "INTEGER DEFAULT 0",
|
||||
"checkpoint_round_at_last_resume": "INTEGER",
|
||||
"last_error_signature": "TEXT",
|
||||
}
|
||||
for name, coltype in new_cols.items():
|
||||
if name not in cols:
|
||||
@@ -1204,6 +1217,40 @@ def _attachments_map(conn, task_ids):
|
||||
return out
|
||||
|
||||
|
||||
def _ensure_task_checkpoints_table(conn):
|
||||
"""幂等建断点表(P0 三件其一):server init_db(PG 与 SQLite
|
||||
两支)与 worker 迁移兜底共用。DOUBLE PRECISION 在两后端都可用。
|
||||
"""
|
||||
conn.execute(
|
||||
"CREATE TABLE IF NOT EXISTS task_checkpoints("
|
||||
"task_id TEXT, "
|
||||
"round_no INTEGER, "
|
||||
"thread_id TEXT, "
|
||||
"goal_status TEXT, "
|
||||
"state_sha TEXT, "
|
||||
"verify_out TEXT, "
|
||||
"created_at DOUBLE PRECISION, "
|
||||
"PRIMARY KEY(task_id, round_no))")
|
||||
|
||||
|
||||
def _ensure_task_steps_table(conn):
|
||||
"""幂等建步骤表(P0 三件其三):timeline 接口直读此表。
|
||||
id 由写入方给 uuid hex(两后端都不依赖自增语义)。
|
||||
"""
|
||||
conn.execute(
|
||||
"CREATE TABLE IF NOT EXISTS task_steps("
|
||||
"id TEXT PRIMARY KEY, "
|
||||
"task_id TEXT, "
|
||||
"step_no INTEGER, "
|
||||
"kind TEXT, "
|
||||
"summary TEXT, "
|
||||
"state_sha TEXT, "
|
||||
"created_at DOUBLE PRECISION)")
|
||||
conn.execute(
|
||||
"CREATE INDEX IF NOT EXISTS idx_task_steps_task"
|
||||
" ON task_steps(task_id, step_no)")
|
||||
|
||||
|
||||
def _ensure_followups_table(conn):
|
||||
"""幂等建后续消息表:委托给 peer_messages(两边同一份 DDL,
|
||||
server init_db、各 followup 接口与 worker 投递路径共用)。
|
||||
@@ -1530,6 +1577,8 @@ def init_db(db_path):
|
||||
_backfill_default_quotas(conn)
|
||||
_ensure_gateway_keys(conn)
|
||||
_ensure_uploads_tables(conn)
|
||||
_ensure_task_checkpoints_table(conn)
|
||||
_ensure_task_steps_table(conn)
|
||||
conn.commit()
|
||||
return
|
||||
conn.execute(
|
||||
@@ -1564,6 +1613,8 @@ def init_db(db_path):
|
||||
_ensure_quotas_table(conn)
|
||||
_backfill_default_quotas(conn)
|
||||
_ensure_uploads_tables(conn)
|
||||
_ensure_task_checkpoints_table(conn)
|
||||
_ensure_task_steps_table(conn)
|
||||
# worker 每轮 poll 都按 status 取任务(见 load_tasks 的
|
||||
# WHERE status IN (...)),无索引时是全表扫描;任务表随
|
||||
# 时间变大后索引是必须的。IF NOT EXISTS 保证老库幂等。
|
||||
@@ -2099,6 +2150,8 @@ class CloudControlPlaneHandler(BaseHTTPRequestHandler):
|
||||
self._create_followup(user_id, rest[: -len("/followups")])
|
||||
elif rest.endswith("/followups") and method == "GET":
|
||||
self._list_followups(user_id, rest[: -len("/followups")])
|
||||
elif rest.endswith("/timeline") and method == "GET":
|
||||
self._task_timeline(user_id, rest[: -len("/timeline")])
|
||||
elif rest.endswith("/chat-history") and method == "GET":
|
||||
self._task_chat_history(
|
||||
user_id, rest[: -len("/chat-history")])
|
||||
@@ -4602,6 +4655,49 @@ class CloudControlPlaneHandler(BaseHTTPRequestHandler):
|
||||
self._send_json(200, {
|
||||
"followups": [_followup_to_dict(r) for r in rows]})
|
||||
|
||||
def _task_timeline(self, user_id, task_id):
|
||||
"""GET /tasks/{id}/timeline(P0 三件其三):一个任务一张
|
||||
时间线。steps 直读 task_steps(claim/resume/round/final),
|
||||
checkpoints 直读 task_checkpoints(每轮断点快照);旧
|
||||
任务(部署前跑完、无步骤记录)返回空数组,不编造。
|
||||
"""
|
||||
row = self._fetch_task(task_id)
|
||||
if row is None:
|
||||
self._send_json(404, {"error": "task not found"})
|
||||
return
|
||||
task = _task_to_dict(row)
|
||||
with _auth_store(self.server.db_path) as store:
|
||||
if not store.check_permission(
|
||||
user_id, task["project_id"], "read"):
|
||||
self._send_json(403, {"error": "forbidden"})
|
||||
return
|
||||
conn = self._db()
|
||||
try:
|
||||
_ensure_task_checkpoints_table(conn)
|
||||
_ensure_task_steps_table(conn)
|
||||
step_rows = conn.execute(
|
||||
"SELECT step_no, kind, summary, state_sha, created_at"
|
||||
" FROM task_steps WHERE task_id = ?"
|
||||
" ORDER BY step_no, created_at, id",
|
||||
(task_id,)).fetchall()
|
||||
cp_rows = conn.execute(
|
||||
"SELECT round_no, thread_id, goal_status, state_sha,"
|
||||
" verify_out, created_at FROM task_checkpoints"
|
||||
" WHERE task_id = ? ORDER BY round_no",
|
||||
(task_id,)).fetchall()
|
||||
finally:
|
||||
conn.close()
|
||||
self._send_json(200, {
|
||||
"task_id": task_id,
|
||||
"steps": [{"step_no": r[0], "kind": r[1], "summary": r[2],
|
||||
"state_sha": r[3], "created_at": r[4]}
|
||||
for r in step_rows],
|
||||
"checkpoints": [{"round_no": r[0], "thread_id": r[1],
|
||||
"goal_status": r[2], "state_sha": r[3],
|
||||
"verify_out": r[4], "created_at": r[5]}
|
||||
for r in cp_rows],
|
||||
})
|
||||
|
||||
def _task_chat_history(self, user_id, task_id):
|
||||
"""GET /tasks/{id}/chat-history:真正的交替历史,一次性
|
||||
按时间交错拼好返回,前端不再猜顺序。
|
||||
@@ -6541,6 +6637,21 @@ class CloudControlPlaneHandler(BaseHTTPRequestHandler):
|
||||
" updated_at = ? WHERE id = ?",
|
||||
(now, task_id),
|
||||
)
|
||||
# P0 三件其三:时间线 pause 步骤(best-effort)。
|
||||
try:
|
||||
_ensure_task_steps_table(conn)
|
||||
_max = conn.execute(
|
||||
"SELECT MAX(step_no) FROM task_steps"
|
||||
" WHERE task_id = ?", (task_id,)).fetchone()
|
||||
conn.execute(
|
||||
"INSERT INTO task_steps(id, task_id, step_no, kind,"
|
||||
" summary, state_sha, created_at)"
|
||||
" VALUES(?, ?, ?, 'pause', '用户暂停', NULL, ?)",
|
||||
(uuid.uuid4().hex, task_id,
|
||||
((_max[0] or 0) + 1 if _max and _max[0] else 1),
|
||||
now))
|
||||
except Exception:
|
||||
pass
|
||||
conn.commit()
|
||||
task = _task_to_dict(
|
||||
conn.execute(
|
||||
|
||||
Reference in new issue
Block a user