长任务 P0 三件：断点进库、错误分类+熔断、Run/Step 时间线

断点进库：task_checkpoints 表（PG 新库 + 老库幂等补表 + 迁移清单
对齐）；worker 在 goal 任务执行期间每 30 秒把磁盘 host-state 断点
镜像进库，终局再补一次；断点续跑认领前，若 host-state 条目丢失
但库里有断点，先按库断点重建最小条目再跑，库里也没有断点时不
造假、走原有拒绝路径。

错误分类+熔断：sandbox 四处解析加永久错误分支（KeyError、配置
缺失等无结论异常退出判 failed 死信，不再判 interrupted；host 已
明言 interrupted 的不受影响，未知错误不分类）；worker 自动恢复
加退避（10 秒起指数退避、封顶 300 秒）与无进展熔断（库断点轮次
连续 3 次不变即标 auto_resume_stalled 等人工）。

时间线：task_steps 表 + host 每轮写 host-steps 步骤文件 + worker
导入；GET /tasks/{id}/timeline 一次返回 steps 与 checkpoints；
认领、暂停、自动恢复、终局都记步骤。旧任务返回空数组，不编造。

单测：cloud/tests/test_p0_three.py 14 项（分类、熔断、镜像重建、
步骤导入去重、timeline 接口）。

cloud/worker.py

@@ -119,6 +119,7 @@ Docker 下 repos_dir 必须放在 extra-mounts 已挂载的目录下，
 """
 
 import argparse
+import hashlib
 import json
 import os
 from datetime import datetime, timezone
@@ -205,6 +206,23 @@ GOAL_MAX_ROUNDS_CAP = 100
 _ALREADY_EXISTS_MARKER = "已存在"
 
 
+def _uuid_hex():
+    """uuid hex（task_steps id 用，两后端都不依赖自增语义）。"""
+    return uuid.uuid4().hex
+
+
+def _worker_host_name(executor, task):
+    """host 任务名：优先 executor._host_name（与 sandbox 同款
+    派生），没有 executor 时回落任务 id（uuid hex 本就是安全名）。"""
+    fn = getattr(executor, "_host_name", None)
+    if callable(fn):
+        try:
+            return fn(task)
+        except Exception:
+            pass
+    return str(task.get("id"))
+
+
 def _default_quota_usd_micros():
     """无配额行时的默认配额（微美元），与 server 侧同值。
 
@@ -868,6 +886,15 @@ class Worker:
         从 goal_rounds_done 下一轮起），同时清旧结果、清
         interrupted_reason、auto_resume_count+1；上限
         self.auto_resume_max 到顶的任务停住并记审计，等人工。
+        P0 三件其二的熔断（与次数上限并存，防确定性失败空转）：
+        - 退避：重排队时写 resume_after = now + min(300,
+          10 * 2**stall)，未到时间不重排队，不再 12 秒一轮烧次数。
+        - 无进展熔断：库中断点轮次（task_checkpoints 最大
+          round_no）与上次重排队时记的基线相同 → stall_count+1；
+          stall 到 3 时停住，interrupted_reason 标
+          auto_resume_stalled 并记审计，等人工（任务 2 的
+          KeyError 空转：分类已把永久错误判 failed；万一分类
+          漏网，无进展熔断是第二道闸）。
         返回自动续跑的任务 id 列表；任何异常只记日志，不炸
         run_loop（恢复是辅助路径，不能带走常驻 worker）。
         """
@@ -878,7 +905,9 @@ class Worker:
                 self._ensure_result_columns(conn)
                 rows = conn.execute(
                     "SELECT id, project_id, created_by, parent_task_id,"
-                    " model, interrupted_reason FROM tasks"
+                    " model, interrupted_reason, resume_after,"
+                    " auto_resume_stall_count,"
+                    " checkpoint_round_at_last_resume FROM tasks"
                     " WHERE status = 'interrupted'"
                     " AND (paused_by_user IS NULL OR paused_by_user = 0)"
                     " AND (interrupted_reason IS NULL OR"
@@ -895,6 +924,8 @@ class Worker:
                 for row in rows:
                     task_id = row[0]
                     project_id = row[1]
+                    if row[6] and now < row[6]:
+                        continue
                     if row[5] == "quota":
                         # 配额中断：现查配额，仍超限或查不到
                         # 就留在 interrupted，下轮再查，不重排队。
@@ -908,6 +939,31 @@ class Worker:
                             _qerr = "quota_check_failed"
                         if _qerr is not None:
                             continue
+                    try:
+                        cp = conn.execute(
+                            "SELECT MAX(round_no) FROM task_checkpoints"
+                            " WHERE task_id = ?", (task_id,)).fetchone()
+                        cp_round = cp[0] if cp and cp[0] is not None else -1
+                    except Exception:
+                        cp_round = -1
+                    prev = row[8]
+                    stall = 0
+                    if prev is not None:
+                        stall = (row[7] or 0) + 1 if cp_round == prev else 0
+                    if stall >= 3:
+                        cur = conn.execute(
+                            "UPDATE tasks SET interrupted_reason ="
+                            " 'auto_resume_stalled',"
+                            " auto_resume_stall_count = ?,"
+                            " updated_at = ?"
+                            " WHERE id = ? AND status = 'interrupted'",
+                            (stall, now, task_id))
+                        if cur.rowcount == 1:
+                            self._audit(
+                                project_id, "worker_auto_resume_stalled",
+                                task_id)
+                        continue
+                    backoff = min(300, 10 * (2 ** stall))
                     cur = conn.execute(
                         "UPDATE tasks SET status = 'queued',"
                         " goal_resume_pending = 1,"
@@ -916,16 +972,23 @@ class Worker:
                         " interrupted_reason = NULL,"
                         " auto_resume_count ="
                         " COALESCE(auto_resume_count, 0) + 1,"
+                        " auto_resume_stall_count = ?,"
+                        " checkpoint_round_at_last_resume = ?,"
+                        " resume_after = ?,"
                         " updated_at = ?"
                         " WHERE id = ? AND status = 'interrupted'"
                         " AND (paused_by_user IS NULL"
                         " OR paused_by_user = 0)",
-                        (now, task_id),
+                        (stall, cp_round, now + backoff, now, task_id),
                     )
                     if cur.rowcount == 1:
                         resumed.append(task_id)
                         self._audit(project_id, "worker_auto_resumed",
                                     task_id)
+                        self._record_step(
+                            conn, task_id, "auto_resume",
+                            "自动续跑（无进展计数 %d，下次退避 %d 秒）"
+                            % (stall, backoff))
                 conn.commit()
             finally:
                 conn.close()
@@ -934,6 +997,249 @@ class Worker:
                       message="auto resume failed: %s" % exc)
         return resumed
 
+    def _record_step(self, conn, task_id, kind, summary, step_no=None,
+                     state_sha=None):
+        """向 task_steps 写一行（P0 三件其三，时间线事实源）。
+
+        best-effort：写失败只记日志，绝不影响调用方（认领/
+        恢复/终局）的正常流程。step_no 未给时按该任务已有
+        最大 step_no + 1 自增（kind=round 的调用方显式传
+        round_no，保证 host 轮次与时间线 step_no 一致）。
+        """
+        try:
+            if step_no is None:
+                row = conn.execute(
+                    "SELECT MAX(step_no) FROM task_steps"
+                    " WHERE task_id = ?", (task_id,)).fetchone()
+                step_no = (row[0] or 0) + 1 if row and row[0] else 1
+            conn.execute(
+                "INSERT INTO task_steps(id, task_id, step_no, kind,"
+                " summary, state_sha, created_at) VALUES(?, ?, ?, ?,"
+                " ?, ?, ?)",
+                (_uuid_hex(), task_id, step_no, kind, summary[:500]
+                 if summary else None, state_sha, time.time()))
+        except Exception as exc:
+            log_event(self.logger, "task_step_record_failed",
+                      message="step record failed: %s" % exc)
+
+    def _checkpoint_round(self, conn, task_id):
+        """库中断点最大轮次（无断点时 -1）；表缺失时也返回 -1。"""
+        try:
+            row = conn.execute(
+                "SELECT MAX(round_no) FROM task_checkpoints"
+                " WHERE task_id = ?", (task_id,)).fetchone()
+            return row[0] if row and row[0] is not None else -1
+        except Exception:
+            return -1
+
+    def _checkpoint_mirror_loop(self, task, stop):
+        """断点镜像线程（P0 三件其一）：goal 任务执行期间每 30 秒
+        把磁盘 host-state 断点镜像进库一次；执行结束由
+        _run_attempt 的 finally 再补一次终局镜像。线程本身绝不
+        抛（_mirror_checkpoint_once 已全量兜底），stop 后即退。
+        """
+        try:
+            self._mirror_checkpoint_once(task)
+        except Exception:
+            pass
+        while not stop.wait(30):
+            try:
+                self._mirror_checkpoint_once(task)
+            except Exception:
+                pass
+
+    def _host_state_path_for_task(self, task):
+        """断点镜像的 host-state 文件路径（P0 三件其一）。
+
+        租户 Docker：tenants/<safe>/work/host-state.json；
+        非租户 Docker：executor extra_mounts 里 dest=/app/work
+        的宿主路径（若无则回落 /opt/codex-app-server/work）；
+        Local：读 executor config 的 state 项（相对配置目录）。
+        找不到时返回 None，镜像静默跳过（不影响任务执行）。
+        """
+        try:
+            project_id = task.get("project_id")
+            if getattr(self, "tenant_isolation", False) and project_id:
+                safe = self._tenant_safe_name(project_id)
+                base = os.path.join(self.tenants_root, safe, "work")
+                return os.path.join(base, "host-state.json")
+            mounts = getattr(self.executor, "extra_mounts", None) or []
+            for item in mounts:
+                src, dst = (item if isinstance(item, tuple)
+                            else (None, None))
+                if dst == "/app/work" and src:
+                    return os.path.join(src, "host-state.json")
+            cfg = getattr(self.executor, "config_path", None)
+            if cfg and os.path.exists(cfg):
+                with open(cfg, encoding="utf-8") as fh:
+                    data = json.load(fh)
+                state = data.get("state") or "host-state.json"
+                if not os.path.isabs(state):
+                    state = os.path.join(
+                        os.path.dirname(os.path.abspath(cfg)), state)
+                return state
+            return os.path.join("/opt/codex-app-server/work",
+                                "host-state.json")
+        except Exception:
+            return None
+
+    def _mirror_checkpoint_once(self, task):
+        """把磁盘 host-state 里的本任务断点镜像进库（单次）。
+
+        读 state → 取 tasks[host 名] 条目 → (task_id, round_no)
+        upsert 进 task_checkpoints（state_sha 是条目 JSON 的
+        sha256，轮次相同但线程/状态变了也能看出来）→ 同时
+        导入同目录 host-steps/<任务>.jsonl 里的 round 步骤。
+        任何异常静默（镜像是辅助路径，执行本身不受影响）。
+        返回镜像到的轮次（-1 = 没镜像到）。
+        """
+        try:
+            state_path = self._host_state_path_for_task(task)
+            if not state_path or not os.path.exists(state_path):
+                return -1
+            with open(state_path, encoding="utf-8") as fh:
+                state = json.load(fh)
+            host_name = _worker_host_name(self.executor, task)
+            entry = (state.get("tasks") or {}).get(host_name)
+            if not isinstance(entry, dict):
+                return -1
+            rounds = int(entry.get("goal_rounds_done") or 0)
+            state_sha = hashlib.sha256(
+                json.dumps(entry, sort_keys=True,
+                           default=str).encode("utf-8")).hexdigest()
+            conn = _db_connect(self.db_path)
+            try:
+                self._ensure_result_columns(conn)
+                existing = conn.execute(
+                    "SELECT 1 FROM task_checkpoints"
+                    " WHERE task_id = ? AND round_no = ?",
+                    (task["id"], rounds)).fetchone()
+                if existing:
+                    conn.execute(
+                        "UPDATE task_checkpoints SET thread_id = ?,"
+                        " goal_status = ?, state_sha = ?,"
+                        " verify_out = ?, created_at = ?"
+                        " WHERE task_id = ? AND round_no = ?",
+                        (entry.get("thread_id"),
+                         entry.get("goal_status"),
+                         state_sha,
+                         (entry.get("goal_last_verify") or "")[:500],
+                         time.time(), task["id"], rounds))
+                else:
+                    conn.execute(
+                        "INSERT INTO task_checkpoints(task_id, round_no,"
+                        " thread_id, goal_status, state_sha, verify_out,"
+                        " created_at) VALUES(?, ?, ?, ?, ?, ?, ?)",
+                        (task["id"], rounds, entry.get("thread_id"),
+                         entry.get("goal_status"), state_sha,
+                         (entry.get("goal_last_verify") or "")[:500],
+                         time.time()))
+                self._import_host_steps(conn, task, state_path)
+                conn.commit()
+            finally:
+                conn.close()
+            return rounds
+        except Exception:
+            return -1
+
+    def _import_host_steps(self, conn, task, state_path):
+        """把 host-steps/<任务>.jsonl 的 round 行导入 task_steps。
+
+        按 (task_id, kind='round', step_no) 去重：已入库的
+        step_no 跳过；文件不存在或格式坏行都跳过不炸。
+        """
+        try:
+            steps_path = os.path.join(
+                os.path.dirname(os.path.abspath(state_path)),
+                "host-steps", "%s.jsonl" % _worker_host_name(
+                    self.executor, task))
+            if not os.path.exists(steps_path):
+                return
+            existing = {row[0] for row in conn.execute(
+                "SELECT step_no FROM task_steps WHERE task_id = ?"
+                " AND kind = 'round'", (task["id"],)).fetchall()}
+            with open(steps_path, encoding="utf-8") as fh:
+                for line in fh:
+                    try:
+                        item = json.loads(line)
+                    except Exception:
+                        continue
+                    step_no = item.get("round")
+                    if not isinstance(step_no, int) or step_no in existing:
+                        continue
+                    conn.execute(
+                        "INSERT INTO task_steps(id, task_id, step_no,"
+                        " kind, summary, state_sha, created_at)"
+                        " VALUES(?, ?, ?, 'round', ?, NULL, ?)",
+                        (_uuid_hex(), task["id"], step_no,
+                         ("第 %d 轮 turn=%s verify=%s" % (
+                             step_no, item.get("turn_status"),
+                             (item.get("verify_out") or "")[:200])),
+                         item.get("ts") or time.time()))
+                    existing.add(step_no)
+        except Exception:
+            pass
+
+    def _rebuild_host_state_from_checkpoint(self, task):
+        """断点丢失时的重建（P0 三件其一，任务 2 类故障的解法）。
+
+        host-state 里没有本任务条目、但库里有断点时：从库里
+        最新断点 + tasks 行的 goal/verify_command/max_rounds
+        拼一个最小 host-state 条目写回（goal_status=interrupted，
+        轮次与 thread_id 以库为准），再由正常 goal-resume
+        路径续跑。拼不出来（库里也无断点）返回 False，
+        调用方走原有拒绝/失败路径，不造假断点。
+        """
+        try:
+            conn = _db_connect(self.db_path)
+            try:
+                self._ensure_result_columns(conn)
+                row = conn.execute(
+                    "SELECT round_no, thread_id, goal_status"
+                    " FROM task_checkpoints WHERE task_id = ?"
+                    " ORDER BY round_no DESC LIMIT 1",
+                    (task["id"],)).fetchone()
+                if not row:
+                    return False
+                trow = conn.execute(
+                    "SELECT goal, verify_command, goal_max_rounds"
+                    " FROM tasks WHERE id = ?", (task["id"],)).fetchone()
+                if not trow:
+                    return False
+            finally:
+                conn.close()
+            state_path = self._host_state_path_for_task(task)
+            if not state_path:
+                return False
+            host_name = _worker_host_name(self.executor, task)
+            if os.path.exists(state_path):
+                with open(state_path, encoding="utf-8") as fh:
+                    state = json.load(fh)
+            else:
+                state = {"version": 1, "tasks": {}}
+            tasks_map = state.setdefault("tasks", {})
+            if host_name in tasks_map:
+                return True
+            tasks_map[host_name] = {
+                "goal": trow[0],
+                "verify_command": trow[1],
+                "goal_max_rounds": trow[2] or 0,
+                "goal_rounds_done": row[0],
+                "thread_id": row[1],
+                "goal_status": "interrupted",
+                "goal_last_verify": "",
+                "rebuilt_from_checkpoint": True,
+            }
+            os.makedirs(os.path.dirname(
+                os.path.abspath(state_path)), exist_ok=True)
+            tmp_path = state_path + ".tmp"
+            with open(tmp_path, "w", encoding="utf-8") as fh:
+                json.dump(state, fh)
+            os.replace(tmp_path, state_path)
+            return True
+        except Exception:
+            return False
+
     def _ensure_result_columns(self, conn):
         """worker 侧的迁移兜底：与 server.migrate_tasks_table 同列。
 
@@ -976,7 +1282,11 @@ class Worker:
                 "paused_by_user": "INTEGER DEFAULT 0",
                 "interrupted_reason": "TEXT",
                 "auto_resume_count": "INTEGER DEFAULT 0",
-                "no_time_limit": "INTEGER DEFAULT 0"}.items():
+                "no_time_limit": "INTEGER DEFAULT 0",
+                "resume_after": "DOUBLE PRECISION DEFAULT 0",
+                "auto_resume_stall_count": "INTEGER DEFAULT 0",
+                "checkpoint_round_at_last_resume": "INTEGER",
+                "last_error_signature": "TEXT"}.items():
             if name not in cols:
                 try:
                     conn.execute(
@@ -984,6 +1294,32 @@ class Worker:
                         % (name, coltype))
                 except Exception:
                     pass
+        # P0 三件的表（worker 直连老库时也补，与 server ensure 同 DDL）。
+        try:
+            conn.execute(
+                "CREATE TABLE IF NOT EXISTS task_checkpoints("
+                "task_id TEXT, "
+                "round_no INTEGER, "
+                "thread_id TEXT, "
+                "goal_status TEXT, "
+                "state_sha TEXT, "
+                "verify_out TEXT, "
+                "created_at DOUBLE PRECISION, "
+                "PRIMARY KEY(task_id, round_no))")
+            conn.execute(
+                "CREATE TABLE IF NOT EXISTS task_steps("
+                "id TEXT PRIMARY KEY, "
+                "task_id TEXT, "
+                "step_no INTEGER, "
+                "kind TEXT, "
+                "summary TEXT, "
+                "state_sha TEXT, "
+                "created_at DOUBLE PRECISION)")
+            conn.execute(
+                "CREATE INDEX IF NOT EXISTS idx_task_steps_task"
+                " ON task_steps(task_id, step_no)")
+        except Exception:
+            pass
 
     def _set_status_if_running(self, task_id, status):
         """仅当任务仍为 running 时改状态，返回是否改成功。
@@ -3084,6 +3420,20 @@ class Worker:
         _watchdog_stop = None
         _watchdog_thread = None
         _quota_watchdog_thread = None
+        # P0 三件其一：goal 任务起断点镜像线程（任何执行器）；
+        # 非 goal 任务没有断点语义，不起。终局镜像在 finally 补。
+        _mirror_stop = None
+        _mirror_thread = None
+        if goal_mode:
+            try:
+                _mirror_stop = threading.Event()
+                _mirror_thread = threading.Thread(
+                    target=self._checkpoint_mirror_loop,
+                    args=(task, _mirror_stop), daemon=True)
+                _mirror_thread.start()
+            except Exception:
+                _mirror_stop = None
+                _mirror_thread = None
         if is_docker and hasattr(self.executor, "kill_container"):
             try:
                 _container_name = self.executor.container_name(task)
@@ -3306,6 +3656,16 @@ class Worker:
                     _watchdog_thread.join(timeout=2)
                 if _quota_watchdog_thread is not None:
                     _quota_watchdog_thread.join(timeout=2)
+            if _mirror_stop is not None:
+                _mirror_stop.set()
+                if _mirror_thread is not None:
+                    _mirror_thread.join(timeout=2)
+                # 终局镜像：执行结束时的断点（最后一轮）必须落库，
+                # 否则中断后库断点比实际少一轮；失败不影响结果写回。
+                try:
+                    self._mirror_checkpoint_once(task)
+                except Exception:
+                    pass
             # 同步路径执行结束：清掉取消用的容器标记（取消路径里
             # 若已清过，这里再清一次无害）。进程模式的 proc 由
             # _execute_via_process 自己的 finally 清，这里只清
@@ -3770,6 +4130,18 @@ class Worker:
                                            error=error)
         else:
             self._audit(project_id, "worker_interrupted", task_id)
+        # P0 三件其三：时间线终局步骤（best-effort）。
+        try:
+            conn = _db_connect(self.db_path)
+            try:
+                self._ensure_result_columns(conn)
+                self._record_step(conn, task_id, "final",
+                                  "终态 %s" % final)
+                conn.commit()
+            finally:
+                conn.close()
+        except Exception:
+            pass
         host_name = self.executor._host_name(task) \
             if hasattr(self.executor, "_host_name") else task_id
         return {"dispatched": True, "task_id": task_id,
@@ -3834,6 +4206,18 @@ class Worker:
         if not self._claim(task_id, executor=self._executor_name()):
             return {"dispatched": False, "reason": "already_claimed",
                     "task_id": task_id}
+        # P0 三件其三：时间线 claim 步骤（best-effort，写失败
+        # 不影响执行；step_no 由 _record_step 自增）。
+        try:
+            conn = _db_connect(self.db_path)
+            try:
+                self._ensure_result_columns(conn)
+                self._record_step(conn, task_id, "claim", "worker 认领")
+                conn.commit()
+            finally:
+                conn.close()
+        except Exception:
+            pass
         task = self.get_task(task_id) or task
         # G10：goal 断点续跑标记——认领后立即清掉库里标记（本轮
         # 已认领为 running，不会重复触发；本轮若再被杀回
@@ -3845,6 +4229,14 @@ class Worker:
         if task.get("goal_resume_pending"):
             task = dict(task)
             task["_goal_resume_breakpoint"] = True
+            # P0 三件其一：断点丢失重建——host-state 里条目已丢
+            # 但库里有断点时，先按库断点拼最小条目写回，再由
+            # 正常 goal-resume 续跑；库里也没有断点时返回 False，
+            # 正常路径继续（host 拒绝 → failed，不造假断点）。
+            try:
+                self._rebuild_host_state_from_checkpoint(task)
+            except Exception:
+                pass
             try:
                 conn = _db_connect(self.db_path)
                 try:
@@ -4460,6 +4852,18 @@ class Worker:
                        else "worker_failed"),
                       task_id=task_id, project_id=task["project_id"],
                       status=final)
+            # P0 三件其三：时间线终局步骤（best-effort）。
+            try:
+                conn = _db_connect(self.db_path)
+                try:
+                    self._ensure_result_columns(conn)
+                    self._record_step(conn, task_id, "final",
+                                      "终态 %s" % final)
+                    conn.commit()
+                finally:
+                    conn.close()
+            except Exception:
+                pass
             if result.get("goal_met") is False:
                 # goal 未达只发 goal-unmet，不再重复发 failed：
                 # 同一结局两条通知会刷屏，digest 偏好下两者又都是