Production-grade: webhook execution, Postgres backup, rate limit, worker timeout/retry

GitHub webhook: delivery-id dedup persisted in webhook_deliveries,
/fix-review creates a queued task with audit (no model run in the
webhook itself), check_run/check_suite parsing without mutating
task state (CI backfill still via pr-checks).
Control plane: Postgres pg_dump backup branch + POST /admin/backup
(maintainer+, password only via PGPASSWORD env), per-key/IP rate
limit (default 120/min, 429), health returns backend + db_ok,
metrics by_status always has all seven status keys.
Worker: per-task timeout (default 1800s, --task-timeout), timeout
is failed not interrupted, limited retries (default 1, backoff,
fallback failures not re-retried), stale codex- container cleanup
on loop/CLI start.
Tests: cloud 271 (267 pass, 4 skipped are pre-existing real-
Postgres tests), host 27 pass.

cloud/worker.py

@@ -46,6 +46,34 @@ queued/running 任务直接标 interrupted（已终态的任务不动），
 执行返回后会先查任务是否已被取消，已被取消的一律保持
 interrupted，不用执行结果覆盖它。
 
+任务级超时与失败重试（生产级，见 Worker.__init__ 参数）：
+- task_timeout（默认 1800 秒，CLI --task-timeout 可调）是
+  worker 级的每任务截止时间，与下面两个参数不要混淆：
+  --docker-timeout 是执行器级的单次 docker run 超时（Docker
+  Executor 自身的 timeout），--max-seconds 是 run_loop 的
+  整体墙钟上限（不是每任务）。三者互不覆盖：Docker 路径下
+  worker 取 min(task_timeout, docker_timeout)（docker_timeout
+  未设时取 task_timeout）作为本次 docker run 的超时，超时后
+  真杀容器、任务置 failed（超时是失败，不是取消，绝不标
+  interrupted）；Local popen 路径下 worker 在轮询循环里到
+  截止时间就杀进程组、同样置 failed；Local 同步路径（假
+  runner / 无超时的 subprocess.run）无法在中途抢占一个已阻塞
+  的调用，超时只能靠执行器自身的 timeout，这是诚实边界，
+  不要假装同步路径也能中途杀掉。
+- max_retries（默认 1 次重试，CLI --max-retries 可调，0 表示
+  不重试）是对模型任务失败的有限重试：取消不重试；已记
+  fallback 事件的失败不做同模型重试（fallback 本身就是模型
+  级的重试机制，再同模型重试会重复烧额度）；其它失败（含
+  worker 超时）重试前按 retry_backoff_seconds（默认 5 秒，
+  测试可注入 0）等待，重试次数记在审计（worker_retry）与
+  最终 error_text（[retries=N] 前缀）里，不另起新表；重试
+  耗尽后任务才最终 failed，成本记录与 usage_turns 只记一次
+  （按最终结果记，不按尝试次数重复记）。
+- 容器清理兜底：Worker.cleanup_stale_containers() 列出
+  codex- 前缀的残留容器并逐个 kill+rm，失败只记日志不炸
+  worker；run_loop 启动时与 CLI 启动时各调一次（run_all /
+  __init__ 里不调，避免单测构造 Worker 时多余 I/O）。
+
 可注入：runner（假 CLI，见 sandbox.LocalExecutor）与 env 映射
 均可注入，单测不跑真实模型回合、不联网、不登录任何服务。
 
@@ -164,7 +192,9 @@ class Worker:
                  runner=None, env=None, env_key=None,
                  codex_config_path=None, model=None, router=None,
                  max_turns=None, max_cost_usd=None, executor=None,
-                 cost_store=None, process_factory=None):
+                 cost_store=None, process_factory=None,
+                 task_timeout=1800, max_retries=1,
+                 retry_backoff_seconds=5.0):
         self.db_path = str(db_path)
         self.host_py_path = host_py_path
         self.config_path = config_path
@@ -198,6 +228,10 @@ class Worker:
         self._running_container = None
         self._loop_stop = False
         self.logger = get_logger("cloud.worker")
+        # 任务级超时 / 失败重试（见模块 docstring 的生产级说明）。
+        self.task_timeout = task_timeout
+        self.max_retries = max_retries
+        self.retry_backoff_seconds = retry_backoff_seconds
 
     # -- 成本存储（惰性建表，避免只认领不执行时也建表）--------------
     @property
@@ -463,6 +497,28 @@ class Worker:
                       project_id=task["project_id"], status="interrupted")
         return claimed
 
+    def cleanup_stale_containers(self):
+        """启动清理兜底：清掉上一轮残留的 codex- 容器。
+
+        只在执行器有 cleanup_stale_containers 时做（Docker）；
+        列出或删除失败只记日志，绝不抛异常炸掉 worker。
+        返回已清理的容器名列表（非 Docker 执行器返回 []）。
+        """
+        cleanup = getattr(self.executor, "cleanup_stale_containers", None)
+        if cleanup is None:
+            return []
+        try:
+            cleaned = cleanup()
+        except Exception as exc:
+            log_event(self.logger, "worker_cleanup_failed",
+                      message="cleanup stale containers failed: %s" % exc)
+            return []
+        if cleaned:
+            log_event(self.logger, "worker_cleanup",
+                      message="cleaned stale containers: %s"
+                              % ",".join(cleaned))
+        return cleaned
+
     def _was_cancelled(self, task_id):
         if task_id in self._cancel_requested:
             return True
@@ -493,12 +549,15 @@ class Worker:
                 is _sandbox_default_runner
                 and hasattr(self.executor, "popen_task"))
 
-    def _execute_via_process(self, task, kind="dispatch"):
+    def _execute_via_process(self, task, kind="dispatch",
+                             deadline_seconds=None):
         """进程模式执行一轮：启动、轮询、取消时杀进程组。
 
         返回与同步路径同款 result dict（status/stdout/stderr/
         host_status/returncode）。取消时返回 status='cancelled'
-        的标记，调用方据此保持 interrupted。
+        的标记，调用方据此保持 interrupted；deadline_seconds
+        到期时杀进程组并返回 status='timeout' 的标记，调用方
+        据此置 failed（超时是失败，不是取消）。
         """
         task_id = task["id"]
         if self.process_factory is not None:
@@ -512,12 +571,19 @@ class Worker:
         except Exception:
             pass
         cancelled = False
+        timed_out = False
+        started = time.time()
         try:
             while proc.poll() is None:
                 if task_id in self._cancel_requested or self._loop_stop:
                     cancelled = True
                     self._kill_proc_group(proc)
                     break
+                if deadline_seconds is not None and \
+                        time.time() - started >= deadline_seconds:
+                    timed_out = True
+                    self._kill_proc_group(proc)
+                    break
                 time.sleep(0.05)
             try:
                 out, err = proc.communicate(timeout=5)
@@ -535,6 +601,11 @@ class Worker:
                     "status": "cancelled", "stdout": out or "",
                     "stderr": err or "", "returncode": proc.returncode,
                     "host_status": None}
+        if timed_out:
+            return {"executor": "local", "task_id": task_id,
+                    "status": "timeout", "stdout": out or "",
+                    "stderr": err or "", "returncode": proc.returncode,
+                    "host_status": None}
         fake = SimpleNamespace(returncode=proc.returncode,
                                stdout=out or "", stderr=err or "")
         if hasattr(self.executor, "_result_from_runner"):
@@ -545,6 +616,85 @@ class Worker:
                 "returncode": proc.returncode, "host_status": None}
 
     # -- 执行 ----------------------------------------------------------
+    def _run_attempt(self, task):
+        """单次执行尝试（不含认领、key 门、重试、成本与最终审计）。
+
+        返回 (result, error)：result 是执行器结果 dict，error 是
+        runner 异常时的文本（无异常时 None）。Docker 路径下把
+        worker 级 task_timeout 与执行器级 docker_timeout 取较小
+        值作为本次 docker run 超时（docker_timeout 未设时取
+        task_timeout）；Local popen 路径下把 task_timeout 作为
+        轮询截止时间；Local 同步路径无法中途抢占，超时只能靠
+        执行器自身 timeout（见模块 docstring 的诚实边界）。
+        调用方负责：取消优先、超时转 failed、重试、成本与审计。
+        """
+        task_id = task["id"]
+        result = None
+        error = None
+        # Docker 同步路径：dispatch 阻塞在 docker run 里时，取消
+        # 线程靠 _running_container 按名杀容器（见 cancel()）。
+        if hasattr(self.executor, "container_name"):
+            try:
+                self._running_container = \
+                    self.executor.container_name(task)
+                self._running_task_id = task_id
+            except Exception:
+                self._running_container = None
+        # Docker 超时取较小值：两个超时都不应静默延长另一个。
+        saved_docker_timeout = None
+        is_docker = isinstance(self.executor, DockerExecutor)
+        if is_docker and self.task_timeout is not None:
+            saved_docker_timeout = self.executor.timeout
+            if saved_docker_timeout is None:
+                self.executor.timeout = self.task_timeout
+            else:
+                try:
+                    self.executor.timeout = min(
+                        float(self.task_timeout),
+                        float(saved_docker_timeout))
+                except (TypeError, ValueError):
+                    self.executor.timeout = self.task_timeout
+        try:
+            if self._use_popen():
+                result = self._execute_via_process(
+                    task, deadline_seconds=self.task_timeout)
+                combined = (result.get("stdout", "") or "") + \
+                    (result.get("stderr", "") or "")
+                if result["status"] not in (
+                        "completed", "cancelled", "timeout") and \
+                        _ALREADY_EXISTS_MARKER in combined:
+                    result = self._execute_via_process(
+                        task, kind="run",
+                        deadline_seconds=self.task_timeout)
+            else:
+                result = self.executor.dispatch_task(task)
+                combined = (result.get("stdout", "") or "") + \
+                    (result.get("stderr", "") or "")
+                if result["status"] != "completed" and \
+                        _ALREADY_EXISTS_MARKER in combined:
+                    result = self.executor.run_task(task)
+        except Exception as exc:  # runner 异常也如实写回 failed
+            result = {"executor": "local", "task_id": task_id,
+                      "status": "failed", "stdout": "", "returncode": None,
+                      "host_status": None, "stderr": ""}
+            error = str(exc)
+        finally:
+            if is_docker:
+                self.executor.timeout = saved_docker_timeout
+            # 同步路径执行结束：清掉取消用的容器标记（取消路径里
+            # 若已清过，这里再清一次无害）。
+            self._running_container = None
+            if self._running_task_id == task_id and \
+                    self._running_proc is None:
+                self._running_task_id = None
+        # Docker 超时标记：执行器已在超时时 kill+rm 容器并返回
+        # failed（stdout 含 timeout），这里把状态统一成 timeout，
+        # 调用方据此置 failed 并记 worker_timeout 审计。
+        if result is not None and result.get("status") == "failed" and \
+                "docker run timeout" in (result.get("stdout", "") or ""):
+            result = dict(result, status="timeout")
+        return result, error
+
     def run_one(self, task=None):
         """认领并执行一个任务，返回结果 dict（见模块 docstring）。
 
@@ -589,41 +739,40 @@ class Worker:
         fallback_count = 0
         result = None
         error = None
-        # Docker 同步路径：dispatch 阻塞在 docker run 里时，取消
-        # 线程靠 _running_container 按名杀容器（见 cancel()）。
-        if hasattr(self.executor, "container_name"):
-            try:
-                self._running_container = \
-                    self.executor.container_name(task)
-                self._running_task_id = task_id
-            except Exception:
-                self._running_container = None
-        try:
-            if self._use_popen():
-                result = self._execute_via_process(task)
-                combined = (result.get("stdout", "") or "") + \
-                    (result.get("stderr", "") or "")
-                if result["status"] not in ("completed", "cancelled") and \
-                        _ALREADY_EXISTS_MARKER in combined:
-                    result = self._execute_via_process(task, kind="run")
-            else:
-                result = self.executor.dispatch_task(task)
-                combined = (result.get("stdout", "") or "") + \
-                    (result.get("stderr", "") or "")
-                if result["status"] != "completed" and \
-                        _ALREADY_EXISTS_MARKER in combined:
-                    result = self.executor.run_task(task)
-        except Exception as exc:  # runner 异常也如实写回 failed
-            result = {"executor": "local", "task_id": task_id,
-                      "status": "failed", "stdout": "", "returncode": None,
-                      "host_status": None, "stderr": ""}
-            error = str(exc)
-        # 同步路径执行结束：清掉取消用的容器标记（取消路径里
-        # 若已清过，这里再清一次无害）。
-        self._running_container = None
-        if self._running_task_id == task_id and \
-                self._running_proc is None:
-            self._running_task_id = None
+        retry_count = 0
+        # 有限重试：取消不重试；已记 fallback 的失败不做同模型重试
+        # （fallback 本身就是模型级重试机制）；其它失败（含 worker
+        # 超时）按 max_retries 重试，退避等待后重跑一次 _run_attempt。
+        for attempt in range(self.max_retries + 1):
+            result, error = self._run_attempt(task)
+            if self._was_cancelled(task_id) or \
+                    result.get("status") == "cancelled":
+                break
+            if result.get("status") == "completed":
+                break
+            host_status_try = result.get("host_status")
+            if self.router is not None and host_status_try is not None \
+                    and is_fallbackable(host_status_try) and \
+                    self.router.next_after(
+                        self.model, host_status_try) is not None:
+                break
+            if attempt < self.max_retries:
+                retry_count += 1
+                self._audit(
+                    task["project_id"], "worker_retry",
+                    "%s: attempt %d failed, retry %d/%d"
+                    % (task_id, attempt + 1, retry_count,
+                       self.max_retries))
+                if self.retry_backoff_seconds:
+                    time.sleep(self.retry_backoff_seconds)
+                continue
+            break
+        # 超时转 failed：超时是失败，不是取消；最终状态必须是
+        # VALID_STATUSES 里的 failed，绝不能把 "timeout" 写进库。
+        if result is not None and result.get("status") == "timeout":
+            result = dict(result, status="failed")
+            error = "task timeout after %s seconds" % self.task_timeout
+            self._audit(task["project_id"], "worker_timeout", task_id)
 
         # 取消优先：执行期间被 cancel 的任务保持 interrupted，
         # 绝不用执行结果覆盖它（见模块 docstring 的取消说明）。
@@ -649,6 +798,8 @@ class Worker:
         if final != "completed":
             if error is None:
                 error = stdout.strip() or "host execution failed"
+            if retry_count:
+                error = "[retries=%d] %s" % (retry_count, error)
             # fallback 事件：只记、不自动重试（见模块 docstring）。
             if self.router is not None and host_status is not None \
                     and is_fallbackable(host_status):
@@ -714,6 +865,9 @@ class Worker:
         _running_task_id 非空），先杀进程组、再标 interrupted，
         不让任务永远卡在 running。
         """
+        # 常驻启动清理兜底：清掉上一轮残留的 codex- 容器，
+        # 失败只记日志（见 cleanup_stale_containers），不炸循环。
+        self.cleanup_stale_containers()
         started = time.time()
         all_results = []
         try:
@@ -807,6 +961,16 @@ def main(argv=None):
                              "用于容器内 host.py 写 work/、host-state.json，"
                              "以及挂载原仓库和 codex-home（同路径挂载，"
                              "容器内配置无需改路径）")
+    parser.add_argument("--task-timeout", type=float, default=1800,
+                        help="每任务截止时间（秒，默认 1800）；超时后真杀"
+                             "容器/进程组、任务置 failed；与 --docker-timeout"
+                             "（单次 docker run 超时）取较小值，与"
+                             " --max-seconds（run_loop 整体墙钟上限）不同")
+    parser.add_argument("--max-retries", type=int, default=1,
+                        help="模型任务失败重试次数（默认 1，0 表示不重试；"
+                             "取消与已记 fallback 的失败不重试）")
+    parser.add_argument("--retry-backoff-seconds", type=float, default=5.0,
+                        help="重试前等待秒数（默认 5）")
     parser.add_argument("--loop", action="store_true",
                         help="常驻循环，每 --interval 秒跑一轮")
     parser.add_argument("--interval", type=int, default=300)
@@ -821,7 +985,12 @@ def main(argv=None):
                     repo=args.repo, env_key=args.env_key,
                     codex_config_path=args.codex_config, model=args.model,
                     max_turns=args.max_turns,
-                    executor=_build_cli_executor(args, host_py))
+                    executor=_build_cli_executor(args, host_py),
+                    task_timeout=args.task_timeout,
+                    max_retries=args.max_retries,
+                    retry_backoff_seconds=args.retry_backoff_seconds)
+    # CLI 启动清理兜底（常驻与单次都做）：清掉上一轮残留容器。
+    worker.cleanup_stale_containers()
     # CLI 起 worker 前先建表/迁移：全新库直接起循环不应因缺表而炸。
     try:
         from cloud.server import init_db as _init_db