cloud: production gaps (worker loop + true cancel, task result columns + migration, DB interface migration, metrics + backup)

cloud/worker.py

@@ -38,13 +38,13 @@
    （user_id="system"，与 server.sweep_stale_tasks 的做法一致），
    审计失败不影响任务状态（best-effort）。
 
-取消：host.py 的 argparse 子命令表里没有 interrupt（turn/interrupt
-只存在于 host.py Server 类内部的 app-server 协议调用，未暴露为
-CLI），所以 worker 的 cancel() 走 server 的 cancel 语义：把
-queued/running 任务直接标 interrupted（已终态的任务不动）。
-进行中的 host.py 子进程不会因此被杀（中途杀进程可能把 host
-state 写一半）；run_one() 在执行返回后会先查任务是否已被取消，
-已被取消的一律保持 interrupted，不用执行结果覆盖它。
+取消：worker 的 cancel() 走 server 的 cancel 语义：把
+queued/running 任务直接标 interrupted（已终态的任务不动），
+且正在跑的子进程会真杀——子进程以进程组启动（start_new_session），
+取消时先 SIGTERM 给整个进程组、等 2 秒、仍活着再 SIGKILL，
+杀完后任务保持 interrupted，执行结果不覆盖它。run_one() 在
+执行返回后会先查任务是否已被取消，已被取消的一律保持
+interrupted，不用执行结果覆盖它。
 
 可注入：runner（假 CLI，见 sandbox.LocalExecutor）与 env 映射
 均可注入，单测不跑真实模型回合、不联网、不登录任何服务。
@@ -54,14 +54,20 @@ import argparse
 import json
 import os
 import re
+import signal
+import subprocess
+import sys
 import time
+from types import SimpleNamespace
 
 from cloud.auth import AuthStore
 from cloud.cost import CostStore, estimate_tokens
 from cloud.db import connect as _db_connect
+from cloud.logging_config import get_logger, log_event, setup_logging
 from cloud.models import ModelRouter, is_fallbackable, record_fallback
 from cloud.queue import budget_exceeded, ready_tasks
 from cloud.sandbox import LocalExecutor
+from cloud.sandbox import _default_runner as _sandbox_default_runner
 
 # 与 server.TERMINAL_STATUSES 对齐：这些状态的任务不再被认领或取消。
 TERMINAL_STATUSES = frozenset(
@@ -152,7 +158,7 @@ class Worker:
                  runner=None, env=None, env_key=None,
                  codex_config_path=None, model=None, router=None,
                  max_turns=None, max_cost_usd=None, executor=None,
-                 cost_store=None):
+                 cost_store=None, process_factory=None):
         self.db_path = str(db_path)
         self.host_py_path = host_py_path
         self.config_path = config_path
@@ -177,6 +183,14 @@ class Worker:
                 repo=repo)
         self._cost_store = cost_store
         self._cancel_requested = set()
+        # 真取消用：正在跑的子进程（Popen 对象）与任务 id。
+        # process_factory 可注入：单测用它返回一个假长跑子进程，
+        # 真实执行时走 LocalExecutor.popen_task（进程组启动）。
+        self.process_factory = process_factory
+        self._running_proc = None
+        self._running_task_id = None
+        self._loop_stop = False
+        self.logger = get_logger("cloud.worker")
 
     # -- 成本存储（惰性建表，避免只认领不执行时也建表）--------------
     @property
@@ -241,6 +255,99 @@ class Worker:
         finally:
             conn.close()
 
+    def _ensure_result_columns(self, conn):
+        """worker 侧的迁移兜底：与 server.migrate_tasks_table 同列。
+
+        worker 的库理论上已由 server.init_db 迁过；这里再查一次
+        是为了 worker 直连老库（测试里手工建的表）时也不炸。
+        """
+        try:
+            cols = {row[1] for row in conn.execute(
+                "PRAGMA table_info(tasks)").fetchall()}
+        except Exception:
+            return
+        for name, coltype in {
+                "result_text": "TEXT", "error_text": "TEXT",
+                "actual_model": "TEXT", "started_at": "REAL",
+                "finished_at": "REAL", "child_pid": "INTEGER"}.items():
+            if name not in cols:
+                try:
+                    conn.execute(
+                        "ALTER TABLE tasks ADD COLUMN %s %s"
+                        % (name, coltype))
+                except Exception:
+                    pass
+
+    def _write_execution_result(self, task_id, result_text=None,
+                                error_text=None, actual_model=None,
+                                started_at=None, finished_at=None,
+                                child_pid=None):
+        """把执行结果写进 tasks 新列；老库缺列时先迁移再写。"""
+        conn = _db_connect(self.db_path)
+        try:
+            self._ensure_result_columns(conn)
+            if child_pid is None:
+                conn.execute(
+                    "UPDATE tasks SET result_text = ?, error_text = ?,"
+                    " actual_model = ?, started_at = ?, finished_at = ?,"
+                    " updated_at = ? WHERE id = ?",
+                    ((result_text or "")[:4000] if result_text else None,
+                     (error_text or "")[:4000] if error_text else None,
+                     actual_model, started_at, finished_at,
+                     time.time(), task_id),
+                )
+            else:
+                conn.execute(
+                    "UPDATE tasks SET result_text = ?, error_text = ?,"
+                    " actual_model = ?, started_at = ?, finished_at = ?,"
+                    " child_pid = ?, updated_at = ? WHERE id = ?",
+                    ((result_text or "")[:4000] if result_text else None,
+                     (error_text or "")[:4000] if error_text else None,
+                     actual_model, started_at, finished_at, child_pid,
+                     time.time(), task_id),
+                )
+            conn.commit()
+        finally:
+            conn.close()
+
+    def _set_child_pid(self, task_id, pid):
+        conn = _db_connect(self.db_path)
+        try:
+            self._ensure_result_columns(conn)
+            conn.execute(
+                "UPDATE tasks SET child_pid = ? WHERE id = ?",
+                (pid, task_id),
+            )
+            conn.commit()
+        finally:
+            conn.close()
+
+    def _kill_proc_group(self, proc):
+        """真杀：先 SIGTERM 给整个进程组，等 2 秒，仍活着再 SIGKILL。
+
+        proc 须是 start_new_session 启动的 Popen（自成进程组），
+        杀组才能连同它拉起的子孙一起杀掉，不只杀父进程。
+        """
+        try:
+            pgid = os.getpgid(proc.pid)
+        except Exception:
+            return
+        try:
+            os.killpg(pgid, signal.SIGTERM)
+        except (ProcessLookupError, PermissionError, OSError):
+            return
+        deadline = time.time() + 2.0
+        while time.time() < deadline:
+            try:
+                os.killpg(pgid, 0)
+            except (ProcessLookupError, PermissionError, OSError):
+                return
+            time.sleep(0.05)
+        try:
+            os.killpg(pgid, signal.SIGKILL)
+        except (ProcessLookupError, PermissionError, OSError):
+            pass
+
     def heartbeat(self, task_id):
         """刷新 running 任务的心跳；任务不在 running 时返回 False。"""
         now = time.time()
@@ -295,11 +402,23 @@ class Worker:
 
     # -- 取消（server cancel 语义，见模块 docstring）-------------------
     def cancel(self, task_id):
-        """把 queued/running 任务标 interrupted；已终态或不存在返回 False。"""
+        """把 queued/running 任务标 interrupted；已终态或不存在返回 False。
+
+        正在跑的子进程要真杀：先杀进程组（见 _kill_proc_group），
+        再标状态；已终态或不存在返回 False。
+        """
         task = self.get_task(task_id)
         if task is None or task["status"] in TERMINAL_STATUSES:
             return False
         self._cancel_requested.add(task_id)
+        if self._running_task_id == task_id and self._running_proc is not None:
+            self._kill_proc_group(self._running_proc)
+            try:
+                self._running_proc.wait(timeout=3)
+            except Exception:
+                pass
+            self._running_proc = None
+            self._running_task_id = None
         conn = _db_connect(self.db_path)
         try:
             cur = conn.execute(
@@ -313,8 +432,16 @@ class Worker:
             claimed = cur.rowcount == 1
         finally:
             conn.close()
+        if not claimed:
+            # 并发竞态：run_one 的轮询已看到本次取消的标记、先一步
+            # 把任务标成 interrupted——中断确实由本次取消触发，返回 True。
+            current = self.get_task(task_id)
+            if current is not None and current["status"] == "interrupted":
+                claimed = True
         if claimed:
             self._audit(task["project_id"], "worker_cancelled", task_id)
+            log_event(self.logger, "worker_cancelled", task_id=task_id,
+                      project_id=task["project_id"], status="interrupted")
         return claimed
 
     def _was_cancelled(self, task_id):
@@ -332,6 +459,67 @@ class Worker:
             return False
         return value is not None and value != ""
 
+    def _use_popen(self):
+        """是否走进程模式：注入了 process_factory，或执行器是
+        默认 runner 的 LocalExecutor（真实 subprocess）。"""
+        if self.process_factory is not None:
+            return True
+        return (isinstance(self.executor, LocalExecutor)
+                and getattr(self.executor, "runner", None)
+                is _sandbox_default_runner
+                and hasattr(self.executor, "popen_task"))
+
+    def _execute_via_process(self, task, kind="dispatch"):
+        """进程模式执行一轮：启动、轮询、取消时杀进程组。
+
+        返回与同步路径同款 result dict（status/stdout/stderr/
+        host_status/returncode）。取消时返回 status='cancelled'
+        的标记，调用方据此保持 interrupted。
+        """
+        task_id = task["id"]
+        if self.process_factory is not None:
+            proc = self.process_factory(task)
+        else:
+            proc = self.executor.popen_task(task, kind=kind)
+        self._running_proc = proc
+        self._running_task_id = task_id
+        try:
+            self._set_child_pid(task_id, getattr(proc, "pid", None))
+        except Exception:
+            pass
+        cancelled = False
+        try:
+            while proc.poll() is None:
+                if task_id in self._cancel_requested or self._loop_stop:
+                    cancelled = True
+                    self._kill_proc_group(proc)
+                    break
+                time.sleep(0.05)
+            try:
+                out, err = proc.communicate(timeout=5)
+            except Exception:
+                try:
+                    proc.kill()
+                except Exception:
+                    pass
+                out, err = proc.communicate()
+        finally:
+            self._running_proc = None
+            self._running_task_id = None
+        if cancelled:
+            return {"executor": "local", "task_id": task_id,
+                    "status": "cancelled", "stdout": out or "",
+                    "stderr": err or "", "returncode": proc.returncode,
+                    "host_status": None}
+        fake = SimpleNamespace(returncode=proc.returncode,
+                               stdout=out or "", stderr=err or "")
+        if hasattr(self.executor, "_result_from_runner"):
+            return self.executor._result_from_runner(task_id, fake)
+        return {"executor": "local", "task_id": task_id,
+                "status": "completed" if proc.returncode == 0 else "failed",
+                "stdout": out or "", "stderr": err or "",
+                "returncode": proc.returncode, "host_status": None}
+
     # -- 执行 ----------------------------------------------------------
     def run_one(self, task=None):
         """认领并执行一个任务，返回结果 dict（见模块 docstring）。
@@ -351,13 +539,21 @@ class Worker:
                     "task_id": task_id}
         task = self.get_task(task_id) or task
         self._audit(task["project_id"], "worker_claim", task_id)
+        log_event(self.logger, "worker_claim", task_id=task_id,
+                  project_id=task["project_id"], status="running")
+        started_at = time.time()
 
         # 缺 key 门：认领之后、调 host.py 之前；缺 key 直接 failed，
         # 不调 runner（假 CLI 单测可断言 runner 未被调用）。
         if not self.key_available():
             error = _missing_key_error(self.env_key)
             self._set_status(task_id, "failed")
+            self._write_execution_result(
+                task_id, error_text=error, actual_model=self.model,
+                started_at=started_at, finished_at=time.time())
             self._audit(task["project_id"], "worker_missing_key", error)
+            log_event(self.logger, "worker_missing_key", task_id=task_id,
+                      project_id=task["project_id"], status="failed")
             return {"dispatched": True, "task_id": task_id,
                     "host_name": self.executor._host_name(task)
                     if hasattr(self.executor, "_host_name") else task_id,
@@ -370,14 +566,20 @@ class Worker:
         result = None
         error = None
         try:
-            result = self.executor.dispatch_task(task)
-            combined = (result.get("stdout", "") or "") + \
-                (result.get("stderr", "") or "")
-            if result["status"] != "completed" and \
-                    _ALREADY_EXISTS_MARKER in combined:
-                # host 任务已存在（上轮 worker 在 dispatch 后、写回前
-                # 崩溃）：改用 run 在已有任务上续跑，不重复建 worktree。
-                result = self.executor.run_task(task)
+            if self._use_popen():
+                result = self._execute_via_process(task)
+                combined = (result.get("stdout", "") or "") + \
+                    (result.get("stderr", "") or "")
+                if result["status"] not in ("completed", "cancelled") and \
+                        _ALREADY_EXISTS_MARKER in combined:
+                    result = self._execute_via_process(task, kind="run")
+            else:
+                result = self.executor.dispatch_task(task)
+                combined = (result.get("stdout", "") or "") + \
+                    (result.get("stderr", "") or "")
+                if result["status"] != "completed" and \
+                        _ALREADY_EXISTS_MARKER in combined:
+                    result = self.executor.run_task(task)
         except Exception as exc:  # runner 异常也如实写回 failed
             result = {"executor": "local", "task_id": task_id,
                       "status": "failed", "stdout": "", "returncode": None,
@@ -386,8 +588,14 @@ class Worker:
 
         # 取消优先：执行期间被 cancel 的任务保持 interrupted，
         # 绝不用执行结果覆盖它（见模块 docstring 的取消说明）。
-        if self._was_cancelled(task_id):
+        if self._was_cancelled(task_id) or result.get("status") == "cancelled":
             self._set_status(task_id, "interrupted")
+            self._write_execution_result(
+                task_id, result_text=(result or {}).get("stdout", ""),
+                error_text="cancelled", actual_model=self.model,
+                started_at=started_at, finished_at=time.time())
+            log_event(self.logger, "worker_cancelled", task_id=task_id,
+                      project_id=task["project_id"], status="interrupted")
             return {"dispatched": True, "task_id": task_id,
                     "host_name": self.executor._host_name(task)
                     if hasattr(self.executor, "_host_name") else task_id,
@@ -417,6 +625,11 @@ class Worker:
                     self._audit(task["project_id"], "worker_fallback",
                                 "%s -> %s" % (self.model, nxt))
         self._set_status(task_id, final)
+        finished_at = time.time()
+        self._write_execution_result(
+            task_id, result_text=stdout, error_text=error,
+            actual_model=self.model, started_at=started_at,
+            finished_at=finished_at)
         self.usage_turns += 1
         # 成本：host 协议不暴露真实 token 数，用 cost.py 的估算结构
         # 记 estimated，绝不标 actual；fallback_count 复用同一记录。
@@ -428,6 +641,11 @@ class Worker:
         self._audit(task["project_id"],
                     "worker_completed" if final == "completed"
                     else "worker_failed", task_id)
+        log_event(self.logger,
+                  "worker_completed" if final == "completed"
+                  else "worker_failed",
+                  task_id=task_id, project_id=task["project_id"],
+                  status=final)
         return {"dispatched": True, "task_id": task_id,
                 "host_name": self.executor._host_name(task)
                 if hasattr(self.executor, "_host_name") else task_id,
@@ -449,14 +667,65 @@ class Worker:
             results.append(result)
         return results
 
+    def run_loop(self, interval=300, max_seconds=None, stop_event=None):
+        """常驻循环：每 interval 秒跑一轮 run_all，空闲也不退出。
+
+        退出条件：stop_event 被置位、_loop_stop 被信号处理置位、
+        或 max_seconds 到期。退出时若有正在跑的任务（进程模式下
+        _running_task_id 非空），先杀进程组、再标 interrupted，
+        不让任务永远卡在 running。
+        """
+        started = time.time()
+        all_results = []
+        try:
+            while not self._loop_stop:
+                if stop_event is not None and stop_event.is_set():
+                    break
+                if max_seconds is not None and \
+                        time.time() - started >= max_seconds:
+                    break
+                try:
+                    results = self.run_all()
+                except Exception as exc:
+                    # 常驻循环一轮炸了不带走整个 worker：记结构化日志，
+                    # 睡完本轮间隔再试下一轮（缺表、锁冲突都属此类）。
+                    log_event(self.logger, "worker_loop_error",
+                              message="worker_loop_error: %s" % exc)
+                    results = []
+                all_results.extend(results)
+                # 按 0.2 秒切片睡，期间可响应 stop_event/_loop_stop。
+                deadline = time.time() + interval
+                while time.time() < deadline:
+                    if self._loop_stop:
+                        break
+                    if stop_event is not None and stop_event.is_set():
+                        break
+                    if max_seconds is not None and \
+                            time.time() - started >= max_seconds:
+                        break
+                    time.sleep(0.2)
+        finally:
+            if self._running_task_id is not None:
+                task_id = self._running_task_id
+                if self._running_proc is not None:
+                    self._kill_proc_group(self._running_proc)
+                    try:
+                        self._running_proc.wait(timeout=3)
+                    except Exception:
+                        pass
+                self._running_proc = None
+                self._running_task_id = None
+                self._set_status(task_id, "interrupted")
+        return all_results
+
 
 def main(argv=None):
-    """单次派发循环：python3 -m cloud.worker --db <库> --host-py <路径>。
+    """worker CLI：默认单次 run_all；--loop 进常驻循环。
 
     只打印任务结果摘要（状态、host 名、错误文本），绝不打印
     任何环境变量的值。
     """
-    parser = argparse.ArgumentParser(description="cloud worker (one pass)")
+    parser = argparse.ArgumentParser(description="cloud worker")
     parser.add_argument("--db", default="cloud-control-plane.db")
     parser.add_argument("--host-py", default=None,
                         help="host.py 路径，缺省为 cloud/ 上一级的 host.py")
@@ -468,7 +737,12 @@ def main(argv=None):
     parser.add_argument("--model", default=None)
     parser.add_argument("--max-turns", type=int, default=None)
     parser.add_argument("--max-tasks", type=int, default=None)
+    parser.add_argument("--loop", action="store_true",
+                        help="常驻循环，每 --interval 秒跑一轮")
+    parser.add_argument("--interval", type=int, default=300)
+    parser.add_argument("--max-seconds", type=float, default=None)
     args = parser.parse_args(argv)
+    setup_logging()
     host_py = args.host_py
     if host_py is None:
         here = os.path.dirname(os.path.abspath(__file__))
@@ -477,8 +751,24 @@ def main(argv=None):
                     repo=args.repo, env_key=args.env_key,
                     codex_config_path=args.codex_config, model=args.model,
                     max_turns=args.max_turns)
+    # CLI 起 worker 前先建表/迁移：全新库直接起循环不应因缺表而炸。
     try:
-        results = worker.run_all(max_tasks=args.max_tasks)
+        from cloud.server import init_db as _init_db
+        _init_db(args.db)
+    except Exception as exc:
+        print("worker db 初始化失败：%s" % exc, file=sys.stderr)
+        raise SystemExit(1)
+    if args.loop:
+        def _on_signal(signum, frame):
+            worker._loop_stop = True
+        signal.signal(signal.SIGTERM, _on_signal)
+        signal.signal(signal.SIGINT, _on_signal)
+    try:
+        if args.loop:
+            results = worker.run_loop(interval=args.interval,
+                                      max_seconds=args.max_seconds)
+        else:
+            results = worker.run_all(max_tasks=args.max_tasks)
     finally:
         worker.close()
     print(json.dumps(