cloud: production gaps (worker loop + true cancel, task result columns + migration, DB interface migration, metrics + backup)

cloud/server.py

@@ -87,6 +87,30 @@ def _connect(db_path):
     return _db_connect(str(db_path))
 
 
+def migrate_tasks_table(conn):
+    """幂等迁移：给老库 tasks 表补新列，不丢数据。
+
+    新列：result_text、error_text、actual_model、started_at、
+    finished_at、child_pid。PRAGMA table_info 查现有列，缺哪个
+    补哪个；已迁移的库再跑一次是 no-op。worker 写执行结果、
+    server 读任务时都依赖这些列存在。
+    """
+    cols = {row[1] for row in conn.execute(
+        "PRAGMA table_info(tasks)").fetchall()}
+    new_cols = {
+        "result_text": "TEXT",
+        "error_text": "TEXT",
+        "actual_model": "TEXT",
+        "started_at": "REAL",
+        "finished_at": "REAL",
+        "child_pid": "INTEGER",
+    }
+    for name, coltype in new_cols.items():
+        if name not in cols:
+            conn.execute(
+                "ALTER TABLE tasks ADD COLUMN %s %s" % (name, coltype))
+
+
 def init_db(db_path):
     """创建 server 自己的 projects/tasks 表（AuthStore 的表由其自建）。"""
     conn = _connect(db_path)
@@ -113,6 +137,7 @@ def init_db(db_path):
             "last_heartbeat REAL, "
             "UNIQUE(project_id, idempotency_key))"
         )
+        migrate_tasks_table(conn)
         conn.commit()
     finally:
         conn.close()
@@ -131,6 +156,14 @@ def _auth_store(db_path):
             pass
 
 
+def _row_get(row, name):
+    """sqlite Row 的容错取值：老库行缺列时返回 None 不炸。"""
+    try:
+        return row[name]
+    except (KeyError, IndexError, TypeError):
+        return None
+
+
 def _task_to_dict(row):
     depends_on = row["depends_on"]
     try:
@@ -151,6 +184,12 @@ def _task_to_dict(row):
         "created_at": row["created_at"],
         "updated_at": row["updated_at"],
         "last_heartbeat": row["last_heartbeat"],
+        "result_text": _row_get(row, "result_text"),
+        "error_text": _row_get(row, "error_text"),
+        "actual_model": _row_get(row, "actual_model"),
+        "started_at": _row_get(row, "started_at"),
+        "finished_at": _row_get(row, "finished_at"),
+        "child_pid": _row_get(row, "child_pid"),
     }
 
 
@@ -306,6 +345,8 @@ class CloudControlPlaneHandler(BaseHTTPRequestHandler):
             self._create_project(user_id)
         elif path == "/projects" and method == "GET":
             self._list_projects(user_id)
+        elif path == "/metrics" and method == "GET":
+            self._metrics(user_id)
         elif path == "/models" and method == "GET":
             self._models_overview(user_id)
         elif path.startswith("/projects/"):
@@ -989,6 +1030,75 @@ class CloudControlPlaneHandler(BaseHTTPRequestHandler):
         swept = sweep_stale_tasks(self.server.db_path, threshold)
         self._send_json(200, {"swept": swept})
 
+    def _metrics(self, user_id):
+        """可观测性：任务计数、成功率、平均耗时、fallback、成本合计。
+
+        权限：任意项目成员即可读（不要求 owner），但只统计调用者
+        是成员的项目——不是全库数字，避免跨项目信息泄露。
+        成本真数与估算分开：cost_records 的 source 列区分。
+        """
+        conn = self._db()
+        try:
+            member_ids = {
+                row["project_id"]
+                for row in conn.execute(
+                    "SELECT project_id FROM project_members WHERE user_id = ?",
+                    (user_id,),
+                ).fetchall()
+            }
+            rows = conn.execute(
+                "SELECT project_id, status, started_at, finished_at"
+                " FROM tasks"
+            ).fetchall()
+            by_status = {}
+            durations = []
+            for row in rows:
+                if row["project_id"] not in member_ids:
+                    continue
+                by_status[row["status"]] = by_status.get(row["status"], 0) + 1
+                if row["started_at"] and row["finished_at"]:
+                    durations.append(row["finished_at"] - row["started_at"])
+            total = sum(by_status.values())
+            done = by_status.get("completed", 0) + by_status.get("merged", 0)
+            failed = by_status.get("failed", 0) + by_status.get("merge-failed", 0)
+            finished = done + failed
+            try:
+                cost_rows = conn.execute(
+                    "SELECT project_id, source, total_tokens, usd_micros,"
+                    " fallback_count FROM cost_records"
+                ).fetchall()
+            except sqlite3.OperationalError:
+                cost_rows = []
+            actual_tokens = estimated_tokens = 0
+            actual_usd = estimated_usd = 0
+            fallback_total = 0
+            for row in cost_rows:
+                if row["project_id"] not in member_ids:
+                    continue
+                if row["source"] == "actual":
+                    actual_tokens += row["total_tokens"] or 0
+                    actual_usd += row["usd_micros"] or 0
+                else:
+                    estimated_tokens += row["total_tokens"] or 0
+                    estimated_usd += row["usd_micros"] or 0
+                fallback_total += row["fallback_count"] or 0
+        finally:
+            conn.close()
+        self._send_json(200, {
+            "total_tasks": total,
+            "by_status": by_status,
+            "success_rate": (done / finished) if finished else None,
+            "avg_duration_seconds": (
+                sum(durations) / len(durations)) if durations else None,
+            "fallback_total": fallback_total,
+            "cost": {
+                "actual_tokens": actual_tokens,
+                "estimated_tokens": estimated_tokens,
+                "actual_usd_micros": actual_usd,
+                "estimated_usd_micros": estimated_usd,
+            },
+        })
+
     # -- OAuth（pre-auth，靠 state + secret，不走 X-API-Key）-----------
     def _oauth_provider(self, path):
         parts = [p for p in path.split("/") if p]