cloud: fill gaps (memory/KB/cost/models endpoints, WAL concurrency, OAuth scaffold, push subscriptions)

cloud/server.py

@@ -45,6 +45,23 @@ from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
 from urllib.parse import parse_qs, urlparse
 
 from cloud.auth import AuthStore
+from cloud.cost import CostStore
+from cloud.db import connect as _db_connect
+from cloud.kb import KBStore
+from cloud.memory import MemoryStore
+from cloud.oauth import (
+    OAuthError,
+    OAuthStateStore,
+    handle_oauth_callback,
+    map_external_identity_to_local_user,
+    start_oauth_login,
+)
+from cloud.push import (
+    create_subscription as _push_create,
+    delete_subscription as _push_delete,
+    dispatch_event as _push_dispatch,
+    list_subscriptions as _push_list,
+)
 from cloud.webhooks import (
     parse_github_pr_comment,
     parse_slack_command,
@@ -59,8 +76,12 @@ DEFAULT_SWEEP_THRESHOLD_SECONDS = 300
 
 
 def _connect(db_path):
-    """每个请求新建一个连接；busy timeout 给并发写留重试窗口。"""
-    return sqlite3.connect(str(db_path), timeout=30)
+    """每个请求新建一个连接；走 cloud.db 的 WAL + busy_timeout。
+
+    单机 SQLite 的上限仍在：WAL + 有限重试只能缓解多线程写冲突，
+    高并发要换 Postgres（见 cloud/README.md）。
+    """
+    return _db_connect(str(db_path))
 
 
 def init_db(db_path):
@@ -236,6 +257,9 @@ class CloudControlPlaneHandler(BaseHTTPRequestHandler):
     def do_POST(self):
         self._dispatch("POST")
 
+    def do_DELETE(self):
+        self._dispatch("DELETE")
+
     def _dispatch(self, method):
         parsed = urlparse(self.path)
         path = parsed.path
@@ -246,6 +270,17 @@ class CloudControlPlaneHandler(BaseHTTPRequestHandler):
         if path == "/health" and method == "GET":
             self._send_json(200, {"ok": True, "service": "cloud-control-plane"})
             return
+        if path.startswith("/oauth/"):
+            # OAuth 登录/回调是 pre-auth 端点（用户还没本站 API key，
+            # 这正是 OAuth 要解决的问题），靠 state + client_secret
+            # 认证，不走 X-API-Key；不要把它当成无认证后门。
+            if method == "GET" and path.endswith("/login"):
+                self._oauth_login(path)
+            elif method == "POST" and path.endswith("/callback"):
+                self._oauth_callback(path)
+            else:
+                self._send_json(404, {"error": "not found"})
+            return
         if path.startswith("/webhooks/"):
             # webhook 靠签名认证（外部服务无法持 X-API-Key），不走 API key。
             if method != "POST":
@@ -268,16 +303,24 @@ class CloudControlPlaneHandler(BaseHTTPRequestHandler):
             self._create_project(user_id)
         elif path == "/projects" and method == "GET":
             self._list_projects(user_id)
+        elif path == "/models" and method == "GET":
+            self._models_overview(user_id)
+        elif path.startswith("/projects/"):
+            self._dispatch_project(user_id, method, path, query)
         elif path == "/tasks" and method == "POST":
             self._create_task(user_id)
         elif path == "/tasks" and method == "GET":
             self._list_tasks(user_id, query)
         elif path == "/admin/sweep" and method == "POST":
             self._admin_sweep()
+        elif path == "/admin/push/dispatch" and method == "POST":
+            self._push_dispatch_endpoint(user_id)
         elif path.startswith("/tasks/"):
             rest = path[len("/tasks/"):]
             if "/" not in rest and method == "GET":
                 self._get_task(user_id, rest)
+            elif rest.endswith("/cost") and method == "GET":
+                self._task_cost(user_id, rest[: -len("/cost")])
             elif rest.endswith("/cancel") and method == "POST":
                 self._cancel_task(user_id, rest[: -len("/cancel")])
             elif rest.endswith("/heartbeat") and method == "POST":
@@ -354,6 +397,396 @@ class CloudControlPlaneHandler(BaseHTTPRequestHandler):
                     if r["id"] in member_ids]
         self._send_json(200, {"projects": projects})
 
+    # -- 项目子资源：记忆 / 知识库 / 成本 -------------------------------
+    def _project_exists(self, project_id):
+        conn = self._db()
+        try:
+            row = conn.execute(
+                "SELECT id FROM projects WHERE id = ?", (project_id,)
+            ).fetchone()
+            return row is not None
+        finally:
+            conn.close()
+
+    def _check_project(self, user_id, project_id, action):
+        """返回 None 表示通过；否则返回 (status, error) 先行响应。"""
+        if not self._project_exists(project_id):
+            return 404, "project not found"
+        with _auth_store(self.server.db_path) as store:
+            if not store.check_permission(user_id, project_id, action):
+                return 403, "forbidden"
+        return None
+
+    def _dispatch_project(self, user_id, method, path, query):
+        rest = path[len("/projects/"):]
+        parts = rest.split("/")
+        project_id = parts[0] if parts else ""
+        if not project_id:
+            self._send_json(404, {"error": "not found"})
+            return
+        # /projects/{id}/memory
+        if len(parts) == 2 and parts[1] == "memory":
+            if method == "POST":
+                err = self._check_project(user_id, project_id, "create_task")
+                if err:
+                    self._send_json(err[0], {"error": err[1]})
+                    return
+                self._memory_create(user_id, project_id)
+            elif method == "GET":
+                err = self._check_project(user_id, project_id, "read")
+                if err:
+                    self._send_json(err[0], {"error": err[1]})
+                    return
+                self._memory_list(project_id, query)
+            else:
+                self._send_json(404, {"error": "not found"})
+            return
+        # /projects/{id}/memory/{mid}/supersede
+        if len(parts) == 4 and parts[1] == "memory" and parts[3] == "supersede":
+            if method != "POST":
+                self._send_json(404, {"error": "not found"})
+                return
+            err = self._check_project(user_id, project_id, "create_task")
+            if err:
+                self._send_json(err[0], {"error": err[1]})
+                return
+            self._memory_supersede(user_id, project_id, parts[2])
+            return
+        # /projects/{id}/kb/index
+        if len(parts) == 3 and parts[1] == "kb" and parts[2] == "index":
+            if method != "POST":
+                self._send_json(404, {"error": "not found"})
+                return
+            err = self._check_project(user_id, project_id, "create_task")
+            if err:
+                self._send_json(err[0], {"error": err[1]})
+                return
+            self._kb_index(user_id, project_id)
+            return
+        # /projects/{id}/kb/search
+        if len(parts) == 3 and parts[1] == "kb" and parts[2] == "search":
+            if method != "GET":
+                self._send_json(404, {"error": "not found"})
+                return
+            err = self._check_project(user_id, project_id, "read")
+            if err:
+                self._send_json(err[0], {"error": err[1]})
+                return
+            self._kb_search(project_id, query)
+            return
+        # /projects/{id}/cost
+        if len(parts) == 2 and parts[1] == "cost":
+            if method != "GET":
+                self._send_json(404, {"error": "not found"})
+                return
+            err = self._check_project(user_id, project_id, "read")
+            if err:
+                self._send_json(err[0], {"error": err[1]})
+                return
+            self._project_cost(project_id, query)
+            return
+        # /projects/{id}/push/subscriptions
+        if len(parts) == 3 and parts[1] == "push" and parts[2] == "subscriptions":
+            if method == "POST":
+                err = self._check_project(user_id, project_id, "create_task")
+                if err:
+                    self._send_json(err[0], {"error": err[1]})
+                    return
+                self._push_create_endpoint(user_id, project_id)
+            elif method == "GET":
+                err = self._check_project(user_id, project_id, "read")
+                if err:
+                    self._send_json(err[0], {"error": err[1]})
+                    return
+                self._send_json(200, {
+                    "subscriptions": _push_list(
+                        self.server.db_path, project_id, user_id=user_id)})
+            else:
+                self._send_json(404, {"error": "not found"})
+            return
+        # /projects/{id}/push/subscriptions/{sid}  DELETE
+        if len(parts) == 4 and parts[1] == "push" and parts[2] == "subscriptions":
+            if method != "DELETE":
+                self._send_json(404, {"error": "not found"})
+                return
+            err = self._check_project(user_id, project_id, "create_task")
+            if err:
+                self._send_json(err[0], {"error": err[1]})
+                return
+            subs = _push_list(self.server.db_path, project_id)
+            mine = [s for s in subs if s["id"] == parts[3]]
+            if not mine:
+                self._send_json(404, {"error": "subscription not found"})
+                return
+            # 只能删自己的订阅，项目 owner 可删任意（owner 角色检查）。
+            with _auth_store(self.server.db_path) as auth:
+                is_owner = auth.check_permission(
+                    user_id, project_id, "manage_project")
+            if mine[0]["user_id"] != user_id and not is_owner:
+                self._send_json(403, {"error": "forbidden"})
+                return
+            deleted = _push_delete(self.server.db_path,
+                                   subscription_id=parts[3])
+            self._send_json(200, {"deleted": bool(deleted)})
+            return
+        self._send_json(404, {"error": "not found"})
+
+    def _memory_create(self, user_id, project_id):
+        body, err = self._read_json()
+        if err:
+            self._send_json(400, {"error": err})
+            return
+        goal = body.get("goal")
+        conclusion = body.get("conclusion")
+        if not goal or not isinstance(goal, str):
+            self._send_json(400, {"error": "goal required"})
+            return
+        if not conclusion or not isinstance(conclusion, str):
+            self._send_json(400, {"error": "conclusion required"})
+            return
+        store = MemoryStore(self.server.db_path)
+        try:
+            mid = store.add_summary(
+                project_id,
+                body.get("thread_id") or "",
+                goal,
+                conclusion,
+                files_changed=body.get("files_changed") or [],
+                open_issues=body.get("open_issues") or [],
+            )
+        finally:
+            try:
+                store.conn.close()
+            except Exception:
+                pass
+        with _auth_store(self.server.db_path) as auth:
+            auth.audit(user_id, project_id, "create_memory", str(mid))
+        self._send_json(201, {"id": mid, "project_id": project_id})
+
+    def _memory_list(self, project_id, query):
+        try:
+            top_k = int((query.get("top_k") or ["5"])[0])
+        except (TypeError, ValueError):
+            self._send_json(400, {"error": "top_k must be an integer"})
+            return
+        store = MemoryStore(self.server.db_path)
+        try:
+            summaries = store.list_summaries(project_id)
+            context = store.build_prompt_context(project_id, top_k=top_k)
+        finally:
+            try:
+                store.conn.close()
+            except Exception:
+                pass
+        self._send_json(200, {
+            "project_id": project_id,
+            "summaries": summaries[:top_k],
+            "prompt_context": context,
+        })
+
+    def _memory_supersede(self, user_id, project_id, mid_raw):
+        try:
+            mid = int(mid_raw)
+        except (TypeError, ValueError):
+            self._send_json(400, {"error": "id must be an integer"})
+            return
+        conn = self._db()
+        try:
+            row = conn.execute(
+                "SELECT project_id FROM summaries WHERE id = ?", (mid,)
+            ).fetchone()
+        finally:
+            conn.close()
+        if row is None or row["project_id"] != project_id:
+            self._send_json(404, {"error": "summary not found"})
+            return
+        store = MemoryStore(self.server.db_path)
+        try:
+            store.deprecate(mid)
+        finally:
+            try:
+                store.conn.close()
+            except Exception:
+                pass
+        with _auth_store(self.server.db_path) as auth:
+            auth.audit(user_id, project_id, "supersede_memory", str(mid))
+        self._send_json(200, {"id": mid, "status": "deprecated"})
+
+    def _kb_index(self, user_id, project_id):
+        body, err = self._read_json()
+        if err:
+            self._send_json(400, {"error": err})
+            return
+        path = body.get("path")
+        content = body.get("content")
+        if not path or not isinstance(path, str):
+            self._send_json(400, {"error": "path required"})
+            return
+        if content is None or not isinstance(content, str):
+            self._send_json(400, {"error": "content required"})
+            return
+        store = KBStore(self.server.db_path)
+        try:
+            doc_id = store.index_file(
+                project_id, path, content,
+                kind=body.get("kind") or "file")
+        except ValueError as e:
+            self._send_json(400, {"error": str(e)})
+            return
+        finally:
+            try:
+                store.conn.close()
+            except Exception:
+                pass
+        with _auth_store(self.server.db_path) as auth:
+            auth.audit(user_id, project_id, "kb_index", path)
+        self._send_json(201, {"id": doc_id, "project_id": project_id,
+                              "path": path})
+
+    def _kb_search(self, project_id, query):
+        q = (query.get("q") or [""])[0]
+        if not q:
+            self._send_json(400, {"error": "q required"})
+            return
+        try:
+            top_k = int((query.get("top_k") or ["5"])[0])
+        except (TypeError, ValueError):
+            self._send_json(400, {"error": "top_k must be an integer"})
+            return
+        store = KBStore(self.server.db_path)
+        try:
+            results = store.search(project_id, q, top_k=top_k)
+        finally:
+            try:
+                store.conn.close()
+            except Exception:
+                pass
+        self._send_json(200, {"project_id": project_id, "query": q,
+                              "results": results})
+
+    def _project_cost(self, project_id, query):
+        store = CostStore(self.server.db_path)
+        try:
+            overall = store.report_by_project(project_id)
+            conn = store._conn
+            conn.row_factory = sqlite3.Row
+            days = (query.get("days") or [None])[0]
+            day = (query.get("day") or [None])[0]
+            daily = []
+            if day:
+                row = conn.execute(
+                    "SELECT day, COALESCE(SUM(total_tokens),0) AS t,"
+                    " COALESCE(SUM(usd_micros),0) AS u, COUNT(*) AS c,"
+                    " COALESCE(SUM(fallback_count),0) AS f"
+                    " FROM cost_records WHERE project_id = ? AND day = ?"
+                    " GROUP BY day",
+                    (project_id, day)).fetchone()
+                if row:
+                    daily = [{"day": row["day"], "total_tokens": row["t"],
+                              "usd_micros": row["u"], "turns": row["c"],
+                              "fallback_count": row["f"]}]
+            elif days is not None:
+                try:
+                    n = int(days)
+                except (TypeError, ValueError):
+                    self._send_json(400, {"error": "days must be an integer"})
+                    return
+                from datetime import date, timedelta
+                cutoff = (date.today() - timedelta(days=max(n - 1, 0))
+                          ).isoformat()
+                rows = conn.execute(
+                    "SELECT day, COALESCE(SUM(total_tokens),0) AS t,"
+                    " COALESCE(SUM(usd_micros),0) AS u, COUNT(*) AS c,"
+                    " COALESCE(SUM(fallback_count),0) AS f"
+                    " FROM cost_records WHERE project_id = ? AND day >= ?"
+                    " GROUP BY day ORDER BY day",
+                    (project_id, cutoff)).fetchall()
+                daily = [{"day": r["day"], "total_tokens": r["t"],
+                          "usd_micros": r["u"], "turns": r["c"],
+                          "fallback_count": r["f"]} for r in rows]
+        finally:
+            try:
+                store._conn.close()
+            except Exception:
+                pass
+        self._send_json(200, {"project_id": project_id, "overall": overall,
+                              "daily": daily})
+
+    # -- 任务成本与模型总览 ----------------------------------------------
+    def _task_cost(self, user_id, task_id):
+        row = self._fetch_task(task_id)
+        if row is None:
+            self._send_json(404, {"error": "task not found"})
+            return
+        task = _task_to_dict(row)
+        with _auth_store(self.server.db_path) as auth:
+            if not auth.check_permission(user_id, task["project_id"], "read"):
+                self._send_json(403, {"error": "forbidden"})
+                return
+        store = CostStore(self.server.db_path)
+        try:
+            report = store.report_by_task(task_id)
+        finally:
+            try:
+                store._conn.close()
+            except Exception:
+                pass
+        self._send_json(200, {"task_id": task_id,
+                              "project_id": task["project_id"], **report})
+
+    def _models_overview(self, user_id):
+        # 优先级列表来自环境变量 MODELS_PRIORITY（逗号分隔），未设时
+        # 用默认列表；这只是控制面的默认配置，不是 provider 的承诺。
+        raw = os.environ.get("MODELS_PRIORITY", "")
+        priority = [m.strip() for m in raw.split(",") if m.strip()] or [
+            "gpt-6.1-sol", "gpt-6-sol", "gpt-5.6-sol"]
+        # 只统计调用者是成员的项目，避免跨项目泄露成本。
+        conn = self._db()
+        try:
+            member_ids = [r["project_id"] for r in conn.execute(
+                "SELECT project_id FROM project_members WHERE user_id = ?",
+                (user_id,)).fetchall()]
+        finally:
+            conn.close()
+        if not member_ids:
+            self._send_json(200, {"priority_list": priority,
+                                  "model_stats": [], "fallback_events": []})
+            return
+        placeholders = ",".join("?" for _ in member_ids)
+        store = CostStore(self.server.db_path)
+        try:
+            conn = store._conn
+            conn.row_factory = sqlite3.Row
+            rows = conn.execute(
+                "SELECT model, COUNT(*) AS turns,"
+                " COALESCE(SUM(total_tokens),0) AS tokens,"
+                " COALESCE(SUM(fallback_count),0) AS fallbacks"
+                " FROM cost_records WHERE project_id IN (%s)"
+                " GROUP BY model ORDER BY model" % placeholders,
+                member_ids).fetchall()
+            fb_rows = conn.execute(
+                "SELECT task_id, project_id, model, fallback_count"
+                " FROM cost_records WHERE project_id IN (%s)"
+                " AND fallback_count > 0"
+                " ORDER BY created_at DESC LIMIT 50" % placeholders,
+                member_ids).fetchall()
+        finally:
+            try:
+                store._conn.close()
+            except Exception:
+                pass
+        self._send_json(200, {
+            "priority_list": priority,
+            "model_stats": [
+                {"model": r["model"], "turns": r["turns"],
+                 "total_tokens": r["tokens"],
+                 "fallback_count": r["fallbacks"]} for r in rows],
+            "fallback_events": [
+                {"task_id": r["task_id"], "project_id": r["project_id"],
+                 "model": r["model"],
+                 "fallback_count": r["fallback_count"]} for r in fb_rows],
+        })
+
     # -- 任务 ----------------------------------------------------------
     def _create_task(self, user_id):
         body, err = self._read_json()
@@ -553,6 +986,121 @@ class CloudControlPlaneHandler(BaseHTTPRequestHandler):
         swept = sweep_stale_tasks(self.server.db_path, threshold)
         self._send_json(200, {"swept": swept})
 
+    # -- OAuth（pre-auth，靠 state + secret，不走 X-API-Key）-----------
+    def _oauth_provider(self, path):
+        parts = [p for p in path.split("/") if p]
+        if len(parts) != 3 or parts[0] != "oauth":
+            return None
+        return parts[1]
+
+    def _oauth_login(self, path):
+        provider = self._oauth_provider(path)
+        if provider is None:
+            self._send_json(404, {"error": "not found"})
+            return
+        query = parse_qs(urlparse(self.path).query)
+        client_id = (query.get("client_id") or [None])[0]
+        redirect_uri = (query.get("redirect_uri") or [None])[0]
+        scope = (query.get("scope") or [None])[0]
+        if not client_id or not redirect_uri:
+            self._send_json(400, {"error": "client_id and redirect_uri required"})
+            return
+        store = OAuthStateStore(self.server.db_path)
+        try:
+            state, url = start_oauth_login(
+                store, provider, client_id, redirect_uri, scope=scope)
+        except OAuthError as e:
+            self._send_json(400, {"error": str(e), "reason": e.reason})
+            return
+        except ValueError as e:
+            self._send_json(400, {"error": str(e)})
+            return
+        finally:
+            try:
+                store.conn.close()
+            except Exception:
+                pass
+        # 只返回 URL，不直接跳转到任何登录页；跳转由调用方决定。
+        self._send_json(200, {"provider": provider, "state": state,
+                              "authorization_url": url})
+
+    def _oauth_callback(self, path):
+        provider = self._oauth_provider(path)
+        if provider is None:
+            self._send_json(404, {"error": "not found"})
+            return
+        body, err = self._read_json()
+        if err:
+            self._send_json(400, {"error": err})
+            return
+        code = body.get("code")
+        state = body.get("state")
+        client_id = body.get("client_id")
+        redirect_uri = body.get("redirect_uri")
+        if not code or not state or not client_id or not redirect_uri:
+            self._send_json(400, {"error": "code, state, client_id and redirect_uri required"})
+            return
+        store = OAuthStateStore(self.server.db_path)
+        try:
+            identity = handle_oauth_callback(
+                store, provider, code, state, client_id, redirect_uri)
+        except OAuthError as e:
+            self._send_json(400, {"error": str(e), "reason": e.reason})
+            return
+        finally:
+            try:
+                store.conn.close()
+            except Exception:
+                pass
+        with _auth_store(self.server.db_path) as auth:
+            local_id, created = map_external_identity_to_local_user(
+                auth, provider, identity)
+            auth.audit(local_id, "", "oauth_login",
+                       "%s:%s" % (provider, identity.get("external_id")))
+        self._send_json(200, {"user_id": local_id, "created": created,
+                              "provider": provider,
+                              "login": identity.get("login"),
+                              "email": identity.get("email")})
+
+    # -- 推送订阅与分发 ---------------------------------------------------
+    def _push_create_endpoint(self, user_id, project_id):
+        body, err = self._read_json()
+        if err:
+            self._send_json(400, {"error": err})
+            return
+        try:
+            sub = _push_create(
+                self.server.db_path, project_id, user_id,
+                body.get("channel"), body.get("target"),
+                body.get("preference") or "instant")
+        except ValueError as e:
+            self._send_json(400, {"error": str(e)})
+            return
+        self._send_json(201, sub)
+
+    def _push_dispatch_endpoint(self, user_id):
+        body, err = self._read_json()
+        if err:
+            self._send_json(400, {"error": err})
+            return
+        event_type = body.get("event_type")
+        project_id = body.get("project_id")
+        if not event_type or not project_id:
+            self._send_json(400, {"error": "event_type and project_id required"})
+            return
+        # 分发者须是该项目的成员（至少可读），否则不能往别人项目里推。
+        with _auth_store(self.server.db_path) as auth:
+            if not auth.check_permission(user_id, project_id, "read"):
+                self._send_json(403, {"error": "forbidden"})
+                return
+        # HTTP 层无可注入 sender：webhook/slack 会报“无 sender”，
+        # webpush/apns/fcm 会报缺凭证——都不静默成功。真实发送端
+        # 由 worker 注入 sender 后调 push.dispatch_event。
+        result = _push_dispatch(
+            self.server.db_path, event_type, project_id,
+            body.get("payload") or {}, sender=None)
+        self._send_json(200, result)
+
     # -- webhooks（签名认证，不走 X-API-Key）---------------------------
     def _handle_github_webhook(self):
         raw = self._read_body()