"""云端控制面 HTTP server(首版)。 零第三方依赖,只用 http.server + sqlite3 + 标准库。 分工与边界(以实际代码为准,不夸大): - 本 server 只做控制面:项目/任务的创建、查询、取消、心跳、 stale 任务回收,以及 GitHub/Slack webhook 的验签与解析。 - 本版不直接跑模型回合。任务状态机与 host.py 对齐: queued -> running -> completed/failed/interrupted/merged/merge-failed, 其中 completed/merged 由 worker/控制面后续调用方更新(例如 worker 调 host.py 跑完回合后回写),本 server 不提供“跑模型”端点。 - 其它已就绪模块(queue 的 order_tasks/ready_tasks、sandbox 的执行器、 memory 的 MemoryStore、kb 的 KBStore、models 的 ModelRouter、 cost 的 CostStore、webhooks 的 NotificationService)本版尚未接成 HTTP 端点,留给 worker 接线时调用;本 server 只接入其中两处: AuthStore(认证/权限/审计)与 webhooks 的验签/解析函数。 - DockerExecutor/CloudExecutor 目前仍是占位(见 sandbox.py), 本 server 不假装已接通容器或云端执行。 导入规则:只用绝对包导入(from cloud.auth import ...)。绝不要把 cloud/ 目录加进 sys.path:标准库已有 queue 模块,会和 cloud/queue.py 撞名,导致 stdlib queue 被错误模块覆盖。 状态库:单一 SQLite 文件。AuthStore 用 users/project_members/ audit_log 三张表;本 server 用 projects/tasks 两张表,表名不冲突。 并发:ThreadingHTTPServer 多线程处理请求,每个请求内新建 SQLite 连接(server 表用新连接,AuthStore 每次请求新建一个实例),绝不在 多线程 handler 间共用一个连接。 启动: python3 -m cloud.server [--host 127.0.0.1] [--port 8765] [--db <路径>] 测试用 create_server(db_path, host, port) 只返回 ThreadingHTTPServer 实例,不自动 serve。 """ import argparse import json import os import sqlite3 import time import uuid from contextlib import contextmanager from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer from urllib.parse import parse_qs, urlparse from cloud.auth import AuthStore from cloud.webhooks import ( parse_github_pr_comment, parse_slack_command, verify_github_signature, verify_slack_signature, ) TERMINAL_STATUSES = frozenset( {"completed", "failed", "interrupted", "merged", "merge-failed"} ) DEFAULT_SWEEP_THRESHOLD_SECONDS = 300 def _connect(db_path): """每个请求新建一个连接;busy timeout 给并发写留重试窗口。""" return sqlite3.connect(str(db_path), timeout=30) def init_db(db_path): """创建 server 自己的 projects/tasks 表(AuthStore 的表由其自建)。""" conn = _connect(db_path) try: conn.execute( "CREATE TABLE IF NOT EXISTS projects(" "id TEXT PRIMARY KEY, " "name TEXT, " "owner_id TEXT, " "created_at REAL)" ) conn.execute( "CREATE TABLE IF NOT EXISTS tasks(" "id TEXT PRIMARY KEY, " "project_id TEXT, " "prompt TEXT, " "status TEXT, " "idempotency_key TEXT, " "depends_on TEXT, " "priority INTEGER, " "created_by TEXT, " "created_at REAL, " "updated_at REAL, " "last_heartbeat REAL, " "UNIQUE(project_id, idempotency_key))" ) conn.commit() finally: conn.close() @contextmanager def _auth_store(db_path): """每请求一个 AuthStore 实例(自带新连接),用完即关闭。""" store = AuthStore(db_path) try: yield store finally: try: store.conn.close() except Exception: pass def _task_to_dict(row): depends_on = row["depends_on"] try: depends_on = json.loads(depends_on) if depends_on else [] except (TypeError, ValueError): depends_on = [] if not isinstance(depends_on, list): depends_on = [depends_on] return { "id": row["id"], "project_id": row["project_id"], "prompt": row["prompt"], "status": row["status"], "idempotency_key": row["idempotency_key"], "depends_on": depends_on, "priority": row["priority"], "created_by": row["created_by"], "created_at": row["created_at"], "updated_at": row["updated_at"], "last_heartbeat": row["last_heartbeat"], } def _project_to_dict(row): return { "id": row["id"], "name": row["name"], "owner_id": row["owner_id"], "created_at": row["created_at"], } def sweep_stale_tasks(db_path, threshold_seconds=300): """把 running 且心跳过期的任务改为 interrupted,返回被扫任务 id 列表。 心跳取 last_heartbeat,无心跳则用 updated_at;早于 now - threshold_seconds 即过期。审计记一条 "sweep_interrupted" (user_id 记 "system",因为回收不是某个用户的操作)。 独立可单测,不依赖 HTTP。 """ now = time.time() cutoff = now - threshold_seconds conn = _connect(db_path) try: conn.row_factory = sqlite3.Row rows = conn.execute( "SELECT id, project_id FROM tasks WHERE status = 'running'" " AND COALESCE(last_heartbeat, updated_at) < ?", (cutoff,), ).fetchall() swept = [] for row in rows: cur = conn.execute( "UPDATE tasks SET status = 'interrupted', updated_at = ?" " WHERE id = ? AND status = 'running'", (now, row["id"]), ) if cur.rowcount: swept.append(row["id"]) conn.commit() stale = [(row["id"], row["project_id"]) for row in rows if row["id"] in swept] finally: conn.close() with _auth_store(db_path) as store: for task_id, project_id in stale: store.audit("system", project_id, "sweep_interrupted", task_id) return swept class CloudControlPlaneHandler(BaseHTTPRequestHandler): """控制面请求处理:除 /health 与 /webhooks/* 外一律要求 X-API-Key。""" protocol_version = "HTTP/1.1" # -- 通用工具 ------------------------------------------------------ def log_message(self, *args): # 测试输出保持干净 pass def _send_json(self, status, obj): body = json.dumps(obj).encode("utf-8") self.send_response(status) self.send_header("Content-Type", "application/json") self.send_header("Content-Length", str(len(body))) self.end_headers() self.wfile.write(body) def _read_body(self): try: length = int(self.headers.get("Content-Length") or 0) except ValueError: length = 0 if length <= 0: return b"" return self.rfile.read(length) def _read_json(self): """返回 (obj, error):空 body 视为 {};非法 JSON 返回 error。""" raw = self._read_body() if not raw: return {}, None try: obj = json.loads(raw.decode("utf-8")) except (ValueError, UnicodeDecodeError): return None, "invalid json" if not isinstance(obj, dict): return None, "body must be a JSON object" return obj, None def _db(self): conn = _connect(self.server.db_path) conn.row_factory = sqlite3.Row return conn def _authenticate(self): """X-API-Key -> user_id;无 key 或 key 错返回 None。""" api_key = self.headers.get("X-API-Key") if not api_key: return None with _auth_store(self.server.db_path) as store: return store.authenticate(api_key) # -- 入口 ---------------------------------------------------------- def do_GET(self): self._dispatch("GET") def do_POST(self): self._dispatch("POST") def _dispatch(self, method): parsed = urlparse(self.path) path = parsed.path if len(path) > 1 and path.endswith("/"): path = path.rstrip("/") query = parse_qs(parsed.query) if path == "/health" and method == "GET": self._send_json(200, {"ok": True, "service": "cloud-control-plane"}) return if path.startswith("/webhooks/"): # webhook 靠签名认证(外部服务无法持 X-API-Key),不走 API key。 if method != "POST": self._send_json(404, {"error": "not found"}) return if path == "/webhooks/github": self._handle_github_webhook() elif path == "/webhooks/slack": self._handle_slack_webhook() else: self._send_json(404, {"error": "not found"}) return user_id = self._authenticate() if user_id is None: self._send_json(401, {"error": "unauthorized"}) return if path == "/projects" and method == "POST": self._create_project(user_id) elif path == "/projects" and method == "GET": self._list_projects(user_id) elif path == "/tasks" and method == "POST": self._create_task(user_id) elif path == "/tasks" and method == "GET": self._list_tasks(user_id, query) elif path == "/admin/sweep" and method == "POST": self._admin_sweep() elif path.startswith("/tasks/"): rest = path[len("/tasks/"):] if "/" not in rest and method == "GET": self._get_task(user_id, rest) elif rest.endswith("/cancel") and method == "POST": self._cancel_task(user_id, rest[: -len("/cancel")]) elif rest.endswith("/heartbeat") and method == "POST": self._heartbeat_task(user_id, rest[: -len("/heartbeat")]) else: self._send_json(404, {"error": "not found"}) else: self._send_json(404, {"error": "not found"}) # -- 项目 ---------------------------------------------------------- def _create_project(self, user_id): body, err = self._read_json() if err: self._send_json(400, {"error": err}) return name = body.get("name") if not name or not isinstance(name, str): self._send_json(400, {"error": "name required"}) return project_id = body.get("id") or uuid.uuid4().hex if not isinstance(project_id, str): self._send_json(400, {"error": "id must be a string"}) return now = time.time() conn = self._db() try: existing = conn.execute( "SELECT id FROM projects WHERE id = ?", (project_id,) ).fetchone() if existing: self._send_json(409, {"error": "project exists"}) return conn.execute( "INSERT INTO projects(id, name, owner_id, created_at)" " VALUES(?, ?, ?, ?)", (project_id, name, user_id, now), ) conn.commit() project = { "id": project_id, "name": name, "owner_id": user_id, "created_at": now, } except sqlite3.IntegrityError: self._send_json(409, {"error": "project exists"}) return finally: conn.close() # AuthStore.create_project 只写成员表(owner 角色),项目记录 # 本身在上面的 projects 表;两者配合才算一个完整项目。 with _auth_store(self.server.db_path) as store: store.create_project(project_id, user_id) store.audit(user_id, project_id, "create_project", name) self._send_json(201, project) def _list_projects(self, user_id): conn = self._db() try: member_ids = { row["project_id"] for row in conn.execute( "SELECT project_id FROM project_members WHERE user_id = ?", (user_id,), ).fetchall() } rows = conn.execute( "SELECT id, name, owner_id, created_at FROM projects" " ORDER BY created_at, id" ).fetchall() finally: conn.close() projects = [_project_to_dict(r) for r in rows if r["id"] in member_ids] self._send_json(200, {"projects": projects}) # -- 任务 ---------------------------------------------------------- def _create_task(self, user_id): body, err = self._read_json() if err: self._send_json(400, {"error": err}) return project_id = body.get("project_id") prompt = body.get("prompt") if not project_id or not isinstance(project_id, str): self._send_json(400, {"error": "project_id required"}) return if not prompt or not isinstance(prompt, str): self._send_json(400, {"error": "prompt required"}) return depends_on = body.get("depends_on") or [] if isinstance(depends_on, str): depends_on = [depends_on] if not isinstance(depends_on, list): self._send_json(400, {"error": "depends_on must be a list"}) return try: priority = int(body.get("priority", 0) or 0) except (TypeError, ValueError): self._send_json(400, {"error": "priority must be an integer"}) return idempotency_key = body.get("idempotency_key") if idempotency_key is not None and not isinstance(idempotency_key, str): self._send_json(400, {"error": "idempotency_key must be a string"}) return with _auth_store(self.server.db_path) as store: if not store.check_permission(user_id, project_id, "create_task"): self._send_json(403, {"error": "forbidden"}) return now = time.time() conn = self._db() try: if idempotency_key is not None: row = conn.execute( "SELECT * FROM tasks WHERE project_id = ?" " AND idempotency_key = ?", (project_id, idempotency_key), ).fetchone() if row: self._send_json(200, _task_to_dict(row)) return task_id = uuid.uuid4().hex try: conn.execute( "INSERT INTO tasks(id, project_id, prompt, status," " idempotency_key, depends_on, priority, created_by," " created_at, updated_at, last_heartbeat)" " VALUES(?, ?, ?, 'queued', ?, ?, ?, ?, ?, ?, NULL)", (task_id, project_id, prompt, idempotency_key, json.dumps(depends_on), priority, user_id, now, now), ) conn.commit() except sqlite3.IntegrityError: # 并发下同幂等键抢建:回读已有任务,不建重复。 row = conn.execute( "SELECT * FROM tasks WHERE project_id = ?" " AND idempotency_key = ?", (project_id, idempotency_key), ).fetchone() if row: self._send_json(200, _task_to_dict(row)) return raise row = conn.execute( "SELECT * FROM tasks WHERE id = ?", (task_id,) ).fetchone() task = _task_to_dict(row) finally: conn.close() with _auth_store(self.server.db_path) as store: store.audit(user_id, project_id, "create_task", task_id) self._send_json(201, task) def _list_tasks(self, user_id, query): project_ids = query.get("project_id") project_id = project_ids[0] if project_ids else None if not project_id: self._send_json(400, {"error": "project_id required"}) return with _auth_store(self.server.db_path) as store: if not store.check_permission(user_id, project_id, "read"): self._send_json(403, {"error": "forbidden"}) return conn = self._db() try: rows = conn.execute( "SELECT * FROM tasks WHERE project_id = ?" " ORDER BY created_at, id", (project_id,), ).fetchall() finally: conn.close() self._send_json(200, {"tasks": [_task_to_dict(r) for r in rows]}) def _fetch_task(self, task_id): conn = self._db() try: return conn.execute( "SELECT * FROM tasks WHERE id = ?", (task_id,) ).fetchone() finally: conn.close() def _get_task(self, user_id, task_id): row = self._fetch_task(task_id) if row is None: self._send_json(404, {"error": "task not found"}) return task = _task_to_dict(row) with _auth_store(self.server.db_path) as store: if not store.check_permission(user_id, task["project_id"], "read"): self._send_json(403, {"error": "forbidden"}) return self._send_json(200, task) def _cancel_task(self, user_id, task_id): row = self._fetch_task(task_id) if row is None: self._send_json(404, {"error": "task not found"}) return task = _task_to_dict(row) with _auth_store(self.server.db_path) as store: if not store.check_permission( user_id, task["project_id"], "create_task"): self._send_json(403, {"error": "forbidden"}) return if task["status"] in TERMINAL_STATUSES: self._send_json(400, {"error": "task already terminal"}) return now = time.time() conn = self._db() try: conn.execute( "UPDATE tasks SET status = 'interrupted', updated_at = ?" " WHERE id = ?", (now, task_id), ) conn.commit() task = _task_to_dict( conn.execute( "SELECT * FROM tasks WHERE id = ?", (task_id,) ).fetchone() ) finally: conn.close() with _auth_store(self.server.db_path) as store: store.audit(user_id, task["project_id"], "cancel_task", task_id) self._send_json(200, task) def _heartbeat_task(self, user_id, task_id): row = self._fetch_task(task_id) if row is None: self._send_json(404, {"error": "task not found"}) return task = _task_to_dict(row) with _auth_store(self.server.db_path) as store: if not store.check_permission( user_id, task["project_id"], "create_task"): self._send_json(403, {"error": "forbidden"}) return now = time.time() new_status = "running" if task["status"] == "queued" else task["status"] conn = self._db() try: conn.execute( "UPDATE tasks SET status = ?, last_heartbeat = ?," " updated_at = ? WHERE id = ?", (new_status, now, now, task_id), ) conn.commit() task = _task_to_dict( conn.execute( "SELECT * FROM tasks WHERE id = ?", (task_id,) ).fetchone() ) finally: conn.close() self._send_json(200, task) # -- 管理 ---------------------------------------------------------- def _admin_sweep(self): body, err = self._read_json() if err: self._send_json(400, {"error": err}) return threshold = body.get( "threshold_seconds", DEFAULT_SWEEP_THRESHOLD_SECONDS) try: threshold = int(threshold) except (TypeError, ValueError): self._send_json(400, {"error": "threshold_seconds must be an integer"}) return swept = sweep_stale_tasks(self.server.db_path, threshold) self._send_json(200, {"swept": swept}) # -- webhooks(签名认证,不走 X-API-Key)--------------------------- def _handle_github_webhook(self): raw = self._read_body() secret = os.environ.get("GITHUB_WEBHOOK_SECRET") if not secret: self._send_json(503, {"error": "github webhook secret not configured"}) return signature = self.headers.get("X-Hub-Signature-256") if not verify_github_signature(secret, raw, signature): self._send_json(401, {"error": "unauthorized"}) return try: payload = json.loads(raw.decode("utf-8")) if raw else {} except (ValueError, UnicodeDecodeError): self._send_json(400, {"error": "invalid json"}) return if not isinstance(payload, dict): self._send_json(400, {"error": "body must be a JSON object"}) return # 只解析不执行:/fix-review 的真正执行由控制面后续调 host.py 完成。 result = parse_github_pr_comment(payload) if result is None: self._send_json(200, {"result": None}) else: self._send_json(200, result) def _handle_slack_webhook(self): raw = self._read_body() secret = os.environ.get("SLACK_SIGNING_SECRET") if not secret: self._send_json(503, {"error": "slack webhook secret not configured"}) return signature = self.headers.get("X-Slack-Signature") timestamp = self.headers.get("X-Slack-Request-Timestamp") if not verify_slack_signature(secret, timestamp, raw, signature): self._send_json(401, {"error": "unauthorized"}) return # 只解析不执行任务:Slack 斜杠命令的执行由控制面后续决定。 content_type = self.headers.get("Content-Type") or "" text = "" if "application/x-www-form-urlencoded" in content_type: parsed = parse_qs(raw.decode("utf-8", "replace")) values = parsed.get("text") text = values[0] if values else "" elif raw: try: payload = json.loads(raw.decode("utf-8")) if isinstance(payload, dict): text = payload.get("text", "") or "" else: text = raw.decode("utf-8", "replace") except (ValueError, UnicodeDecodeError): text = raw.decode("utf-8", "replace") self._send_json(200, parse_slack_command(text)) def create_server(db_path, host="127.0.0.1", port=8765): """建 server 实例(不自动 serve,供测试与 __main__ 共用)。""" db_path = str(db_path) init_db(db_path) # 确保 AuthStore 的三张表也存在,后续请求才能直接查 project_members。 with _auth_store(db_path): pass server = ThreadingHTTPServer((host, port), CloudControlPlaneHandler) server.db_path = db_path server.daemon_threads = True return server def main(argv=None): parser = argparse.ArgumentParser(description="cloud control plane") parser.add_argument("--host", default="127.0.0.1") parser.add_argument("--port", type=int, default=8765) parser.add_argument("--db", default="cloud-control-plane.db") args = parser.parse_args(argv) server = create_server(args.db, args.host, args.port) print( "cloud-control-plane listening on %s:%d db=%s" % (args.host, server.server_address[1], args.db), flush=True, ) try: server.serve_forever() except KeyboardInterrupt: pass if __name__ == "__main__": main()