Files
codex-app-server/cloud/server.py
T

645 lines
24 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
"""云端控制面 HTTP server(首版)。
零第三方依赖,只用 http.server + sqlite3 + 标准库。
分工与边界(以实际代码为准,不夸大):
- 本 server 只做控制面:项目/任务的创建、查询、取消、心跳、
stale 任务回收,以及 GitHub/Slack webhook 的验签与解析。
- 本版不直接跑模型回合。任务状态机与 host.py 对齐:
queued -> running -> completed/failed/interrupted/merged/merge-failed,
其中 completed/merged 由 worker/控制面后续调用方更新(例如 worker
调 host.py 跑完回合后回写),本 server 不提供“跑模型”端点。
- 其它已就绪模块(queue 的 order_tasks/ready_tasks、sandbox 的执行器、
memory 的 MemoryStore、kb 的 KBStore、models 的 ModelRouter、
cost 的 CostStore、webhooks 的 NotificationService)本版尚未接成
HTTP 端点,留给 worker 接线时调用;本 server 只接入其中两处:
AuthStore(认证/权限/审计)与 webhooks 的验签/解析函数。
- DockerExecutor/CloudExecutor 目前仍是占位(见 sandbox.py),
本 server 不假装已接通容器或云端执行。
导入规则:只用绝对包导入(from cloud.auth import ...)。绝不要把
cloud/ 目录加进 sys.path:标准库已有 queue 模块,会和 cloud/queue.py
撞名,导致 stdlib queue 被错误模块覆盖。
状态库:单一 SQLite 文件。AuthStore 用 users/project_members/
audit_log 三张表;本 server 用 projects/tasks 两张表,表名不冲突。
并发:ThreadingHTTPServer 多线程处理请求,每个请求内新建 SQLite
连接(server 表用新连接,AuthStore 每次请求新建一个实例),绝不在
多线程 handler 间共用一个连接。
启动:
python3 -m cloud.server [--host 127.0.0.1] [--port 8765] [--db <路径>]
测试用 create_server(db_path, host, port) 只返回 ThreadingHTTPServer
实例,不自动 serve。
"""
import argparse
import json
import os
import sqlite3
import time
import uuid
from contextlib import contextmanager
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
from urllib.parse import parse_qs, urlparse
from cloud.auth import AuthStore
from cloud.webhooks import (
parse_github_pr_comment,
parse_slack_command,
verify_github_signature,
verify_slack_signature,
)
TERMINAL_STATUSES = frozenset(
{"completed", "failed", "interrupted", "merged", "merge-failed"}
)
DEFAULT_SWEEP_THRESHOLD_SECONDS = 300
def _connect(db_path):
"""每个请求新建一个连接;busy timeout 给并发写留重试窗口。"""
return sqlite3.connect(str(db_path), timeout=30)
def init_db(db_path):
"""创建 server 自己的 projects/tasks 表(AuthStore 的表由其自建)。"""
conn = _connect(db_path)
try:
conn.execute(
"CREATE TABLE IF NOT EXISTS projects("
"id TEXT PRIMARY KEY, "
"name TEXT, "
"owner_id TEXT, "
"created_at REAL)"
)
conn.execute(
"CREATE TABLE IF NOT EXISTS tasks("
"id TEXT PRIMARY KEY, "
"project_id TEXT, "
"prompt TEXT, "
"status TEXT, "
"idempotency_key TEXT, "
"depends_on TEXT, "
"priority INTEGER, "
"created_by TEXT, "
"created_at REAL, "
"updated_at REAL, "
"last_heartbeat REAL, "
"UNIQUE(project_id, idempotency_key))"
)
conn.commit()
finally:
conn.close()
@contextmanager
def _auth_store(db_path):
"""每请求一个 AuthStore 实例(自带新连接),用完即关闭。"""
store = AuthStore(db_path)
try:
yield store
finally:
try:
store.conn.close()
except Exception:
pass
def _task_to_dict(row):
depends_on = row["depends_on"]
try:
depends_on = json.loads(depends_on) if depends_on else []
except (TypeError, ValueError):
depends_on = []
if not isinstance(depends_on, list):
depends_on = [depends_on]
return {
"id": row["id"],
"project_id": row["project_id"],
"prompt": row["prompt"],
"status": row["status"],
"idempotency_key": row["idempotency_key"],
"depends_on": depends_on,
"priority": row["priority"],
"created_by": row["created_by"],
"created_at": row["created_at"],
"updated_at": row["updated_at"],
"last_heartbeat": row["last_heartbeat"],
}
def _project_to_dict(row):
return {
"id": row["id"],
"name": row["name"],
"owner_id": row["owner_id"],
"created_at": row["created_at"],
}
def sweep_stale_tasks(db_path, threshold_seconds=300):
"""把 running 且心跳过期的任务改为 interrupted,返回被扫任务 id 列表。
心跳取 last_heartbeat,无心跳则用 updated_at;早于
now - threshold_seconds 即过期。审计记一条 "sweep_interrupted"
(user_id 记 "system",因为回收不是某个用户的操作)。
独立可单测,不依赖 HTTP。
"""
now = time.time()
cutoff = now - threshold_seconds
conn = _connect(db_path)
try:
conn.row_factory = sqlite3.Row
rows = conn.execute(
"SELECT id, project_id FROM tasks WHERE status = 'running'"
" AND COALESCE(last_heartbeat, updated_at) < ?",
(cutoff,),
).fetchall()
swept = []
for row in rows:
cur = conn.execute(
"UPDATE tasks SET status = 'interrupted', updated_at = ?"
" WHERE id = ? AND status = 'running'",
(now, row["id"]),
)
if cur.rowcount:
swept.append(row["id"])
conn.commit()
stale = [(row["id"], row["project_id"]) for row in rows
if row["id"] in swept]
finally:
conn.close()
with _auth_store(db_path) as store:
for task_id, project_id in stale:
store.audit("system", project_id, "sweep_interrupted", task_id)
return swept
class CloudControlPlaneHandler(BaseHTTPRequestHandler):
"""控制面请求处理:除 /health 与 /webhooks/* 外一律要求 X-API-Key。"""
protocol_version = "HTTP/1.1"
# -- 通用工具 ------------------------------------------------------
def log_message(self, *args): # 测试输出保持干净
pass
def _send_json(self, status, obj):
body = json.dumps(obj).encode("utf-8")
self.send_response(status)
self.send_header("Content-Type", "application/json")
self.send_header("Content-Length", str(len(body)))
self.end_headers()
self.wfile.write(body)
def _read_body(self):
try:
length = int(self.headers.get("Content-Length") or 0)
except ValueError:
length = 0
if length <= 0:
return b""
return self.rfile.read(length)
def _read_json(self):
"""返回 (obj, error):空 body 视为 {};非法 JSON 返回 error。"""
raw = self._read_body()
if not raw:
return {}, None
try:
obj = json.loads(raw.decode("utf-8"))
except (ValueError, UnicodeDecodeError):
return None, "invalid json"
if not isinstance(obj, dict):
return None, "body must be a JSON object"
return obj, None
def _db(self):
conn = _connect(self.server.db_path)
conn.row_factory = sqlite3.Row
return conn
def _authenticate(self):
"""X-API-Key -> user_id;无 key 或 key 错返回 None。"""
api_key = self.headers.get("X-API-Key")
if not api_key:
return None
with _auth_store(self.server.db_path) as store:
return store.authenticate(api_key)
# -- 入口 ----------------------------------------------------------
def do_GET(self):
self._dispatch("GET")
def do_POST(self):
self._dispatch("POST")
def _dispatch(self, method):
parsed = urlparse(self.path)
path = parsed.path
if len(path) > 1 and path.endswith("/"):
path = path.rstrip("/")
query = parse_qs(parsed.query)
if path == "/health" and method == "GET":
self._send_json(200, {"ok": True, "service": "cloud-control-plane"})
return
if path.startswith("/webhooks/"):
# webhook 靠签名认证(外部服务无法持 X-API-Key),不走 API key。
if method != "POST":
self._send_json(404, {"error": "not found"})
return
if path == "/webhooks/github":
self._handle_github_webhook()
elif path == "/webhooks/slack":
self._handle_slack_webhook()
else:
self._send_json(404, {"error": "not found"})
return
user_id = self._authenticate()
if user_id is None:
self._send_json(401, {"error": "unauthorized"})
return
if path == "/projects" and method == "POST":
self._create_project(user_id)
elif path == "/projects" and method == "GET":
self._list_projects(user_id)
elif path == "/tasks" and method == "POST":
self._create_task(user_id)
elif path == "/tasks" and method == "GET":
self._list_tasks(user_id, query)
elif path == "/admin/sweep" and method == "POST":
self._admin_sweep()
elif path.startswith("/tasks/"):
rest = path[len("/tasks/"):]
if "/" not in rest and method == "GET":
self._get_task(user_id, rest)
elif rest.endswith("/cancel") and method == "POST":
self._cancel_task(user_id, rest[: -len("/cancel")])
elif rest.endswith("/heartbeat") and method == "POST":
self._heartbeat_task(user_id, rest[: -len("/heartbeat")])
else:
self._send_json(404, {"error": "not found"})
else:
self._send_json(404, {"error": "not found"})
# -- 项目 ----------------------------------------------------------
def _create_project(self, user_id):
body, err = self._read_json()
if err:
self._send_json(400, {"error": err})
return
name = body.get("name")
if not name or not isinstance(name, str):
self._send_json(400, {"error": "name required"})
return
project_id = body.get("id") or uuid.uuid4().hex
if not isinstance(project_id, str):
self._send_json(400, {"error": "id must be a string"})
return
now = time.time()
conn = self._db()
try:
existing = conn.execute(
"SELECT id FROM projects WHERE id = ?", (project_id,)
).fetchone()
if existing:
self._send_json(409, {"error": "project exists"})
return
conn.execute(
"INSERT INTO projects(id, name, owner_id, created_at)"
" VALUES(?, ?, ?, ?)",
(project_id, name, user_id, now),
)
conn.commit()
project = {
"id": project_id,
"name": name,
"owner_id": user_id,
"created_at": now,
}
except sqlite3.IntegrityError:
self._send_json(409, {"error": "project exists"})
return
finally:
conn.close()
# AuthStore.create_project 只写成员表(owner 角色),项目记录
# 本身在上面的 projects 表;两者配合才算一个完整项目。
with _auth_store(self.server.db_path) as store:
store.create_project(project_id, user_id)
store.audit(user_id, project_id, "create_project", name)
self._send_json(201, project)
def _list_projects(self, user_id):
conn = self._db()
try:
member_ids = {
row["project_id"]
for row in conn.execute(
"SELECT project_id FROM project_members WHERE user_id = ?",
(user_id,),
).fetchall()
}
rows = conn.execute(
"SELECT id, name, owner_id, created_at FROM projects"
" ORDER BY created_at, id"
).fetchall()
finally:
conn.close()
projects = [_project_to_dict(r) for r in rows
if r["id"] in member_ids]
self._send_json(200, {"projects": projects})
# -- 任务 ----------------------------------------------------------
def _create_task(self, user_id):
body, err = self._read_json()
if err:
self._send_json(400, {"error": err})
return
project_id = body.get("project_id")
prompt = body.get("prompt")
if not project_id or not isinstance(project_id, str):
self._send_json(400, {"error": "project_id required"})
return
if not prompt or not isinstance(prompt, str):
self._send_json(400, {"error": "prompt required"})
return
depends_on = body.get("depends_on") or []
if isinstance(depends_on, str):
depends_on = [depends_on]
if not isinstance(depends_on, list):
self._send_json(400, {"error": "depends_on must be a list"})
return
try:
priority = int(body.get("priority", 0) or 0)
except (TypeError, ValueError):
self._send_json(400, {"error": "priority must be an integer"})
return
idempotency_key = body.get("idempotency_key")
if idempotency_key is not None and not isinstance(idempotency_key, str):
self._send_json(400, {"error": "idempotency_key must be a string"})
return
with _auth_store(self.server.db_path) as store:
if not store.check_permission(user_id, project_id, "create_task"):
self._send_json(403, {"error": "forbidden"})
return
now = time.time()
conn = self._db()
try:
if idempotency_key is not None:
row = conn.execute(
"SELECT * FROM tasks WHERE project_id = ?"
" AND idempotency_key = ?",
(project_id, idempotency_key),
).fetchone()
if row:
self._send_json(200, _task_to_dict(row))
return
task_id = uuid.uuid4().hex
try:
conn.execute(
"INSERT INTO tasks(id, project_id, prompt, status,"
" idempotency_key, depends_on, priority, created_by,"
" created_at, updated_at, last_heartbeat)"
" VALUES(?, ?, ?, 'queued', ?, ?, ?, ?, ?, ?, NULL)",
(task_id, project_id, prompt, idempotency_key,
json.dumps(depends_on), priority, user_id, now, now),
)
conn.commit()
except sqlite3.IntegrityError:
# 并发下同幂等键抢建:回读已有任务,不建重复。
row = conn.execute(
"SELECT * FROM tasks WHERE project_id = ?"
" AND idempotency_key = ?",
(project_id, idempotency_key),
).fetchone()
if row:
self._send_json(200, _task_to_dict(row))
return
raise
row = conn.execute(
"SELECT * FROM tasks WHERE id = ?", (task_id,)
).fetchone()
task = _task_to_dict(row)
finally:
conn.close()
with _auth_store(self.server.db_path) as store:
store.audit(user_id, project_id, "create_task", task_id)
self._send_json(201, task)
def _list_tasks(self, user_id, query):
project_ids = query.get("project_id")
project_id = project_ids[0] if project_ids else None
if not project_id:
self._send_json(400, {"error": "project_id required"})
return
with _auth_store(self.server.db_path) as store:
if not store.check_permission(user_id, project_id, "read"):
self._send_json(403, {"error": "forbidden"})
return
conn = self._db()
try:
rows = conn.execute(
"SELECT * FROM tasks WHERE project_id = ?"
" ORDER BY created_at, id",
(project_id,),
).fetchall()
finally:
conn.close()
self._send_json(200, {"tasks": [_task_to_dict(r) for r in rows]})
def _fetch_task(self, task_id):
conn = self._db()
try:
return conn.execute(
"SELECT * FROM tasks WHERE id = ?", (task_id,)
).fetchone()
finally:
conn.close()
def _get_task(self, user_id, task_id):
row = self._fetch_task(task_id)
if row is None:
self._send_json(404, {"error": "task not found"})
return
task = _task_to_dict(row)
with _auth_store(self.server.db_path) as store:
if not store.check_permission(user_id, task["project_id"], "read"):
self._send_json(403, {"error": "forbidden"})
return
self._send_json(200, task)
def _cancel_task(self, user_id, task_id):
row = self._fetch_task(task_id)
if row is None:
self._send_json(404, {"error": "task not found"})
return
task = _task_to_dict(row)
with _auth_store(self.server.db_path) as store:
if not store.check_permission(
user_id, task["project_id"], "create_task"):
self._send_json(403, {"error": "forbidden"})
return
if task["status"] in TERMINAL_STATUSES:
self._send_json(400, {"error": "task already terminal"})
return
now = time.time()
conn = self._db()
try:
conn.execute(
"UPDATE tasks SET status = 'interrupted', updated_at = ?"
" WHERE id = ?",
(now, task_id),
)
conn.commit()
task = _task_to_dict(
conn.execute(
"SELECT * FROM tasks WHERE id = ?", (task_id,)
).fetchone()
)
finally:
conn.close()
with _auth_store(self.server.db_path) as store:
store.audit(user_id, task["project_id"], "cancel_task", task_id)
self._send_json(200, task)
def _heartbeat_task(self, user_id, task_id):
row = self._fetch_task(task_id)
if row is None:
self._send_json(404, {"error": "task not found"})
return
task = _task_to_dict(row)
with _auth_store(self.server.db_path) as store:
if not store.check_permission(
user_id, task["project_id"], "create_task"):
self._send_json(403, {"error": "forbidden"})
return
now = time.time()
new_status = "running" if task["status"] == "queued" else task["status"]
conn = self._db()
try:
conn.execute(
"UPDATE tasks SET status = ?, last_heartbeat = ?,"
" updated_at = ? WHERE id = ?",
(new_status, now, now, task_id),
)
conn.commit()
task = _task_to_dict(
conn.execute(
"SELECT * FROM tasks WHERE id = ?", (task_id,)
).fetchone()
)
finally:
conn.close()
self._send_json(200, task)
# -- 管理 ----------------------------------------------------------
def _admin_sweep(self):
body, err = self._read_json()
if err:
self._send_json(400, {"error": err})
return
threshold = body.get(
"threshold_seconds", DEFAULT_SWEEP_THRESHOLD_SECONDS)
try:
threshold = int(threshold)
except (TypeError, ValueError):
self._send_json(400, {"error": "threshold_seconds must be an integer"})
return
swept = sweep_stale_tasks(self.server.db_path, threshold)
self._send_json(200, {"swept": swept})
# -- webhooks(签名认证,不走 X-API-Key)---------------------------
def _handle_github_webhook(self):
raw = self._read_body()
secret = os.environ.get("GITHUB_WEBHOOK_SECRET")
if not secret:
self._send_json(503, {"error": "github webhook secret not configured"})
return
signature = self.headers.get("X-Hub-Signature-256")
if not verify_github_signature(secret, raw, signature):
self._send_json(401, {"error": "unauthorized"})
return
try:
payload = json.loads(raw.decode("utf-8")) if raw else {}
except (ValueError, UnicodeDecodeError):
self._send_json(400, {"error": "invalid json"})
return
if not isinstance(payload, dict):
self._send_json(400, {"error": "body must be a JSON object"})
return
# 只解析不执行:/fix-review 的真正执行由控制面后续调 host.py 完成。
result = parse_github_pr_comment(payload)
if result is None:
self._send_json(200, {"result": None})
else:
self._send_json(200, result)
def _handle_slack_webhook(self):
raw = self._read_body()
secret = os.environ.get("SLACK_SIGNING_SECRET")
if not secret:
self._send_json(503, {"error": "slack webhook secret not configured"})
return
signature = self.headers.get("X-Slack-Signature")
timestamp = self.headers.get("X-Slack-Request-Timestamp")
if not verify_slack_signature(secret, timestamp, raw, signature):
self._send_json(401, {"error": "unauthorized"})
return
# 只解析不执行任务:Slack 斜杠命令的执行由控制面后续决定。
content_type = self.headers.get("Content-Type") or ""
text = ""
if "application/x-www-form-urlencoded" in content_type:
parsed = parse_qs(raw.decode("utf-8", "replace"))
values = parsed.get("text")
text = values[0] if values else ""
elif raw:
try:
payload = json.loads(raw.decode("utf-8"))
if isinstance(payload, dict):
text = payload.get("text", "") or ""
else:
text = raw.decode("utf-8", "replace")
except (ValueError, UnicodeDecodeError):
text = raw.decode("utf-8", "replace")
self._send_json(200, parse_slack_command(text))
def create_server(db_path, host="127.0.0.1", port=8765):
"""建 server 实例(不自动 serve,供测试与 __main__ 共用)。"""
db_path = str(db_path)
init_db(db_path)
# 确保 AuthStore 的三张表也存在,后续请求才能直接查 project_members。
with _auth_store(db_path):
pass
server = ThreadingHTTPServer((host, port), CloudControlPlaneHandler)
server.db_path = db_path
server.daemon_threads = True
return server
def main(argv=None):
parser = argparse.ArgumentParser(description="cloud control plane")
parser.add_argument("--host", default="127.0.0.1")
parser.add_argument("--port", type=int, default=8765)
parser.add_argument("--db", default="cloud-control-plane.db")
args = parser.parse_args(argv)
server = create_server(args.db, args.host, args.port)
print(
"cloud-control-plane listening on %s:%d db=%s"
% (args.host, server.server_address[1], args.db),
flush=True,
)
try:
server.serve_forever()
except KeyboardInterrupt:
pass
if __name__ == "__main__":
main()