diff --git a/cloud/README.md b/cloud/README.md index 1c8731a..bfd46cf 100644 --- a/cloud/README.md +++ b/cloud/README.md @@ -159,6 +159,45 @@ client_secret（只走环境变量），不是无认证后门。 没有返回 404）后恢复原来的 oauth 账号登录。查看自己的绑定： `GET /oauth/{provider}/bindings`（须持本站 key，只看得到自己的）。 +### GitHub App 一键安装（已实现安装一半；App-token API 动作未做） + +目标：把手动建 webhook（填 URL、secret、选事件）变成“点一下 +安装”。流程用 GitHub manifest 流程（GitHub 文档标准流程）： + +1. 操作方持本站 key 调 `POST /github-app/setup/state`（body 可含 + org），返回 `{state, setup_url}`；要求环境变量 + `GITHUB_APP_PUBLIC_BASE`（公网域名，如 `https://api.diante.me`， + manifest 里的 hook / 回调地址都由它拼出）。 +2. 用户打开 setup_url：表单页含 manifest JSON（App 名默认 + `codex-app-server`，可由 `GITHUB_APP_NAME` / `GITHUB_APP_URL` + 改；权限 contents:write、pull_requests:write、issues:read， + 事件 issue_comment、pull_request；默认私有 App，只有所有者 + 能安装，`GITHUB_APP_PUBLIC=1` 可翻成公开）与 state，点提交 + 到 GitHub 建 App 页（个人账号 `/settings/apps/new`，组织账号 + `/organizations/{org}/settings/apps/new`），用户确认 Create。 +3. GitHub 跳回 `GET /github-app/callback?code=...&state=...`： + server 验 setup state（一次性、1 小时过期，manifest code 同样 + 1 小时内有效）→ POST + `api.github.com/app-manifests/{code}/conversions` 换回 id、slug、 + webhook_secret、pem → 存库里 `github_app_config` 表（单行）。 + 成功页只显示 App 身份与安装链接 + `https://github.com/apps//installations/new`，绝不显示 + webhook_secret 与 pem。 +4. 用户点安装：installation 事件到 `/webhooks/github`，记 + `github_app_installations` 表（created/added 增行、removed 减 + 行、deleted 清安装），`GET /github-app/status`（须持本站 key） + 可查配置与安装列表，同样不含 secret/pem。 + +验签与去重：webhook 验签同时接受 App secret（库里）与 env 里 +`GITHUB_WEBHOOK_SECRET`（手动 webhook，过渡期共存）；App 装好 +后应删掉手动 webhook，避免同一事件送两遍；即使共存， +/fix-review 也有内容去重（repo + PR + 评论 id， +`github_fix_review_dedup` 表），只建一次任务。 + +诚实边界：pem 虽然存下来了，但本阶段不用它签 JWT、不用它换 +installation token；推分支、建 PR 仍走 gh + PAT（见“云端派发与 +自动 PR”），App-token API 动作是下一阶段。 + ### 推送凭证清单 - webhook / slack：需要可注入 sender（worker 侧），HTTP 层没有 diff --git a/cloud/github_app.py b/cloud/github_app.py new file mode 100644 index 0000000..c3b1c56 --- /dev/null +++ b/cloud/github_app.py @@ -0,0 +1,412 @@ +"""GitHub App 一键安装：manifest 建 App + 安装记录 + App webhook。 + +范围（诚实边界）：本模块只做“安装”这一半——manifest 生成、 +建 App 回调换凭证、安装事件记录、App webhook 验签。App 的 +pem 虽然存下来了，但本模块不用它签 JWT、不用它换 installation +token：推分支、建 PR 目前仍走 gh + PAT（见 cloud/pr.py）， +App-token API 动作是下一阶段，不在本模块假装已做。 + +流程（GitHub manifest 流程，见 GitHub 文档）： + POST /github-app/setup/state（持本站 key）-> state + setup_url + -> 用户打开 setup 页（表单含 manifest JSON + state）点提交 + -> GitHub 建 App 页（用户确认 Create） + -> GitHub 跳回 /github-app/callback?code=...&state=... + -> server 验 setup state（一次性、1 小时过期）-> 换 code + （POST api.github.com/app-manifests/{code}/conversions， + 无需认证，1 小时内有效）-> 存 App 配置 + -> 成功页给安装链接（https://github.com/apps// + installations/new），用户点安装 -> installation 事件 + 到 /webhooks/github -> 记安装记录。 + +凭证存放：App 配置存数据库 github_app_config 表（单行）， +不进仓库、不进日志、不出现在任何 API 响应里；webhook_secret +与 pem 只在 server 进程内读（验签用 secret，JWT 阶段用 pem）。 +与 env 里 GITHUB_WEBHOOK_SECRET 的关系：手动 webhook 仍用 +env secret，App webhook 用库里 secret，验签时两个都试（过渡 +期共存）；App 装好后应删掉手动 webhook，避免同一事件送两 +遍（/fix-review 另有内容去重兜底，见 claim_fix_review）。 + +零第三方依赖，只用 Python 标准库；换 code 与 userinfo 均可 +注入 fake http 客户端做单测。 +""" + +import json +import secrets +import time +import urllib.request +from cloud.db import connect as _db_connect + +# setup state 有效期：与 manifest code 的 1 小时对齐。 +SETUP_STATE_TTL_SECONDS = 3600 + +# manifest 默认值：App 名与主页；hook / 回调地址由部署的公网 +# 域名（GITHUB_APP_PUBLIC_BASE）拼出，不硬编码 diante.me。 +DEFAULT_APP_NAME = "codex-app-server" +DEFAULT_APP_URL = "https://github.com/openqj/codex-app-server" + +# App 权限与事件：与当前 PAT 能力对齐（读仓库、推分支、建 PR、 +# 读 issue），不多要；webhook 只订阅 /fix-review 与 PR 相关事件。 +APP_PERMISSIONS = { + "contents": "write", + "pull_requests": "write", + "issues": "read", +} +APP_EVENTS = ["issue_comment", "pull_request"] + + +class AppError(Exception): + """App 流程失败：换 code 失败、响应缺字段等。 + + reason 取值：``exchange_failed`` / ``config_incomplete`` / + ``state_invalid`` / ``state_expired``。 + """ + + def __init__(self, message, reason=None): + super().__init__(message) + self.reason = reason + + +def build_app_manifest(public_base, app_name=DEFAULT_APP_NAME, + app_url=DEFAULT_APP_URL, public=False): + """拼 manifest dict：GitHub 建 App 表单直接提交这份 JSON。 + + public=False 是单租户部署的默认（只有 App 所有者能安装）； + 多租户 SaaS 形态才需要 public=True（改环境变量翻转）。 + """ + if not public_base or not isinstance(public_base, str): + raise ValueError("public_base required") + base = public_base.rstrip("/") + return { + "name": app_name, + "url": app_url, + "hook_attributes": {"url": base + "/webhooks/github"}, + "redirect_url": base + "/github-app/callback", + "callback_urls": [base + "/github-app/callback"], + "public": bool(public), + "default_permissions": dict(APP_PERMISSIONS), + "default_events": list(APP_EVENTS), + } + + +class SetupStateStore: + """setup state 存储：一次性、1 小时过期，防回调被冒用。 + + 表 github_app_setup_states：state（主键）、created_at。 + state 只由持本站 key 调 POST /github-app/setup/state 的 + 操作方签发；回调验到即删，同一 state 第二次回调必然失败。 + """ + + def __init__(self, db_path): + self.db_path = str(db_path) + self.conn = _db_connect(self.db_path) + self.conn.execute( + "CREATE TABLE IF NOT EXISTS github_app_setup_states(" + "state TEXT PRIMARY KEY, created_at REAL)" + ) + self.conn.commit() + + def create_state(self): + state = secrets.token_urlsafe(32) + self.conn.execute( + "INSERT INTO github_app_setup_states(state, created_at)" + " VALUES(?, ?)", + (state, time.time()), + ) + self.conn.commit() + return state + + def consume_state(self, state, now=None, + ttl_seconds=SETUP_STATE_TTL_SECONDS): + """校验并消耗 state：成功返回 True，失败返回 False。 + + 不存在 -> False；过期 -> 删行并返回 False；成功 -> 删行 + 并返回 True。``now`` 只为单测注入时间。 + """ + if not state: + return False + if now is None: + now = time.time() + row = self.conn.execute( + "SELECT created_at FROM github_app_setup_states" + " WHERE state = ?", + (state,), + ).fetchone() + if row is None: + return False + self.conn.execute( + "DELETE FROM github_app_setup_states WHERE state = ?", + (state,), + ) + self.conn.commit() + if now > row[0] + ttl_seconds: + return False + return True + + +def _urllib_post_json(url, headers): + """默认的换 code 实现：POST 空 body，返回解析后的 dict。""" + request = urllib.request.Request(url, data=b"", method="POST", + headers=headers) + with urllib.request.urlopen(request, timeout=30) as response: + raw = response.read().decode("utf-8", "replace") + try: + parsed = json.loads(raw) + except ValueError as exc: + raise AppError("manifest conversion response is not JSON: %s" + % (exc,), reason="exchange_failed") from exc + if not isinstance(parsed, dict): + raise AppError("manifest conversion response is not an object", + reason="exchange_failed") + return parsed + + +def exchange_manifest_code(code, http_client=None): + """用 manifest code 换 App 配置：返回 {id, slug, client_id, + webhook_secret, pem, html_url}（缺关键字段即 AppError）。 + + http_client 注入约定（单测用）：``http_client(url, headers) + -> dict``。真实路径无需认证（code 本身就是一次性凭证）， + 1 小时内必须完成。 + """ + if not code or not isinstance(code, str): + raise AppError("code required", reason="exchange_failed") + url = "https://api.github.com/app-manifests/%s/conversions" % code + headers = {"Accept": "application/vnd.github+json", + "User-Agent": "codex-app-server"} + client = http_client if http_client is not None else _urllib_post_json + try: + result = client(url, headers) + except AppError: + raise + except Exception as exc: + raise AppError( + "manifest conversion failed: %s" % (exc,), + reason="exchange_failed", + ) from exc + if not isinstance(result, dict) or result.get("error"): + raise AppError("manifest conversion failed: %r" % (result,), + reason="exchange_failed") + missing = [k for k in ("id", "webhook_secret", "pem") + if not result.get(k)] + if missing: + raise AppError( + "manifest conversion response missing: %s" % ",".join(missing), + reason="config_incomplete") + return { + "app_id": result.get("id"), + "slug": result.get("slug"), + "client_id": result.get("client_id"), + "webhook_secret": result.get("webhook_secret"), + "pem": result.get("pem"), + "html_url": result.get("html_url"), + } + + +def save_app_config(conn, config): + """存 App 配置（单行，id 恒为 1；重复 setup 直接覆盖）。 + + 调用方须保证 config 含 webhook_secret 与 pem；本函数不校验 + 它们的值，只负责落库。secret/pem 绝不出现在返回值里。 + """ + conn.execute( + "CREATE TABLE IF NOT EXISTS github_app_config(" + "id INTEGER PRIMARY KEY CHECK(id = 1), " + "app_id TEXT, slug TEXT, client_id TEXT, " + "webhook_secret TEXT, pem TEXT, html_url TEXT, created_at REAL)" + ) + conn.execute( + "INSERT OR REPLACE INTO github_app_config(id, app_id, slug," + " client_id, webhook_secret, pem, html_url, created_at)" + " VALUES(1, ?, ?, ?, ?, ?, ?, ?)", + (config.get("app_id"), config.get("slug"), + config.get("client_id"), config.get("webhook_secret"), + config.get("pem"), config.get("html_url"), time.time()), + ) + conn.commit() + + +def get_app_config(conn): + """取 App 配置 dict，未配置返回 None。 + + 返回里含 webhook_secret 与 pem——调用方只许在 server 进程内 + 用（验签 / JWT），绝不放进任何 API 响应；公开状态接口须 + 另外拼不含 secret 的视图。 + """ + conn.execute( + "CREATE TABLE IF NOT EXISTS github_app_config(" + "id INTEGER PRIMARY KEY CHECK(id = 1), " + "app_id TEXT, slug TEXT, client_id TEXT, " + "webhook_secret TEXT, pem TEXT, html_url TEXT, created_at REAL)" + ) + row = conn.execute( + "SELECT app_id, slug, client_id, webhook_secret, pem, html_url," + " created_at FROM github_app_config WHERE id = 1", + ).fetchone() + if row is None: + return None + return { + "app_id": row[0], "slug": row[1], "client_id": row[2], + "webhook_secret": row[3], "pem": row[4], "html_url": row[5], + "created_at": row[6], + } + + +def public_app_status(conn): + """公开状态视图：不含 secret/pem，只给配置与否 + 身份信息。""" + config = get_app_config(conn) + if config is None: + return {"configured": False, "app_id": None, "slug": None, + "html_url": None} + return {"configured": True, "app_id": config["app_id"], + "slug": config["slug"], "html_url": config["html_url"]} + + +def _ensure_installations_table(conn): + conn.execute( + "CREATE TABLE IF NOT EXISTS github_app_installations(" + "installation_id TEXT, account_login TEXT, " + "repo_full_name TEXT, created_at REAL, " + "UNIQUE(installation_id, repo_full_name))" + ) + conn.commit() + + +def record_installation_repos(conn, installation_id, account_login, repos): + """记安装的仓库行（按 (installation_id, repo) 去重，多次 + 同步不翻倍）。repos 为 full_name 字符串列表。""" + _ensure_installations_table(conn) + now = time.time() + for repo in repos or []: + conn.execute( + "INSERT OR IGNORE INTO github_app_installations(" + "installation_id, account_login, repo_full_name, created_at)" + " VALUES(?, ?, ?, ?)", + (str(installation_id), account_login, repo, now), + ) + conn.commit() + + +def remove_installation_repos(conn, installation_id, repos): + """删安装的部分仓库行（removed 事件用）。""" + _ensure_installations_table(conn) + for repo in repos or []: + conn.execute( + "DELETE FROM github_app_installations" + " WHERE installation_id = ? AND repo_full_name = ?", + (str(installation_id), repo), + ) + conn.commit() + + +def remove_installation(conn, installation_id): + """删整个安装的全部仓库行（deleted / suspended 后的清理用， + 返回删掉的行数）。""" + _ensure_installations_table(conn) + cur = conn.execute( + "DELETE FROM github_app_installations WHERE installation_id = ?", + (str(installation_id),), + ) + conn.commit() + return cur.rowcount + + +def list_installations(conn): + """列出全部安装行（按 installation_id / repo 排序）。""" + _ensure_installations_table(conn) + rows = conn.execute( + "SELECT installation_id, account_login, repo_full_name, created_at" + " FROM github_app_installations" + " ORDER BY installation_id, repo_full_name", + ).fetchall() + return [{"installation_id": r[0], "account_login": r[1], + "repo_full_name": r[2], "created_at": r[3]} for r in rows] + + +def parse_installation_event(payload): + """解析 installation 事件，返回动作 dict 或 None。 + + 只认 action 在 created / deleted / added / removed / + new_permissions_accepted 且带 installation 的 payload；其它 + 事件返回 None（调用方继续走原有解析）。repositories 取 + full_name：created 用 payload repositories，added 用 + repositories_added，removed 用 repositories_removed。 + """ + if not isinstance(payload, dict): + return None + action = payload.get("action") + if action not in ("created", "deleted", "added", "removed", + "new_permissions_accepted"): + return None + installation = payload.get("installation") + if not isinstance(installation, dict): + return None + account = installation.get("account") or {} + + def names(key): + out = [] + for repo in payload.get(key) or []: + if isinstance(repo, dict) and repo.get("full_name"): + out.append(repo["full_name"]) + return out + + if action == "added": + repos = names("repositories_added") + elif action == "removed": + repos = names("repositories_removed") + elif action == "deleted": + repos = [] + else: + repos = names("repositories") + return { + "action": action, + "installation_id": installation.get("id"), + "account_login": account.get("login"), + "repos": repos, + } + + +def claim_fix_review(conn, repo, pr_number, comment_id): + """ /fix-review 内容去重：同一仓库 + PR + 评论只建一次任务。 + + 手动 webhook 与 App webhook 共存时，同一评论会送两遍， + delivery_id 不同、内容相同——delivery 去重拦不住，内容去重 + 才能拦。返回 (is_new, task_id)：首次认领 True；已认领过 + 返回 False 与当初建的 task_id（可能为 None：认领方还没来 + 得及把 task_id 写回，任务正在建）。comment_id 为 None 时 + 无法去重，直接放行（不阻塞合法任务）。 + """ + conn.execute( + "CREATE TABLE IF NOT EXISTS github_fix_review_dedup(" + "repo TEXT, pr_number INTEGER, comment_id TEXT, " + "task_id TEXT, created_at REAL, " + "UNIQUE(repo, pr_number, comment_id))" + ) + if comment_id is None: + return True, None + cur = conn.execute( + "INSERT OR IGNORE INTO github_fix_review_dedup(" + "repo, pr_number, comment_id, task_id, created_at)" + " VALUES(?, ?, ?, NULL, ?)", + (repo, pr_number, str(comment_id), time.time()), + ) + conn.commit() + if cur.rowcount > 0: + return True, None + row = conn.execute( + "SELECT task_id FROM github_fix_review_dedup" + " WHERE repo = ? AND pr_number = ? AND comment_id = ?", + (repo, pr_number, str(comment_id)), + ).fetchone() + return False, row[0] if row else None + + +def set_fix_review_task_id(conn, repo, pr_number, comment_id, task_id): + """认领后把 task_id 写回内容去重行（供重复送达时回读）。""" + if comment_id is None: + return + conn.execute( + "UPDATE github_fix_review_dedup SET task_id = ?" + " WHERE repo = ? AND pr_number = ? AND comment_id = ?", + (task_id, repo, pr_number, str(comment_id)), + ) + conn.commit() diff --git a/cloud/server.py b/cloud/server.py index f51201e..9cf2dd7 100644 --- a/cloud/server.py +++ b/cloud/server.py @@ -48,7 +48,7 @@ import uuid from collections import deque from contextlib import contextmanager from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer -from urllib.parse import parse_qs, urlparse +from urllib.parse import parse_qs, quote, urlparse from cloud.auth import AuthStore from cloud.cost import CostStore @@ -85,12 +85,30 @@ from cloud.push import ( dispatch_event as _push_dispatch, list_subscriptions as _push_list, ) +from cloud.github_app import ( + DEFAULT_APP_NAME, + DEFAULT_APP_URL, + AppError, + SetupStateStore, + build_app_manifest, + claim_fix_review, + exchange_manifest_code, + get_app_config, + list_installations, + parse_installation_event, + public_app_status, + record_installation_repos, + remove_installation, + remove_installation_repos, + save_app_config, + set_fix_review_task_id, +) from cloud.webhooks import ( WebhookDeliveryStore, parse_github_check_run, parse_github_pr_comment, parse_slack_command, - verify_github_signature, + verify_github_signature_any, verify_slack_signature, ) @@ -559,6 +577,22 @@ class CloudControlPlaneHandler(BaseHTTPRequestHandler): else: self._send_json(404, {"error": "not found"}) return + if path.startswith("/github-app/"): + # GitHub App 一键安装：setup/state 与 status 须持本站 key + # （在处理器内验）；setup 表单页与 callback 靠 setup + # state 凭证（一次性、1 小时），不走 X-API-Key（浏览器 + # 回调带不了请求头，state 就是凭证）。 + if method == "POST" and path == "/github-app/setup/state": + self._github_app_setup_state() + elif method == "GET" and path == "/github-app/setup": + self._github_app_setup_page() + elif method == "GET" and path == "/github-app/callback": + self._github_app_callback() + elif method == "GET" and path == "/github-app/status": + self._github_app_status() + else: + self._send_json(404, {"error": "not found"}) + return if path.startswith("/webhooks/"): # webhook 靠签名认证（外部服务无法持 X-API-Key），不走 API key。 if method != "POST": @@ -2042,6 +2076,196 @@ class CloudControlPlaneHandler(BaseHTTPRequestHandler): return self._send_json(200, {"unbound": True, "provider": provider}) + # -- GitHub App 一键安装 ------------------------------------------ + def _github_app_public_base(self): + base = os.environ.get("GITHUB_APP_PUBLIC_BASE") or "" + return base.rstrip("/") or None + + def _github_app_error_page(self, reason, message): + """App 流程错误页：只含 reason 与提示，不含任何 secret、 + pem、code、state（message 来自流程错误，本身不含密钥， + 仍做 HTML 转义）。""" + return ( + "" + "GitHub App setup failed" + "

GitHub App setup failed

" + "

reason: %s

%s

" + "" + % (html.escape(str(reason or "error")), + html.escape(str(message)))) + + def _github_app_setup_state(self): + """签发 setup state：须持本站 key（只有操作方能发起建 App）。 + + 要求环境变量 GITHUB_APP_PUBLIC_BASE（公网域名，如 + https://api.diante.me）：manifest 里的 hook / 回调地址 + 都由它拼出。返回 {state, setup_url}，setup_url 是用户 + 在浏览器里打开的表单页；state 一次性、1 小时过期。 + """ + user_id = self._authenticate() + if user_id is None: + self._send_json(401, {"error": "unauthorized"}) + return + base = self._github_app_public_base() + if not base: + self._send_json(400, { + "error": "GITHUB_APP_PUBLIC_BASE not configured"}) + return + body, err = self._read_json() + if err: + body = {} + org = (body or {}).get("org") + store = SetupStateStore(self.server.db_path) + try: + state = store.create_state() + finally: + try: + store.conn.close() + except Exception: + pass + setup_url = base + "/github-app/setup?state=" + state + if org: + setup_url += "&org=" + quote(str(org)) + self._send_json(200, {"state": state, "setup_url": setup_url}) + + def _github_app_setup_page(self): + """建 App 表单页：公开页，但 state 必须是操作方签发的。 + + 无 state 或 state 未签发时不渲染表单（state 是回调的 + 唯一凭证，公开页自己现发 state 等于谁都能覆盖 App 配置）。 + 表单按 GitHub manifest 文档 POST 到 github.com 的建 App + 页（个人账号 /settings/apps/new，组织账号 + /organizations/{org}/settings/apps/new），manifest JSON + 由 JS 填进隐藏字段；用户在 GitHub 页确认 Create 后跳回 + 本站 /github-app/callback。 + """ + query = parse_qs(urlparse(self.path).query) + state = (query.get("state") or [None])[0] + org = (query.get("org") or [None])[0] + if not state: + self._send_html(400, self._github_app_error_page( + "missing_state", + "setup state required: ask the operator to issue one")) + return + base = self._github_app_public_base() + if not base: + self._send_html(400, self._github_app_error_page( + "not_configured", "GITHUB_APP_PUBLIC_BASE not configured")) + return + manifest = build_app_manifest( + base, + app_name=os.environ.get("GITHUB_APP_NAME") or DEFAULT_APP_NAME, + app_url=os.environ.get("GITHUB_APP_URL") or DEFAULT_APP_URL, + public=os.environ.get("GITHUB_APP_PUBLIC") == "1") + manifest_json = json.dumps(manifest).replace("<", "\\u003c") + if org: + action = ("https://github.com/organizations/%s" + "/settings/apps/new?state=%s" + % (quote(str(org)), + quote(state))) + else: + action = ("https://github.com/settings/apps/new?state=%s" + % quote(state)) + perms = ", ".join("%s: %s" % kv + for kv in manifest["default_permissions"].items()) + events = ", ".join(manifest["default_events"]) + page = ( + "" + "Create GitHub App" + "

Create GitHub App

" + "

App name: %s

" + "

Webhook URL: %s

" + "

Permissions: %s

" + "

Events: %s

" + "

Click Submit to create the app on GitHub. GitHub will " + "redirect back here to finish setup.

" + "
" + "" + "" + "
" + "" + "" + % (html.escape(manifest["name"]), + html.escape(manifest["hook_attributes"]["url"]), + html.escape(perms), html.escape(events), + html.escape(action), manifest_json)) + self._send_html(200, page) + + def _github_app_callback(self): + """建 App 回调：验 setup state -> 换 code -> 存配置。 + + state 一次性、1 小时过期；换 code 失败或响应缺字段时 + 报错误页，不存任何东西。成功页只显示 App 身份（slug、 + id、主页）与安装链接，绝不显示 webhook_secret 与 pem + （两者只在库里，server 进程内用）。 + """ + query = parse_qs(urlparse(self.path).query) + code = (query.get("code") or [None])[0] + state = (query.get("state") or [None])[0] + if not code or not state: + self._send_html(400, self._github_app_error_page( + "missing_params", "code and state required")) + return + store = SetupStateStore(self.server.db_path) + try: + ok = store.consume_state(state) + finally: + try: + store.conn.close() + except Exception: + pass + if not ok: + self._send_html(400, self._github_app_error_page( + "invalid_state", "setup state invalid or expired")) + return + http_client = getattr(self.server, "github_app_exchange_client", + None) + try: + config = exchange_manifest_code(code, http_client=http_client) + except AppError as e: + self._send_html(400, self._github_app_error_page( + e.reason or "exchange_failed", str(e))) + return + conn = self._db() + try: + save_app_config(conn, config) + finally: + conn.close() + slug = config.get("slug") or "" + page = ( + "" + "GitHub App created" + "

GitHub App created

" + "

App: %s

" + "

App ID: %s

" + "

Install it on your repositories:

" + "

" + "Install this GitHub App

" + "

After installing, webhooks from the app are verified " + "automatically. API actions (push, PR) still use the " + "existing PAT until the App-token phase; you may remove " + "the manual webhook once the app is installed.

" + "" + % (html.escape(slug), + html.escape(str(config.get("app_id"))), + html.escape(slug))) + self._send_html(200, page) + + def _github_app_status(self): + """App 配置与安装状态：须持本站 key；绝不返回 secret/pem。""" + user_id = self._authenticate() + if user_id is None: + self._send_json(401, {"error": "unauthorized"}) + return + conn = self._db() + try: + status = public_app_status(conn) + status["installations"] = list_installations(conn) + finally: + conn.close() + self._send_json(200, status) + # -- 推送订阅与分发 --------------------------------------------------- def _push_create_endpoint(self, user_id, project_id): body, err = self._read_json() @@ -2088,23 +2312,41 @@ class CloudControlPlaneHandler(BaseHTTPRequestHandler): 顺序是故意的：secret 未配 503、签名无效 401（两者都在 去重之前，避免未验签的重复送达污染去重表）；同一 X-GitHub-Delivery 第二次送达直接回 200 duplicate，不 - 重复建任务、不重复记审计。/fix-review 解析成功后在 - 控制面建一个 queued 任务（prompt 含 repo、PR 号、评论 - 原文、作者），由 worker 认领执行——webhook 本身不跑模型。 - 任务归属：按项目名 == repo full_name 匹配项目；没有匹配 - 项目时不凭空建项目、不建任务，返回 task_id 为 None 并 - 说明原因（worker 的 repo 仍是配置里的 repo，不是任意 - GitHub 仓库的自动克隆，见 README）。check_run / - check_suite 只做通知解析，返回解析结果而不建任务、不改 - 云端任务状态（CI 状态回填由 pr-checks 主动拉取）。 + 重复建任务、不重复记审计。验签接受两个 secret：App + webhook secret（库里，App 装好后）与 env 里 + GITHUB_WEBHOOK_SECRET（手动 webhook，过渡期共存；App + 装好、手动 webhook 删掉后自然不再被命中）。 + /fix-review 解析成功后在控制面建一个 queued 任务（prompt + 含 repo、PR 号、评论原文、作者），由 worker 认领执行—— + webhook 本身不跑模型。任务归属：按项目名 == repo + full_name 匹配项目；没有匹配项目时不凭空建项目、不建 + 任务，返回 task_id 为 None 并说明原因（worker 的 repo + 仍是配置里的 repo，不是任意 GitHub 仓库的自动克隆， + 见 README）。check_run / check_suite 只做通知解析， + 返回解析结果而不建任务、不改云端任务状态（CI 状态回填 + 由 pr-checks 主动拉取）。installation 事件记安装记录 + （created/added 增行、removed 减行、deleted 清安装）， + 不建任务。手动 webhook 与 App webhook 共存时同一评论 + 会送两遍（delivery_id 不同）：/fix-review 另有内容去重 + （repo + PR + 评论 id），只建一次任务。 """ raw = self._read_body() - secret = os.environ.get("GITHUB_WEBHOOK_SECRET") - if not secret: + app_secret = None + conn = self._db() + try: + config = get_app_config(conn) + if config is not None: + app_secret = config.get("webhook_secret") + finally: + conn.close() + secrets = [s for s in + (app_secret, os.environ.get("GITHUB_WEBHOOK_SECRET")) + if s] + if not secrets: self._send_json(503, {"error": "github webhook secret not configured"}) return signature = self.headers.get("X-Hub-Signature-256") - if not verify_github_signature(secret, raw, signature): + if not verify_github_signature_any(secrets, raw, signature): self._send_json(401, {"error": "unauthorized"}) return try: @@ -2131,6 +2373,42 @@ class CloudControlPlaneHandler(BaseHTTPRequestHandler): store.close() except Exception: pass + # installation 事件：记安装记录，不建任务。 + if event == "installation": + inst = parse_installation_event(payload) + if inst is None: + self._send_json(200, {"result": None}) + return + conn = self._db() + try: + action = inst["action"] + if action == "created": + record_installation_repos( + conn, inst["installation_id"], + inst["account_login"], inst["repos"]) + elif action == "added": + record_installation_repos( + conn, inst["installation_id"], + inst["account_login"], inst["repos"]) + elif action == "removed": + remove_installation_repos( + conn, inst["installation_id"], inst["repos"]) + elif action == "deleted": + remove_installation(conn, inst["installation_id"]) + # new_permissions_accepted：权限已接受，无需改行。 + finally: + conn.close() + with _auth_store(self.server.db_path) as auth: + auth.audit("system", "", "github_app_installation", + "%s:%s" % (inst["installation_id"], + inst["action"])) + self._send_json(200, { + "result": "installation_recorded", + "installation_id": inst["installation_id"], + "action": inst["action"], + "repos": inst["repos"], + }) + return # check_run / check_suite：只解析，不建任务、不改状态。 check = parse_github_check_run(payload) if check is not None: @@ -2163,6 +2441,17 @@ class CloudControlPlaneHandler(BaseHTTPRequestHandler): error="no matching project for repo")) return project_id = project_row[0] + # 内容去重：手动 webhook 与 App webhook 共存时同一评论 + # 送两遍（delivery_id 不同），只建一次任务；重复送达 + # 回读当初的 task_id，不重复记审计。 + comment_id = (payload.get("comment") or {}).get("id") + is_new, existing_task_id = claim_fix_review( + conn, repo_name, result.get("pr_number"), comment_id) + if not is_new: + self._send_json(200, dict( + result, task_id=existing_task_id, + project_id=project_id, duplicate=True)) + return task_id = uuid.uuid4().hex now = time.time() prompt = ( @@ -2196,6 +2485,9 @@ class CloudControlPlaneHandler(BaseHTTPRequestHandler): duplicate=True)) return raise + set_fix_review_task_id( + conn, repo_name, result.get("pr_number"), comment_id, + task_id) finally: conn.close() with _auth_store(self.server.db_path) as auth: diff --git a/cloud/tests/test_github_app.py b/cloud/tests/test_github_app.py new file mode 100644 index 0000000..9adcaf0 --- /dev/null +++ b/cloud/tests/test_github_app.py @@ -0,0 +1,439 @@ +"""GitHub App 一键安装单测：manifest、setup state、换 code、 +配置、安装记录、内容去重、双 secret 验签、HTTP 端点。 + +不联网：换 code 挂 server.github_app_exchange_client 的 fake； +测试里出现的 secret、pem、key 一律是 fake 占位，绝无真实 +secret、真实 GitHub 账号。 +""" + +import hashlib +import hmac +import http.client +import json +import os +import sqlite3 +import sys +import tempfile +import threading +import unittest +from pathlib import Path + +try: + from cloud.auth import AuthStore + from cloud.db import connect as _db_connect + from cloud.github_app import ( + AppError, + SetupStateStore, + build_app_manifest, + claim_fix_review, + exchange_manifest_code, + get_app_config, + list_installations, + parse_installation_event, + public_app_status, + record_installation_repos, + remove_installation, + remove_installation_repos, + save_app_config, + set_fix_review_task_id, + ) + from cloud.server import create_server + from cloud.webhooks import verify_github_signature_any +except ImportError: + sys.path.insert(0, str(Path(__file__).resolve().parents[2])) + from cloud.auth import AuthStore + from cloud.db import connect as _db_connect + from cloud.github_app import ( + AppError, + SetupStateStore, + build_app_manifest, + claim_fix_review, + exchange_manifest_code, + get_app_config, + list_installations, + parse_installation_event, + public_app_status, + record_installation_repos, + remove_installation, + remove_installation_repos, + save_app_config, + set_fix_review_task_id, + ) + from cloud.server import create_server + from cloud.webhooks import verify_github_signature_any + +BASE = "https://example.invalid" + + +def _fake_exchange(result=None): + def fake(url, headers): + assert url.startswith( + "https://api.github.com/app-manifests/"), url + return dict(result or { + "id": 424242, "slug": "fake-codex-app", + "client_id": "Iv1.fake", "webhook_secret": "fake-app-secret", + "pem": "-----BEGIN RSA PRIVATE KEY-----\nfake-pem\n" + "-----END RSA PRIVATE KEY-----\n", + "html_url": "https://github.com/apps/fake-codex-app"}) + return fake + + +class ManifestTestCase(unittest.TestCase): + def test_manifest_urls_and_permissions(self): + m = build_app_manifest(BASE) + self.assertEqual(m["hook_attributes"]["url"], + BASE + "/webhooks/github") + self.assertEqual(m["redirect_url"], BASE + "/github-app/callback") + self.assertEqual(m["default_permissions"]["contents"], "write") + self.assertEqual(m["default_permissions"]["pull_requests"], "write") + self.assertEqual(m["default_permissions"]["issues"], "read") + self.assertIn("issue_comment", m["default_events"]) + self.assertIn("pull_request", m["default_events"]) + self.assertFalse(m["public"]) + + def test_manifest_requires_base(self): + with self.assertRaises(ValueError): + build_app_manifest("") + + +class SetupStateTestCase(unittest.TestCase): + def setUp(self): + self.tmp = tempfile.TemporaryDirectory() + self.db_path = os.path.join(self.tmp.name, "control.db") + + def tearDown(self): + self.tmp.cleanup() + + def test_consume_once_and_expired(self): + store = SetupStateStore(self.db_path) + state = store.create_state() + self.assertTrue(store.consume_state(state)) + self.assertFalse(store.consume_state(state)) + store.conn.close() + + def test_expired_state_rejected(self): + store = SetupStateStore(self.db_path) + state = store.create_state() + # 注入“1 小时之后”的 now：state 应过期且被删。 + self.assertFalse(store.consume_state( + state, now=9999999999.0)) + self.assertFalse(store.consume_state(state)) + store.conn.close() + + +class ExchangeTestCase(unittest.TestCase): + def test_exchange_ok(self): + config = exchange_manifest_code( + "fake-code", http_client=_fake_exchange()) + self.assertEqual(config["app_id"], 424242) + self.assertEqual(config["slug"], "fake-codex-app") + self.assertEqual(config["webhook_secret"], "fake-app-secret") + + def test_exchange_missing_fields(self): + with self.assertRaises(AppError) as ctx: + exchange_manifest_code( + "fake-code", + http_client=_fake_exchange({"id": 1, "pem": "x"})) + self.assertEqual(ctx.exception.reason, "config_incomplete") + + def test_exchange_http_error(self): + def boom(url, headers): + raise RuntimeError("boom") + with self.assertRaises(AppError) as ctx: + exchange_manifest_code("fake-code", http_client=boom) + self.assertEqual(ctx.exception.reason, "exchange_failed") + + +class ConfigAndInstallationTestCase(unittest.TestCase): + def setUp(self): + self.tmp = tempfile.TemporaryDirectory() + self.db_path = os.path.join(self.tmp.name, "control.db") + self.conn = _db_connect(self.db_path) + + def tearDown(self): + self.conn.close() + self.tmp.cleanup() + + def test_config_roundtrip_and_public_status(self): + self.assertIsNone(get_app_config(self.conn)) + save_app_config(self.conn, { + "app_id": 7, "slug": "fake-app", "client_id": "Iv1.x", + "webhook_secret": "fake-secret", "pem": "fake-pem", + "html_url": "https://github.com/apps/fake-app"}) + config = get_app_config(self.conn) + self.assertEqual(config["slug"], "fake-app") + status = public_app_status(self.conn) + self.assertTrue(status["configured"]) + self.assertNotIn("webhook_secret", status) + self.assertNotIn("pem", status) + + def test_installations_created_added_removed_deleted(self): + record_installation_repos(self.conn, 11, "octo", + ["octo/a", "octo/b"]) + record_installation_repos(self.conn, 11, "octo", ["octo/a"]) + rows = list_installations(self.conn) + self.assertEqual(len(rows), 2) # 重复同步不翻倍 + record_installation_repos(self.conn, 11, "octo", ["octo/c"]) + remove_installation_repos(self.conn, 11, ["octo/b"]) + repos = [r["repo_full_name"] for r in list_installations(self.conn)] + self.assertEqual(sorted(repos), ["octo/a", "octo/c"]) + removed = remove_installation(self.conn, 11) + self.assertEqual(removed, 2) + self.assertEqual(list_installations(self.conn), []) + + def test_parse_installation_event(self): + payload = {"action": "created", + "installation": {"id": 11, "account": {"login": "octo"}}, + "repositories": [{"full_name": "octo/a"}]} + inst = parse_installation_event(payload) + self.assertEqual(inst["installation_id"], 11) + self.assertEqual(inst["repos"], ["octo/a"]) + added = dict(payload, action="added", repositories_added=[ + {"full_name": "octo/b"}]) + self.assertEqual(parse_installation_event(added)["repos"], + ["octo/b"]) + self.assertIsNone(parse_installation_event( + {"action": "created"})) + self.assertIsNone(parse_installation_event( + {"action": "opened", "installation": {"id": 1}})) + + +class DedupTestCase(unittest.TestCase): + def setUp(self): + self.tmp = tempfile.TemporaryDirectory() + self.db_path = os.path.join(self.tmp.name, "control.db") + self.conn = _db_connect(self.db_path) + + def tearDown(self): + self.conn.close() + self.tmp.cleanup() + + def test_content_dedup(self): + is_new, task_id = claim_fix_review( + self.conn, "octo/a", 3, 99) + self.assertTrue(is_new) + self.assertIsNone(task_id) + set_fix_review_task_id(self.conn, "octo/a", 3, 99, "task-1") + is_new, task_id = claim_fix_review( + self.conn, "octo/a", 3, 99) + self.assertFalse(is_new) + self.assertEqual(task_id, "task-1") + # 另一个评论 id 不受影响。 + self.assertTrue(claim_fix_review( + self.conn, "octo/a", 3, 100)[0]) + # 无评论 id 时无法去重，直接放行。 + self.assertTrue(claim_fix_review( + self.conn, "octo/a", 3, None)[0]) + + +class SignatureAnyTestCase(unittest.TestCase): + def _sig(self, secret, raw): + return "sha256=" + hmac.new( + secret.encode(), raw, hashlib.sha256).hexdigest() + + def test_any_secret_matches(self): + raw = b'{"zen":"hi"}' + self.assertTrue(verify_github_signature_any( + ["app-secret", "legacy-secret"], raw, + self._sig("legacy-secret", raw))) + self.assertFalse(verify_github_signature_any( + ["app-secret", "legacy-secret"], raw, + self._sig("wrong-secret", raw))) + self.assertFalse(verify_github_signature_any([], raw, "sha256=x")) + + +class AppHttpTestCase(unittest.TestCase): + def setUp(self): + self.tmp = tempfile.TemporaryDirectory() + self.db_path = os.path.join(self.tmp.name, "control.db") + self.auth = AuthStore(self.db_path) + self.user_a, self.key_a = self.auth.create_user("user-a") + self.server = create_server(self.db_path, "127.0.0.1", 0) + self.port = self.server.server_address[1] + self.thread = threading.Thread( + target=self.server.serve_forever, daemon=True) + self.thread.start() + self.server.github_app_exchange_client = _fake_exchange() + + def tearDown(self): + self.server.shutdown() + self.server.server_close() + self.thread.join(timeout=5) + try: + self.auth.conn.close() + except Exception: + pass + self.tmp.cleanup() + os.environ.pop("GITHUB_APP_PUBLIC_BASE", None) + os.environ.pop("GITHUB_WEBHOOK_SECRET", None) + + def _req(self, method, path, body=None, key=None, headers=None): + conn = http.client.HTTPConnection("127.0.0.1", self.port, timeout=10) + try: + hdrs = dict(headers or {}) + payload = None + if body is not None: + payload = body if isinstance(body, str) else json.dumps(body) + hdrs.setdefault("Content-Type", "application/json") + if key is not None: + hdrs["X-API-Key"] = key + conn.request(method, path, body=payload, headers=hdrs) + resp = conn.getresponse() + raw = resp.read().decode() + try: + parsed = json.loads(raw) + except ValueError: + parsed = raw + return resp.status, parsed + finally: + conn.close() + + def _sig(self, secret, raw): + return "sha256=" + hmac.new( + secret.encode(), raw, hashlib.sha256).hexdigest() + + def _webhook(self, event, payload, secret, delivery): + raw = json.dumps(payload) + return self._req( + "POST", "/webhooks/github", body=raw, + headers={"X-GitHub-Event": event, + "X-GitHub-Delivery": delivery, + "X-Hub-Signature-256": self._sig(secret, raw.encode())}) + + def _issue_setup_state(self): + os.environ["GITHUB_APP_PUBLIC_BASE"] = BASE + status, body = self._req( + "POST", "/github-app/setup/state", {}, key=self.key_a) + self.assertEqual(status, 200, body) + return body["state"], body["setup_url"] + + def test_setup_state_requires_auth_and_base(self): + status, _ = self._req("POST", "/github-app/setup/state", {}) + self.assertEqual(status, 401) + os.environ.pop("GITHUB_APP_PUBLIC_BASE", None) + status, body = self._req( + "POST", "/github-app/setup/state", {}, key=self.key_a) + self.assertEqual(status, 400, body) + + def test_setup_page_and_callback_flow(self): + state, setup_url = self._issue_setup_state() + self.assertTrue(setup_url.startswith(BASE + "/github-app/setup")) + # 无 state 不渲染表单。 + status, page = self._req("GET", "/github-app/setup") + self.assertEqual(status, 400, page) + # 有 state 渲染表单：manifest 与 GitHub 建 App 地址都在。 + status, page = self._req( + "GET", "/github-app/setup?state=%s" % state) + self.assertEqual(status, 200, page) + self.assertIn("github.com/settings/apps/new", page) + self.assertIn("codex-app-server", page) + self.assertIn("/webhooks/github", page) + # 回调：坏 state 失败；好 state 换 code 存配置。 + status, page = self._req( + "GET", "/github-app/callback?code=x&state=nope") + self.assertEqual(status, 400, page) + status, page = self._req( + "GET", "/github-app/callback?code=fake-code&state=%s" % state) + self.assertEqual(status, 200, page) + self.assertIn("fake-codex-app", page) + self.assertIn("/installations/new", page) + self.assertNotIn("fake-app-secret", page) + self.assertNotIn("fake-pem", page) + # 同一 state 第二次回调必然失败。 + status, page = self._req( + "GET", "/github-app/callback?code=fake-code&state=%s" % state) + self.assertEqual(status, 400, page) + # 状态接口：已配置、无 secret。 + status, body = self._req( + "GET", "/github-app/status", key=self.key_a) + self.assertEqual(status, 200, body) + self.assertTrue(body["configured"]) + self.assertEqual(body["slug"], "fake-codex-app") + self.assertNotIn("webhook_secret", body) + self.assertNotIn("pem", body) + + def test_webhook_dual_secret_and_installation(self): + # 先存一个 App 配置（secret 为 fake-app-secret）。 + conn = _db_connect(self.db_path) + try: + save_app_config(conn, { + "app_id": 424242, "slug": "fake-codex-app", + "client_id": "Iv1.fake", "webhook_secret": "fake-app-secret", + "pem": "fake-pem", + "html_url": "https://github.com/apps/fake-codex-app"}) + finally: + conn.close() + os.environ["GITHUB_WEBHOOK_SECRET"] = "legacy-secret" + ping = {"zen": "hi"} + # App secret 与 legacy secret 都能验过；错 secret 401。 + status, _ = self._webhook( + "ping", ping, "fake-app-secret", "d-app-1") + self.assertEqual(status, 200) + status, _ = self._webhook( + "ping", ping, "legacy-secret", "d-legacy-1") + self.assertEqual(status, 200) + status, _ = self._webhook("ping", ping, "wrong", "d-wrong-1") + self.assertEqual(status, 401) + # installation 事件记安装记录。 + inst_payload = { + "action": "created", + "installation": {"id": 11, "account": {"login": "octo"}}, + "repositories": [{"full_name": "octo/a"}, + {"full_name": "octo/b"}]} + status, body = self._webhook( + "installation", inst_payload, "fake-app-secret", "d-inst-1") + self.assertEqual(status, 200, body) + self.assertEqual(body["result"], "installation_recorded") + status, body = self._req( + "GET", "/github-app/status", key=self.key_a) + self.assertEqual(status, 200, body) + self.assertEqual(len(body["installations"]), 2) + self.assertEqual( + body["installations"][0]["repo_full_name"], "octo/a") + + def test_webhook_no_secret_is_503(self): + os.environ.pop("GITHUB_WEBHOOK_SECRET", None) + status, _ = self._webhook( + "ping", {"zen": "hi"}, "anything", "d-none-1") + self.assertEqual(status, 503) + + def test_fix_review_content_dedup_over_http(self): + # 项目名 == 仓库 full_name，/fix-review 才能建任务。 + conn = sqlite3.connect(self.db_path) + try: + conn.execute( + "INSERT INTO projects(id, name, owner_id, created_at," + " repo_url, auto_pr) VALUES(?, ?, ?, 0, NULL, 0)", + ("proj-1", "octo/a", self.user_a)) + conn.commit() + finally: + conn.close() + conn = _db_connect(self.db_path) + try: + save_app_config(conn, { + "app_id": 424242, "slug": "fake-codex-app", + "client_id": "Iv1.fake", "webhook_secret": "fake-app-secret", + "pem": "fake-pem", + "html_url": "https://github.com/apps/fake-codex-app"}) + finally: + conn.close() + payload = { + "action": "created", + "comment": {"id": 777, "body": "please /fix-review this", + "user": {"login": "octo"}}, + "issue": {"number": 5}, + "repository": {"full_name": "octo/a"}} + # 同一评论送两遍，delivery_id 不同（模拟手动 + App 双通道）。 + status, first = self._webhook( + "issue_comment", payload, "fake-app-secret", "d-fix-1") + self.assertEqual(status, 200, first) + self.assertIsNotNone(first.get("task_id"), first) + status, second = self._webhook( + "issue_comment", payload, "fake-app-secret", "d-fix-2") + self.assertEqual(status, 200, second) + self.assertTrue(second.get("duplicate"), second) + self.assertEqual(second.get("task_id"), first.get("task_id")) + + +if __name__ == "__main__": + unittest.main() diff --git a/cloud/webhooks.py b/cloud/webhooks.py index ae53f07..e860359 100644 --- a/cloud/webhooks.py +++ b/cloud/webhooks.py @@ -60,6 +60,20 @@ def verify_github_signature(secret, payload_bytes, signature_header): return hmac.compare_digest(expected, signature_header) +def verify_github_signature_any(secrets, payload_bytes, signature_header): + """多 secret 验签：任一 secret 对上即 True，全不对或无 secret + 即 False。过渡期手动 webhook（env secret）与 App webhook + （库里 secret）共存时用；App 装好、手动 webhook 删掉后， + env secret 自然不再被命中，不用改代码。""" + if not secrets: + return False + for secret in secrets: + if secret and verify_github_signature( + secret, payload_bytes, signature_header): + return True + return False + + def parse_github_pr_comment(payload): """解析 GitHub issue_comment 事件，返回 fix-review 触发点或 None。