diff --git a/cloud/console.html b/cloud/console.html index 5cf1265..6e21687 100644 --- a/cloud/console.html +++ b/cloud/console.html @@ -160,7 +160,7 @@ -
协调员格式：[{"name":"后端","prompt":"做后端"},{"name":"前端","prompt":"做前端"}]；提交后 worker 并行建子任务、等齐、合流汇总。不能与 goal/计划模式同用。
+
协调员格式：[{"name":"后端","prompt":"做后端"},{"name":"前端","prompt":"做前端"}]；提交后 worker 并行建子任务、等齐、合流汇总。不能与 goal/计划模式同用。协调员与 goal/计划互斥：勾了 goal 或计划模式时不要填协调员 JSON，提交前会校验并拦截。子任务费用记父任务创建人账；父任务创建时按分工数预留配额，余额不足时整个父任务被挡下。
- +
+
本基线只登记评审需求（requested），不自动跑评审；评审是否执行、结果如何，以评审队列里的记录为准。
点任务行查看详情后，点“刷新队列”查看评审记录

实时链路

@@ -286,7 +287,7 @@

运行时审批队列

-
待审批项需人工点允许或拒绝；超时默认拒绝。本队列的决定会留回执；实时请求的事前拦截待 host 接入后生效，在此之前实时行为不变。
+
待审批项需人工点记录允许或记录拒绝；超时默认拒绝。当前版本只留回执，不拦截实时行为（实时请求的事前拦截待 host 接入后生效，在此之前实时行为不变）。

给其他任务发消息

@@ -354,7 +355,12 @@ async function api(path, method, body) { var msg = (data && data.error) ? data.error : ("HTTP " + r.status); if (r.status === 401) msg += "（key 未填或无效）"; if (r.status === 429) msg += "（触发限流，稍后再试）"; - if (r.status === 402) msg += "（本月配额已用完，联系管理员调整）"; + if (r.status === 402) { + // 402 要区分两种满：服务端在响应里带哪个上限字段就是哪个满。 + if (data && data.unpriced_token_cap !== undefined) msg += "（无价 token 已满，联系管理员调整）"; + else if (data && data.error === "coordinator_quota_reserve_exceeded") msg = "协调员分工预留不足：剩余美元配额不够覆盖全部分工，子任务费用记父任务创建人账，整个父任务已被挡下（未建任何子任务）"; + else msg += "（美元配额已满，联系管理员调整）"; + } throw new Error(msg); } return data; @@ -394,13 +400,22 @@ async function loadQuota() { try { var d = await api("/users/me/quota", "GET"); var used = (d.used_usd_micros / 1000000).toFixed(2); + var usdLine; if (d.quota_usd_micros === null || d.quota_usd_micros === undefined) { - el.textContent = "本月用量 $" + used + "（不限额）"; + usdLine = "美元：本月用量 $" + used + "（不限额）"; } else { var q = (d.quota_usd_micros / 1000000).toFixed(2); var rem = (d.remaining_usd_micros / 1000000).toFixed(2); - el.textContent = "本月用量 $" + used + " / $" + q + "（剩余 $" + rem + "）"; + usdLine = "美元：本月用量 $" + used + " / $" + q + "（剩余 $" + rem + "）"; } + // 两条进度并列：美元一条，无价 token 一条；402 时按哪条满区分报错。 + var tokLine = "无价 token："; + if (d.unpriced_token_cap === undefined || d.unpriced_tokens_used === undefined) { + tokLine += "未返回（旧服务端）"; + } else { + tokLine += d.unpriced_tokens_used + " / " + d.unpriced_token_cap + " tokens"; + } + el.textContent = usdLine + " ｜ " + tokLine; } catch (e) { el.textContent = "配额加载失败：" + e.message; } } @@ -453,8 +468,8 @@ async function loadApprovals(taskId) { } var btns = ""; if (a.status === "pending") { - btns = " " + - " "; + btns = " " + + " "; } return "[" + esc(a.kind) + "｜" + esc(a.status) + "] " + esc(a.summary) + receipt + btns; }).join("
"); @@ -465,13 +480,20 @@ async function decideApproval(approvalId, decision) { var verb = decision === "approved" ? "approve" : "deny"; try { await api("/tasks/" + encodeURIComponent(currentTask.id) + "/approvals/" + encodeURIComponent(approvalId) + "/" + verb, "POST", {}); - $("#approvalMsg").innerHTML = "已" + (decision === "approved" ? "允许" : "拒绝") + ""; + $("#approvalMsg").innerHTML = "已记录" + (decision === "approved" ? "允许" : "拒绝") + "（当前版本只留回执，不拦截实时行为）"; loadApprovals(currentTask.id); } catch (e) { $("#approvalMsg").innerHTML = "决定失败：" + esc(e.message) + ""; } } async function loadWorkers() { try { var d = await api("/workers", "GET"); + // 非全局管理员只拿到聚合数字（服务端 P1 收紧）：不展示 worker 清单。 + if (d.workers === undefined && d.workers_total !== undefined) { + if (!d.workers_total) { $("#workers").textContent = "无 worker 注册"; return; } + $("#workers").textContent = "在线 " + d.workers_online + " / 离线 " + d.workers_offline + + "（共 " + d.workers_total + "；完整 worker 清单仅全局管理员可见）"; + return; + } var list = d.workers || d || []; if (!list.length) { $("#workers").textContent = "无 worker 注册"; return; } $("#workers").innerHTML = list.map(function (w) { @@ -759,7 +781,7 @@ async function postReview() { await api("/tasks/" + encodeURIComponent(currentTask.id) + "/reviews", "POST", body); $("#reviewMsg").innerHTML = "评审已登记（requested）"; loadReviews(currentTask.id); - } catch (e) { $("#reviewMsg").innerHTML = "发起评审失败：" + esc(e.message) + ""; } + } catch (e) { $("#reviewMsg").innerHTML = "登记评审需求失败：" + esc(e.message) + ""; } } async function submitReviewFindings(reviewId) { if (!currentTask) return; @@ -778,8 +800,19 @@ async function showDetail(taskId) { document.querySelectorAll("#taskRows tr").forEach(function (tr) { tr.classList.toggle("selected", tr.getAttribute("data-id") === taskId); }); + // 每个任务状态配一句中文解释（只解释状态语义，不改后端状态机）。 + var STATUS_EXPLAIN = { + queued: "排队中，尚未被 worker 认领", + running: "运行中，模型回合进行中", + completed: "模型回合结束、改动在 worktree 里尚未合并", + failed: "模型回合失败，可重试", + interrupted: "已被取消/中断，可重试", + merged: "控制面已标记 merged，不代表 git 主分支已合入", + "merge-failed": "合并落盘失败，可重试" + }; $("#detail").innerHTML = - "状态 " + esc(t.status) + " ｜ " + + "状态 " + esc(t.status) + "（" + + esc(STATUS_EXPLAIN[t.status] || "未知状态") + "） ｜ " + "executor " + esc(t.executor || "") + " ｜ 主机 " + esc(t.worktree_host || "") + " ｜ 模型 " + esc(t.model || "默认") + " ｜ 强度 " + esc(t.effort || "默认") + " ｜ 权限 " + esc(t.approval || "默认") + @@ -800,7 +833,35 @@ async function showDetail(taskId) { pre.style.display = "block"; $("#detailActions").style.display = "flex"; $("#btnFork").style.display = TRACE_TERMINAL.indexOf(t.status) >= 0 ? "" : "none"; - $("#actionMsg").textContent = ""; + // 按钮按状态 + 当前用户项目角色禁用并写明原因（前端禁用只是 + // 防误点，真正鉴权仍在服务端每次请求时做；角色来自项目列表 + // 的 my_role 字段，未知角色时按最严（仅取消可用）处理）。 + (function () { + var proj = null; + (projectsCache || []).forEach(function (p) { if (p.id === t.project_id) proj = p; }); + var role = proj ? proj.my_role : null; + var isMaintainer = (role === "maintainer" || role === "owner"); + var canCreate = (role === "contributor" || isMaintainer); + var terminal = TRACE_TERMINAL.indexOf(t.status) >= 0; + var rules = [ + { btn: "#btnRetry", ok: ["failed", "interrupted", "merge-failed"].indexOf(t.status) >= 0 && isMaintainer, + why: "仅 failed/interrupted/merge-failed 可重试" + (isMaintainer ? "" : "，且仅 maintainer 可重试") }, + { btn: "#btnCancel", ok: !terminal && canCreate, + why: terminal ? "终态任务不可取消" : (canCreate ? "" : "仅项目成员（contributor 及以上）可取消") }, + { btn: "#btnMerge", ok: t.status === "completed" && isMaintainer, + why: "仅 completed 可合并" + (isMaintainer ? "" : "，且仅 maintainer 可合并") }, + { btn: "#btnPr", ok: t.status === "merged" && isMaintainer, + why: "仅 merged 可提 PR" + (isMaintainer ? "" : "，且仅 maintainer 可提 PR") } + ]; + var reasons = []; + rules.forEach(function (r) { + var b = $(r.btn); + b.disabled = !r.ok; + b.title = r.ok ? "" : r.why; + if (!r.ok && r.why) reasons.push(b.textContent.replace(/（.*$/, "") + "：不可用——" + r.why); + }); + $("#actionMsg").textContent = reasons.length ? reasons.join("；") : ""; + })(); $("#costBox").innerHTML = ""; $("#forkForm").style.display = "none"; $("#forkMsgSpan").textContent = ""; @@ -1062,6 +1123,12 @@ $("#createTask").onclick = async function () { try { body.coordinator_parts = JSON.parse(coordRaw); } catch (e) { $("#taskMsg").textContent = "协调员子任务 JSON 解析失败"; return; } if (!Array.isArray(body.coordinator_parts)) { $("#taskMsg").textContent = "协调员子任务必须是数组"; return; } + // 协调员与 goal/计划互斥：提交前校验并在表单处写明冲突 + // （服务端同样 400，但不要等到服务端才发现）。 + if ($("#goalMode").checked || $("#planMode").checked) { + $("#taskMsg").textContent = "协调员任务不能与 goal/计划模式同用：协调员任务本身不跑模型回合，请取消 goal/计划勾选，或清空协调员 JSON"; + return; + } } var atts = gatherAttachments(); if (atts.length) { diff --git a/cloud/coordinator.py b/cloud/coordinator.py index f3405e0..791e2a6 100644 --- a/cloud/coordinator.py +++ b/cloud/coordinator.py @@ -16,6 +16,15 @@ 超时时同样取消挂起的子任务并标 failed，报告里写清哪几项 没完成。取消路径的协调员标 interrupted，不发完成/失败推送 （与普通任务的取消口径一致）。 +- 孤儿收口（P1）：协调员本身死亡（worker 进程死、VM 回收） + 时没有“取消者”替它取消子任务，正在跑的子任务会无声继续 + 烧钱。worker 侧两道收口：① 协调员取消/超时终局时，cancel + 之后仍未终态的子任务由 _coord_mark_orphans 直接标 + interrupted 并记审计；② 常驻循环每轮的 + sweep_orphaned_coordinator_children 把“父已终态（或父行 + 已失踪）但子仍未终态”的子任务统一标 interrupted 并记审计 + （父仍在跑时绝不动子任务）。本模块只提供纯函数 + nonterminal_children 供 worker 判断，DB 写回在 worker 侧。 本模块只放纯逻辑（校验、解析、汇总、建表），不碰 DB 之外的 任何东西，单测不需要 worker/server。 @@ -128,6 +137,28 @@ def ensure_columns(conn): pass +TERMINAL_STATUSES = frozenset( + {"completed", "failed", "interrupted", "merged", "merge-failed"}) + + +def nonterminal_children(children): + """未终态的子任务（P1 孤儿判断用，纯函数）。 + + children 为 [{name, id, status, ...}]（worker 的 _coord_child_rows + 拼出来的 merged 形态，或 _coord_sync_children 的 [{name, id}] + 补上 status 后的形态）；返回其中 status 不在终态集里的项 + （缺 status 按未终态算，绝不把“查不到”当成“已完成”）。 + worker 的孤儿标记与取消/超时终局都以它为准。 + """ + out = [] + for ch in children or []: + if not isinstance(ch, dict): + continue + if ch.get("status") not in TERMINAL_STATUSES: + out.append(ch) + return out + + def build_report(children): """合流汇总：children 为 [{name, id, status, result_text, error_text}]，返回 (report_text, overall_status)。 diff --git a/cloud/cost.py b/cloud/cost.py index 0c119f8..1510a17 100644 --- a/cloud/cost.py +++ b/cloud/cost.py @@ -18,6 +18,28 @@ may charge differently. Override per model with with no price record ``usd_micros=None`` and reports count their tokens in ``unpriced_actual_tokens`` instead of pricing them at zero. +cache_write_input_tokens: EXPLICITLY NOT BILLED HERE (P1 decision, +not an oversight). host.py collects ``cacheWriteInputTokens`` from +upstream tokenUsage events and sandbox.py carries it in usage dicts +for observability, but it is NOT added to ``total_tokens`` (upstream +total is input + output), NOT stored in ``cost_records`` (``record`` +takes input/output/cached only), NOT priced by ``price_usd_micros``, +and NOT counted toward the unpriced-token cap (worker counts input + +output from ``source="actual"`` rows only). Rationale: no price-table +entry and no relay bill evidence treats cache creation as a +separately billed token stream for the OpenAI-style relay models in +use; cache_write here is prompt-cache bookkeeping detail, not a +second charge on top of input. If a future provider bills cache +creation separately, the price table must gain an explicit +cache_write rate FIRST -- only then may ``record``/pricing count it. +Until then, counting it would double-charge input tokens that are +already priced (or inflate the unpriced cap with non-spend). + +Unpriced cap scope (P1): ``unpriced_actual_tokens`` and the worker's +cap query count ``source="actual"`` rows only. Estimated rows +(``source="estimated"``, character-count fallback when no usage +events arrive) are never real spend and must not trip the cap. + Dead end, documented so nobody retries it: the app-server's account/usage/read method requires ChatGPT account authentication and fails on a relay API key (2026-10-26 probe: ``codex account @@ -98,6 +120,9 @@ def price_usd_micros(model, input_tokens, cached_input_tokens, output rate. Reasoning tokens are already inside output_tokens (provider total = input + output), never added twice. Returns None when the model has no price entry (caller records usd as unknown). + cache_write_input_tokens is deliberately NOT a parameter here: + it is not billed separately (see module docstring); callers must + not add it to input_tokens to “count it anyway”. """ rates = model_prices().get(model) if rates is None: diff --git a/cloud/push.py b/cloud/push.py index 9002c17..3fa2ba3 100644 --- a/cloud/push.py +++ b/cloud/push.py @@ -37,15 +37,22 @@ 发送端是后续待接线的工作，见模块末尾的缺口说明。 - 本模块不跑真实模型回合。webhook/slack 的真实发送只 POST 到订阅者自己填的 target URL（创建订阅已要求项目 - create_task 权限；能建订阅的成员本来就能经任务执行代码， - 因此不对 target 做内网屏蔽——屏蔽既拦不住提权，也会误伤 - 本地接收器）。密钥、token、证书一律不由本模块读取、写入 - 或返回。 + create_task 权限）。P1 收紧后：真实发送层对 target 做 SSRF + 校验——loopback、私有网段、链路本地、云元数据地址一律不 + POST；主机名先 DNS 解析再按解析出的 IP 判，解析失败拒绝 + （见 validate_push_target）。本地接收器如需放行，须运维显式 + 置 CODEX_PUSH_ALLOW_PRIVATE_TARGETS=1（默认关闭，测试用 + 本地接收器也须经此开关，不在代码里默认放行 loopback）。 + 密钥、token、证书一律不由本模块读取、写入或返回。 """ +import ipaddress import json +import os +import socket import sqlite3 import urllib.error +import urllib.parse import urllib.request from cloud.db import connect as _db_connect from cloud.db import PgConnection as _PgConnection @@ -67,6 +74,7 @@ __all__ = [ "send_webhook", "send_slack", "dispatch_event", + "validate_push_target", ] CHANNELS = ("webhook", "slack", "webpush", "apns", "fcm") @@ -311,6 +319,91 @@ def _require_http_url(target, channel): return target +# 云元数据主机名：即使解析出的 IP 恰好不在私有段，也直接挡。 +_BLOCKED_HOSTNAMES = frozenset({"metadata.google.internal"}) + + +def _allow_private_targets(): + """运维显式开关：仅 CODEX_PUSH_ALLOW_PRIVATE_TARGETS=1 时放行 + 内网目标（本地接收器场景）。默认关闭；开关只影响 IP 段判定， + 不影响 scheme 校验。""" + return os.environ.get( + "CODEX_PUSH_ALLOW_PRIVATE_TARGETS", "").strip() == "1" + + +def _default_resolver(hostname): + """默认 DNS 解析：返回 IP 字符串列表；解析失败抛异常， + 由 validate_push_target 转成“拒绝”。""" + infos = socket.getaddrinfo(hostname, None) + out = [] + for info in infos: + sockaddr = info[4] if len(info) > 4 else None + if sockaddr and sockaddr[0]: + out.append(str(sockaddr[0])) + return out + + +def _ip_is_blocked(ip_text): + """非全局单播 IP 一律算内网/特殊地址：loopback、私有网段、 + 链路本地（169.254/16，含云元数据 169.254.169.254）、保留段、 + 组播、未指定地址都在 is_global=False 里。""" + try: + return not ipaddress.ip_address(ip_text).is_global + except ValueError: + return True + + +def validate_push_target(target, resolver=None, check_dns=True): + """SSRF 校验：通过返回 None，否则返回原因字符串（不抛）。 + + 顺序：scheme（http/https）-> 主机名存在 -> 元数据主机名 -> + IP 字面量直接判段 -> （check_dns 时）DNS 解析后逐 IP 判段， + 解析失败或解析出空结果都拒绝（DNS 重绑定下解析时与发送时 + 仍可能不一致，这是本层能做的上限，见残留说明）。 + check_dns=False 时只做字面量判定：供直接调 send_* 但不想 + 做 DNS 的调用方用；dispatch 的真实发送路径一律 + check_dns=True，显式 sender 路径不走本预检（见 dispatch）。 + """ + if not isinstance(target, str) or not ( + target.startswith("http://") or target.startswith("https://")): + return "target must be an http(s) URL" + try: + host = urllib.parse.urlparse(target).hostname + except ValueError: + return "target host is invalid" + if not host: + return "target host required" + lowered = host.lower() + if lowered in _BLOCKED_HOSTNAMES or lowered.endswith( + ".metadata.google.internal"): + return "target host is a cloud metadata address" + if _allow_private_targets(): + return None + try: + ipaddress.ip_address(host) + is_literal = True + except ValueError: + is_literal = False + if is_literal: + if _ip_is_blocked(host): + return "target resolves to a non-public address: %s" % host + return None + if not check_dns: + return None + resolve = resolver or _default_resolver + try: + ips = resolve(host) + except Exception: + return "DNS resolution failed for target host" + if not ips: + return "DNS resolution failed for target host" + for ip_text in ips: + if _ip_is_blocked(ip_text): + return ("target host resolves to a non-public address: %s" + % ip_text) + return None + + def send_webhook(message): """webhook 通道真实发送：POST JSON 到订阅的 target URL。 @@ -319,6 +412,11 @@ def send_webhook(message): RuntimeError（dispatch 记入 errors，不算 sent）。 """ target = _require_http_url(message["target"], "webhook") + blocked = validate_push_target( + target, resolver=message.get("_resolver")) + if blocked is not None: + raise RuntimeError( + "webhook delivery failed: target blocked: %s" % blocked) payload = message["payload"] if isinstance( message["payload"], dict) else {} ok, detail = _post_json(target, { @@ -340,6 +438,11 @@ def send_slack(message): 事件附 error 摘要（截断 500 字）。非 2xx 抛 RuntimeError。 """ target = _require_http_url(message["target"], "slack") + blocked = validate_push_target( + target, resolver=message.get("_resolver")) + if blocked is not None: + raise RuntimeError( + "slack delivery failed: target blocked: %s" % blocked) payload = message["payload"] if isinstance( message["payload"], dict) else {} text = "[codex] task %s %s (project %s)" % ( @@ -357,7 +460,8 @@ def send_slack(message): DEFAULT_SENDERS = {"webhook": send_webhook, "slack": send_slack} -def dispatch_event(db, event_type, project_id, payload, sender=None): +def dispatch_event(db, event_type, project_id, payload, sender=None, + resolver=None): """分发一次状态变化事件，只发本项目的订阅。 参数： @@ -418,6 +522,7 @@ def dispatch_event(db, event_type, project_id, payload, sender=None): }] return result + real_send = sender is None if sender is None: # 内置真实发送端：webhook/slack 真 POST；显式 sender # 传入时不走这里（注入优先），webpush/apns/fcm 永远 @@ -445,6 +550,21 @@ def dispatch_event(db, event_type, project_id, payload, sender=None): result["errors"].append( dict(base, reason=MISSING_CREDENTIAL_REASONS[sub["channel"]])) continue + # SSRF 预检（真实发送路径，发送前挡，不 POST）：含 DNS + # 的全量校验。显式 sender 路径不预检：注入的 sender 是 + # 调用方自控的投递（测试 seam），不是服务端发起的 HTTP， + # 挡它不增加生产安全性；生产服务端一律走 sender=None + # 的真实发送，必经本预检与 send_webhook/send_slack 的 + # 二次校验。原因文案含 "delivery failed"，与旧调用方 + # 对投递失败的断言兼容。 + if real_send: + blocked = validate_push_target( + sub["target"], resolver=resolver, check_dns=True) + if blocked is not None: + result["errors"].append( + dict(base, reason="%s delivery failed: target blocked:" + " %s" % (sub["channel"], blocked))) + continue # webhook/slack：sender=None 时已换成内置真实发送端； # 显式 sender 里没有该通道时明确报错，不静默成功。 channel_sender = _resolve_channel_sender(sender, sub["channel"]) diff --git a/cloud/sandbox.py b/cloud/sandbox.py index a05ecf0..e27677e 100644 --- a/cloud/sandbox.py +++ b/cloud/sandbox.py @@ -423,6 +423,15 @@ class LocalExecutor(Executor): rounds = task.get("goal_max_rounds") if rounds is None: rounds = 3 + # P1 轮次上限（与 worker 的 GOAL_MAX_ROUNDS_CAP 同值 100）： + # 有限轮次超限直接 loud fail，不静默截断；goal_mode 的 0 + #（不限轮次）放行，上界是配额；旧验收形态的 0 同样拒绝 + #（0 在旧形态里表示不限轮次，P1 已禁用无上限验收循环）。 + if rounds is not None and rounds > 100: + raise ValueError("goal_max_rounds 超过上限 100 轮") + if rounds == 0 and verify and not task.get("goal_mode"): + raise ValueError( + "旧验收形态 goal_max_rounds=0 表示不限轮次，已禁用") cmd = self._base_command(model=task.get("model"), effort=task.get("effort"), approval=task.get("approval"), @@ -440,26 +449,35 @@ class LocalExecutor(Executor): return self._goal_result_from_runner( task["id"], self._run_local(cmd, extra_env)) - def goal_resume_task(self, task, goal_text, extra_env=None, - model_provider=None) -> dict: - """调 host.py goal-resume --goal-text：在旧线程上开新目标 - 循环（旧任务接续 goal 模式，云端 /continue 带 goal_mode=1 - 时走这里）。命令：host.py goal-resume --task - --goal-text <新目标> --max-rounds 0（新语义不限轮次）。 + def goal_resume_task(self, task, goal_text=None, extra_env=None, + model_provider=None, max_rounds=None) -> dict: + """调 host.py goal-resume，分两种调用（P1，与 worker、 + host 三处一致，绝不能混用）： + - 断点续跑：goal_text=None（worker 的 _resume_kind= + "breakpoint" 时传 None）。命令不带 --goal-text、默认 + 也不带 --max-rounds：host 沿用原 goal + 原 verify_command + + 原 goal_max_rounds，从断点轮次续跑。传了 --goal-text + 就会走 host 的开新目标分支、把原 verify 覆盖成 None， + 所以断点调用绝不能顺手把续跑消息塞进 goal_text。 + - 开新目标：goal_text 为新目标文本（云端 /continue 带 + goal_mode=1 时走这里）。命令带 --goal-text 与 + --max-rounds（默认 0，新语义不限轮次，上界是配额）。 不传 --repo：goal-resume 操作已有 host 任务，不新建 worktree。结论解析与 goal_task 同款。 """ if not isinstance(task, dict) or task.get("id") is None: raise ValueError("任务缺少 id") - if not goal_text: - raise ValueError("goal-resume 缺少目标文本") cmd = self._base_command(model=task.get("model"), effort=task.get("effort"), approval=task.get("approval"), model_provider=model_provider) - cmd += ["goal-resume", "--task", self._host_name(task), - "--goal-text", goal_text, - "--max-rounds", "0"] + cmd += ["goal-resume", "--task", self._host_name(task)] + if goal_text: + cmd += ["--goal-text", goal_text, + "--max-rounds", + str(max_rounds if max_rounds is not None else 0)] + elif max_rounds is not None: + cmd += ["--max-rounds", str(max_rounds)] return self._goal_result_from_runner( task["id"], self._run_local(cmd, extra_env)) @@ -469,7 +487,9 @@ class LocalExecutor(Executor): 解析用完整 stdout（结论在末尾），存库的 stdout 才截尾； returncode 非 0 一律 failed；为 0 时无结论也判 failed， 绝不把“没打印结论”当成达标。host 任务已存在时（同任务 - /retry 撞名，goal 没有 run 续跑语义）错误写明请新建任务。 + /retry 撞名，goal 没有 run 续跑语义）错误写明“goal 失败 + 请用继续而非重试”（P1：重试在已存在 host-state 上必失败， + 断点续跑才是正确路径，见 worker 的回落逻辑）。 """ returncode = getattr(result, "returncode", 1) stdout_full = getattr(result, "stdout", "") or "" @@ -478,8 +498,10 @@ class LocalExecutor(Executor): combined = stdout_full + stderr if "已存在" in combined: met, goal_error = None, ( - "host 任务已存在：goal 任务不支持同任务重试，" - "请新建任务") + "host 任务已存在：goal 失败请用继续而非重试——" + "同任务重试会撞已存在 host-state，worker 会先" + "尝试断点续跑，断点续跑仍失败时请用继续（不要" + "重排同一任务）") status = "failed" elif returncode != 0: status = "failed" @@ -516,9 +538,11 @@ class LocalExecutor(Executor): killpg 才能连同子孙一起杀掉。注意：只有默认 runner（真实 subprocess）时才用本方法；单测注入的假 runner 走同步路径， 不走这里。kind=goal 时建 goal 命令、kind=goal-resume 时建 - goal-resume 命令（旧任务接续 goal 模式，目标文本取本轮 - prompt；结论解析由 worker 的 _execute_via_process 按 kind - 分发到 _goal_result_from_runner）。goal 语义与同步 goal_task + goal-resume 命令（结论解析由 worker 的 _execute_via_process + 按 kind 分发到 _goal_result_from_runner）。goal-resume 分流 + （P1）：_resume_kind="breakpoint" 时不传 --goal-text（断点 + 续跑，沿用原 goal+verify），其它情况才把本轮 prompt 当 + --goal-text（开新目标）。goal 语义与同步 goal_task 一致：新形态（goal_mode）无验收命令时 --verify 不传、 --max-rounds 传 0（0 不能用 or 兜底成 3）；旧形态仍传验收 命令与轮次。seed_dir 只对 dispatch/goal 追加（run 续跑与 @@ -545,6 +569,11 @@ class LocalExecutor(Executor): rounds = task.get("goal_max_rounds") if rounds is None: rounds = 3 + if rounds is not None and rounds > 100: + raise ValueError("goal_max_rounds 超过上限 100 轮") + if rounds == 0 and verify and not task.get("goal_mode"): + raise ValueError( + "旧验收形态 goal_max_rounds=0 表示不限轮次，已禁用") cmd += ["goal", "--name", self._host_name(task), "--task", goal_text] if verify: @@ -556,12 +585,15 @@ class LocalExecutor(Executor): if seed_dir: cmd += ["--seed-dir", seed_dir] elif kind == "goal-resume": - # 旧任务接续 goal：host 侧用 --goal-text 在旧线程上开 - # 新目标循环（新语义，不限轮次）；task["prompt"] 在续跑 - # 时已被 worker 换成续跑消息。 - cmd += ["goal-resume", "--task", self._host_name(task), - "--goal-text", task["prompt"], - "--max-rounds", "0"] + # P1 分流（与 goal_resume_task 同款契约）： + # _resume_kind="breakpoint" 时不传 --goal-text（host + # 沿用原 goal+verify 断点续跑，绝不能用续跑消息覆盖 + # 原 verify）；其它情况（开新目标）才用 task["prompt"] + #（worker 已换成续跑消息）当 --goal-text，--max-rounds 0。 + cmd += ["goal-resume", "--task", self._host_name(task)] + if task.get("_resume_kind") != "breakpoint": + cmd += ["--goal-text", task["prompt"], + "--max-rounds", "0"] else: cmd += ["dispatch", "--name", self._host_name(task), "--task", task["prompt"]] @@ -1101,6 +1133,12 @@ class DockerExecutor(Executor): rounds = task.get("goal_max_rounds") if rounds is None: rounds = 3 + # P1 轮次上限（与 Local goal_task 同款，不静默截断）。 + if rounds is not None and rounds > 100: + raise ValueError("goal_max_rounds 超过上限 100 轮") + if rounds == 0 and verify and not task.get("goal_mode"): + raise ValueError( + "旧验收形态 goal_max_rounds=0 表示不限轮次，已禁用") inner_args = ["goal", "--name", self._host_name(task), "--task", goal_text] if verify: @@ -1150,8 +1188,10 @@ class DockerExecutor(Executor): combined = stdout_full + stderr if "已存在" in combined: met, goal_error = None, ( - "host 任务已存在：goal 任务不支持同任务重试，" - "请新建任务") + "host 任务已存在：goal 失败请用继续而非重试——" + "同任务重试会撞已存在 host-state，worker 会先" + "尝试断点续跑，断点续跑仍失败时请用继续（不要" + "重排同一任务）") status = "failed" elif returncode != 0: status = "failed" @@ -1181,22 +1221,29 @@ class DockerExecutor(Executor): "usage": _parse_goal_usage(stdout_full), } - def goal_resume_task(self, task, goal_text, timeout=None, + def goal_resume_task(self, task, goal_text=None, timeout=None, extra_mounts=None, network=None, - extra_env=None, model_provider=None) -> dict: - """容器内调 host.py goal-resume --goal-text：在旧线程上开 - 新目标循环（旧任务接续 goal 模式）。解析与 goal_task 同款 - （完整 stdout 结论、存库截尾），但没有“已存在”分支： - goal-resume 本来就是操作已有 host 任务，不存在 dispatch - 撞名问题；returncode 非 0 或无结论一律 failed。 + extra_env=None, model_provider=None, + max_rounds=None) -> dict: + """容器内调 host.py goal-resume，分两种调用（P1，与 Local + goal_resume_task 同款契约）：断点续跑传 goal_text=None + （不带 --goal-text，host 沿用原 goal+verify+轮次），开新 + 目标才传 goal_text（带 --goal-text 与 --max-rounds，默认 0）。 + 解析与 goal_task 同款（完整 stdout 结论、存库截尾），但 + 没有“已存在”分支：goal-resume 本来就是操作已有 host 任务， + 不存在 dispatch 撞名问题；returncode 非 0 或无结论一律 failed。 """ if not isinstance(task, dict) or task.get("id") is None: raise ValueError("任务缺少 id") - if not goal_text: - raise ValueError("goal-resume 缺少目标文本") + _resume_args = ["goal-resume", "--task", self._host_name(task)] + if goal_text: + _resume_args += ["--goal-text", goal_text, "--max-rounds", + str(max_rounds + if max_rounds is not None else 0)] + elif max_rounds is not None: + _resume_args += ["--max-rounds", str(max_rounds)] inner = self._inner_host_command( - "goal-resume", "--task", self._host_name(task), - "--goal-text", goal_text, "--max-rounds", "0", + *_resume_args, model=task.get("model"), effort=task.get("effort"), approval=task.get("approval"), model_provider=model_provider) diff --git a/cloud/server.py b/cloud/server.py index c7c2025..4e77fd3 100644 --- a/cloud/server.py +++ b/cloud/server.py @@ -113,12 +113,15 @@ from cloud.oauth import ( ) from cloud.relay import RelayError as _RelayError from cloud.relay import get_relay_models as _get_relay_models +from cloud.sub2api import Sub2apiError as _Sub2apiError from cloud.sub2api import ensure_gateway_keys_table as _ensure_gateway_keys from cloud.sub2api import ensure_user_gateway_key as _ensure_sub2api_key from cloud.sub2api import gateway_name as _gateway_name from cloud.sub2api import gateway_origin as _gateway_origin from cloud.sub2api import get_user_gateway_key as _get_sub2api_key from cloud.sub2api import list_gateway_models as _list_sub2api_models +from cloud.sub2api import rotate_user_gateway_key as _rotate_sub2api_key +from cloud.sub2api import validate_or_rotate_gateway_key as _validate_sub2api_key from cloud.repos import validate_repo_url from cloud.push import ( create_subscription as _push_create, @@ -167,6 +170,31 @@ TRACE_STREAM_MAX_SECONDS = 3600 TRACE_STREAM_POLL_SECONDS = 1.0 TRACE_DEFAULT_DIR = "/opt/codex-app-server/work/host-traces" +# P1 控制面限额（建项目时校验，超限 400 并在错误里写明数值）： +# 单用户项目数上限、项目名长度上限、自定义项目 id 长度上限。 +# 均可经环境变量调（运维侧），缺省值见下方；改动只影响新项目， +# 已有超限项目不受影响（不删、不改名）。 +def _env_int(name, default): + try: + value = int(os.environ.get(name, "") or default) + except (TypeError, ValueError): + return default + return value if value > 0 else default + + +MAX_PROJECTS_PER_USER = _env_int("CODEX_MAX_PROJECTS_PER_USER", 50) +MAX_PROJECT_NAME_LEN = _env_int("CODEX_MAX_PROJECT_NAME_LEN", 100) +MAX_PROJECT_ID_LEN = _env_int("CODEX_MAX_PROJECT_ID_LEN", 64) +# 协调员父任务预留：每分工至少预留的美元额度（微美元）。 +# 父任务准入只查一次时，N 个分工可解锁 N 倍消费；本预留要求 +# 剩余配额 >= 分工数 × 本值，否则父任务直接 402（见 +# _check_coordinator_reserve）。缺省 $1/分工，可经环境变量调； +# 它是准入下限估计，不是实际计费——实际花费仍按子任务 +# created_by=父任务创建人记到创建人账上（worker 建子任务时 +# created_by 直接取父任务创建人，成本 JOIN 据此归属）。 +COORDINATOR_RESERVE_PER_PART_USD_MICROS = _env_int( + "CODEX_COORDINATOR_RESERVE_PER_PART_USD_MICROS", 1000000) + def _trace_file_name(task_id): """云任务 id -> 链路文件名：与 cloud.sandbox 的 _host_name @@ -1451,6 +1479,9 @@ class CloudControlPlaneHandler(BaseHTTPRequestHandler): else: self._send_json(404, {"error": "not found"}) return + if path == "/users/me/gateway/rotate" and method == "POST": + self._user_gateway_rotate() + return if path.startswith("/users/me/keys"): # API keys 管理：全部只操作认证用户自己的 key。 if method == "GET" and path == "/users/me/keys": @@ -1830,10 +1861,35 @@ class CloudControlPlaneHandler(BaseHTTPRequestHandler): if not name or not isinstance(name, str): self._send_json(400, {"error": "name required"}) return + if len(name) > MAX_PROJECT_NAME_LEN: + self._send_json(400, { + "error": "project name too long: max %d characters" + % MAX_PROJECT_NAME_LEN}) + return project_id = body.get("id") or uuid.uuid4().hex if not isinstance(project_id, str): self._send_json(400, {"error": "id must be a string"}) return + if len(project_id) > MAX_PROJECT_ID_LEN: + self._send_json(400, { + "error": "project id too long: max %d characters" + % MAX_PROJECT_ID_LEN}) + return + # 单用户项目数上限：按 projects.owner_id 计（成员身份加入 + # 的项目不计入自己的上限，避免“被拉进很多项目就建不了 + # 项目”）；超限 400 且不建项目。 + _count_conn = self._db() + try: + _owned = _count_conn.execute( + "SELECT COUNT(*) FROM projects WHERE owner_id = ?", + (user_id,)).fetchone()[0] + finally: + _count_conn.close() + if _owned >= MAX_PROJECTS_PER_USER: + self._send_json(400, { + "error": "project limit exceeded: max %d projects" + " per user" % MAX_PROJECTS_PER_USER}) + return # repo_url 可选：不传时为 None（老行为不变）；传了则与 worker # 共用 cloud.repos.validate_repo_url 的同一套校验，不合法时 # 400 且不建项目，绝不把带凭证的 URL 存进库。 @@ -1947,43 +2003,78 @@ class CloudControlPlaneHandler(BaseHTTPRequestHandler): self._send_json(200, {"project": _project_to_dict(row)}) def _github_repos_list(self, user_id): - """GET /github/repos：列出本服务 GitHub App 安装记录里的仓 - 库（installation_repositories 事件同步进 github_app_ - installations 表），给控制台建项目/补仓库时的下拉菜单用。 - 任何已认证用户可列（只含仓库名，不含密钥）；选了哪个仓库 - 只影响调用者自己有管理权的项目（PATCH 仍按 owner 校验）。 + """GET /github/repos：只返回调用者成员项目对应的仓库。 + + P1 收紧前返回全部安装记录（任一已认证用户可看到全部 + 安装仓库名，属跨租户信息泄露）。现按 project_members + 过滤：只保留调用者是成员的项目的 repo_url 对应的仓库 + （clone_url 归一化后比对：去 .git 后缀、比对小写）； + 调用者无成员项目或项目未填 repo_url 时返回空列表。 手填 URL 的能力不变——没进 App 安装的公开仓库仍可手动填。""" conn = self._db() try: rows = list_installations(conn) + member_ids = { + row["project_id"] + for row in conn.execute( + "SELECT project_id FROM project_members" + " WHERE user_id = ?", + (user_id,)).fetchall() + } + allowed = set() + for row in conn.execute( + "SELECT id, repo_url FROM projects").fetchall(): + if row["id"] not in member_ids or not row["repo_url"]: + continue + url = str(row["repo_url"]).strip().lower() + if url.endswith(".git"): + url = url[:-4] + allowed.add(url) finally: conn.close() - repos = [{ - "installation_id": r["installation_id"], - "account_login": r["account_login"], - "repo_full_name": r["repo_full_name"], - "clone_url": "https://github.com/%s.git" % r["repo_full_name"], - } for r in rows] + repos = [] + for r in rows: + clone_url = "https://github.com/%s.git" % r["repo_full_name"] + normalized = clone_url.lower() + if normalized.endswith(".git"): + normalized = normalized[:-4] + if normalized not in allowed: + continue + repos.append({ + "installation_id": r["installation_id"], + "account_login": r["account_login"], + "repo_full_name": r["repo_full_name"], + "clone_url": clone_url, + }) self._send_json(200, {"repos": repos}) def _list_projects(self, user_id): conn = self._db() try: - member_ids = { - row["project_id"] - for row in conn.execute( - "SELECT project_id FROM project_members WHERE user_id = ?", - (user_id,), - ).fetchall() - } + member_rows = conn.execute( + "SELECT project_id, role FROM project_members" + " WHERE user_id = ?", + (user_id,)).fetchall() + member_ids = {row["project_id"] for row in member_rows} + roles = {row["project_id"]: row["role"] + for row in member_rows} rows = conn.execute( "SELECT id, name, owner_id, created_at, repo_url, auto_pr" " FROM projects ORDER BY created_at, id" ).fetchall() finally: conn.close() - projects = [_project_to_dict(r) for r in rows - if r["id"] in member_ids] + projects = [] + for r in rows: + if r["id"] not in member_ids: + continue + # 附加字段 my_role（只增字段，不改已有成功语义）： + # 控制台据此按角色禁用重试/合并/提 PR 等按钮；服务端 + # 权限仍以每次请求的 check_permission 为准，前端禁用 + # 只是防误点，不是鉴权。 + item = _project_to_dict(r) + item["my_role"] = roles.get(r["id"]) + projects.append(item) self._send_json(200, {"projects": projects}) # -- 项目子资源：记忆 / 知识库 / 成本 ------------------------------- @@ -2119,6 +2210,11 @@ class CloudControlPlaneHandler(BaseHTTPRequestHandler): return deleted = _push_delete(self.server.db_path, subscription_id=parts[3]) + if deleted: + with _auth_store(self.server.db_path) as store: + store.audit( + user_id, project_id, "push_subscription_deleted", + "subscription_id=%s" % parts[3]) self._send_json(200, {"deleted": bool(deleted)}) return # /projects/{id}/automations @@ -2389,31 +2485,33 @@ class CloudControlPlaneHandler(BaseHTTPRequestHandler): # 缓存（见 cloud/relay.py）。单测可经 server.relay_fetcher # 注入假取数函数；中转挂了或未配 key 时 502，不编造列表。 # 网关为 sub2api 时：模型目录因用户而异（key 绑定的组 - # 决定），取调用者自己的网关 key 现查；无 key 时返回空 - # 列表 + provisioning 标记（控制台据此提示开通中），不 - # 在页面加载时悄悄开通（开通在建号与任务时发生）。 + # 决定）。P1 收紧：首次调模型目录即做真正的懒开通/ + # 健康检查，不再只回 provisioning: pending——无行则开通， + # 旧 key 经 /v1/models 实测已死（401/403，含余额为 0 的 + # 403）则轮换重开；网关本身不可用时回 502、保留旧行。 + # key 明文绝不进响应、日志、缓存。 if _gateway_name() == "sub2api": conn = self._db() try: - row = _get_sub2api_key(conn, user_id) try: - conn.commit() - except Exception: - pass + status, api_key, _sub, _kid = _validate_sub2api_key( + conn, user_id) + except _Sub2apiError as exc: + self._send_json(502, {"error": str(exc)}) + return + except Exception as exc: + self._send_json(502, {"error": str(exc)}) + return finally: conn.close() - if row is None or not row.get("api_key"): - self._send_json(200, {"models": [], "gateway": "sub2api", - "provisioning": "pending"}) - return try: - models = _list_sub2api_models(_gateway_origin(), - row["api_key"]) + models = _list_sub2api_models(_gateway_origin(), api_key) except Exception as exc: self._send_json(502, {"error": str(exc)}) return self._send_json(200, {"models": models, "gateway": "sub2api", - "cached": False, "updated_at": None}) + "cached": False, "updated_at": None, + "provisioning": status}) return fetcher = getattr(self.server, "relay_fetcher", None) try: @@ -2647,6 +2745,12 @@ class CloudControlPlaneHandler(BaseHTTPRequestHandler): self._send_json(400, { "error": "goal_max_rounds must be >= 1"}) return + # P1：旧验收形态轮次设上限 100（与 worker/sandbox 两道门 + # 一致；新 goal_mode=1 不走本分支，仍由配额兜底）。 + if goal_max_rounds > 100: + self._send_json(400, { + "error": "goal_max_rounds 超过上限 100 轮"}) + return # 协调员：coordinator_parts 非空时任务本身不跑模型回合， # worker 认领后按分工建并行子任务、等齐、合流；与 goal/ # plan/验收命令语义冲突（那些都是“本任务跑模型”的意思）， @@ -2673,6 +2777,33 @@ class CloudControlPlaneHandler(BaseHTTPRequestHandler): return if self._reject_if_over_quota(user_id): return + # 协调员预留（P1）：父任务准入只查一次时，N 个分工会 + # 解锁 N 倍消费。父任务创建时要求剩余美元配额至少覆盖 + # 分工数 × 每分工预留额，否则直接 402、父任务不建； + # 子任务归属 = 父任务创建人（worker 建子任务时 + # created_by 直接取父任务创建人，成本 JOIN 据此归属， + # 子任务认领时 worker 侧配额门再查一次）。不限额用户 + # （quota 为 None）不受此预留约束。 + if coord_parts and not self._check_coordinator_reserve( + user_id, len(coord_parts)): + return + # sub2api 懒开通/健康检查（P1）：建任务前先真校验网关 + # key——无行则开通，死 key 则轮换重开；网关不可用时 + # 502 且不建任务（不把死 key/待开通当成已开通发给 + # worker）。xcmapi 网关下跳过本段。 + if _gateway_name() == "sub2api": + _gw_conn = self._db() + try: + try: + _validate_sub2api_key(_gw_conn, user_id) + except _Sub2apiError as exc: + self._send_json(502, {"error": str(exc)}) + return + except Exception as exc: + self._send_json(502, {"error": str(exc)}) + return + finally: + _gw_conn.close() # 环境绑定：environment_id 非空时必须是本项目、已 ready # 的环境；creating/failed/archived 一律拒绝并把状态写进 # 报错（creating 时用户就知道该等 worker 准备好再建任务， @@ -2836,6 +2967,15 @@ class CloudControlPlaneHandler(BaseHTTPRequestHandler): conn.close() with _auth_store(self.server.db_path) as store: store.audit(user_id, project_id, "create_task", task_id) + if coord_parts: + # 创建响应里写明子任务配额归属（与 worker 建子任务时 + # created_by=父任务创建人一致），不是只写文档宣称。 + task["quota_attribution"] = ( + "coordinator subtasks are charged to the parent" + " task creator") + task["quota_reserved_usd_micros"] = ( + len(coord_parts) + * COORDINATOR_RESERVE_PER_PART_USD_MICROS) self._send_json(201, task) def _list_tasks(self, user_id, query): @@ -4091,6 +4231,34 @@ class CloudControlPlaneHandler(BaseHTTPRequestHandler): return True return False + def _check_coordinator_reserve(self, user_id, n_parts): + """协调员预留准入：通过返回 True；被挡时已回 402 返回 False。 + + 剩余美元配额（quota - used，quota 为 None = 不限额时放行） + 必须 >= 分工数 × COORDINATOR_RESERVE_PER_PART_USD_MICROS。 + 这是准入下限估计，不是计费承诺：子任务实际花费仍按 + created_by=父任务创建人全额记创建人账；父任务被挡时 + 一个子任务也不会建，不存在“第 2 个子任务才被挡”的 + 半截状态。""" + if not n_parts or n_parts <= 1: + return True + quota, used, month = self._quota_status(user_id) + if quota is None: + return True + need = n_parts * COORDINATOR_RESERVE_PER_PART_USD_MICROS + if quota - used < need: + self._send_json(402, { + "error": "coordinator_quota_reserve_exceeded", + "month": month, + "quota_usd_micros": quota, + "used_usd_micros": used, + "coordinator_parts": n_parts, + "reserved_usd_micros": need, + "quota_attribution": "coordinator subtasks are charged" + " to the parent task creator"}) + return False + return True + def _audit_admin(self, user_id, action, detail): """admin 调用审计（成功与被拒都记，失败绝不挡主流程）。""" try: @@ -4125,10 +4293,17 @@ class CloudControlPlaneHandler(BaseHTTPRequestHandler): def _user_quota_self(self, user_id): quota, used, month = self._quota_status(user_id) - self._send_json(200, - self._quota_payload( - user_id, quota, used, month, - self._quota_unpriced_tokens(user_id))) + # 合并取舍（P1 rebase 到 d254240）：main 的 _quota_payload + # 已带 unpriced_actual_tokens，P1 的附加字段也全留，供控制台 + # 同时显示美元与无价 token 两条进度；402 时控制台据 + # 响应字段区分“美元配额已满”与“无价 token 已满”。 + payload = self._quota_payload( + user_id, quota, used, month, + self._quota_unpriced_tokens(user_id)) + payload["unpriced_tokens_used"] = self._quota_unpriced_tokens( + user_id, month) + payload["unpriced_token_cap"] = unpriced_token_cap() + self._send_json(200, payload) def _admin_quota_get(self, user_id, query): if not self._is_global_admin(user_id): @@ -4204,10 +4379,20 @@ class CloudControlPlaneHandler(BaseHTTPRequestHandler): # -- worker 池 ----------------------------------------------------- def _workers_list(self, user_id): - """列出注册 worker：任何已认证用户可读（worker 是基础设施 - 信息，不属于某个项目的数据，不按项目成员过滤）。""" + """列出注册 worker：全局管理员看完整清单；其他已认证用户 + 只看在线/离线聚合（P1 收紧：worker_id、执行器、容量、 + 心跳时间等内部细节不对普通用户暴露，控制台的 Workers + 区据此只显示聚合数字）。""" workers = _registry_list(self.server.db_path) - self._send_json(200, {"workers": workers}) + if self._is_global_admin(user_id): + self._send_json(200, {"workers": workers}) + return + online = sum(1 for w in workers if w.get("alive")) + self._send_json(200, { + "workers_online": online, + "workers_offline": len(workers) - online, + "workers_total": len(workers), + }) def _workers_heartbeat(self, user_id): """worker 心跳：body 含 worker_id（必填，缺了 400）与可选 @@ -4429,11 +4614,15 @@ class CloudControlPlaneHandler(BaseHTTPRequestHandler): """POST /admin/backup：备份当前后端数据库，返回路径与规模信息。 权限：只认 CODEX_GLOBAL_ADMINS（见 _is_global_admin）， - 未配置时一律 403。返回只含 backend、备份文件路径、 - 文件大小与任务行数，绝不返回数据库密码、备份文件内容 - 或任何凭据。out_dir 可由 body 指定，否则走环境变量 - CLOUD_BACKUP_DIR，未设则用 cloud-backups。成功与被拒 - 都记审计（审计里只记路径与规模，不记库内容）。 + 未配置时一律 403；非运维调用方根本到不了这里，绝对 + 路径只回给全局管理员。out_dir 限白名单目录（P1 收紧）： + 白名单根来自 CLOUD_BACKUP_ROOT（逗号分隔多根），未设 + 则 CLOUD_BACKUP_DIR，未设则固定根 cloud-backups；请求 + 的 out_dir 经实路径解析后必须等于某根或在其下，含 .. + 逃逸、指向 /、/etc、/opt 等系统目录的一律 400（白名单 + 外的绝对路径即使存在也不放行）。备份文件落盘后 chmod + 0600。成功与被拒都记审计（调用人由审计的 user_id 列 + 记，detail 含目标目录与大小，不记库内容）。 """ if not self._is_global_admin(user_id): self._audit_admin(user_id, "admin_backup_denied", "backup") @@ -4443,11 +4632,33 @@ class CloudControlPlaneHandler(BaseHTTPRequestHandler): if err: self._send_json(400, {"error": err}) return - out_dir = body.get("out_dir") or os.environ.get( - "CLOUD_BACKUP_DIR") or "cloud-backups" - if not isinstance(out_dir, str) or not out_dir: + requested = body.get("out_dir") + if requested is not None and ( + not isinstance(requested, str) or not requested): self._send_json(400, {"error": "out_dir must be a string"}) return + roots_raw = os.environ.get("CLOUD_BACKUP_ROOT", "") + if roots_raw.strip(): + roots = [os.path.realpath(p.strip()) + for p in roots_raw.split(",") if p.strip()] + else: + roots = [os.path.realpath( + os.environ.get("CLOUD_BACKUP_DIR") or "cloud-backups")] + if requested: + candidate = os.path.realpath(requested) + else: + candidate = roots[0] + allowed = any(candidate == root + or candidate.startswith(root + os.sep) + for root in roots) + if not allowed: + self._audit_admin( + user_id, "admin_backup_denied", + "out_dir outside backup roots: %s" % requested) + self._send_json(400, { + "error": "out_dir outside allowed backup roots"}) + return + out_dir = candidate backend = _db_backend_name() try: if backend == "postgres": @@ -4456,6 +4667,10 @@ class CloudControlPlaneHandler(BaseHTTPRequestHandler): else: from cloud.backup import backup_db as _backup_sqlite dest = _backup_sqlite(self.server.db_path, out_dir, keep=3) + try: + os.chmod(dest, 0o600) + except OSError: + pass size_bytes = os.path.getsize(dest) conn = self._db() try: @@ -4468,8 +4683,8 @@ class CloudControlPlaneHandler(BaseHTTPRequestHandler): return self._audit_admin( user_id, "admin_backup", - "backend=%s path=%s size=%d tasks=%d" - % (backend, dest, size_bytes, task_count)) + "caller=%s backend=%s out_dir=%s path=%s size=%d tasks=%d" + % (user_id, backend, out_dir, dest, size_bytes, task_count)) self._send_json(200, { "backend": backend, "path": dest, "size_bytes": size_bytes, "task_count": task_count, @@ -5165,6 +5380,9 @@ class CloudControlPlaneHandler(BaseHTTPRequestHandler): label = "api-key" with _auth_store(self.server.db_path) as auth: key_id, api_key_plain = auth.create_api_key(user_id, label) + # 审计只记 key_id 与 label，绝不记明文与哈希。 + auth.audit(user_id, "", "api_key_created", + "key_id=%s label=%s" % (key_id, label)) self._send_json(201, {"id": key_id, "label": label, "hint": api_key_plain[-4:], "api_key": api_key_plain}) @@ -5182,6 +5400,9 @@ class CloudControlPlaneHandler(BaseHTTPRequestHandler): return with _auth_store(self.server.db_path) as auth: ok = auth.revoke_api_key(user_id, key_id) + if ok: + auth.audit(user_id, "", "api_key_revoked", + "key_id=%s" % key_id) if not ok: self._send_json(404, {"error": "key not found"}) return @@ -5199,8 +5420,45 @@ class CloudControlPlaneHandler(BaseHTTPRequestHandler): keep_hash = hashlib.sha256(api_key.encode()).hexdigest() with _auth_store(self.server.db_path) as auth: count = auth.revoke_other_api_keys(user_id, keep_hash) + # 只记撤销数，不记任何 key 明文、哈希或保留 key 的身份。 + auth.audit(user_id, "", "api_key_revoke_others", + "revoked_count=%d" % count) self._send_json(200, {"revoked_count": count}) + def _user_gateway_rotate(self): + """POST /users/me/gateway/rotate：仅限本人重新开通/轮换 + 网关 key。非 sub2api 网关时 400；网关不可用时 502 且 + 旧行不动（rotate 内部先删旧行再重开，只在重开失败时 + 旧 key 已丢——调用方须知：轮换是显式动作，失败后须 + 重试本端点重新开通）。审计只记 sub2api_user_id 与 + key_id，绝不记 key 明文；响应不含 key。""" + user_id = self._authenticate() + if user_id is None: + self._send_json(401, {"error": "unauthorized"}) + return + if _gateway_name() != "sub2api": + self._send_json(400, {"error": "gateway is not sub2api"}) + return + conn = self._db() + try: + try: + status, _key, sub_id, key_id = _rotate_sub2api_key( + conn, user_id) + except _Sub2apiError as exc: + self._send_json(502, {"error": str(exc)}) + return + except Exception as exc: + self._send_json(502, {"error": str(exc)}) + return + finally: + conn.close() + with _auth_store(self.server.db_path) as store: + store.audit(user_id, "", "gateway_key_rotated", + "sub2api_user_id=%s key_id=%s" + % (sub_id, key_id)) + self._send_json(200, {"rotated": True, "provisioning": status, + "sub2api_user_id": sub_id}) + # -- 推送订阅与分发 --------------------------------------------------- def _push_create_endpoint(self, user_id, project_id): body, err = self._read_json() @@ -5215,6 +5473,12 @@ class CloudControlPlaneHandler(BaseHTTPRequestHandler): except ValueError as e: self._send_json(400, {"error": str(e)}) return + # 审计只记订阅 id 与通道：target 全文不记（Slack webhook + # URL 本身含 secret 路径，记进审计等于把 secret 落盘）。 + with _auth_store(self.server.db_path) as store: + store.audit(user_id, project_id, "push_subscription_created", + "subscription_id=%s channel=%s" + % (sub["id"], sub["channel"])) self._send_json(201, sub) def _push_dispatch_endpoint(self, user_id): @@ -5228,8 +5492,9 @@ class CloudControlPlaneHandler(BaseHTTPRequestHandler): self._send_json(400, {"error": "event_type and project_id required"}) return # P0 安全批后：/admin/push/dispatch 是管理端点，只认 - # CODEX_GLOBAL_ADMINS，不再只凭项目 read 就能让服务端 - # 向订阅目标（可能含内网地址）真 POST。被拒与成功都记审计。 + # CODEX_GLOBAL_ADMINS——该口径已强于 P1 要求的 create_task， + # 不再降级。被拒与成功都记审计；目标 SSRF 校验在 + # push.dispatch_event / 真实发送层做。 if not self._is_global_admin(user_id): self._audit_admin(user_id, "admin_push_denied", "dispatch %s %s" % (event_type, project_id)) @@ -5258,11 +5523,21 @@ class CloudControlPlaneHandler(BaseHTTPRequestHandler): """创建自动化定义：只登记定义与到期状态，不跑模型、 不另起调度器(执行器后续走 claim 门，见 automations 模块文档)。body：name、prompt 必填；interval_seconds - 可选(默认 86400，最小 60)；repo_url 可选。""" + 可选(默认 86400，最小 60)；repo_url 可选。 + + P1 配额门：创建时先过与建任务同一道配额门（美元配额 + 与无价 token cap，超限 402），避免“定义免费建、到期 + 无限跑”；单自动化的 spend 上限即创建人配额本身—— + 每次到期产出的任务以 created_by=创建人建任务，走 + POST /tasks 的配额门（触发路径若绕过服务端建任务， + 则不在本门覆盖内，见残留）。本服务端没有自动化触发 + 端点，触发侧的门不在本次 server.py 改动内。""" body, err = self._read_json() if err: self._send_json(400, {"error": err}) return + if self._reject_if_over_quota(user_id): + return interval = body.get("interval_seconds", 86400) if interval is None: interval = 86400 diff --git a/cloud/sub2api.py b/cloud/sub2api.py index 64e0865..ffe1753 100644 --- a/cloud/sub2api.py +++ b/cloud/sub2api.py @@ -441,8 +441,28 @@ def list_gateway_models(origin, api_key, request=None): return out -def ensure_user_gateway_key(conn, user_id, admin=None, request=None): - """幂等开通：已有行直接返回 key；否则建号->登录->发 key->授余额->入库。 +def gateway_key_health(origin, api_key, request=None): + """真健康检查：用已有 key 调一次 /v1/models。 + + 返回 True（200，可用）/ False（401/403：key 已失效，或余额 + 为 0——网关在余额为 0 时鉴权层直接 403，两者都算死 key）/ + "unknown"（其它状态或网络异常：网关本身不可用时不据此轮换， + 避免把一次抖动当成 key 死亡、反复删号重建）。 + """ + req = request or _default_request + try: + st, _data = req("GET", origin + "/v1/models", token=api_key) + except Exception: + return "unknown" + if st == 200: + return True + if st in (401, 403): + return False + return "unknown" + + +def _create_fresh_gateway_key(conn, user_id, admin=None, request=None): + """建号->登录->发 key->授余额->入库，返回 (key, sub_id, key_id)。 入库失败时尽力清理刚建的 key 与账号（避免孤儿 key），再抛原错； 授余额失败同样走这条清理（余额为 0 的 key 在网关侧直接 403， @@ -451,11 +471,9 @@ def ensure_user_gateway_key(conn, user_id, admin=None, request=None): （同一进程内第二次起免管理员登录，见 shared_admin_session）。 调用方负责 conn 的提交语义：本函数在入库后 commit（开通是独立事务，不跟调用方的业务事务绑在一起）。 + 调用方须保证调用前本用户无有效行（ensure/rotate 先处理 + 旧行），否则会主键冲突（并发撞号除外，走轮询复用）。 """ - ensure_gateway_keys_table(conn) - row = get_user_gateway_key(conn, user_id) - if row is not None: - return row["api_key"] origin = gateway_origin() if not origin: raise Sub2apiError("SUB2API_BASE_URL 未配置，无法开通网关账号") @@ -476,9 +494,16 @@ def ensure_user_gateway_key(conn, user_id, admin=None, request=None): time.sleep(2) row = get_user_gateway_key(conn, user_id) if row is not None: - return row["api_key"] + # 合并修复（P1 rebase 到 d254240）：main 的 #92 竞态 + # 分支原在单体 ensure 里只回一个 key 字符串；P1 + # 抽出 _create_fresh 后契约是 3 元组，4 个调用点 + # 都按 3 元组解包，裸字符串会被当成字符序列解包 + # 而 ValueError。这里按契约回 3 元组，key_id 未知 + # 故为 None（与 validate_or_rotate 的 ok 路一致）。 + return (row["api_key"], row["sub2api_user_id"], None) raise key_id = None + user_token = None try: user_token, refresh = user_login(origin, email, password, request=req) @@ -497,7 +522,7 @@ def ensure_user_gateway_key(conn, user_id, admin=None, request=None): (user_id, sub_id, key, refresh, now, now)) conn.commit() except Exception: - if key_id is not None: + if key_id is not None and user_token is not None: try: user_delete_key(origin, user_token, key_id, request=req) except Exception: @@ -507,4 +532,72 @@ def ensure_user_gateway_key(conn, user_id, admin=None, request=None): except Exception: pass raise + return key, sub_id, key_id + + +def ensure_user_gateway_key(conn, user_id, admin=None, request=None): + """幂等开通：已有行直接返回 key；否则走 _create_fresh 开通。 + + 只保证“行存在”，不保证 key 仍有效：要校验有效性用 + validate_or_rotate_gateway_key（死 key 会触发轮换重开）。 + """ + ensure_gateway_keys_table(conn) + row = get_user_gateway_key(conn, user_id) + if row is not None: + return row["api_key"] + key, _sub_id, _key_id = _create_fresh_gateway_key( + conn, user_id, admin=admin, request=request) return key + + +def validate_or_rotate_gateway_key(conn, user_id, admin=None, request=None): + """懒开通/健康检查合一：返回 (status, key, sub_id, key_id)。 + + status：ok（旧 key 经 /v1/models 实测可用）/ provisioned + （无行，新开通）/ rotated（旧 key 经实测已死：401/403， + 含余额为 0 的 403；删旧行后重开，死 key 不再被当成已开通）。 + 网关本身不可用（健康检查 unknown）时抛 Sub2apiError，调用方 + 应回 502、保留旧行不轮换；绝不在此时把“待开通”当成可用。 + """ + ensure_gateway_keys_table(conn) + row = get_user_gateway_key(conn, user_id) + origin = gateway_origin() + if not origin: + raise Sub2apiError("SUB2API_BASE_URL 未配置，无法开通网关账号") + if row is None or not row.get("api_key"): + key, sub_id, key_id = _create_fresh_gateway_key( + conn, user_id, admin=admin, request=request) + return "provisioned", key, sub_id, key_id + health = gateway_key_health(origin, row["api_key"], request=request) + if health is True: + return "ok", row["api_key"], row.get("sub2api_user_id"), None + if health is False: + conn.execute( + "DELETE FROM user_gateway_keys WHERE user_id = ?", + (user_id,)) + conn.commit() + key, sub_id, key_id = _create_fresh_gateway_key( + conn, user_id, admin=admin, request=request) + return "rotated", key, sub_id, key_id + raise Sub2apiError("sub2api 网关健康检查暂不可用，保留现有 key") + + +def rotate_user_gateway_key(conn, user_id, admin=None, request=None): + """仅限本人调用的重新开通/轮换：删旧行后重开。 + + 无旧行时直接走开通（与 validate 的 provisioned 等价）；返回 + (status, key, sub_id, key_id)，调用方记审计时只可用 sub_id / + key_id，绝不记 key 明文。旧 key 即使只是“疑似失效”也会被 + 换掉：轮换是用户显式动作，不以健康检查为前提。 + """ + ensure_gateway_keys_table(conn) + row = get_user_gateway_key(conn, user_id) + if row is not None: + conn.execute( + "DELETE FROM user_gateway_keys WHERE user_id = ?", + (user_id,)) + conn.commit() + key, sub_id, key_id = _create_fresh_gateway_key( + conn, user_id, admin=admin, request=request) + return ("rotated" if row is not None else "provisioned", + key, sub_id, key_id) diff --git a/cloud/tests/test_caps_server.py b/cloud/tests/test_caps_server.py index b1d7206..c98fe97 100644 --- a/cloud/tests/test_caps_server.py +++ b/cloud/tests/test_caps_server.py @@ -33,6 +33,9 @@ class CapsServerTestCase(unittest.TestCase): self.auth = AuthStore(self.db_path) self.owner_id, self.owner_key = self.auth.create_user("owner") self.viewer_id, self.viewer_key = self.auth.create_user("viewer") + # P1：完整 workers 清单只限全局管理员；owner 在此测试里即管理员。 + self._old_admins = os.environ.get("CODEX_GLOBAL_ADMINS") + os.environ["CODEX_GLOBAL_ADMINS"] = self.owner_id self.server = create_server(self.db_path, "127.0.0.1", 0) self.port = self.server.server_address[1] self.thread = threading.Thread( @@ -48,6 +51,10 @@ class CapsServerTestCase(unittest.TestCase): except Exception: pass self.tmp.cleanup() + if getattr(self, "_old_admins", None) is None: + os.environ.pop("CODEX_GLOBAL_ADMINS", None) + else: + os.environ["CODEX_GLOBAL_ADMINS"] = self._old_admins def _req(self, method, path, body=None, key=None): conn = http.client.HTTPConnection("127.0.0.1", self.port, timeout=10) @@ -82,11 +89,21 @@ class CapsServerTestCase(unittest.TestCase): "capacity": 4, "running": 2}, self.owner_key) self.assertEqual(status, 200, body) - # 读权限：任何已认证用户可读（viewer 也能看到基础设施行）。 - workers = self._workers(self.viewer_key) + # P1：完整清单只限全局管理员（owner 在此测试里即管理员）。 + workers = self._workers(self.owner_key) self.assertEqual(workers["w1"]["capacity"], 4) self.assertEqual(workers["w1"]["running"], 2) + def test_workers_list_non_admin_gets_aggregate_only(self): + self._req("POST", "/workers/heartbeat", + {"worker_id": "w1", "executor": "local", + "capacity": 4, "running": 2}, + self.owner_key) + status, body = self._req("GET", "/workers", key=self.viewer_key) + self.assertEqual(status, 200, body) + self.assertNotIn("workers", body) + self.assertIn("workers_total", body) + def test_heartbeat_without_running_preserves_old(self): self._req("POST", "/workers/heartbeat", {"worker_id": "w1", "capacity": 2, "running": 1}, diff --git a/cloud/tests/test_cloud_dispatch_pr.py b/cloud/tests/test_cloud_dispatch_pr.py index 8d27c62..1f62d2e 100644 --- a/cloud/tests/test_cloud_dispatch_pr.py +++ b/cloud/tests/test_cloud_dispatch_pr.py @@ -184,7 +184,16 @@ class WorkerRegistryTestCase(_ServerCase): self.assertEqual(status, 200, body) self.assertTrue(body["alive"]) self.assertEqual(body["executor"], "docker") - status, body = self._req("GET", "/workers", None, self.owner_key) + # P1：完整 workers 清单只限全局管理员，本测试里 owner 即管理员。 + old_admins = os.environ.get("CODEX_GLOBAL_ADMINS") + os.environ["CODEX_GLOBAL_ADMINS"] = self.owner_id + try: + status, body = self._req("GET", "/workers", None, self.owner_key) + finally: + if old_admins is None: + os.environ.pop("CODEX_GLOBAL_ADMINS", None) + else: + os.environ["CODEX_GLOBAL_ADMINS"] = old_admins self.assertEqual(status, 200, body) self.assertEqual(len(body["workers"]), 1) self.assertTrue(body["workers"][0]["alive"]) diff --git a/cloud/tests/test_github_repos.py b/cloud/tests/test_github_repos.py index a56a857..889776f 100644 --- a/cloud/tests/test_github_repos.py +++ b/cloud/tests/test_github_repos.py @@ -95,13 +95,23 @@ class ServerGithubReposTestCase(unittest.TestCase): def test_list_returns_seeded_installations(self): self._seed_installations() + # P1：repos 只返回调用者成员项目对应的仓库；先把项目仓库 + # 绑到其中一个已安装仓库，再断言只返回该仓库，非成员为空。 + s, d = _req(self.port, "PATCH", + "/projects/" + self.project_id, + {"repo_url": "https://github.com/openqj/codex-app-server.git"}, + self.owner_key) + self.assertEqual(s, 200, d) s, d = _req(self.port, "GET", "/github/repos", key=self.owner_key) self.assertEqual(s, 200, d) names = [r["repo_full_name"] for r in d["repos"]] - self.assertEqual(names, ["openqj/codex-app-server", "openqj/demo"]) + self.assertEqual(names, ["openqj/codex-app-server"]) self.assertEqual( d["repos"][0]["clone_url"], "https://github.com/openqj/codex-app-server.git") + s, d = _req(self.port, "GET", "/github/repos", key=self.other_key) + self.assertEqual(s, 200, d) + self.assertEqual(d["repos"], []) def test_patch_repo_url_owner(self): url = "https://github.com/openqj/demo.git" diff --git a/cloud/tests/test_p1_security_server.py b/cloud/tests/test_p1_security_server.py new file mode 100644 index 0000000..2196586 --- /dev/null +++ b/cloud/tests/test_p1_security_server.py @@ -0,0 +1,598 @@ +"""P1 控制面/控制台单测（server.py / console.html / sub2api.py / push.py）。 + +只覆盖本批 P1 控制面改动，不跑真实模型回合、不联网、不碰生产： +server 侧走本地临时端口 + 临时 SQLite；网关侧用假 request / +假 admin（key、token 全是测试假值）；推送 DNS 用注入的 +resolver。绝不改 worker.py / coordinator.py / cost.py / host.py。 +""" + +import http.client +import json +import os +import sqlite3 +import stat +import sys +import tempfile +import threading +import unittest +from pathlib import Path +from unittest import mock + +try: + from cloud.auth import AuthStore + from cloud.push import dispatch_event, validate_push_target + from cloud.push import create_subscription + from cloud.server import create_server + from cloud import server as server_mod + from cloud.sub2api import Sub2apiError + from cloud.sub2api import ensure_gateway_keys_table + from cloud.sub2api import get_user_gateway_key + from cloud.sub2api import rotate_user_gateway_key + from cloud.sub2api import validate_or_rotate_gateway_key +except ImportError: + sys.path.insert(0, str(Path(__file__).resolve().parents[2])) + from cloud.auth import AuthStore + from cloud.push import dispatch_event, validate_push_target + from cloud.push import create_subscription + from cloud.server import create_server + from cloud import server as server_mod + from cloud.sub2api import Sub2apiError + from cloud.sub2api import ensure_gateway_keys_table + from cloud.sub2api import get_user_gateway_key + from cloud.sub2api import rotate_user_gateway_key + from cloud.sub2api import validate_or_rotate_gateway_key + + +def _req(port, method, path, body=None, key=None, raw_body=None): + conn = http.client.HTTPConnection("127.0.0.1", port, timeout=10) + hdrs = {} + if raw_body is not None: + payload = raw_body + elif body is not None: + payload = json.dumps(body).encode("utf-8") + hdrs["Content-Type"] = "application/json" + else: + payload = None + if key is not None: + hdrs["X-API-Key"] = key + try: + conn.request(method, path, body=payload, headers=hdrs) + resp = conn.getresponse() + data = resp.read() + finally: + conn.close() + try: + parsed = json.loads(data.decode("utf-8")) if data else None + except ValueError: + parsed = data.decode("utf-8", "replace") + return resp.status, parsed + + +class _FakeGatewayAdmin: + """假网关管理员：建号/授余额只记调用，不联网。""" + + def __init__(self): + self.created = [] + self.deleted = [] + self.balances = [] + + def create_user(self, email, password, username): + sub_id = 1000 + len(self.created) + self.created.append((email, username)) + return sub_id + + def delete_user(self, sub_user_id): + self.deleted.append(sub_user_id) + return True + + def set_balance(self, sub_user_id, amount): + self.balances.append((sub_user_id, amount)) + + def resolve_group_id(self): + return 2 + + +class _FakeGatewayRequest: + """假网关 HTTP：models 状态可调，其余走固定成功路径。""" + + def __init__(self, models_status=200): + self.models_status = models_status + self.calls = [] + + def __call__(self, method, url, body=None, token=None): + self.calls.append((method, url)) + if url.endswith("/v1/models"): + return self.models_status, {"data": []} + if url.endswith("/api/v1/auth/login"): + return 200, {"data": {"access_token": "fake-user-token", + "refresh_token": "fake-refresh", + "expires_in": 3600}} + if url.endswith("/api/v1/keys"): + return 201, {"data": {"id": "fake-key-id-1", + "key": "fake-gateway-key-1"}} + if "/balance" in url: + return 200, {"data": {}} + return 200, {"data": {}} + + +class Sub2apiP1TestCase(unittest.TestCase): + def setUp(self): + self.conn = sqlite3.connect(":memory:") + ensure_gateway_keys_table(self.conn) + self.admin = _FakeGatewayAdmin() + # 假 origin：请求本身全是 fake，不产生真实网络调用。 + self._old_origin = os.environ.get("SUB2API_BASE_URL") + os.environ["SUB2API_BASE_URL"] = "http://127.0.0.1:9" + + def tearDown(self): + self.conn.close() + if self._old_origin is None: + os.environ.pop("SUB2API_BASE_URL", None) + else: + os.environ["SUB2API_BASE_URL"] = self._old_origin + + def _seed_row(self, key="fake-old-key"): + self.conn.execute( + "INSERT INTO user_gateway_keys(user_id, sub2api_user_id," + " api_key, refresh_token, created_at, updated_at)" + " VALUES(?, ?, ?, ?, ?, ?)", + ("u1", 42, key, "fake-refresh", 1.0, 1.0)) + self.conn.commit() + + def test_validate_no_row_provisions(self): + req = _FakeGatewayRequest() + status, key, sub_id, key_id = validate_or_rotate_gateway_key( + self.conn, "u1", admin=self.admin, request=req) + self.assertEqual(status, "provisioned") + self.assertEqual(key, "fake-gateway-key-1") + self.assertEqual(key_id, "fake-key-id-1") + self.assertIsNotNone(get_user_gateway_key(self.conn, "u1")) + + def test_validate_healthy_key_not_rotated(self): + self._seed_row() + req = _FakeGatewayRequest(models_status=200) + status, key, _sub, _kid = validate_or_rotate_gateway_key( + self.conn, "u1", admin=self.admin, request=req) + self.assertEqual(status, "ok") + self.assertEqual(key, "fake-old-key") + self.assertEqual(self.admin.created, []) + + def test_validate_dead_key_rotates_not_pending(self): + self._seed_row() + req = _FakeGatewayRequest(models_status=403) + status, key, _sub, _kid = validate_or_rotate_gateway_key( + self.conn, "u1", admin=self.admin, request=req) + self.assertEqual(status, "rotated") + self.assertEqual(key, "fake-gateway-key-1") + row = get_user_gateway_key(self.conn, "u1") + self.assertEqual(row["api_key"], "fake-gateway-key-1") + self.assertNotEqual(row["api_key"], "fake-old-key") + + def test_validate_zero_balance_403_rotates(self): + # 余额为 0 时网关鉴权层回 403：与 401 同属死 key，必须轮换。 + self._seed_row() + req = _FakeGatewayRequest(models_status=403) + status, _key, _sub, _kid = validate_or_rotate_gateway_key( + self.conn, "u1", admin=self.admin, request=req) + self.assertEqual(status, "rotated") + + def test_validate_gateway_down_keeps_old_row(self): + self._seed_row() + req = _FakeGatewayRequest(models_status=500) + with self.assertRaises(Sub2apiError): + validate_or_rotate_gateway_key( + self.conn, "u1", admin=self.admin, request=req) + row = get_user_gateway_key(self.conn, "u1") + self.assertEqual(row["api_key"], "fake-old-key") + + def test_rotate_replaces_row(self): + self._seed_row() + req = _FakeGatewayRequest() + status, key, sub_id, key_id = rotate_user_gateway_key( + self.conn, "u1", admin=self.admin, request=req) + self.assertEqual(status, "rotated") + self.assertEqual(key_id, "fake-key-id-1") + self.assertEqual( + get_user_gateway_key(self.conn, "u1")["api_key"], key) + + +class PushP1TestCase(unittest.TestCase): + def setUp(self): + self.conn = sqlite3.connect(":memory:") + + def tearDown(self): + self.conn.close() + + def test_literal_private_and_loopback_blocked(self): + for target in ( + "http://127.0.0.1:9/hook", + "http://10.0.0.5/hook", + "http://172.16.4.4/hook", + "http://192.168.1.10/hook", + "http://169.254.169.254/latest/meta-data/", + "http://[::1]/hook"): + self.assertIsNotNone( + validate_push_target(target), target) + + def test_metadata_hostname_blocked(self): + self.assertIsNotNone(validate_push_target( + "http://metadata.google.internal/hook")) + + def test_hostname_resolved_to_private_blocked(self): + err = validate_push_target( + "https://hooks.example.com/hook", + resolver=lambda host: ["10.1.2.3"]) + self.assertIsNotNone(err) + + def test_hostname_public_resolves_ok(self): + err = validate_push_target( + "https://hooks.example.com/hook", + resolver=lambda host: ["93.184.216.34"]) + self.assertIsNone(err) + + def test_dns_failure_rejected(self): + def _boom(host): + raise OSError("no dns in test") + self.assertIsNotNone(validate_push_target( + "https://hooks.example.com/hook", resolver=_boom)) + + def test_dispatch_loopback_never_posts(self): + # 真实发送路径（sender=None，生产服务端走的路径）： + # loopback 目标在预检即被挡，不产生任何 POST。 + create_subscription(self.conn, "proj-a", "user-1", "webhook", + "http://127.0.0.1:9/hook", "instant") + result = dispatch_event(self.conn, "failed", "proj-a", + {"task_id": "t1"}, sender=None) + self.assertEqual(result["sent"], []) + self.assertEqual(len(result["errors"]), 1) + self.assertIn("blocked", result["errors"][0]["reason"]) + + +class ServerP1TestCase(unittest.TestCase): + def setUp(self): + self.tmp = tempfile.TemporaryDirectory() + self.db_path = os.path.join(self.tmp.name, "control.db") + self.auth = AuthStore(self.db_path) + self.owner_id, self.owner_key = self.auth.create_user("owner") + self.member_id, self.member_key = self.auth.create_user("member") + self.outsider_id, self.outsider_key = self.auth.create_user( + "outsider") + self._old_admins = os.environ.get("CODEX_GLOBAL_ADMINS") + self._old_gateway = os.environ.get("CLOUD_GATEWAY") + self._old_backup_root = os.environ.get("CLOUD_BACKUP_ROOT") + self._old_backup_dir = os.environ.get("CLOUD_BACKUP_DIR") + os.environ["CODEX_GLOBAL_ADMINS"] = self.owner_id + os.environ.pop("CLOUD_GATEWAY", None) + os.environ.pop("CLOUD_BACKUP_ROOT", None) + os.environ.pop("CLOUD_BACKUP_DIR", None) + self._old_max_projects = server_mod.MAX_PROJECTS_PER_USER + self.server = create_server(self.db_path, "127.0.0.1", 0) + self.port = self.server.server_address[1] + self.thread = threading.Thread( + target=self.server.serve_forever, daemon=True) + self.thread.start() + s, proj = _req(self.port, "POST", "/projects", + {"name": "owner/repo"}, self.owner_key) + self.assertEqual(s, 201, proj) + self.project_id = proj["id"] + self.auth.set_member( + self.project_id, self.owner_id, self.member_id, "maintainer") + + def tearDown(self): + self.server.shutdown() + self.server.server_close() + self.thread.join(timeout=5) + try: + self.auth.conn.close() + except Exception: + pass + self.tmp.cleanup() + server_mod.MAX_PROJECTS_PER_USER = self._old_max_projects + if self._old_admins is None: + os.environ.pop("CODEX_GLOBAL_ADMINS", None) + else: + os.environ["CODEX_GLOBAL_ADMINS"] = self._old_admins + if self._old_gateway is None: + os.environ.pop("CLOUD_GATEWAY", None) + else: + os.environ["CLOUD_GATEWAY"] = self._old_gateway + if self._old_backup_root is None: + os.environ.pop("CLOUD_BACKUP_ROOT", None) + else: + os.environ["CLOUD_BACKUP_ROOT"] = self._old_backup_root + if self._old_backup_dir is None: + os.environ.pop("CLOUD_BACKUP_DIR", None) + else: + os.environ["CLOUD_BACKUP_DIR"] = self._old_backup_dir + # 恢复猴子补丁过的 server 模块函数（relay/rotate 相关测试用）。 + for name in ("_validate_sub2api_key", "_list_sub2api_models", + "_rotate_sub2api_key"): + try: + mock.patch.object(server_mod, name).stop() + except Exception: + pass + + # -- helpers --------------------------------------------------- + def _audit_actions(self, user_id=None): + conn = sqlite3.connect(self.db_path) + try: + rows = conn.execute( + "SELECT action, detail FROM audit_log" + + (" WHERE user_id = ?" if user_id else ""), + (user_id,) if user_id else ()).fetchall() + finally: + conn.close() + return rows + + def _set_quota(self, target, quota): + return _req(self.port, "POST", "/admin/quotas", + {"user_id": target, "quota_usd_micros": quota}, + self.owner_key) + + # -- 备份 ------------------------------------------------------- + def test_backup_whitelist_and_mode(self): + root = os.path.join(self.tmp.name, "backup-root") + os.environ["CLOUD_BACKUP_ROOT"] = root + out_dir = os.path.join(root, "daily") + s, body = _req(self.port, "POST", "/admin/backup", + {"out_dir": out_dir}, self.owner_key) + self.assertEqual(s, 200, body) + self.assertTrue(os.path.exists(body["path"])) + self.assertEqual(stat.S_IMODE(os.stat(body["path"]).st_mode), + 0o600) + details = [d for _a, d in self._audit_actions(self.owner_id)] + self.assertTrue(any("admin_backup" in a + for a, _d in self._audit_actions( + self.owner_id))) + self.assertTrue(any(out_dir in (d or "") + for d in details)) + # 白名单外：系统目录与 .. 逃逸一律 400。 + for bad in ("/etc", os.path.join(root, "..", "escape")): + s, body = _req(self.port, "POST", "/admin/backup", + {"out_dir": bad}, self.owner_key) + self.assertEqual(s, 400, (bad, body)) + + # -- workers 聚合 ------------------------------------------------ + def test_workers_admin_full_member_aggregate_only(self): + s, _b = _req(self.port, "POST", "/workers/heartbeat", + {"worker_id": "w-p1", "executor": "docker", + "capacity": 2, "running": 0}, self.owner_key) + self.assertEqual(s, 200, _b) + s, body = _req(self.port, "GET", "/workers", key=self.owner_key) + self.assertEqual(s, 200) + self.assertIn("workers", body) + self.assertTrue(any(w.get("worker_id") == "w-p1" + for w in body["workers"])) + s, body = _req(self.port, "GET", "/workers", key=self.member_key) + self.assertEqual(s, 200) + self.assertNotIn("workers", body) + self.assertNotIn("w-p1", json.dumps(body)) + self.assertEqual(body["workers_total"], 1) + self.assertEqual(body["workers_online"], 1) + self.assertEqual(body["workers_offline"], 0) + + # -- github repos 过滤 ------------------------------------------- + def test_github_repos_filtered_by_member_projects(self): + s, proj = _req(self.port, "POST", "/projects", + {"name": "with-repo", + "repo_url": "https://github.com/acme/alpha.git"}, + self.owner_key) + self.assertEqual(s, 201, proj) + self.auth.set_member(proj["id"], self.owner_id, self.member_id, + "maintainer") + conn = sqlite3.connect(self.db_path) + try: + conn.execute( + "CREATE TABLE IF NOT EXISTS github_app_installations(" + "installation_id TEXT, account_login TEXT," + " repo_full_name TEXT, created_at REAL," + " UNIQUE(installation_id, repo_full_name))") + conn.execute( + "INSERT OR IGNORE INTO github_app_installations" + " VALUES(?, ?, ?, ?)", ("7", "acme", "acme/alpha", 1.0)) + conn.execute( + "INSERT OR IGNORE INTO github_app_installations" + " VALUES(?, ?, ?, ?)", ("7", "acme", "acme/beta", 1.0)) + conn.commit() + finally: + conn.close() + s, body = _req(self.port, "GET", "/github/repos", + key=self.member_key) + self.assertEqual(s, 200, body) + names = [r["repo_full_name"] for r in body["repos"]] + self.assertEqual(names, ["acme/alpha"]) + s, body = _req(self.port, "GET", "/github/repos", + key=self.outsider_key) + self.assertEqual(s, 200, body) + self.assertEqual(body["repos"], []) + + # -- key 审计 ----------------------------------------------------- + def test_api_key_audit_only_id_and_label(self): + s, body = _req(self.port, "POST", "/users/me/keys", + {"label": "p1-test"}, self.owner_key) + self.assertEqual(s, 201, body) + key_id = body["id"] + plain = body["api_key"] + s, _b = _req(self.port, "DELETE", + "/users/me/keys/" + key_id, key=self.owner_key) + self.assertEqual(s, 200, _b) + s, body = _req(self.port, "POST", + "/users/me/keys/revoke-others", {}, + self.owner_key) + self.assertEqual(s, 200, body) + actions = {a for a, _d in self._audit_actions(self.owner_id)} + self.assertIn("api_key_created", actions) + self.assertIn("api_key_revoked", actions) + self.assertIn("api_key_revoke_others", actions) + blob = json.dumps(self._audit_actions(self.owner_id)) + self.assertNotIn(plain, blob) + + # -- 项目上限 ------------------------------------------------------ + def test_project_limits(self): + s, body = _req(self.port, "POST", "/projects", + {"name": "x" * 101}, self.owner_key) + self.assertEqual(s, 400, body) + self.assertIn("100", body["error"]) + s, body = _req(self.port, "POST", "/projects", + {"name": "ok", "id": "y" * 65}, self.owner_key) + self.assertEqual(s, 400, body) + self.assertIn("64", body["error"]) + server_mod.MAX_PROJECTS_PER_USER = 1 + # setUp 已建 1 个 owner 项目，限额 1 时再建即超限。 + s, body = _req(self.port, "POST", "/projects", + {"name": "second"}, self.owner_key) + self.assertEqual(s, 400, body) + self.assertIn("max 1", body["error"]) + + # -- 协调员预留 ---------------------------------------------------- + def test_coordinator_reserve_blocks_parent(self): + s, _b = self._set_quota(self.owner_id, 1000000) + self.assertEqual(s, 200, _b) + parts = [{"name": "a", "prompt": "做 a"}, + {"name": "b", "prompt": "做 b"}, + {"name": "c", "prompt": "做 c"}] + s, body = _req(self.port, "POST", "/tasks", + {"project_id": self.project_id, "prompt": "协调", + "coordinator_parts": parts}, self.owner_key) + self.assertEqual(s, 402, body) + self.assertEqual(body["error"], + "coordinator_quota_reserve_exceeded") + self.assertEqual(body["coordinator_parts"], 3) + s, _b = self._set_quota(self.owner_id, 10000000) + self.assertEqual(s, 200, _b) + s, body = _req(self.port, "POST", "/tasks", + {"project_id": self.project_id, "prompt": "协调", + "coordinator_parts": parts}, self.owner_key) + self.assertEqual(s, 201, body) + self.assertIn("parent task creator", + body.get("quota_attribution", "")) + self.assertEqual(body.get("quota_reserved_usd_micros"), 3000000) + + # -- 自动化配额门 ---------------------------------------------------- + def test_automation_create_quota_gate(self): + s, _b = self._set_quota(self.owner_id, 0) + self.assertEqual(s, 200, _b) + s, body = _req(self.port, "POST", + "/projects/%s/automations" % self.project_id, + {"name": "nightly", "prompt": "巡检"}, + self.owner_key) + self.assertEqual(s, 402, body) + s, _b = self._set_quota(self.owner_id, 10000000) + self.assertEqual(s, 200, _b) + s, body = _req(self.port, "POST", + "/projects/%s/automations" % self.project_id, + {"name": "nightly", "prompt": "巡检"}, + self.owner_key) + self.assertEqual(s, 201, body) + + # -- sub2api 懒开通 / 轮换端点 -------------------------------------- + def test_relay_models_lazy_provision_not_just_pending(self): + os.environ["CLOUD_GATEWAY"] = "sub2api" + orig_validate = server_mod._validate_sub2api_key + orig_list = server_mod._list_sub2api_models + try: + server_mod._validate_sub2api_key = ( + lambda conn, uid: ("provisioned", "fake-key", 7, 9)) + server_mod._list_sub2api_models = ( + lambda origin, key: ["model-a"]) + s, body = _req(self.port, "GET", "/relay/models", + key=self.owner_key) + self.assertEqual(s, 200, body) + self.assertEqual(body["models"], ["model-a"]) + self.assertEqual(body["provisioning"], "provisioned") + server_mod._validate_sub2api_key = ( + lambda conn, uid: (_ for _ in ()).throw( + Sub2apiError("网关暂不可用"))) + s, body = _req(self.port, "GET", "/relay/models", + key=self.owner_key) + self.assertEqual(s, 502, body) + finally: + server_mod._validate_sub2api_key = orig_validate + server_mod._list_sub2api_models = orig_list + + def test_gateway_rotate_endpoint(self): + s, body = _req(self.port, "POST", "/users/me/gateway/rotate", + {}, self.owner_key) + self.assertEqual(s, 400, body) + os.environ["CLOUD_GATEWAY"] = "sub2api" + orig_rotate = server_mod._rotate_sub2api_key + try: + server_mod._rotate_sub2api_key = ( + lambda conn, uid: ("rotated", "fake-key", 7, 9)) + s, body = _req(self.port, "POST", "/users/me/gateway/rotate", + {}, self.owner_key) + self.assertEqual(s, 200, body) + self.assertTrue(body["rotated"]) + self.assertNotIn("api_key", body) + actions = {a for a, _d in self._audit_actions(self.owner_id)} + self.assertIn("gateway_key_rotated", actions) + finally: + server_mod._rotate_sub2api_key = orig_rotate + + # -- 推送审计与分发 SSRF ---------------------------------------------- + def test_push_audit_and_dispatch_ssrf(self): + s, sub = _req(self.port, "POST", + "/projects/%s/push/subscriptions" % self.project_id, + {"channel": "webhook", + "target": "https://example.com/hook", + "preference": "instant"}, self.owner_key) + self.assertEqual(s, 201, sub) + actions = {a for a, _d in self._audit_actions(self.owner_id)} + self.assertIn("push_subscription_created", actions) + s, _b = _req(self.port, "DELETE", + "/projects/%s/push/subscriptions/%s" + % (self.project_id, sub["id"]), key=self.owner_key) + self.assertEqual(s, 200, _b) + actions = {a for a, _d in self._audit_actions(self.owner_id)} + self.assertIn("push_subscription_deleted", actions) + # 内网目标：分发不 POST、sent 为空、错误含 delivery failed。 + s, sub = _req(self.port, "POST", + "/projects/%s/push/subscriptions" % self.project_id, + {"channel": "webhook", + "target": "http://127.0.0.1:9/hook", + "preference": "instant"}, self.owner_key) + self.assertEqual(s, 201, sub) + s, body = _req(self.port, "POST", "/admin/push/dispatch", + {"event_type": "failed", + "project_id": self.project_id, + "payload": {"task_id": "t1"}}, self.owner_key) + self.assertEqual(s, 200, body) + self.assertEqual(body["sent"], []) + self.assertEqual(len(body["errors"]), 1) + self.assertIn("delivery failed", body["errors"][0]["reason"]) + actions = {a for a, _d in self._audit_actions(self.owner_id)} + self.assertIn("admin_push_dispatch", actions) + + # -- 配额双字段与控制台文案 -------------------------------------------- + def test_quota_payload_has_unpriced_fields(self): + s, body = _req(self.port, "GET", "/users/me/quota", + key=self.owner_key) + self.assertEqual(s, 200, body) + self.assertIn("unpriced_tokens_used", body) + self.assertIn("unpriced_token_cap", body) + + def test_console_p1_markers(self): + conn = http.client.HTTPConnection( + "127.0.0.1", self.port, timeout=10) + try: + conn.request("GET", "/console") + resp = conn.getresponse() + text = resp.read().decode("utf-8") + finally: + conn.close() + self.assertEqual(resp.status, 200) + for marker in ( + "登记评审需求", + "记录允许", + "记录拒绝", + "当前版本只留回执，不拦截实时行为", + "模型回合结束、改动在 worktree 里尚未合并", + "控制面已标记 merged，不代表 git 主分支已合入", + "协调员与 goal/计划互斥", + "仅 failed/interrupted/merge-failed 可重试", + "仅 maintainer 可合并"): + self.assertIn(marker, text) + + +if __name__ == "__main__": + unittest.main() diff --git a/cloud/tests/test_p1_security_worker.py b/cloud/tests/test_p1_security_worker.py new file mode 100644 index 0000000..8cf2d4c --- /dev/null +++ b/cloud/tests/test_p1_security_worker.py @@ -0,0 +1,534 @@ +"""P1 执行面针对性单测（只测本路改动，不跑全量三套）。 + +运行（在 ~/workspace/codex-app-server-p1security 下）： + python3 -m unittest discover -s cloud/tests -p "test_p1_security_worker.py" -v + +说明：只用注入的假 runner 与测试假值，不跑真实模型回合、不联网、 +不登录任何服务；env 里的值只是占位用的假名，绝不是真实密钥。 +本文件只归本路（执行面）所有，不碰 test_p1_security_server.py。 +""" + +import json +import os +import sqlite3 +import sys +import tempfile +import threading +import time +import unittest +from pathlib import Path +from types import SimpleNamespace +from unittest import mock + +try: + import cloud.sandbox as sandbox_mod + from cloud.cost import CostStore + from cloud.pr_exec import enqueue_pr_job + from cloud.sandbox import DockerExecutor, LocalExecutor + from cloud.server import init_db + from cloud.worker import Worker, _code_signature +except ImportError: # discover -s cloud/tests 时把项目根目录补进 sys.path + sys.path.insert(0, str(Path(__file__).resolve().parents[2])) + import cloud.sandbox as sandbox_mod + from cloud.cost import CostStore + from cloud.pr_exec import enqueue_pr_job + from cloud.sandbox import DockerExecutor, LocalExecutor + from cloud.server import init_db + from cloud.worker import Worker, _code_signature + +FAKE_ENV = {"XCM_API_KEY": "test-placeholder"} + + +def _completed_runner(cmd): + if "dispatch" in cmd: + name = cmd[cmd.index("--name") + 1] + else: + name = cmd[cmd.index("--task") + 1] + return SimpleNamespace( + returncode=0, + stdout=json.dumps({"task": name, "status": "completed", + "text": "done"}), + stderr="") + + +def _usage_runner(usage, calls): + def fake(cmd): + calls.append(list(cmd)) + return SimpleNamespace( + returncode=0, + stdout=json.dumps({"status": "completed", "text": "done", + "usage": usage}), + stderr="") + return fake + + +def _insert_task(db_path, task_id, created_by="tester", status="queued", + project_id="p1", **cols): + now = time.time() + conn = sqlite3.connect(db_path) + try: + conn.execute( + "INSERT INTO tasks(id, project_id, prompt, status," + " idempotency_key, depends_on, priority, created_by," + " created_at, updated_at, last_heartbeat)" + " VALUES(?, ?, ?, ?, NULL, '[]', 0, ?, ?, ?, NULL)", + (task_id, project_id, "do work", status, created_by, now, now), + ) + for key, value in cols.items(): + conn.execute( + "UPDATE tasks SET %s = ? WHERE id = ?" % key, + (value, task_id)) + conn.commit() + finally: + conn.close() + + +def _insert_quota(db_path, user_id, micros): + conn = sqlite3.connect(db_path) + try: + conn.execute( + "CREATE TABLE IF NOT EXISTS user_quotas(" + "user_id TEXT PRIMARY KEY, quota_usd_micros BIGINT," + " updated_at DOUBLE PRECISION)") + conn.execute( + "INSERT INTO user_quotas(user_id, quota_usd_micros, updated_at)" + " VALUES(?, ?, ?)", + (user_id, micros, time.time())) + conn.commit() + finally: + conn.close() + + +def _insert_cost(db_path, task_id, model, input_tokens, output_tokens, + micros, source="actual"): + _insert_task(db_path, task_id, status="completed") + store = CostStore(db_path) + try: + store.record(task_id, "p1", model, input_tokens=input_tokens, + output_tokens=output_tokens, usd_micros=micros, + source=source) + finally: + try: + store._conn.close() + except Exception: + pass + + +def _status(db_path, task_id): + conn = sqlite3.connect(db_path) + try: + row = conn.execute( + "SELECT status FROM tasks WHERE id = ?", (task_id,)).fetchone() + finally: + conn.close() + return row[0] if row else None + + +def _error_text(db_path, task_id): + conn = sqlite3.connect(db_path) + try: + row = conn.execute( + "SELECT error_text FROM tasks WHERE id = ?", + (task_id,)).fetchone() + finally: + conn.close() + return row[0] if row else None + + +class P1WorkerTestCase(unittest.TestCase): + def setUp(self): + self.tmp = tempfile.TemporaryDirectory() + self.db_path = os.path.join(self.tmp.name, "control.db") + init_db(self.db_path) + self.calls = [] + + def tearDown(self): + self.tmp.cleanup() + + def _worker(self, runner=None, **kwargs): + kwargs.setdefault("env", FAKE_ENV) + return Worker(self.db_path, "/fake/host.py", + runner=runner or _completed_runner, **kwargs) + + # 1. 无配额行按默认 $10 额度（不再是不限额） + def test_missing_quota_row_uses_default_quota(self): + _insert_cost(self.db_path, "donor-a", "gpt-6.1-sol", + 1, 0, 11000000) + _insert_cost(self.db_path, "donor-b", "gpt-6.1-sol", + 1, 0, 9000000) + worker = self._worker() + over = worker._quota_exceeded( + {"created_by": "tester", "project_id": "p1"}, None) + # donor-a 归 tester：已花 $11 > 默认 $10，挡住。 + self.assertIsNotNone(over) + self.assertIn("quota_exceeded", over) + # donor-b 同样归 tester 会把总额推得更高，单独测另一用户： + _insert_task(self.db_path, "donor-c", created_by="rich", + status="completed") + store = CostStore(self.db_path) + try: + store.record("donor-c", "p1", "gpt-6.1-sol", input_tokens=1, + output_tokens=0, usd_micros=9000000, + source="actual") + finally: + try: + store._conn.close() + except Exception: + pass + under = worker._quota_exceeded( + {"created_by": "rich", "project_id": "p1"}, None) + self.assertIsNone(under) + + # 2. 显式 NULL 配额行仍是不限额 + def test_explicit_null_quota_row_stays_unlimited(self): + _insert_quota(self.db_path, "tester", None) + _insert_cost(self.db_path, "donor", "gpt-6.1-sol", + 1, 0, 999000000) + worker = self._worker() + self.assertIsNone(worker._quota_exceeded( + {"created_by": "tester", "project_id": "p1"}, None)) + + # 3. 属主为空拒绝新回合（不再返回 None 放行） + def test_ownerless_task_refused(self): + worker = self._worker() + for owner in (None, ""): + err = worker._quota_exceeded( + {"created_by": owner, "project_id": "p1"}, None) + self.assertIsNotNone(err) + self.assertIn("quota_exceeded", err) + + # 4. 协调员子任务属主跟父任务走 + def test_child_inherits_parent_owner(self): + _insert_quota(self.db_path, "tester", 100) + _insert_cost(self.db_path, "donor", "gpt-6.1-sol", + 1, 0, 200) + _insert_task(self.db_path, "parent", created_by="tester") + _insert_task(self.db_path, "parent2", created_by="fresh") + worker = self._worker() + blocked = worker._quota_exceeded( + {"created_by": None, "parent_task_id": "parent", + "project_id": "p1"}, None) + self.assertIsNotNone(blocked) + self.assertIn("quota_exceeded", blocked) + # fresh 用户无配额行、未花钱：默认额度内放行。 + self.assertIsNone(worker._quota_exceeded( + {"created_by": None, "parent_task_id": "parent2", + "project_id": "p1"}, None)) + + # 5. 查库失败 fail-closed（新回合拒绝，不自动放行） + def test_quota_db_failure_is_fail_closed(self): + worker = self._worker() + with mock.patch("cloud.worker._db_connect", + side_effect=RuntimeError("test db down")): + err = worker._quota_exceeded( + {"created_by": "tester", "project_id": "p1"}, None) + self.assertIsNotNone(err) + self.assertTrue(err.startswith("quota_check_failed:")) + + # 6. Docker 配额看门狗：命中杀容器、未命中不杀 + def test_docker_quota_watchdog(self): + killed = [] + + def fake(cmd, timeout=None): + killed.append(list(cmd)) + return SimpleNamespace(returncode=0, stdout="", stderr="") + + executor = DockerExecutor(runner=fake) + worker = Worker(self.db_path, "/fake/host.py", + env=FAKE_ENV, executor=executor) + # 未命中：0.2 秒后停表，kill 不应被调。 + stop = threading.Event() + threading.Timer(0.2, stop.set).start() + worker._docker_quota_watchdog( + "T1", "codex-test", stop, lambda: False, interval=0.05) + self.assertEqual(killed, []) + # 命中：立即杀容器并记命中标记。 + stop2 = threading.Event() + worker._docker_quota_watchdog( + "T2", "codex-test", stop2, lambda: True, interval=0.05) + self.assertTrue(any("kill" in c for c in killed)) + self.assertTrue(worker._take_quota_hit("T2")) + # 命中已消费，第二次取不到（不污染下一轮）。 + self.assertFalse(worker._take_quota_hit("T2")) + + # 7. usage_cost_usd 按实际成本累加，max_cost_usd 不再是死参数 + def test_usage_cost_accumulates_and_budget_trips(self): + _insert_task(self.db_path, "T1") + worker = self._worker( + runner=_usage_runner( + {"input_tokens": 10000, "cached_input_tokens": 0, + "output_tokens": 1000, "total_tokens": 11000}, + self.calls), + model="gpt-6.1-sol", max_cost_usd=0.01) + self.assertFalse(worker._budget_exceeded()) + result = worker.run_one() + self.assertEqual(result["status"], "completed") + # 10000*$2 + 1000*$10 = $0.03，超过 $0.01 上限。 + self.assertAlmostEqual(worker.usage_cost_usd, 0.03, places=6) + self.assertTrue(worker._budget_exceeded()) + conn = sqlite3.connect(self.db_path) + try: + row = conn.execute( + "SELECT usd_micros, source FROM cost_records" + " WHERE task_id = 'T1'").fetchone() + finally: + conn.close() + self.assertEqual(row, (30000, "actual")) + + # 8. 取消分支先记成本再返回（与配额中断收尾同款记账点） + def test_cancel_branch_records_cost(self): + worker = self._worker() + task = {"id": "T1", "project_id": "p1", "prompt": "do work"} + with worker._db_lock: + cost_id = worker._record_usage_cost( + task, "gpt-6.1-sol", + {"input_tokens": 10000, "cached_input_tokens": 0, + "output_tokens": 1000, "total_tokens": 11000}, + "done", 0) + self.assertIsNotNone(cost_id) + self.assertAlmostEqual(worker.usage_cost_usd, 0.03, places=6) + # 无用量时回落估算，不累加 usage_cost_usd。 + with worker._db_lock: + worker._record_usage_cost( + {"id": "T2", "project_id": "p1", "prompt": "do work"}, + "gpt-6.1-sol", None, "done", 0) + self.assertAlmostEqual(worker.usage_cost_usd, 0.03, places=6) + + # 9. 估算行不计入无价 token 上限 + def test_estimated_rows_excluded_from_unpriced_cap(self): + _insert_cost(self.db_path, "donor-est", "gpt-5.6-terra", + 100000, 0, None, source="estimated") + worker = self._worker() + old = os.environ.get("CLOUD_QUOTA_UNPRICED_TOKEN_CAP") + os.environ["CLOUD_QUOTA_UNPRICED_TOKEN_CAP"] = "100" + try: + # 10 万估算 token 在上限 100 下仍放行（估算不是真花费）。 + self.assertIsNone(worker._quota_exceeded( + {"created_by": "tester", "project_id": "p1"}, None)) + _insert_cost(self.db_path, "donor-act", "gpt-5.6-terra", + 75, 75, None, source="actual") + err = worker._quota_exceeded( + {"created_by": "tester", "project_id": "p1"}, None) + self.assertIsNotNone(err) + self.assertIn("无价模型 token", err) + finally: + if old is None: + os.environ.pop("CLOUD_QUOTA_UNPRICED_TOKEN_CAP", None) + else: + os.environ["CLOUD_QUOTA_UNPRICED_TOKEN_CAP"] = old + + # 10. cache_write 不计入无价上限（只 input+output 计） + def test_cache_write_excluded_from_unpriced_cap(self): + worker = self._worker() + old = os.environ.get("CLOUD_QUOTA_UNPRICED_TOKEN_CAP") + os.environ["CLOUD_QUOTA_UNPRICED_TOKEN_CAP"] = "100" + try: + usage = {"input_tokens": 10, "output_tokens": 5, + "cache_write_input_tokens": 9999999, + "total_tokens": 15} + self.assertIsNone(worker._quota_exceeded( + {"created_by": "tester", "project_id": "p1", + "model": "gpt-5.6-terra"}, usage)) + usage2 = {"input_tokens": 200, "output_tokens": 0, + "cache_write_input_tokens": 0, + "total_tokens": 200} + err = worker._quota_exceeded( + {"created_by": "tester", "project_id": "p1", + "model": "gpt-5.6-terra"}, usage2) + self.assertIsNotNone(err) + finally: + if old is None: + os.environ.pop("CLOUD_QUOTA_UNPRICED_TOKEN_CAP", None) + else: + os.environ["CLOUD_QUOTA_UNPRICED_TOKEN_CAP"] = old + + # 11. 协调员子任务继承父任务模型/强度，分工显式值优先 + def test_coordinator_child_inherits_model_effort(self): + worker = self._worker() + task = {"id": "coord", "project_id": "p1", "prompt": "coord", + "priority": 0, "created_by": "tester", "model": "gpt-6.1-sol", + "effort": "high", "executor": None, "environment_id": None} + children = worker._coord_sync_children(task, [ + {"name": "a", "prompt": "do a", "model": None, + "effort": None, "executor": None}, + {"name": "b", "prompt": "do b", "model": "gpt-5.6-terra", + "effort": "low", "executor": None}, + ]) + self.assertEqual(len(children), 2) + conn = sqlite3.connect(self.db_path) + try: + rows = {r[0]: (r[1], r[2]) for r in conn.execute( + "SELECT coordinator_part, model, effort FROM tasks" + " WHERE parent_task_id = 'coord'").fetchall()} + finally: + conn.close() + self.assertEqual(rows["a"], ("gpt-6.1-sol", "high")) + self.assertEqual(rows["b"], ("gpt-5.6-terra", "low")) + + # 12. 断点续跑：Local/Docker/popen 三路都不传 --goal-text + def test_goal_resume_breakpoint_omits_goal_text(self): + calls = [] + + def fake(cmd): + calls.append(list(cmd)) + return SimpleNamespace( + returncode=0, + stdout="goal met after rounds\n" + "usage input=10 cached=0 cache_write=0 output=5 " + "reasoning=0 total=15 completions=1\n", + stderr="") + + task = {"id": "T1", "project_id": "p1", "prompt": "orig goal", + "verify_command": "true", "goal_max_rounds": 3} + local = LocalExecutor("/fake/host.py", runner=fake) + local.goal_resume_task(task, None) + self.assertIn("goal-resume", calls[-1]) + self.assertNotIn("--goal-text", calls[-1]) + docker_calls = [] + + def dfake(cmd, timeout=None): + docker_calls.append(list(cmd)) + return SimpleNamespace( + returncode=0, + stdout="goal met after rounds\n" + "usage input=10 cached=0 cache_write=0 output=5 " + "reasoning=0 total=15 completions=1\n", + stderr="") + + docker = DockerExecutor(runner=dfake, env=FAKE_ENV) + with mock.patch("cloud.sandbox.shutil.which", + return_value="/usr/bin/docker"): + docker.goal_resume_task(task, None) + self.assertIn("goal-resume", docker_calls[-1]) + self.assertNotIn("--goal-text", docker_calls[-1]) + with mock.patch.object(sandbox_mod.subprocess, "Popen") as popen: + popen.return_value = SimpleNamespace(pid=9999) + local.popen_task( + {"id": "T1", "prompt": "resume msg", + "_resume_kind": "breakpoint"}) + # popen_task 默认 kind=dispatch；显式测 goal-resume： + local.popen_task( + {"id": "T1", "prompt": "resume msg", + "_resume_kind": "breakpoint"}, kind="goal-resume") + cmd = popen.call_args[0][0] + self.assertIn("goal-resume", cmd) + self.assertNotIn("--goal-text", cmd) + + # 13. 开新目标：goal-resume 带 --goal-text（旧行为保留） + def test_goal_resume_new_goal_keeps_goal_text(self): + calls = [] + + def fake(cmd): + calls.append(list(cmd)) + return SimpleNamespace( + returncode=0, + stdout="goal met after rounds\n" + "usage input=10 cached=0 cache_write=0 output=5 " + "reasoning=0 total=15 completions=1\n", + stderr="") + + task = {"id": "T1", "project_id": "p1", "prompt": "new goal"} + local = LocalExecutor("/fake/host.py", runner=fake) + local.goal_resume_task(task, "new goal") + self.assertIn("--goal-text", calls[-1]) + self.assertIn("new goal", calls[-1]) + + # 14. goal 轮次上限：sandbox loud fail + worker 认领后 failed + def test_goal_max_rounds_cap(self): + task = {"id": "T1", "project_id": "p1", "prompt": "g", + "verify_command": "true", "goal_max_rounds": 101} + local = LocalExecutor("/fake/host.py", runner=_completed_runner) + with self.assertRaises(ValueError): + local.goal_task(task) + _insert_task(self.db_path, "T1", verify_command="true", + goal_max_rounds=101) + worker = self._worker() + result = worker.run_one() + self.assertEqual(result["status"], "failed") + self.assertIn("上限", result["error"]) + self.assertEqual(_status(self.db_path, "T1"), "failed") + self.assertEqual(self.calls, []) + + # 15. PR 非 local：显式 needs_local_worker，任务与排队都不动 + def test_pr_non_local_needs_local_worker(self): + conn = sqlite3.connect(self.db_path) + try: + job_id = enqueue_pr_job(conn, "T1", "p1", "test-host") + finally: + conn.close() + executor = DockerExecutor(runner=lambda cmd, timeout=None: + SimpleNamespace(returncode=0, stdout="", + stderr="")) + worker = Worker(self.db_path, "/fake/host.py", env=FAKE_ENV, + executor=executor, host_name="test-host") + results = worker._run_pr_jobs() + self.assertEqual(len(results), 1) + self.assertEqual(results[0]["status"], "needs_local_worker") + self.assertEqual(results[0]["pr_job_id"], job_id) + conn = sqlite3.connect(self.db_path) + try: + row = conn.execute( + "SELECT status FROM pr_jobs WHERE id = ?", + (job_id,)).fetchone() + finally: + conn.close() + self.assertEqual(row[0], "queued") + + # 16. 多文件签名 + 签名缺失/已变时认领 fail-closed + def test_code_signature_multi_file_and_claim_refusal(self): + sig = _code_signature("/fake/host.py") + self.assertIsNotNone(sig) + self.assertEqual(len(sig), 6) + for entry in sig: + self.assertEqual(len(entry), 2) + _insert_task(self.db_path, "T1") + worker = self._worker() + worker._code_signature = ("stale-signature",) + result = worker.run_one() + self.assertFalse(result["dispatched"]) + self.assertEqual(result["reason"], "already_claimed") + self.assertEqual(_status(self.db_path, "T1"), "queued") + worker._code_signature = None + result = worker.run_one() + self.assertFalse(result["dispatched"]) + self.assertEqual(_status(self.db_path, "T1"), "queued") + + # 17. goal 首轮撞已存在 → 断点续跑一次，不直接 failed + def test_goal_already_exists_falls_back_to_breakpoint(self): + calls = [] + + def fake(cmd): + calls.append(list(cmd)) + if "goal-resume" in cmd: + return SimpleNamespace( + returncode=0, + stdout="round 1: turn=completed verify_rc=0 " + "verify_out=\ngoal met after rounds\n" + "usage input=10 cached=0 cache_write=0 output=5 " + "reasoning=0 total=15 completions=1\n", + stderr="") + if "goal" in cmd: + return SimpleNamespace( + returncode=0, + stdout="host 任务已存在：T1 已存在", + stderr="") + return _completed_runner(cmd) + + _insert_task(self.db_path, "T1", verify_command="true", + goal_max_rounds=2) + worker = self._worker(runner=fake) + result = worker.run_one() + self.assertEqual(result["status"], "completed") + # 注意：cmd 是元素列表，"goal" in cmd 只匹配恰好等于 + # "goal" 的元素，goal-resume 命令里并没有 "goal" 元素， + # 所以按“任一元素含 goal 子串”过滤。 + goal_cmds = [c for c in calls + if any("goal" in e for e in c)] + self.assertTrue(any("goal-resume" in c for c in goal_cmds)) + resume_cmd = [c for c in goal_cmds if "goal-resume" in c][0] + self.assertNotIn("--goal-text", resume_cmd) + self.assertEqual(_status(self.db_path, "T1"), "completed") + + +if __name__ == "__main__": + unittest.main() diff --git a/cloud/tests/test_pr_jobs.py b/cloud/tests/test_pr_jobs.py index 93274a6..5701207 100644 --- a/cloud/tests/test_pr_jobs.py +++ b/cloud/tests/test_pr_jobs.py @@ -307,7 +307,12 @@ class WorkerPrJobsTestCase(unittest.TestCase): enqueue_pr_job(conn, "t1", "p1", worker.host_name) finally: conn.close() - self.assertEqual(worker._run_pr_jobs(), []) + # P1：Docker worker 不再静默返回空，改为显式 needs_local_worker， + # 任务保持 queued 等 local worker 认领。 + jobs = worker._run_pr_jobs() + self.assertEqual(len(jobs), 1) + self.assertEqual(jobs[0]["status"], "needs_local_worker") + self.assertEqual(jobs[0]["task_id"], "t1") def test_autoscaler_forces_host_name(self): scaler = Autoscaler(self.db_path, []) diff --git a/cloud/tests/test_prod_webhook_exec.py b/cloud/tests/test_prod_webhook_exec.py index 5540bde..06523af 100644 --- a/cloud/tests/test_prod_webhook_exec.py +++ b/cloud/tests/test_prod_webhook_exec.py @@ -41,6 +41,9 @@ class WebhookExecTestCase(unittest.TestCase): self.owner_id, self.owner_key = self.auth.create_user("owner") self._old_admins = os.environ.get("CODEX_GLOBAL_ADMINS") os.environ["CODEX_GLOBAL_ADMINS"] = self.owner_id + # P1：备份 out_dir 限白名单根；本测试的 tmp 目录即白名单根。 + self._old_backup_root = os.environ.get("CLOUD_BACKUP_ROOT") + os.environ["CLOUD_BACKUP_ROOT"] = self.tmp.name self.viewer_id, self.viewer_key = self.auth.create_user("viewer") self.server = create_server(self.db_path, "127.0.0.1", 0) self.port = self.server.server_address[1] @@ -67,6 +70,10 @@ class WebhookExecTestCase(unittest.TestCase): os.environ.pop("CODEX_GLOBAL_ADMINS", None) else: os.environ["CODEX_GLOBAL_ADMINS"] = self._old_admins + if getattr(self, "_old_backup_root", None) is None: + os.environ.pop("CLOUD_BACKUP_ROOT", None) + else: + os.environ["CLOUD_BACKUP_ROOT"] = self._old_backup_root def _req(self, method, path, body=None, key=None, headers=None, raw_body=None): diff --git a/cloud/tests/test_push.py b/cloud/tests/test_push.py index fcda543..6cdf61a 100644 --- a/cloud/tests/test_push.py +++ b/cloud/tests/test_push.py @@ -336,6 +336,11 @@ class TestRealSenders(PushTestCase): def setUp(self): super().setUp() + # P1 SSRF 门默认挡 loopback；本地录制服务器测试显式放行， + # 生产默认值仍关闭（不要把本开关当成生产默认）。 + self._old_push_allow = os.environ.get( + "CODEX_PUSH_ALLOW_PRIVATE_TARGETS") + os.environ["CODEX_PUSH_ALLOW_PRIVATE_TARGETS"] = "1" _RecordingHandler.requests = [] _RecordingHandler.status = 200 self.server = http.server.ThreadingHTTPServer( @@ -349,6 +354,11 @@ class TestRealSenders(PushTestCase): self.server.shutdown() self.server.server_close() self.thread.join(timeout=5) + if getattr(self, "_old_push_allow", None) is None: + os.environ.pop("CODEX_PUSH_ALLOW_PRIVATE_TARGETS", None) + else: + os.environ["CODEX_PUSH_ALLOW_PRIVATE_TARGETS"] = ( + self._old_push_allow) super().tearDown() def _url(self, path="/hook"): diff --git a/cloud/tests/test_queue_sandbox.py b/cloud/tests/test_queue_sandbox.py index 5618d86..42607d2 100644 --- a/cloud/tests/test_queue_sandbox.py +++ b/cloud/tests/test_queue_sandbox.py @@ -331,7 +331,8 @@ class TestGoalExecutors(unittest.TestCase): {"id": "t1", "prompt": "do it", "verify_command": "true"}) self.assertEqual(result["status"], "failed") - self.assertIn("请新建任务", result["goal_error"]) + # P1 新语义：goal 失败请用继续而非重试（旧“请新建任务”已废）。 + self.assertIn("请用继续而非重试", result["goal_error"]) def test_local_goal_task_without_verify_is_llm_decided(self): # 新语义：没有验收命令不再抛错——走模型自决分支： diff --git a/cloud/tests/test_server_goal_notify.py b/cloud/tests/test_server_goal_notify.py index 082b501..877686f 100644 --- a/cloud/tests/test_server_goal_notify.py +++ b/cloud/tests/test_server_goal_notify.py @@ -114,6 +114,11 @@ class _Listener: class GoalPrNotifyTestCase(unittest.TestCase): def setUp(self): + # P1 SSRF 门默认挡 loopback；本地监听器测试显式放行， + # 生产默认值仍关闭。 + self._old_push_allow = os.environ.get( + "CODEX_PUSH_ALLOW_PRIVATE_TARGETS") + os.environ["CODEX_PUSH_ALLOW_PRIVATE_TARGETS"] = "1" self.tmp = tempfile.TemporaryDirectory() self.db_path = os.path.join(self.tmp.name, "control.db") self.auth = AuthStore(self.db_path) @@ -133,6 +138,11 @@ class GoalPrNotifyTestCase(unittest.TestCase): self.server.shutdown() self.server.server_close() self.thread.join(timeout=5) + if getattr(self, "_old_push_allow", None) is None: + os.environ.pop("CODEX_PUSH_ALLOW_PRIVATE_TARGETS", None) + else: + os.environ["CODEX_PUSH_ALLOW_PRIVATE_TARGETS"] = ( + self._old_push_allow) try: self.auth.conn.close() except Exception: diff --git a/cloud/tests/test_worker.py b/cloud/tests/test_worker.py index 7c5f50b..c9c7f80 100644 --- a/cloud/tests/test_worker.py +++ b/cloud/tests/test_worker.py @@ -74,6 +74,11 @@ def _completed_runner(cmd): class WorkerTestCase(unittest.TestCase): def setUp(self): + # P1 SSRF 门默认挡 loopback；通知测试用本地监听器，显式放行， + # 生产默认值仍关闭。 + self._old_push_allow = os.environ.get( + "CODEX_PUSH_ALLOW_PRIVATE_TARGETS") + os.environ["CODEX_PUSH_ALLOW_PRIVATE_TARGETS"] = "1" self.tmp = tempfile.TemporaryDirectory() self.db_path = os.path.join(self.tmp.name, "control.db") init_db(self.db_path) @@ -81,6 +86,11 @@ class WorkerTestCase(unittest.TestCase): self.calls = [] def tearDown(self): + if getattr(self, "_old_push_allow", None) is None: + os.environ.pop("CODEX_PUSH_ALLOW_PRIVATE_TARGETS", None) + else: + os.environ["CODEX_PUSH_ALLOW_PRIVATE_TARGETS"] = ( + self._old_push_allow) self.tmp.cleanup() def _worker(self, runner=None, **kwargs): diff --git a/cloud/worker.py b/cloud/worker.py index d9cf973..e4b28b4 100644 --- a/cloud/worker.py +++ b/cloud/worker.py @@ -139,6 +139,7 @@ from cloud.coordinator import DEFAULT_POLL_INTERVAL as _COORD_POLL_INTERVAL from cloud.coordinator import DEFAULT_TIMEOUT_SECONDS as _COORD_TIMEOUT from cloud.coordinator import ensure_columns as _coord_ensure_columns from cloud.coordinator import parse_parts as _coord_parse_parts +from cloud.coordinator import nonterminal_children as _coord_nonterminal from cloud.cost import CostStore, estimate_tokens, price_usd_micros, \ sum_usage, unpriced_token_cap from cloud.db import connect as _db_connect @@ -170,19 +171,68 @@ DEFAULT_ENV_KEY = "XCM_API_KEY" # gpt-6.1-sol 路由当日实测持续 503，terra 同日实测 1 秒完成； # 任务未指定模型时 worker 与 host.py 都落到这里。 DEFAULT_MODEL = "gpt-5.6-terra" +# P1 执行面：goal 轮次上限（有限轮次形态）。goal_mode=1 的 0 轮 +# （模型宣告才结束）不在此限——它的上界是配额（配额看门狗与 +# 认领/边界门），不是轮次；旧验收形态的 goal_max_rounds 必须 +# 在 1..GOAL_MAX_ROUNDS_CAP 之间，超限的任务 worker 在认领后 +# 直接 failed（建任务时的 400 校验在 server.py，属另一路文件， +# 本路只做执行侧防线，见 run_one 的 goal 上限门）。 +GOAL_MAX_ROUNDS_CAP = 100 +# P1 执行面：无配额行时的默认配额（微美元），与 server.py +# _default_quota_usd_micros() 同值（$10/月，环境变量 +# CLOUD_DEFAULT_QUOTA_USD_MICROS 可覆盖）。P0 前“无行 = +# 不限额”已废：worker 侧查不到行时按此默认额度算，显式 +# NULL 行才是不限额（与服务端 _quota_status 同口径）。 # host.py dispatch 的原话是 "task 已存在: ，先换名或 discard 它" #（英文 task + 已存在），取共有子串做判断，不依赖标点。 _ALREADY_EXISTS_MARKER = "已存在" -def _code_signature(): - """本文件签名（mtime_ns, size）：部署 staging 换目录后必变； - 取不到时返回 None（此时自退检查跳过，绝不误杀）。""" +def _default_quota_usd_micros(): + """无配额行时的默认配额（微美元），与 server 侧同值。 + + server.py 的 _default_quota_usd_micros() 在另一路文件里， + worker 不跨模块 import 它（避免循环 import），这里复刻 + 同款解析：环境变量 CLOUD_DEFAULT_QUOTA_USD_MICROS，非法 + 值回落 10_000_000。""" try: - st = os.stat(os.path.abspath(__file__)) - return (st.st_mtime_ns, st.st_size) + return max(0, int(os.environ.get( + "CLOUD_DEFAULT_QUOTA_USD_MICROS", "10000000"))) + except (TypeError, ValueError): + return 10000000 + + +def _code_signature(host_py_path=None): + """多文件签名：worker/sandbox/coordinator/cost/server/host + 各取 (mtime_ns, size) 组成的元组；任一文件取不到即 None。 + + P1 执行面扩面：旧版只签 worker.py，部署只换了 sandbox 或 + host.py 时旧 worker 会继续用旧代码抢任务。host.py 取 + host_py_path（真实文件时用它，否则用 cloud/ 上一级的 + host.py）；单测常传 /fake/host.py 这种不存在的路径，此时 + 回落到仓库 host.py，不因此判签名缺失。取不到时返回 None， + 调用方必须 fail-closed（不认领、不执行），绝不按“没变” + 放行——旧版“取不到当没变”是 fail-open，已废。""" + here = os.path.dirname(os.path.abspath(__file__)) + candidates = [ + os.path.join(here, "worker.py"), + os.path.join(here, "sandbox.py"), + os.path.join(here, "coordinator.py"), + os.path.join(here, "cost.py"), + os.path.join(here, "server.py"), + ] + if host_py_path and os.path.isfile(host_py_path): + candidates.append(os.path.abspath(host_py_path)) + else: + candidates.append(os.path.join(os.path.dirname(here), "host.py")) + sig = [] + try: + for path in candidates: + st = os.stat(path) + sig.append((st.st_mtime_ns, st.st_size)) except OSError: return None + return tuple(sig) def _missing_key_error(env_key): @@ -447,16 +497,46 @@ class Worker: # 注入的普通 dict 同样可读环境变量（无该键时默认 1）。 self.capacity = resolve_capacity(capacity, env=self._env) # 常驻启动时的代码签名：run_loop 每轮边界比对，变了就自退 - # （见 run_loop 的代码变化自退检查）。 - self._code_signature = _code_signature() + # （见 run_loop 的代码变化自退检查）；_claim 每次认领前 + # 也比对（见 _code_signature_valid），不要只在轮次边界查。 + self._code_signature = _code_signature(host_py_path) + # 配额看门狗命中过的任务（Docker 同步路径用）：看门狗杀 + # 容器后，同步返回的 failed 要被 _run_attempt 改写成 + # quota_exceeded，否则 run_one 会把它当普通失败重试， + # 配额中断就续跑下去了。受 _quota_hit_lock 保护。 + self._quota_hit_tasks = set() + self._quota_hit_lock = threading.Lock() def _code_changed(self): - """启动后 worker.py 是否被换过（部署换目录/手动改代码）： - 签名取不到时一律当没变，不误杀。""" - now = _code_signature() + """启动后关键代码是否被换过（部署换目录/手动改代码）： + 签名取不到时返回 False（是否“变了”无从判断），但这 + 不等于放行——放行与否看 _code_signature_valid。""" + now = _code_signature(self.host_py_path) return (now is not None and self._code_signature is not None and now != self._code_signature) + def _code_signature_valid(self): + """签名是否可信且未变：启动签名与现签名都必须取得到 + 且相等，否则 False。False 时 _claim 拒绝认领、run_loop + 自退（75）、_run_pr_jobs 不碰任务——fail-closed，绝不 + 让签名缺失的 worker 抢任务（旧版取不到当没变是 fail-open）。""" + now = _code_signature(self.host_py_path) + return (now is not None and self._code_signature is not None + and now == self._code_signature) + + def _mark_quota_hit(self, task_id): + with self._quota_hit_lock: + self._quota_hit_tasks.add(task_id) + + def _take_quota_hit(self, task_id): + """取走一次配额看门狗命中（命中即消费，避免旧命中污染 + 同一任务的下一轮）：返回 True 表示本轮确被看门狗杀过。""" + with self._quota_hit_lock: + if task_id in self._quota_hit_tasks: + self._quota_hit_tasks.discard(task_id) + return True + return False + def register(self): """向控制面注册本 worker（写 workers 表，失败只记日志）。 @@ -573,7 +653,17 @@ class Worker: 里清（本地 _execute_via_process / Docker 同步路径）， 认领后提前返回的三道门（缺 key / 仓库 / 配额）在 run_one 里显式清占位；CAS 语义本身一句未改。 + P1 签名门：占位之前先比对多文件签名，签名缺失或已变 + 时直接返回 False（fail-closed，不抢任务、不占槽）， + 并记日志；不要只在 run_loop 轮次边界查——并行池线程 + 直接调 run_one/_claim，轮次边界查不到它们。 """ + if not self._code_signature_valid(): + log_event(self.logger, "worker_claim_refused_code_signature", + task_id=task_id, + message="code signature missing or changed," + " refusing claim") + return False with self._running_lock: already_running = task_id in self._running if not already_running and \ @@ -865,14 +955,51 @@ class Worker: _check_unpriced_token_cap 同口径）：USD 配额不计量无价 模型，默认模型 terra 无价，若无此上限则配额对默认模型 永远不生效；有配额行的用户当月无价 token（input + - output，已记账 + 本轮）达到上限即停。无配额行 = 不限额。 - 查询本身异常时按不限额处理（fail-open 并记一条警告）： - 配额是第二道网，绝不能因查询抖动把全部任务挡在门外； - 服务端建任务/分支/重试时的准入仍是第一道。 + output，已记账 + 本轮）达到上限即停。无价上限只计 + source='actual' 的行：估算行（estimated）不是真实花费， + 计入上限会把“没花钱”算成“花超了”，且与 cost.py + _report 的 unpriced_actual_tokens 口径一致。 + P1 执行面三处收口（旧版三处都是 fail-open，已废）： + - 属主：任务 created_by 为空时先查父任务（协调员子任务 + 行可能无 created_by，属主跟父任务走）；仍查不到时 + 拒绝新回合，不再返回 None 放行。created_by 字面为 + "system" 的系统任务按普通属主算（无行时同样走默认 + 额度，不另设无限额的系统预算）。 + - 无配额行：不再是不限额，按默认配额 + _default_quota_usd_micros()（$10/月）算；只有显式 + NULL 行才是不限额（与 P0 的服务端口径一致，管理员 + 显式设 NULL 的行仍被尊重）。 + - 查库失败：返回 "quota_check_failed: ..."（fail-closed）， + 绝不返回 None 放行。调用方分流：新回合边界（认领门、 + 重试/fallback/后续消息边界）见此串即拒绝新回合并记 + worker_quota_stop 审计；执行中轮询（popen 轮询、 + Docker 配额看门狗）见此串只记警告、不中途杀当前轮 + ——查询抖动不应杀掉已在跑的回合，但下一轮边界仍会 + 拒绝（见 run_one 的 _quota_hit 过滤）。查库失败的 + 审计由边界调用方记，这里只记日志，避免轮询高频写审计。 """ owner = (task or {}).get("created_by") + if not owner and (task or {}).get("parent_task_id"): + # 协调员子任务属主跟父任务走（与 _gateway_context + # 同款回查）；查不到时不猜，直接走下面的拒绝分支。 + try: + conn = _db_connect(self.db_path) + try: + row = conn.execute( + "SELECT created_by FROM tasks WHERE id = ?", + (task["parent_task_id"],)).fetchone() + finally: + conn.close() + if row is not None: + try: + owner = row[0] + except (TypeError, IndexError): + owner = None + except Exception: + owner = None if not owner: - return None + return ("quota_exceeded: 任务缺少属主，无法查配额，" + "拒绝新回合（请归属真实用户后再排队）") try: month = datetime.now(timezone.utc).strftime("%Y-%m") conn = _db_connect(self.db_path) @@ -887,19 +1014,33 @@ class Worker: " WHERE user_id = ?", (owner,)).fetchone() if row is None: - return None - quota = row[0] + # P1：无行按默认额度，不是不限额（P0 服务端 + # 已在建号/登录时补行，worker 侧仍可能遇到 + # 老库未补行的用户，这里按默认额度兜底， + # 不写行——写行是服务端的职责，worker 不抢）。 + quota = _default_quota_usd_micros() + else: + quota = row[0] if quota is None: return None # cost_records 由 CostStore 懒建：全新库上它还不存 # 在，直接 JOIN 会报错——先实例化一次建表再查（与 # 服务端 _quota_status 同款做法），否则 quota=0 #（一分钱也不能花）在全新库上会被误放行。 + # P1 补：建表必须落在本连接上，不只 CostStore 的 + # 连接上——文件库两者指向同一文件所以旧代码碰巧 + # 能用，:memory: 库两者是不同空库，本连接查 JOIN + # 会直接报错并（P1 fail-closed 后）误拒全部任务。 _cost_store = CostStore(self.db_path) try: _cost_store._conn.close() except Exception: pass + try: + conn.execute(CostStore._COST_DDL) + conn.commit() + except Exception: + pass used_row = conn.execute( "SELECT COALESCE(SUM(c.usd_micros), 0)" " FROM cost_records c JOIN tasks t" @@ -912,7 +1053,7 @@ class Worker: " FROM cost_records c JOIN tasks t" " ON c.task_id = t.id" " WHERE t.created_by = ? AND c.day LIKE ?" - " AND c.usd_micros IS NULL", + " AND c.usd_micros IS NULL AND c.source = 'actual'", (owner, month + "%")).fetchone() unpriced_used = unpriced_row[0] if unpriced_row else 0 finally: @@ -940,11 +1081,15 @@ class Worker: % (unpriced_used, cap, month)) return None except Exception as exc: + # P1 fail-closed：查库失败不再返回 None。调用方按 + # 前缀分流（见本函数 docstring）：quota_check_failed + # 在新回合边界 = 拒绝，在执行中轮询 = 只警告不杀轮。 try: self.logger.warning("quota check failed: %s", exc) except Exception: pass - return None + return ("quota_check_failed: 配额查询失败，拒绝新回合" + "（查库异常，非配额用完）") def _ready_tasks(self): """当前就绪任务列表（pick_next 的批量版）。 @@ -1123,6 +1268,41 @@ class Worker: pass return + def _docker_quota_watchdog(self, task_id, container_name, + stop_event, quota_check, interval=30.0): + """Docker 同步路径的配额看门狗（P1，复用取消看门狗模式）。 + + 同步路径阻塞在 docker run 里，run_one 的边界检查要等 + 本轮返回才跑得到——配额在别处被花完时，本轮会继续烧 + 到自然结束。本线程每 interval 秒（默认 30 秒，不超过 + 30 秒）用 quota_check 查一次已记账配额：命中时按容器 + 名真杀容器并记 _quota_hit_tasks，_run_attempt 在同步 + 返回后把结果改写成 quota_exceeded，run_one 走统一的 + 配额中断收尾（interrupted + 配额原因 + 已花成本照记， + 下一轮不再续跑）。查库失败时 quota_check 返回 False + （run_one 的 _quota_hit 会把 quota_check_failed 前缀 + 过滤掉，见 _quota_exceeded docstring），本线程只继续 + 轮询、不杀当前轮；下一轮边界仍会拒绝。诚实边界：看 + 门狗只能看到已记账用量 + 往轮累计用量，本轮容器内尚 + 未回传的 token 它看不到——那部分在结果返回后由边界 + 门拦住，不会悄悄续跑下一轮。 + """ + while not stop_event.wait(interval): + try: + over = bool(quota_check()) if quota_check else False + except Exception: + over = False + if not over: + continue + self._mark_quota_hit(task_id) + try: + kill = getattr(self.executor, "kill_container", None) + if kill is not None and container_name: + kill(container_name) + except Exception: + pass + return + def cleanup_stale_containers(self): """启动清理兜底：清掉上一轮残留的 codex- 容器。 @@ -2069,6 +2249,7 @@ class Worker: # 按容器名真杀；非 Docker 路径不启动，避免空线程）。 _watchdog_stop = None _watchdog_thread = None + _quota_watchdog_thread = None if is_docker and hasattr(self.executor, "kill_container"): try: _container_name = self.executor.container_name(task) @@ -2081,6 +2262,17 @@ class Worker: args=(task_id, _container_name, _watchdog_stop), daemon=True) _watchdog_thread.start() + # P1 配额看门狗：只在同步 Docker 路径起（进程 + # 模式的 _execute_via_process 已每 30 秒轮询 + # quota_check，不重复起）。quota_check 为 None + # 时不起（无配额检查可做，不空转线程）。 + if quota_check is not None and not self._use_popen(): + _quota_watchdog_thread = threading.Thread( + target=self._docker_quota_watchdog, + args=(task_id, _container_name, + _watchdog_stop, quota_check), + daemon=True) + _quota_watchdog_thread.start() try: gateway_provider, gateway_env = self._ensure_gateway(task) if self._use_popen(): @@ -2123,8 +2315,14 @@ class Worker: prep_err) if goal_mode: if task.get("_resume"): + # P1 断点续跑传 None（不传 --goal-text， + # host 沿用原 goal+verify）；开新目标 + # 才传本轮 prompt 当新 goal_text。 + _resume_text = None if task.get( + "_resume_kind") == "breakpoint" \ + else task["prompt"] result = self.executor.goal_resume_task( - task, task["prompt"], + task, _resume_text, timeout=effective_docker_timeout, extra_mounts=tenant_mounts, network=tenant_network, @@ -2165,8 +2363,11 @@ class Worker: if task.get("_resume") and isinstance( self.executor, LocalExecutor): + _resume_text = None if task.get( + "_resume_kind") == "breakpoint" \ + else task["prompt"] result = self.executor.goal_resume_task( - task, task["prompt"], + task, _resume_text, extra_env=gateway_env, model_provider=gateway_provider) elif isinstance(self.executor, LocalExecutor): @@ -2220,6 +2421,8 @@ class Worker: _watchdog_stop.set() if _watchdog_thread is not None: _watchdog_thread.join(timeout=2) + if _quota_watchdog_thread is not None: + _quota_watchdog_thread.join(timeout=2) # 同步路径执行结束：清掉取消用的容器标记（取消路径里 # 若已清过，这里再清一次无害）。进程模式的 proc 由 # _execute_via_process 自己的 finally 清，这里只清 @@ -2236,6 +2439,18 @@ class Worker: if result is not None and result.get("status") == "failed" and \ "docker run timeout" in (result.get("stdout", "") or ""): result = dict(result, status="timeout") + # P1 配额看门狗命中改写：同步路径被看门狗杀容器后， + # 执行器只返回普通 failed，run_one 无法区分“模型失败” + # 与“配额杀容器”——命中时改写成 quota_exceeded，run_one + # 走统一配额中断收尾（interrupted + 配额原因 + 成本照记， + # 不重试、不续跑下一轮）。改写前先取走命中标记，避免 + # 旧命中污染同一任务的后续尝试；超时标记优先于配额 + # 改写的情况不存在（看门狗杀容器时执行器不会同时报超时）。 + if result is not None and self._take_quota_hit(task_id): + result = dict(result, status="quota_exceeded") + if error is None: + error = ("quota_exceeded: Docker 配额看门狗已杀容器" + "（配额中断，下一轮不再续跑）") return result, error def _attempt_error(self, result, error, token): @@ -2384,7 +2599,12 @@ class Worker: task.get("priority", 0) or 0, task["created_by"], now, now, part.get("executor") or task.get("executor"), - part.get("model"), part.get("effort"), + # P1：分工未指定模型/强度时继承父任务（旧版 + # 直接写 part 的 None，子任务会落到 worker + # 默认模型/强度，与父任务的显式选择脱节， + # 既改成本也改行为，且排查时看不出继承关系）。 + part.get("model") or task.get("model"), + part.get("effort") or task.get("effort"), task.get("environment_id"), task["id"], name)) self._audit( task["project_id"], "coordinator_child_created", @@ -2422,6 +2642,109 @@ class Worker: conn.close() return rows + def _coord_mark_orphans(self, task, child_ids): + """把仍未终态的子任务标 interrupted 并记审计（P1 孤儿收口）。 + + 只对 CAS 改到行的子任务记审计（rowcount==1）：已被别的 + worker 终态化的子任务不动，避免覆盖它的结果。返回被标 + 记的子任务 id 列表。正在跑的子任务被标 interrupted 后， + 跑它的 worker 会在轮询里看到库状态并杀进程/容器（popen + 每 1 秒查库、Docker 取消看门狗每 1 秒查库），不会无声 + 继续烧钱；若那个 worker 也已死，子任务至少已是终态， + 不会再被认领烧钱。 + """ + marked = [] + rows = self._coord_child_rows(child_ids) + pending = _coord_nonterminal([ + {"id": cid, "status": (rows.get(cid) or {}).get("status")} + for cid in child_ids]) + if not pending: + return marked + conn = _db_connect(self.db_path) + try: + for ch in pending: + cur = conn.execute( + "UPDATE tasks SET status = 'interrupted'," + " updated_at = ? WHERE id = ? AND status NOT IN" + " ('completed', 'failed', 'interrupted', 'merged'," + " 'merge-failed')", + (time.time(), ch["id"])) + if cur.rowcount == 1: + marked.append(ch["id"]) + self._audit( + task["project_id"], "coordinator_child_orphaned", + "%s -> %s (parent %s)" + % (ch["id"], "interrupted", task["id"])) + conn.commit() + finally: + conn.close() + return marked + + def sweep_orphaned_coordinator_children(self): + """常驻循环每轮的孤儿扫尾（P1）：父已终态或父行失踪、子 + 仍未终态时，把子标 interrupted 并记审计。 + + 父仍在 running/queued 时绝不动子（协调员等齐期间子任务 + 未终态是正常状态）。父行失踪（被删）同样算孤儿：没有 + 父会再来合流，子任务继续跑只是烧钱。返回被标记的 + (parent_id, child_id) 对列表；查库失败时返回 [] 且只记 + 日志，不炸循环。 + """ + marked = [] + try: + conn = _db_connect(self.db_path) + try: + rows = conn.execute( + "SELECT id, parent_task_id, project_id FROM tasks" + " WHERE parent_task_id IS NOT NULL AND status NOT IN" + " ('completed', 'failed', 'interrupted', 'merged'," + " 'merge-failed')").fetchall() + parents = {} + for row in rows: + try: + pid = row["parent_task_id"] + except (KeyError, IndexError, TypeError): + continue + if pid not in parents: + prow = conn.execute( + "SELECT status FROM tasks WHERE id = ?", + (pid,)).fetchone() + try: + parents[pid] = prow["status"] \ + if prow is not None else None + except (KeyError, IndexError, TypeError): + parents[pid] = None + for row in rows: + try: + cid = row["id"] + pid = row["parent_task_id"] + proj = row["project_id"] + except (KeyError, IndexError, TypeError): + continue + pstatus = parents.get(pid) + if pstatus is not None and \ + pstatus not in TERMINAL_STATUSES: + continue + cur = conn.execute( + "UPDATE tasks SET status = 'interrupted'," + " updated_at = ? WHERE id = ? AND status NOT IN" + " ('completed', 'failed', 'interrupted', 'merged'," + " 'merge-failed')", + (time.time(), cid)) + if cur.rowcount == 1: + marked.append((pid, cid)) + self._audit( + proj, "coordinator_child_orphaned", + "%s -> interrupted (parent %s, parent" + " status %s)" % (cid, pid, pstatus)) + conn.commit() + finally: + conn.close() + except Exception as exc: + log_event(self.logger, "worker_orphan_sweep_failed", + message="orphan sweep failed: %s" % exc) + return marked + def _run_coordinator(self, task, started_at, poll_interval=None, timeout=None): """协调员执行：建子任务 → 等齐 → 合流汇总写回。 @@ -2460,6 +2783,7 @@ class Worker: timed_out = True break time.sleep(poll_interval) + orphaned = [] if cancelled or timed_out: rows = self._coord_child_rows(child_ids) for cid in child_ids: @@ -2469,6 +2793,15 @@ class Worker: self.cancel(cid) except Exception: pass + # P1：cancel() 对已终态/不存在的子任务返回 False， + # 跨 worker 在跑的子任务也可能因竞态仍未终态—— + # cancel 之后仍未终态的一律直接标 interrupted 并记 + # 审计（_coord_mark_orphans），绝不留孤儿无声烧钱； + # 标记结果写进汇总报告，调用方在详情里看得到。 + try: + orphaned = self._coord_mark_orphans(task, child_ids) + except Exception: + orphaned = [] rows = self._coord_child_rows(child_ids) merged = [] for ch in children: @@ -2479,6 +2812,9 @@ class Worker: "result_text": row.get("result_text"), "error_text": row.get("error_text")}) report, overall = _coord_build_report(merged) + if orphaned: + report += ("\n\n孤儿子任务已标 interrupted：%s" + % "、".join(orphaned)) if cancelled: final = "interrupted" error = "cancelled" @@ -2531,6 +2867,47 @@ class Worker: "executor": self._executor_name(), "stdout": report, "error": error, "events": [], "cost_record_id": None} + def _record_usage_cost(self, task, actual_model, usage_total, + stdout, fallback_count): + """记一条任务成本并累加 worker 级用量（调用方须持 _db_lock）。 + + P1 成本修齐：旧版 max_cost_usd 参数存在但 usage_cost_usd + 从未累加——_budget_exceeded 的成本门永远是 0，等于没设。 + 本函数是唯一记账点（终局块与取消分支共用）：actual 行 + 按价目表算 usd_micros（无价模型为 None），有价行才累加 + usage_cost_usd（无价行不折算成 0 美元，仍受无价上限管）； + 无用量时回落字符估算记 estimated，绝不冒充 actual。 + cache_write_input_tokens 不进这里：它只在 usage dict 里 + 做观测（见 cost.py 的不计费声明），不加进 total、不用 + 于定价。返回 cost record id。 + """ + task_id = task["id"] + if usage_total and (usage_total.get("total_tokens") or 0) > 0: + micros = price_usd_micros( + actual_model, usage_total.get("input_tokens", 0), + usage_total.get("cached_input_tokens", 0), + usage_total.get("output_tokens", 0)) + cost_id = self.costs.record( + task_id, task["project_id"], actual_model, + input_tokens=usage_total.get("input_tokens", 0), + output_tokens=usage_total.get("output_tokens", 0), + cached_input_tokens=usage_total.get( + "cached_input_tokens", 0), + usd_micros=micros, + source="actual", fallback_count=fallback_count) + else: + micros = None + total = estimate_tokens(task.get("prompt") or "", stdout) + cost_id = self.costs.record( + task_id, task["project_id"], actual_model, + input_tokens=total, output_tokens=0, usd_micros=None, + source="estimated", fallback_count=fallback_count) + if micros is not None: + # max_cost_usd 不再是死参数：只有有价花费才累加， + # _budget_exceeded 据此在下一认领前挡住超支 worker。 + self.usage_cost_usd += micros / 1000000.0 + return cost_id + def run_one(self, task=None): """认领并执行一个任务，返回结果 dict（见模块 docstring）。 @@ -2552,11 +2929,32 @@ class Worker: # 成它（只换内存里的副本，原 prompt 不动）；_resume 标记 # 让 _run_attempt 在 goal 模式下走 goal-resume 而不是 goal # （goal 会撞已存在，只有 goal-resume 能在旧线程上开新循环）。 + # P1 goal-resume 分流（断点 vs 开新目标，三处一致的 worker 侧）： + # - 旧验收形态（verify_command 非空）：一律断点续跑—— + # 绝不能把续跑消息当新 goal_text 传给 host.py，否则 + # host 的 goal-resume --goal-text 分支会用 verify=None + # 开新循环，把原 verify_command 覆盖掉；断点调用不传 + # --goal-text，host 沿用原 goal + 原 verify + 原轮次， + # 从 goal_rounds_done 的下一轮续跑。续跑消息本轮不 + # 当 prompt 用（断点语义就是续原目标），跑完照常清掉。 + # - goal_mode=1（无验收）：开新目标——续跑消息就是新 + # goal_text（服务端 /continue 带 goal_mode=1 时已把 + # task.goal 写成该消息，这里与之对齐）。 + # - 非 goal 任务：沿用旧行为（prompt 换成续跑消息，走 + # 已存在续跑 run 路径）。 resume_message = task.get("resume_message") if resume_message: task = dict(task) - task["prompt"] = resume_message - task["_resume"] = True + if task.get("verify_command"): + task["_resume"] = True + task["_resume_kind"] = "breakpoint" + elif task.get("goal_mode"): + task["prompt"] = resume_message + task["_resume"] = True + task["_resume_kind"] = "new-goal" + else: + task["prompt"] = resume_message + task["_resume"] = True self._audit(task["project_id"], "worker_claim", "%s executor=%s" % (task_id, self._executor_name())) # log_event 只接受 task_id/project_id/status/message 四个 @@ -2647,6 +3045,56 @@ class Worker: "stdout": "", "error": quota_error, "events": [], "cost_record_id": None} + # P1 goal 轮次上限门（执行侧防线）：旧验收形态的 + # goal_max_rounds 超过 GOAL_MAX_ROUNDS_CAP 时直接 failed， + # 不进 host.py goal 循环烧轮次；goal_mode=1 的 0（模型宣告 + # 才结束）豁免轮次上限（上界是配额），但 0 配上旧验收命令 + # 时同样 failed（服务端建任务时 0 本就 400，老行若有 0 + # 验收形态属于脏数据，不按不限轮次跑）。建任务时的 400 + # 校验在 server.py（另一路文件），这里是执行侧第二道门， + # 错误文本写明上限与“请新建合规任务”，不静默截断轮次 + # （截断会改任务语义，调用方看不出被改过）。 + _goal_rounds = task.get("goal_max_rounds") + if (task.get("verify_command") or task.get("goal_mode")) \ + and _goal_rounds is not None: + _goal_cap_error = None + try: + _goal_rounds_int = int(_goal_rounds) + except (TypeError, ValueError): + _goal_rounds_int = None + if _goal_rounds_int is not None \ + and _goal_rounds_int > GOAL_MAX_ROUNDS_CAP: + _goal_cap_error = ( + "goal_max_rounds 超过上限 %d 轮（当前 %d），" + "请新建轮次合规的任务" % ( + GOAL_MAX_ROUNDS_CAP, _goal_rounds_int)) + elif _goal_rounds_int == 0 and task.get("verify_command") \ + and not task.get("goal_mode"): + _goal_cap_error = ( + "旧验收形态 goal_max_rounds=0 表示不限轮次，" + "已禁用（上限 %d 轮），请新建轮次合规的任务" + % GOAL_MAX_ROUNDS_CAP) + if _goal_cap_error is not None: + self._set_status(task_id, "failed") + self._write_execution_result( + task_id, error_text=_goal_cap_error, + actual_model=self.model, + started_at=started_at, finished_at=time.time(), + only_if_status="failed") + self._audit(task["project_id"], "worker_goal_cap_rejected", + "%s: %s" % (task_id, _goal_cap_error)) + log_event(self.logger, "worker_goal_cap_rejected", + task_id=task_id, project_id=task["project_id"], + status="failed") + self._running_clear(task_id) + return {"dispatched": True, "task_id": task_id, + "host_name": self.executor._host_name(task) + if hasattr(self.executor, "_host_name") else task_id, + "status": "failed", + "executor": self._executor_name(), + "stdout": "", "error": _goal_cap_error, "events": [], + "cost_record_id": None} + # 协调员分支：不跑模型回合，改为按分工建并行子任务、等齐、 # 合流汇总。已过 key 门与配额门；仓库门已对协调员任务本身 # 跑过（项目仓库解析失败会在前面直接 failed，不会建一堆 @@ -2721,7 +3169,14 @@ class Worker: def _quota_hit(): # 闭包读当前值：usage_total 与 task 在重试/fallback # 时会被重绑，轮询掐断要按最新值判断。 - return self._quota_exceeded(task, usage_total) is not None + # P1 分流：执行中轮询只认真正的 quota_exceeded， + # quota_check_failed（查库失败）不杀当前轮——查询 + # 抖动不应杀掉已在跑的回合；但它在重试/fallback/ + # 后续消息等新回合边界仍是拒绝（那些边界直接调 + # _quota_exceeded，不走这个过滤闭包）。 + err = self._quota_exceeded(task, usage_total) + return err is not None and \ + not err.startswith("quota_check_failed:") # 分支任务：先 fork 再跑首轮（见 _run_fork）；首轮失败 # 不自动重试/fallback，失败后走重试入口重跑首轮（已存在 @@ -2763,6 +3218,39 @@ class Worker: # 喂回，或新形态的模型自决多轮），worker 级重试/fallback # 不再叠加——同任务重跑会撞 host 任务名已存在，第二 # 次尝试必失败且污染错误文本。 + # P1 唯一例外：goal 首轮撞“已存在”时（重试入口 + # 重排同一 goal 任务的典型死法），不再直接 failed， + # 而是走一次断点续跑（goal-resume 不传 --goal-text， + # 沿用原 goal+verify，从断点续跑）；断点续跑的 + # 结果即本轮结果（成功则完成，仍失败则以续跑的 + # 错误为准，不再拼“已存在”）。已是续跑的任务 + # （_resume）不再二次回落，避免断点循环。 + # 分支任务（forked_from）不走这里：fork 首轮的 + # 已存在语义是 dispatch 续跑，已在 _run_attempt + # 内处理，不叠加 goal 断点。 + if not task.get("_resume") and not task.get("forked_from"): + _combined = ((result or {}).get("stdout", "") or "") \ + + ((result or {}).get("stderr", "") or "") + _goal_err = (result or {}).get("goal_error") or "" + if _ALREADY_EXISTS_MARKER in _combined or \ + _ALREADY_EXISTS_MARKER in _goal_err or \ + _ALREADY_EXISTS_MARKER in (error or ""): + self._audit(task["project_id"], + "worker_goal_resume_fallback", + "%s: goal 已存在，走断点续跑" % task_id) + _resume_task = dict(task) + _resume_task["_resume"] = True + _resume_task["_resume_kind"] = "breakpoint" + result, error = self._run_attempt( + _resume_task, quota_check=_quota_hit) + usage_total = sum_usage( + usage_total, (result or {}).get("usage")) + self._deliver_peer_messages(task, result) + if (result or {}).get("status") == "quota_exceeded": + quota_error = self._quota_exceeded( + task, usage_total) or \ + "quota_exceeded: 本月配额已用完" + quota_stop = True break if result.get("status") == "completed": break @@ -2900,6 +3388,18 @@ class Worker: turn_id=(result or {}).get("turn_id"), root_turn_id=(result or {}).get("root_turn_id"), only_if_status="interrupted") + # P1：取消分支先记一条成本再返回（与配额中断收尾 + # 同款记账点）。旧版这里直接返回 cost_record_id=None， + # 被取消前已花的模型费用就从账上丢了；取消不等于没 + # 花钱。记账失败不挡取消返回（costs.record 异常时 + # cost_id 置 None，任务状态仍是 interrupted）。 + try: + with self._db_lock: + _cancel_cost_id = self._record_usage_cost( + task, actual_model, usage_total, + (result or {}).get("stdout", ""), fallback_count) + except Exception: + _cancel_cost_id = None log_event(self.logger, "worker_cancelled", task_id=task_id, project_id=task["project_id"], status="interrupted") return {"dispatched": True, "task_id": task_id, @@ -2908,7 +3408,7 @@ class Worker: "status": "interrupted", "executor": self._executor_name(), "stdout": (result or {}).get("stdout", ""), "error": "cancelled", "events": [], - "cost_record_id": None} + "cost_record_id": _cancel_cost_id} stdout = result.get("stdout", "") or "" host_status = result.get("host_status") @@ -2979,32 +3479,12 @@ class Worker: self._cleanup_task_seed(task) with self._db_lock: self.usage_turns += 1 - # 成本：执行器带回上游真实用量时记 actual（input/output/ - # cached 全是实测值，usd 按价目表算、无价模型记 None， - # 报表里进 unpriced_actual_tokens 而不是 0 美元）； - # 没带回用量时回落字符估算记 estimated，绝不冒充 actual。 - # fallback_count 复用同一记录；模型记最后真正跑的那个 - # （fallback 成功时就是 fallback 模型）。cost 记录共用 - # 一条连接，并发下必须持锁写，否则两个线程共用同一 - # sqlite 连接会直接报错。 - if usage_total and (usage_total.get("total_tokens") or 0) > 0: - cost_id = self.costs.record( - task_id, task["project_id"], actual_model, - input_tokens=usage_total.get("input_tokens", 0), - output_tokens=usage_total.get("output_tokens", 0), - cached_input_tokens=usage_total.get( - "cached_input_tokens", 0), - usd_micros=price_usd_micros( - actual_model, usage_total.get("input_tokens", 0), - usage_total.get("cached_input_tokens", 0), - usage_total.get("output_tokens", 0)), - source="actual", fallback_count=fallback_count) - else: - total = estimate_tokens(task.get("prompt") or "", stdout) - cost_id = self.costs.record( - task_id, task["project_id"], actual_model, - input_tokens=total, output_tokens=0, usd_micros=None, - source="estimated", fallback_count=fallback_count) + # 成本记账走 _record_usage_cost（终局与取消分支共用 + # 的唯一记账点）：actual/estimated 分流、usage_cost_usd + # 累加都在里面；cost 记录共用一条连接，并发下必须 + # 持锁写，否则两个线程共用同一 sqlite 连接会直接报错。 + cost_id = self._record_usage_cost( + task, actual_model, usage_total, stdout, fallback_count) if final_written: self._audit(task["project_id"], "worker_quota_stop" if quota_stop else @@ -3064,9 +3544,65 @@ class Worker: 发 review-requested/merge-failed，这里只写回任务终态。 跳过（PrSkipped，如任务已不是 merged）与落盘失败都记 failed + error，任务本身保持 merged、可重调 /pr 再排队。 + P1 非 local 通道：旧版非 local worker 直接返回 []， + 排队的 pr 任务看起来像“没人处理”且无任何痕迹—— + 静默未完成。现在改为显式 needs_local_worker：只读 + peek 本主机 queued 的 pr 任务，逐条记审计 + 日志并 + 返回 needs_local_worker 条目，任务本身保持 merged、 + pr 任务保持 queued（留给 local worker 认领），绝不 + 把它标 failed（failed 会让调用方以为 PR 已失败，且 + 任务仍可重调 /pr 的语义会被混淆）。为什么不直接把 + pr_jobs 行写成 needs_local_worker 状态：pr_jobs 的认 + 领只认 queued（claim_pr_job 在 pr_exec.py，本路不改 + 该文件），写成新状态后 local worker 也认领不到，任务 + 会被永久搁置；真要落库状态，需 pr_exec.py 的认领同 + 步改成含 needs_local_worker 或 local 重排队——该改动 + 留给协调员（见汇报残留），本路不伪造一个“写了状态 + 却没人能接”的通道。签名无效时同样不碰任务（返回 []， + 只记日志），fail-closed。 """ - if self._executor_name() != "local": + if not self._code_signature_valid(): + log_event(self.logger, "worker_pr_refused_code_signature", + message="code signature missing or changed," + " refusing pr jobs") return [] + if self._executor_name() != "local": + results = [] + try: + from cloud.pr_exec import ensure_pr_jobs_table + conn = _db_connect(self.db_path) + try: + ensure_pr_jobs_table(conn) + rows = conn.execute( + "SELECT id, task_id, project_id FROM pr_jobs" + " WHERE host = ? AND status = 'queued'", + (self.host_name,)).fetchall() + finally: + conn.close() + for row in rows: + try: + job_id = row["id"] + task_id = row["task_id"] + project_id = row["project_id"] + except (KeyError, IndexError, TypeError): + continue + reason = ("PR 执行需要 local worker（host-worktrees" + " 只在主机上可见），本 worker 为 %s，" + "任务保持 queued 等 local worker 认领" + % self._executor_name()) + self._audit(project_id, "pr_job_needs_local_worker", + "%s: %s" % (job_id, reason)) + log_event(self.logger, "pr_job_needs_local_worker", + task_id=task_id, status="needs_local_worker", + message=reason) + results.append({"pr_job_id": job_id, + "task_id": task_id, + "status": "needs_local_worker", + "error": reason}) + except Exception as exc: + log_event(self.logger, "pr_job_peek_failed", + message="pr job peek failed: %s" % exc) + return results results = [] while True: conn = _db_connect(self.db_path) @@ -3312,8 +3848,12 @@ class Worker: # 任务；退出码 75（EX_TEMPFAIL，失败类）：unit 托管 # 的由 systemd on-failure 重拉新版，autoscaler 托管 # 的由 reconcile 重拉，孤儿直接消失。单次 run_all - # 不走这里（进程本来就退出）。 - if self._code_changed(): + # 不走这里（进程本来就退出）。P1 fail-closed： + # 签名缺失同样自退（旧版只在“变了”时退，取不到 + # 签名时继续跑是 fail-open）；每次认领前的签名门 + # 在 _claim 里（池线程不走轮次边界也拦得住）。 + if self._code_changed() or \ + not self._code_signature_valid(): log_event(self.logger, "worker_code_changed_exit", message="worker code changed, exiting " "for restart") @@ -3336,6 +3876,15 @@ class Worker: except Exception as exc: log_event(self.logger, "worker_sweep_failed", message="worker sweep failed: %s" % exc) + # P1 孤儿扫尾（接在陈旧回收之后）：协调员本身 + # 被回收标 interrupted 后，它的未终态子任务 + # 在这里被统一标 interrupted；协调员仍在跑时 + # 本函数不动子任务，见 sweep 方法 docstring。 + try: + self.sweep_orphaned_coordinator_children() + except Exception as exc: + log_event(self.logger, "worker_orphan_sweep_failed", + message="orphan sweep failed: %s" % exc) # 每轮结束写心跳：哪怕本轮一个任务都没派，worker 活着 # 这件事本身也要被 server 看到（文件心跳 + 注册表心跳）。 self._write_heartbeat() diff --git a/host.py b/host.py index 03139a5..0cec2db 100644 --- a/host.py +++ b/host.py @@ -2164,9 +2164,11 @@ def main(): p = sub.add_parser("goal-resume") p.add_argument("--task", required=True) p.add_argument("--goal-text", default=None, - help="新目标文本：旧任务接续 goal 模式时用——跳过状态与" + help="新目标文本：仅开新目标时用——跳过状态与" "goal 记录守卫，在旧线程上从第 1 轮开新目标循环" - "（新语义：无验收、不传 --max-rounds 时不限轮次）") + "（新语义：无验收、不传 --max-rounds 时不限轮次）。" + "断点续跑绝不能传它（不传时沿用原 goal+原验收命令），" + "传了会把原验收命令覆盖成 None") p.add_argument("--max-rounds", type=int, default=None, help="不传时沿用原任务的 goal_max_rounds；传了则覆盖总轮次") p.add_argument("--verify-timeout", type=int, default=600, @@ -2811,12 +2813,23 @@ def main(): verify_timeout=args.verify_timeout) elif args.cmd == "goal-resume": t = state["tasks"][args.task] + # P1 goal-resume 契约（与 cloud/sandbox.py、cloud/worker.py + # 三处一致，绝不能混用）： + # - 开新目标：调用方传 --goal-text 时走下面分支，在旧线程 + # 上从第 1 轮开新循环（verify=None，新语义）。这个分支 + # 只能由“开新目标”的调用方用，断点续跑绝不能传 --goal-text + # ——传了就会把原 verify_command 覆盖成 None。 + # - 断点续跑：调用方不传 --goal-text 时走下面守卫分支， + # 沿用原 goal + 原 verify_command + 原 goal_max_rounds， + # 从 goal_rounds_done 的下一轮续跑，verify 原样传回 + # run_goal_loop（run_goal_loop 会把它再写回同一值， + # 不是覆盖成 None）。 if args.goal_text: # 旧任务接续 goal 模式（云端 /continue 带 goal_mode=1 # 时走这里）：旧任务已终局、没有 goal 记录是正常情况， # 不能按断点续跑的守卫拒绝。只要线程还在，就在旧线程 # 上从第 1 轮开新目标循环（新语义：无验收、不传轮次 - # 时不限轮次，模型宣告完成才结束）。 + # 时不限轮次，模型宣告完成才结束，上界是云端配额）。 if not t.get("thread_id"): print(f"goal-resume 拒绝：{args.task} 没有会话线程，" f"不能接续") @@ -2851,6 +2864,12 @@ def main(): max_rounds = t["goal_max_rounds"] else: max_rounds = 3 + # P1：断点续跑时旧验收形态的 0 轮次（脏数据，0 在旧形态 + # 里表示不限轮次）不按不限轮次跑，钳制到 100 轮（与云端 + # GOAL_MAX_ROUNDS_CAP 同值）；新形态（verify 为空）的 0 + # 保持不限轮次，上界是云端配额。 + if max_rounds == 0 and t.get("verify_command"): + max_rounds = 100 srv.resume_thread(t["thread_id"]) last_turn = (t.get("turns") or [{}])[-1] last_result = last_turn.get("result") or {}