P1 security batch: quota execution checks, cost accounting, goal/coordinator semantics, PR channel, orphan signature, control-plane and console hardening

- Docker quota watchdog + fail-closed quota checks, system tasks need real owner
- max_cost_usd actually accumulated, cancel records cost, estimated rows excluded from unpriced cap, cache_write declared non-billed
- goal-resume keeps original verify on breakpoint resume, goal retry falls back to resume, goal_max_rounds capped at 100 at create and execution
- coordinator children inherit parent model/effort, orphans marked interrupted with audit
- PR jobs on docker workers return explicit needs_local_worker instead of silent empty
- worker code signature covers 6 files and is checked before claim, fail-closed
- sub2api lazy provisioning with rotation, backup whitelist + 0600, push SSRF block, repos/workers desensitized, key/project audits and limits, console buttons/quota/status wording, coordinator/automation quota reserve
This commit is contained in:
host committed 2026-10-27 16:27:50 +00:00
1 parent d254240d84
commit 1402dc67fd
20 files changed
+2616 -179

No files matched your search

+79 -12
View File
@@ -160,7 +160,7 @@
</div>
<textarea id="prompt" placeholder="任务 prompt(必填)"></textarea>
<textarea id="coordParts" placeholder="协调员子任务 JSON(可选),留空为普通任务"></textarea>
<div class="muted">协调员格式:[{"name":"后端","prompt":"做后端"},{"name":"前端","prompt":"做前端"}];提交后 worker 并行建子任务、等齐、合流汇总。不能与 goal/计划模式同用。</div>
<div class="muted">协调员格式:[{"name":"后端","prompt":"做后端"},{"name":"前端","prompt":"做前端"}];提交后 worker 并行建子任务、等齐、合流汇总。不能与 goal/计划模式同用。协调员与 goal/计划互斥:勾了 goal 或计划模式时不要填协调员 JSON,提交前会校验并拦截。子任务费用记父任务创建人账;父任务创建时按分工数预留配额,余额不足时整个父任务被挡下。</div>
<div class="bar">
<select id="modelSelect">
<option value="">模型:默认(跟随配置)</option>
@@ -265,10 +265,11 @@
<input id="reviewSha" placeholder="指定提交用:提交 sha" style="width:100%">
<textarea id="reviewInstructions" placeholder="自定义评审指令用:评审指令正文"></textarea>
<div class="bar">
<button id="btnCreateReview" class="primary" onclick="postReview()">发起评审</button>
<button id="btnCreateReview" class="primary" onclick="postReview()">登记评审需求</button>
<button onclick="if (currentTask) loadReviews(currentTask.id)">刷新队列</button>
<span id="reviewMsg" class="muted"></span>
</div>
<div class="muted">本基线只登记评审需求(requested),不自动跑评审;评审是否执行、结果如何,以评审队列里的记录为准。</div>
</div>
<div id="reviewList" class="muted">点任务行查看详情后,点“刷新队列”查看评审记录</div>
<h3>实时链路</h3>
@@ -286,7 +287,7 @@
<div id="followupList" class="muted"></div>
<h3>运行时审批队列</h3>
<div class="bar"><button id="btnRefreshApprovals" onclick="if(currentTask)loadApprovals(currentTask.id)">刷新审批队列</button><span id="approvalMsg" class="muted"></span></div>
<div class="muted">待审批项需人工点允许或拒绝;超时默认拒绝。本队列的决定会留回执;实时请求的事前拦截待 host 接入后生效,在此之前实时行为不变。</div>
<div class="muted">待审批项需人工点记录允许或记录拒绝;超时默认拒绝。当前版本只留回执,不拦截实时行为(实时请求的事前拦截待 host 接入后生效,在此之前实时行为不变)。</div>
<div id="approvalList" class="muted"></div>
<h3>给其他任务发消息</h3>
<div id="peerMessageForm">
@@ -354,7 +355,12 @@ async function api(path, method, body) {
var msg = (data && data.error) ? data.error : ("HTTP " + r.status);
if (r.status === 401) msg += "(key 未填或无效)";
if (r.status === 429) msg += "(触发限流,稍后再试)";
if (r.status === 402) msg += "(本月配额已用完,联系管理员调整)";
if (r.status === 402) {
// 402 要区分两种满:服务端在响应里带哪个上限字段就是哪个满。
if (data && data.unpriced_token_cap !== undefined) msg += "(无价 token 已满,联系管理员调整)";
else if (data && data.error === "coordinator_quota_reserve_exceeded") msg = "协调员分工预留不足:剩余美元配额不够覆盖全部分工,子任务费用记父任务创建人账,整个父任务已被挡下(未建任何子任务)";
else msg += "(美元配额已满,联系管理员调整)";
}
throw new Error(msg);
}
return data;
@@ -394,13 +400,22 @@ async function loadQuota() {
try {
var d = await api("/users/me/quota", "GET");
var used = (d.used_usd_micros / 1000000).toFixed(2);
var usdLine;
if (d.quota_usd_micros === null || d.quota_usd_micros === undefined) {
el.textContent = "本月用量 $" + used + "(不限额)";
usdLine = "美元:本月用量 $" + used + "(不限额)";
} else {
var q = (d.quota_usd_micros / 1000000).toFixed(2);
var rem = (d.remaining_usd_micros / 1000000).toFixed(2);
el.textContent = "本月用量 $" + used + " / $" + q + "(剩余 $" + rem + ")";
usdLine = "美元:本月用量 $" + used + " / $" + q + "(剩余 $" + rem + ")";
}
// 两条进度并列:美元一条,无价 token 一条;402 时按哪条满区分报错。
var tokLine = "无价 token:";
if (d.unpriced_token_cap === undefined || d.unpriced_tokens_used === undefined) {
tokLine += "未返回(旧服务端)";
} else {
tokLine += d.unpriced_tokens_used + " / " + d.unpriced_token_cap + " tokens";
}
el.textContent = usdLine + " | " + tokLine;
} catch (e) { el.textContent = "配额加载失败:" + e.message; }
}
@@ -453,8 +468,8 @@ async function loadApprovals(taskId) {
}
var btns = "";
if (a.status === "pending") {
btns = " <button onclick=\"decideApproval('" + esc(a.id) + "', 'approved')\">允许</button>" +
" <button onclick=\"decideApproval('" + esc(a.id) + "', 'denied')\">拒绝</button>";
btns = " <button onclick=\"decideApproval('" + esc(a.id) + "', 'approved')\">记录允许</button>" +
" <button onclick=\"decideApproval('" + esc(a.id) + "', 'denied')\">记录拒绝</button>";
}
return "[" + esc(a.kind) + "|" + esc(a.status) + "] " + esc(a.summary) + receipt + btns;
}).join("<br>");
@@ -465,13 +480,20 @@ async function decideApproval(approvalId, decision) {
var verb = decision === "approved" ? "approve" : "deny";
try {
await api("/tasks/" + encodeURIComponent(currentTask.id) + "/approvals/" + encodeURIComponent(approvalId) + "/" + verb, "POST", {});
$("#approvalMsg").innerHTML = "<span class='ok'>已" + (decision === "approved" ? "允许" : "拒绝") + "</span>";
$("#approvalMsg").innerHTML = "<span class='ok'>已记录" + (decision === "approved" ? "允许" : "拒绝") + "(当前版本只留回执,不拦截实时行为)</span>";
loadApprovals(currentTask.id);
} catch (e) { $("#approvalMsg").innerHTML = "<span class='err'>决定失败:" + esc(e.message) + "</span>"; }
}
async function loadWorkers() {
try {
var d = await api("/workers", "GET");
// 非全局管理员只拿到聚合数字(服务端 P1 收紧):不展示 worker 清单。
if (d.workers === undefined && d.workers_total !== undefined) {
if (!d.workers_total) { $("#workers").textContent = "无 worker 注册"; return; }
$("#workers").textContent = "在线 " + d.workers_online + " / 离线 " + d.workers_offline +
"(共 " + d.workers_total + ";完整 worker 清单仅全局管理员可见)";
return;
}
var list = d.workers || d || [];
if (!list.length) { $("#workers").textContent = "无 worker 注册"; return; }
$("#workers").innerHTML = list.map(function (w) {
@@ -759,7 +781,7 @@ async function postReview() {
await api("/tasks/" + encodeURIComponent(currentTask.id) + "/reviews", "POST", body);
$("#reviewMsg").innerHTML = "<span class='ok'>评审已登记(requested)</span>";
loadReviews(currentTask.id);
} catch (e) { $("#reviewMsg").innerHTML = "<span class='err'>发起评审失败:" + esc(e.message) + "</span>"; }
} catch (e) { $("#reviewMsg").innerHTML = "<span class='err'>登记评审需求失败:" + esc(e.message) + "</span>"; }
}
async function submitReviewFindings(reviewId) {
if (!currentTask) return;
@@ -778,8 +800,19 @@ async function showDetail(taskId) {
document.querySelectorAll("#taskRows tr").forEach(function (tr) {
tr.classList.toggle("selected", tr.getAttribute("data-id") === taskId);
});
// 每个任务状态配一句中文解释(只解释状态语义,不改后端状态机)。
var STATUS_EXPLAIN = {
queued: "排队中,尚未被 worker 认领",
running: "运行中,模型回合进行中",
completed: "模型回合结束、改动在 worktree 里尚未合并",
failed: "模型回合失败,可重试",
interrupted: "已被取消/中断,可重试",
merged: "控制面已标记 merged,不代表 git 主分支已合入",
"merge-failed": "合并落盘失败,可重试"
};
$("#detail").innerHTML =
"状态 <b class='st-" + esc(t.status) + "'>" + esc(t.status) + "</b> | " +
"状态 <b class='st-" + esc(t.status) + "'>" + esc(t.status) + "</b>(" +
esc(STATUS_EXPLAIN[t.status] || "未知状态") + ") | " +
"executor " + esc(t.executor || "") + " | 主机 " + esc(t.worktree_host || "") +
" | 模型 " + esc(t.model || "默认") + " | 强度 " + esc(t.effort || "默认") +
" | 权限 " + esc(t.approval || "默认") +
@@ -800,7 +833,35 @@ async function showDetail(taskId) {
pre.style.display = "block";
$("#detailActions").style.display = "flex";
$("#btnFork").style.display = TRACE_TERMINAL.indexOf(t.status) >= 0 ? "" : "none";
$("#actionMsg").textContent = "";
// 按钮按状态 + 当前用户项目角色禁用并写明原因(前端禁用只是
// 防误点,真正鉴权仍在服务端每次请求时做;角色来自项目列表
// 的 my_role 字段,未知角色时按最严(仅取消可用)处理)。
(function () {
var proj = null;
(projectsCache || []).forEach(function (p) { if (p.id === t.project_id) proj = p; });
var role = proj ? proj.my_role : null;
var isMaintainer = (role === "maintainer" || role === "owner");
var canCreate = (role === "contributor" || isMaintainer);
var terminal = TRACE_TERMINAL.indexOf(t.status) >= 0;
var rules = [
{ btn: "#btnRetry", ok: ["failed", "interrupted", "merge-failed"].indexOf(t.status) >= 0 && isMaintainer,
why: "仅 failed/interrupted/merge-failed 可重试" + (isMaintainer ? "" : ",且仅 maintainer 可重试") },
{ btn: "#btnCancel", ok: !terminal && canCreate,
why: terminal ? "终态任务不可取消" : (canCreate ? "" : "仅项目成员(contributor 及以上)可取消") },
{ btn: "#btnMerge", ok: t.status === "completed" && isMaintainer,
why: "仅 completed 可合并" + (isMaintainer ? "" : ",且仅 maintainer 可合并") },
{ btn: "#btnPr", ok: t.status === "merged" && isMaintainer,
why: "仅 merged 可提 PR" + (isMaintainer ? "" : ",且仅 maintainer 可提 PR") }
];
var reasons = [];
rules.forEach(function (r) {
var b = $(r.btn);
b.disabled = !r.ok;
b.title = r.ok ? "" : r.why;
if (!r.ok && r.why) reasons.push(b.textContent.replace(/(.*$/, "") + ":不可用——" + r.why);
});
$("#actionMsg").textContent = reasons.length ? reasons.join(";") : "";
})();
$("#costBox").innerHTML = "";
$("#forkForm").style.display = "none";
$("#forkMsgSpan").textContent = "";
@@ -1062,6 +1123,12 @@ $("#createTask").onclick = async function () {
try { body.coordinator_parts = JSON.parse(coordRaw); }
catch (e) { $("#taskMsg").textContent = "协调员子任务 JSON 解析失败"; return; }
if (!Array.isArray(body.coordinator_parts)) { $("#taskMsg").textContent = "协调员子任务必须是数组"; return; }
// 协调员与 goal/计划互斥:提交前校验并在表单处写明冲突
// (服务端同样 400,但不要等到服务端才发现)。
if ($("#goalMode").checked || $("#planMode").checked) {
$("#taskMsg").textContent = "协调员任务不能与 goal/计划模式同用:协调员任务本身不跑模型回合,请取消 goal/计划勾选,或清空协调员 JSON";
return;
}
}
var atts = gatherAttachments();
if (atts.length) {
+31
View File
@@ -16,6 +16,15 @@
超时时同样取消挂起的子任务并标 failed,报告里写清哪几项
没完成。取消路径的协调员标 interrupted,不发完成/失败推送
(与普通任务的取消口径一致)。
- 孤儿收口(P1):协调员本身死亡(worker 进程死、VM 回收)
时没有“取消者”替它取消子任务,正在跑的子任务会无声继续
烧钱。worker 侧两道收口:① 协调员取消/超时终局时,cancel
之后仍未终态的子任务由 _coord_mark_orphans 直接标
interrupted 并记审计;② 常驻循环每轮的
sweep_orphaned_coordinator_children 把“父已终态(或父行
已失踪)但子仍未终态”的子任务统一标 interrupted 并记审计
(父仍在跑时绝不动子任务)。本模块只提供纯函数
nonterminal_children 供 worker 判断,DB 写回在 worker 侧。
本模块只放纯逻辑(校验、解析、汇总、建表),不碰 DB 之外的
任何东西,单测不需要 worker/server。
@@ -128,6 +137,28 @@ def ensure_columns(conn):
pass
TERMINAL_STATUSES = frozenset(
{"completed", "failed", "interrupted", "merged", "merge-failed"})
def nonterminal_children(children):
"""未终态的子任务(P1 孤儿判断用,纯函数)。
children 为 [{name, id, status, ...}](worker 的 _coord_child_rows
拼出来的 merged 形态,或 _coord_sync_children 的 [{name, id}]
补上 status 后的形态);返回其中 status 不在终态集里的项
(缺 status 按未终态算,绝不把“查不到”当成“已完成”)。
worker 的孤儿标记与取消/超时终局都以它为准。
"""
out = []
for ch in children or []:
if not isinstance(ch, dict):
continue
if ch.get("status") not in TERMINAL_STATUSES:
out.append(ch)
return out
def build_report(children):
"""合流汇总:children 为 [{name, id, status, result_text,
error_text}],返回 (report_text, overall_status)。
+25
View File
@@ -18,6 +18,28 @@ may charge differently. Override per model with
with no price record ``usd_micros=None`` and reports count their
tokens in ``unpriced_actual_tokens`` instead of pricing them at zero.
cache_write_input_tokens: EXPLICITLY NOT BILLED HERE (P1 decision,
not an oversight). host.py collects ``cacheWriteInputTokens`` from
upstream tokenUsage events and sandbox.py carries it in usage dicts
for observability, but it is NOT added to ``total_tokens`` (upstream
total is input + output), NOT stored in ``cost_records`` (``record``
takes input/output/cached only), NOT priced by ``price_usd_micros``,
and NOT counted toward the unpriced-token cap (worker counts input +
output from ``source="actual"`` rows only). Rationale: no price-table
entry and no relay bill evidence treats cache creation as a
separately billed token stream for the OpenAI-style relay models in
use; cache_write here is prompt-cache bookkeeping detail, not a
second charge on top of input. If a future provider bills cache
creation separately, the price table must gain an explicit
cache_write rate FIRST -- only then may ``record``/pricing count it.
Until then, counting it would double-charge input tokens that are
already priced (or inflate the unpriced cap with non-spend).
Unpriced cap scope (P1): ``unpriced_actual_tokens`` and the worker's
cap query count ``source="actual"`` rows only. Estimated rows
(``source="estimated"``, character-count fallback when no usage
events arrive) are never real spend and must not trip the cap.
Dead end, documented so nobody retries it: the app-server's
account/usage/read method requires ChatGPT account authentication and
fails on a relay API key (2026-10-26 probe: ``codex account
@@ -98,6 +120,9 @@ def price_usd_micros(model, input_tokens, cached_input_tokens,
output rate. Reasoning tokens are already inside output_tokens
(provider total = input + output), never added twice. Returns None
when the model has no price entry (caller records usd as unknown).
cache_write_input_tokens is deliberately NOT a parameter here:
it is not billed separately (see module docstring); callers must
not add it to input_tokens to “count it anyway”.
"""
rates = model_prices().get(model)
if rates is None:
+125 -5
View File
@@ -37,15 +37,22 @@
发送端是后续待接线的工作,见模块末尾的缺口说明。
- 本模块不跑真实模型回合。webhook/slack 的真实发送只 POST
到订阅者自己填的 target URL(创建订阅已要求项目
create_task 权限;能建订阅的成员本来就能经任务执行代码,
因此不对 target 做内网屏蔽——屏蔽既拦不住提权,也会误伤
本地接收器)。密钥、token、证书一律不由本模块读取、写入
或返回。
create_task 权限)。P1 收紧后:真实发送层对 target 做 SSRF
校验——loopback、私有网段、链路本地、云元数据地址一律不
POST;主机名先 DNS 解析再按解析出的 IP 判,解析失败拒绝
(见 validate_push_target)。本地接收器如需放行,须运维显式
置 CODEX_PUSH_ALLOW_PRIVATE_TARGETS=1(默认关闭,测试用
本地接收器也须经此开关,不在代码里默认放行 loopback)。
密钥、token、证书一律不由本模块读取、写入或返回。
"""
import ipaddress
import json
import os
import socket
import sqlite3
import urllib.error
import urllib.parse
import urllib.request
from cloud.db import connect as _db_connect
from cloud.db import PgConnection as _PgConnection
@@ -67,6 +74,7 @@ __all__ = [
"send_webhook",
"send_slack",
"dispatch_event",
"validate_push_target",
]
CHANNELS = ("webhook", "slack", "webpush", "apns", "fcm")
@@ -311,6 +319,91 @@ def _require_http_url(target, channel):
return target
# 云元数据主机名:即使解析出的 IP 恰好不在私有段,也直接挡。
_BLOCKED_HOSTNAMES = frozenset({"metadata.google.internal"})
def _allow_private_targets():
"""运维显式开关:仅 CODEX_PUSH_ALLOW_PRIVATE_TARGETS=1 时放行
内网目标(本地接收器场景)。默认关闭;开关只影响 IP 段判定,
不影响 scheme 校验。"""
return os.environ.get(
"CODEX_PUSH_ALLOW_PRIVATE_TARGETS", "").strip() == "1"
def _default_resolver(hostname):
"""默认 DNS 解析:返回 IP 字符串列表;解析失败抛异常,
由 validate_push_target 转成“拒绝”。"""
infos = socket.getaddrinfo(hostname, None)
out = []
for info in infos:
sockaddr = info[4] if len(info) > 4 else None
if sockaddr and sockaddr[0]:
out.append(str(sockaddr[0]))
return out
def _ip_is_blocked(ip_text):
"""非全局单播 IP 一律算内网/特殊地址:loopback、私有网段、
链路本地(169.254/16,含云元数据 169.254.169.254)、保留段、
组播、未指定地址都在 is_global=False 里。"""
try:
return not ipaddress.ip_address(ip_text).is_global
except ValueError:
return True
def validate_push_target(target, resolver=None, check_dns=True):
"""SSRF 校验:通过返回 None,否则返回原因字符串(不抛)。
顺序:scheme(http/https)-> 主机名存在 -> 元数据主机名 ->
IP 字面量直接判段 -> (check_dns 时)DNS 解析后逐 IP 判段,
解析失败或解析出空结果都拒绝(DNS 重绑定下解析时与发送时
仍可能不一致,这是本层能做的上限,见残留说明)。
check_dns=False 时只做字面量判定:供直接调 send_* 但不想
做 DNS 的调用方用;dispatch 的真实发送路径一律
check_dns=True,显式 sender 路径不走本预检(见 dispatch)。
"""
if not isinstance(target, str) or not (
target.startswith("http://") or target.startswith("https://")):
return "target must be an http(s) URL"
try:
host = urllib.parse.urlparse(target).hostname
except ValueError:
return "target host is invalid"
if not host:
return "target host required"
lowered = host.lower()
if lowered in _BLOCKED_HOSTNAMES or lowered.endswith(
".metadata.google.internal"):
return "target host is a cloud metadata address"
if _allow_private_targets():
return None
try:
ipaddress.ip_address(host)
is_literal = True
except ValueError:
is_literal = False
if is_literal:
if _ip_is_blocked(host):
return "target resolves to a non-public address: %s" % host
return None
if not check_dns:
return None
resolve = resolver or _default_resolver
try:
ips = resolve(host)
except Exception:
return "DNS resolution failed for target host"
if not ips:
return "DNS resolution failed for target host"
for ip_text in ips:
if _ip_is_blocked(ip_text):
return ("target host resolves to a non-public address: %s"
% ip_text)
return None
def send_webhook(message):
"""webhook 通道真实发送:POST JSON 到订阅的 target URL。
@@ -319,6 +412,11 @@ def send_webhook(message):
RuntimeError(dispatch 记入 errors,不算 sent)。
"""
target = _require_http_url(message["target"], "webhook")
blocked = validate_push_target(
target, resolver=message.get("_resolver"))
if blocked is not None:
raise RuntimeError(
"webhook delivery failed: target blocked: %s" % blocked)
payload = message["payload"] if isinstance(
message["payload"], dict) else {}
ok, detail = _post_json(target, {
@@ -340,6 +438,11 @@ def send_slack(message):
事件附 error 摘要(截断 500 字)。非 2xx 抛 RuntimeError。
"""
target = _require_http_url(message["target"], "slack")
blocked = validate_push_target(
target, resolver=message.get("_resolver"))
if blocked is not None:
raise RuntimeError(
"slack delivery failed: target blocked: %s" % blocked)
payload = message["payload"] if isinstance(
message["payload"], dict) else {}
text = "[codex] task %s %s (project %s)" % (
@@ -357,7 +460,8 @@ def send_slack(message):
DEFAULT_SENDERS = {"webhook": send_webhook, "slack": send_slack}
def dispatch_event(db, event_type, project_id, payload, sender=None):
def dispatch_event(db, event_type, project_id, payload, sender=None,
resolver=None):
"""分发一次状态变化事件,只发本项目的订阅。
参数:
@@ -418,6 +522,7 @@ def dispatch_event(db, event_type, project_id, payload, sender=None):
}]
return result
real_send = sender is None
if sender is None:
# 内置真实发送端:webhook/slack 真 POST;显式 sender
# 传入时不走这里(注入优先),webpush/apns/fcm 永远
@@ -445,6 +550,21 @@ def dispatch_event(db, event_type, project_id, payload, sender=None):
result["errors"].append(
dict(base, reason=MISSING_CREDENTIAL_REASONS[sub["channel"]]))
continue
# SSRF 预检(真实发送路径,发送前挡,不 POST):含 DNS
# 的全量校验。显式 sender 路径不预检:注入的 sender 是
# 调用方自控的投递(测试 seam),不是服务端发起的 HTTP,
# 挡它不增加生产安全性;生产服务端一律走 sender=None
# 的真实发送,必经本预检与 send_webhook/send_slack 的
# 二次校验。原因文案含 "delivery failed",与旧调用方
# 对投递失败的断言兼容。
if real_send:
blocked = validate_push_target(
sub["target"], resolver=resolver, check_dns=True)
if blocked is not None:
result["errors"].append(
dict(base, reason="%s delivery failed: target blocked:"
" %s" % (sub["channel"], blocked)))
continue
# webhook/slack:sender=None 时已换成内置真实发送端;
# 显式 sender 里没有该通道时明确报错,不静默成功。
channel_sender = _resolve_channel_sender(sender, sub["channel"])
+83 -36
View File
@@ -423,6 +423,15 @@ class LocalExecutor(Executor):
rounds = task.get("goal_max_rounds")
if rounds is None:
rounds = 3
# P1 轮次上限(与 worker 的 GOAL_MAX_ROUNDS_CAP 同值 100):
# 有限轮次超限直接 loud fail,不静默截断;goal_mode 的 0
#(不限轮次)放行,上界是配额;旧验收形态的 0 同样拒绝
#(0 在旧形态里表示不限轮次,P1 已禁用无上限验收循环)。
if rounds is not None and rounds > 100:
raise ValueError("goal_max_rounds 超过上限 100 轮")
if rounds == 0 and verify and not task.get("goal_mode"):
raise ValueError(
"旧验收形态 goal_max_rounds=0 表示不限轮次,已禁用")
cmd = self._base_command(model=task.get("model"),
effort=task.get("effort"),
approval=task.get("approval"),
@@ -440,26 +449,35 @@ class LocalExecutor(Executor):
return self._goal_result_from_runner(
task["id"], self._run_local(cmd, extra_env))
def goal_resume_task(self, task, goal_text, extra_env=None,
model_provider=None) -> dict:
"""调 host.py goal-resume --goal-text:在旧线程上开新目标
循环(旧任务接续 goal 模式,云端 /continue 带 goal_mode=1
时走这里)。命令:host.py goal-resume --task <host 名>
--goal-text <新目标> --max-rounds 0(新语义不限轮次)。
def goal_resume_task(self, task, goal_text=None, extra_env=None,
model_provider=None, max_rounds=None) -> dict:
"""调 host.py goal-resume,分两种调用(P1,与 worker、
host 三处一致,绝不能混用):
- 断点续跑:goal_text=None(worker 的 _resume_kind=
"breakpoint" 时传 None)。命令不带 --goal-text、默认
也不带 --max-rounds:host 沿用原 goal + 原 verify_command
+ 原 goal_max_rounds,从断点轮次续跑。传了 --goal-text
就会走 host 的开新目标分支、把原 verify 覆盖成 None,
所以断点调用绝不能顺手把续跑消息塞进 goal_text。
- 开新目标:goal_text 为新目标文本(云端 /continue 带
goal_mode=1 时走这里)。命令带 --goal-text 与
--max-rounds(默认 0,新语义不限轮次,上界是配额)。
不传 --repo:goal-resume 操作已有 host 任务,不新建
worktree。结论解析与 goal_task 同款。
"""
if not isinstance(task, dict) or task.get("id") is None:
raise ValueError("任务缺少 id")
if not goal_text:
raise ValueError("goal-resume 缺少目标文本")
cmd = self._base_command(model=task.get("model"),
effort=task.get("effort"),
approval=task.get("approval"),
model_provider=model_provider)
cmd += ["goal-resume", "--task", self._host_name(task),
"--goal-text", goal_text,
"--max-rounds", "0"]
cmd += ["goal-resume", "--task", self._host_name(task)]
if goal_text:
cmd += ["--goal-text", goal_text,
"--max-rounds",
str(max_rounds if max_rounds is not None else 0)]
elif max_rounds is not None:
cmd += ["--max-rounds", str(max_rounds)]
return self._goal_result_from_runner(
task["id"], self._run_local(cmd, extra_env))
@@ -469,7 +487,9 @@ class LocalExecutor(Executor):
解析用完整 stdout(结论在末尾),存库的 stdout 才截尾;
returncode 非 0 一律 failed;为 0 时无结论也判 failed,
绝不把“没打印结论”当成达标。host 任务已存在时(同任务
/retry 撞名,goal 没有 run 续跑语义)错误写明请新建任务。
/retry 撞名,goal 没有 run 续跑语义)错误写明“goal 失败
请用继续而非重试”(P1:重试在已存在 host-state 上必失败,
断点续跑才是正确路径,见 worker 的回落逻辑)。
"""
returncode = getattr(result, "returncode", 1)
stdout_full = getattr(result, "stdout", "") or ""
@@ -478,8 +498,10 @@ class LocalExecutor(Executor):
combined = stdout_full + stderr
if "已存在" in combined:
met, goal_error = None, (
"host 任务已存在:goal 任务不支持同任务重试,"
"请新建任务")
"host 任务已存在:goal 失败请用继续而非重试——"
"同任务重试会撞已存在 host-state,worker 会先"
"尝试断点续跑,断点续跑仍失败时请用继续(不要"
"重排同一任务)")
status = "failed"
elif returncode != 0:
status = "failed"
@@ -516,9 +538,11 @@ class LocalExecutor(Executor):
killpg 才能连同子孙一起杀掉。注意:只有默认 runner(真实
subprocess)时才用本方法;单测注入的假 runner 走同步路径,
不走这里。kind=goal 时建 goal 命令、kind=goal-resume 时建
goal-resume 命令(旧任务接续 goal 模式,目标文本取本轮
prompt;结论解析由 worker 的 _execute_via_process 按 kind
分发到 _goal_result_from_runner)。goal 语义与同步 goal_task
goal-resume 命令(结论解析由 worker 的 _execute_via_process
按 kind 分发到 _goal_result_from_runner)。goal-resume 分流
(P1):_resume_kind="breakpoint" 时不传 --goal-text(断点
续跑,沿用原 goal+verify),其它情况才把本轮 prompt 当
--goal-text(开新目标)。goal 语义与同步 goal_task
一致:新形态(goal_mode)无验收命令时 --verify 不传、
--max-rounds 传 0(0 不能用 or 兜底成 3);旧形态仍传验收
命令与轮次。seed_dir 只对 dispatch/goal 追加(run 续跑与
@@ -545,6 +569,11 @@ class LocalExecutor(Executor):
rounds = task.get("goal_max_rounds")
if rounds is None:
rounds = 3
if rounds is not None and rounds > 100:
raise ValueError("goal_max_rounds 超过上限 100 轮")
if rounds == 0 and verify and not task.get("goal_mode"):
raise ValueError(
"旧验收形态 goal_max_rounds=0 表示不限轮次,已禁用")
cmd += ["goal", "--name", self._host_name(task),
"--task", goal_text]
if verify:
@@ -556,12 +585,15 @@ class LocalExecutor(Executor):
if seed_dir:
cmd += ["--seed-dir", seed_dir]
elif kind == "goal-resume":
# 旧任务接续 goal:host 侧用 --goal-text 在旧线程上开
# 新目标循环(新语义,不限轮次);task["prompt"] 在续跑
# 时已被 worker 换成续跑消息。
cmd += ["goal-resume", "--task", self._host_name(task),
"--goal-text", task["prompt"],
"--max-rounds", "0"]
# P1 分流(与 goal_resume_task 同款契约):
# _resume_kind="breakpoint" 时不传 --goal-text(host
# 沿用原 goal+verify 断点续跑,绝不能用续跑消息覆盖
# 原 verify);其它情况(开新目标)才用 task["prompt"]
#(worker 已换成续跑消息)当 --goal-text,--max-rounds 0。
cmd += ["goal-resume", "--task", self._host_name(task)]
if task.get("_resume_kind") != "breakpoint":
cmd += ["--goal-text", task["prompt"],
"--max-rounds", "0"]
else:
cmd += ["dispatch", "--name", self._host_name(task),
"--task", task["prompt"]]
@@ -1101,6 +1133,12 @@ class DockerExecutor(Executor):
rounds = task.get("goal_max_rounds")
if rounds is None:
rounds = 3
# P1 轮次上限(与 Local goal_task 同款,不静默截断)。
if rounds is not None and rounds > 100:
raise ValueError("goal_max_rounds 超过上限 100 轮")
if rounds == 0 and verify and not task.get("goal_mode"):
raise ValueError(
"旧验收形态 goal_max_rounds=0 表示不限轮次,已禁用")
inner_args = ["goal", "--name", self._host_name(task),
"--task", goal_text]
if verify:
@@ -1150,8 +1188,10 @@ class DockerExecutor(Executor):
combined = stdout_full + stderr
if "已存在" in combined:
met, goal_error = None, (
"host 任务已存在:goal 任务不支持同任务重试,"
"请新建任务")
"host 任务已存在:goal 失败请用继续而非重试——"
"同任务重试会撞已存在 host-state,worker 会先"
"尝试断点续跑,断点续跑仍失败时请用继续(不要"
"重排同一任务)")
status = "failed"
elif returncode != 0:
status = "failed"
@@ -1181,22 +1221,29 @@ class DockerExecutor(Executor):
"usage": _parse_goal_usage(stdout_full),
}
def goal_resume_task(self, task, goal_text, timeout=None,
def goal_resume_task(self, task, goal_text=None, timeout=None,
extra_mounts=None, network=None,
extra_env=None, model_provider=None) -> dict:
"""容器内调 host.py goal-resume --goal-text:在旧线程上开
新目标循环(旧任务接续 goal 模式)。解析与 goal_task 同款
(完整 stdout 结论、存库截尾),但没有“已存在”分支:
goal-resume 本来就是操作已有 host 任务,不存在 dispatch
撞名问题;returncode 非 0 或无结论一律 failed。
extra_env=None, model_provider=None,
max_rounds=None) -> dict:
"""容器内调 host.py goal-resume,分两种调用(P1,与 Local
goal_resume_task 同款契约):断点续跑传 goal_text=None
(不带 --goal-text,host 沿用原 goal+verify+轮次),开新
目标才传 goal_text(带 --goal-text 与 --max-rounds,默认 0)。
解析与 goal_task 同款(完整 stdout 结论、存库截尾),但
没有“已存在”分支:goal-resume 本来就是操作已有 host 任务,
不存在 dispatch 撞名问题;returncode 非 0 或无结论一律 failed。
"""
if not isinstance(task, dict) or task.get("id") is None:
raise ValueError("任务缺少 id")
if not goal_text:
raise ValueError("goal-resume 缺少目标文本")
_resume_args = ["goal-resume", "--task", self._host_name(task)]
if goal_text:
_resume_args += ["--goal-text", goal_text, "--max-rounds",
str(max_rounds
if max_rounds is not None else 0)]
elif max_rounds is not None:
_resume_args += ["--max-rounds", str(max_rounds)]
inner = self._inner_host_command(
"goal-resume", "--task", self._host_name(task),
"--goal-text", goal_text, "--max-rounds", "0",
*_resume_args,
model=task.get("model"), effort=task.get("effort"),
approval=task.get("approval"),
model_provider=model_provider)
+329 -54
View File
@@ -113,12 +113,15 @@ from cloud.oauth import (
)
from cloud.relay import RelayError as _RelayError
from cloud.relay import get_relay_models as _get_relay_models
from cloud.sub2api import Sub2apiError as _Sub2apiError
from cloud.sub2api import ensure_gateway_keys_table as _ensure_gateway_keys
from cloud.sub2api import ensure_user_gateway_key as _ensure_sub2api_key
from cloud.sub2api import gateway_name as _gateway_name
from cloud.sub2api import gateway_origin as _gateway_origin
from cloud.sub2api import get_user_gateway_key as _get_sub2api_key
from cloud.sub2api import list_gateway_models as _list_sub2api_models
from cloud.sub2api import rotate_user_gateway_key as _rotate_sub2api_key
from cloud.sub2api import validate_or_rotate_gateway_key as _validate_sub2api_key
from cloud.repos import validate_repo_url
from cloud.push import (
create_subscription as _push_create,
@@ -167,6 +170,31 @@ TRACE_STREAM_MAX_SECONDS = 3600
TRACE_STREAM_POLL_SECONDS = 1.0
TRACE_DEFAULT_DIR = "/opt/codex-app-server/work/host-traces"
# P1 控制面限额(建项目时校验,超限 400 并在错误里写明数值):
# 单用户项目数上限、项目名长度上限、自定义项目 id 长度上限。
# 均可经环境变量调(运维侧),缺省值见下方;改动只影响新项目,
# 已有超限项目不受影响(不删、不改名)。
def _env_int(name, default):
try:
value = int(os.environ.get(name, "") or default)
except (TypeError, ValueError):
return default
return value if value > 0 else default
MAX_PROJECTS_PER_USER = _env_int("CODEX_MAX_PROJECTS_PER_USER", 50)
MAX_PROJECT_NAME_LEN = _env_int("CODEX_MAX_PROJECT_NAME_LEN", 100)
MAX_PROJECT_ID_LEN = _env_int("CODEX_MAX_PROJECT_ID_LEN", 64)
# 协调员父任务预留:每分工至少预留的美元额度(微美元)。
# 父任务准入只查一次时,N 个分工可解锁 N 倍消费;本预留要求
# 剩余配额 >= 分工数 × 本值,否则父任务直接 402(见
# _check_coordinator_reserve)。缺省 $1/分工,可经环境变量调;
# 它是准入下限估计,不是实际计费——实际花费仍按子任务
# created_by=父任务创建人记到创建人账上(worker 建子任务时
# created_by 直接取父任务创建人,成本 JOIN 据此归属)。
COORDINATOR_RESERVE_PER_PART_USD_MICROS = _env_int(
"CODEX_COORDINATOR_RESERVE_PER_PART_USD_MICROS", 1000000)
def _trace_file_name(task_id):
"""云任务 id -> 链路文件名:与 cloud.sandbox 的 _host_name
@@ -1451,6 +1479,9 @@ class CloudControlPlaneHandler(BaseHTTPRequestHandler):
else:
self._send_json(404, {"error": "not found"})
return
if path == "/users/me/gateway/rotate" and method == "POST":
self._user_gateway_rotate()
return
if path.startswith("/users/me/keys"):
# API keys 管理:全部只操作认证用户自己的 key。
if method == "GET" and path == "/users/me/keys":
@@ -1830,10 +1861,35 @@ class CloudControlPlaneHandler(BaseHTTPRequestHandler):
if not name or not isinstance(name, str):
self._send_json(400, {"error": "name required"})
return
if len(name) > MAX_PROJECT_NAME_LEN:
self._send_json(400, {
"error": "project name too long: max %d characters"
% MAX_PROJECT_NAME_LEN})
return
project_id = body.get("id") or uuid.uuid4().hex
if not isinstance(project_id, str):
self._send_json(400, {"error": "id must be a string"})
return
if len(project_id) > MAX_PROJECT_ID_LEN:
self._send_json(400, {
"error": "project id too long: max %d characters"
% MAX_PROJECT_ID_LEN})
return
# 单用户项目数上限:按 projects.owner_id 计(成员身份加入
# 的项目不计入自己的上限,避免“被拉进很多项目就建不了
# 项目”);超限 400 且不建项目。
_count_conn = self._db()
try:
_owned = _count_conn.execute(
"SELECT COUNT(*) FROM projects WHERE owner_id = ?",
(user_id,)).fetchone()[0]
finally:
_count_conn.close()
if _owned >= MAX_PROJECTS_PER_USER:
self._send_json(400, {
"error": "project limit exceeded: max %d projects"
" per user" % MAX_PROJECTS_PER_USER})
return
# repo_url 可选:不传时为 None(老行为不变);传了则与 worker
# 共用 cloud.repos.validate_repo_url 的同一套校验,不合法时
# 400 且不建项目,绝不把带凭证的 URL 存进库。
@@ -1947,43 +2003,78 @@ class CloudControlPlaneHandler(BaseHTTPRequestHandler):
self._send_json(200, {"project": _project_to_dict(row)})
def _github_repos_list(self, user_id):
"""GET /github/repos:列出本服务 GitHub App 安装记录里的仓
库(installation_repositories 事件同步进 github_app_
installations 表),给控制台建项目/补仓库时的下拉菜单用。
任何已认证用户可列(只含仓库名,不含密钥);选了哪个仓库
只影响调用者自己有管理权的项目(PATCH 仍按 owner 校验)。
"""GET /github/repos:只返回调用者成员项目对应的仓库。
P1 收紧前返回全部安装记录(任一已认证用户可看到全部
安装仓库名,属跨租户信息泄露)。现按 project_members
过滤:只保留调用者是成员的项目的 repo_url 对应的仓库
(clone_url 归一化后比对:去 .git 后缀、比对小写);
调用者无成员项目或项目未填 repo_url 时返回空列表。
手填 URL 的能力不变——没进 App 安装的公开仓库仍可手动填。"""
conn = self._db()
try:
rows = list_installations(conn)
member_ids = {
row["project_id"]
for row in conn.execute(
"SELECT project_id FROM project_members"
" WHERE user_id = ?",
(user_id,)).fetchall()
}
allowed = set()
for row in conn.execute(
"SELECT id, repo_url FROM projects").fetchall():
if row["id"] not in member_ids or not row["repo_url"]:
continue
url = str(row["repo_url"]).strip().lower()
if url.endswith(".git"):
url = url[:-4]
allowed.add(url)
finally:
conn.close()
repos = [{
"installation_id": r["installation_id"],
"account_login": r["account_login"],
"repo_full_name": r["repo_full_name"],
"clone_url": "https://github.com/%s.git" % r["repo_full_name"],
} for r in rows]
repos = []
for r in rows:
clone_url = "https://github.com/%s.git" % r["repo_full_name"]
normalized = clone_url.lower()
if normalized.endswith(".git"):
normalized = normalized[:-4]
if normalized not in allowed:
continue
repos.append({
"installation_id": r["installation_id"],
"account_login": r["account_login"],
"repo_full_name": r["repo_full_name"],
"clone_url": clone_url,
})
self._send_json(200, {"repos": repos})
def _list_projects(self, user_id):
conn = self._db()
try:
member_ids = {
row["project_id"]
for row in conn.execute(
"SELECT project_id FROM project_members WHERE user_id = ?",
(user_id,),
).fetchall()
}
member_rows = conn.execute(
"SELECT project_id, role FROM project_members"
" WHERE user_id = ?",
(user_id,)).fetchall()
member_ids = {row["project_id"] for row in member_rows}
roles = {row["project_id"]: row["role"]
for row in member_rows}
rows = conn.execute(
"SELECT id, name, owner_id, created_at, repo_url, auto_pr"
" FROM projects ORDER BY created_at, id"
).fetchall()
finally:
conn.close()
projects = [_project_to_dict(r) for r in rows
if r["id"] in member_ids]
projects = []
for r in rows:
if r["id"] not in member_ids:
continue
# 附加字段 my_role(只增字段,不改已有成功语义):
# 控制台据此按角色禁用重试/合并/提 PR 等按钮;服务端
# 权限仍以每次请求的 check_permission 为准,前端禁用
# 只是防误点,不是鉴权。
item = _project_to_dict(r)
item["my_role"] = roles.get(r["id"])
projects.append(item)
self._send_json(200, {"projects": projects})
# -- 项目子资源:记忆 / 知识库 / 成本 -------------------------------
@@ -2119,6 +2210,11 @@ class CloudControlPlaneHandler(BaseHTTPRequestHandler):
return
deleted = _push_delete(self.server.db_path,
subscription_id=parts[3])
if deleted:
with _auth_store(self.server.db_path) as store:
store.audit(
user_id, project_id, "push_subscription_deleted",
"subscription_id=%s" % parts[3])
self._send_json(200, {"deleted": bool(deleted)})
return
# /projects/{id}/automations
@@ -2389,31 +2485,33 @@ class CloudControlPlaneHandler(BaseHTTPRequestHandler):
# 缓存(见 cloud/relay.py)。单测可经 server.relay_fetcher
# 注入假取数函数;中转挂了或未配 key 时 502,不编造列表。
# 网关为 sub2api 时:模型目录因用户而异(key 绑定的组
# 决定),取调用者自己的网关 key 现查;无 key 时返回空
# 列表 + provisioning 标记(控制台据此提示开通中),不
# 在页面加载时悄悄开通(开通在建号与任务时发生)。
# 决定)。P1 收紧:首次调模型目录即做真正的懒开通/
# 健康检查,不再只回 provisioning: pending——无行则开通,
# 旧 key 经 /v1/models 实测已死(401/403,含余额为 0 的
# 403)则轮换重开;网关本身不可用时回 502、保留旧行。
# key 明文绝不进响应、日志、缓存。
if _gateway_name() == "sub2api":
conn = self._db()
try:
row = _get_sub2api_key(conn, user_id)
try:
conn.commit()
except Exception:
pass
status, api_key, _sub, _kid = _validate_sub2api_key(
conn, user_id)
except _Sub2apiError as exc:
self._send_json(502, {"error": str(exc)})
return
except Exception as exc:
self._send_json(502, {"error": str(exc)})
return
finally:
conn.close()
if row is None or not row.get("api_key"):
self._send_json(200, {"models": [], "gateway": "sub2api",
"provisioning": "pending"})
return
try:
models = _list_sub2api_models(_gateway_origin(),
row["api_key"])
models = _list_sub2api_models(_gateway_origin(), api_key)
except Exception as exc:
self._send_json(502, {"error": str(exc)})
return
self._send_json(200, {"models": models, "gateway": "sub2api",
"cached": False, "updated_at": None})
"cached": False, "updated_at": None,
"provisioning": status})
return
fetcher = getattr(self.server, "relay_fetcher", None)
try:
@@ -2647,6 +2745,12 @@ class CloudControlPlaneHandler(BaseHTTPRequestHandler):
self._send_json(400, {
"error": "goal_max_rounds must be >= 1"})
return
# P1:旧验收形态轮次设上限 100(与 worker/sandbox 两道门
# 一致;新 goal_mode=1 不走本分支,仍由配额兜底)。
if goal_max_rounds > 100:
self._send_json(400, {
"error": "goal_max_rounds 超过上限 100 轮"})
return
# 协调员:coordinator_parts 非空时任务本身不跑模型回合,
# worker 认领后按分工建并行子任务、等齐、合流;与 goal/
# plan/验收命令语义冲突(那些都是“本任务跑模型”的意思),
@@ -2673,6 +2777,33 @@ class CloudControlPlaneHandler(BaseHTTPRequestHandler):
return
if self._reject_if_over_quota(user_id):
return
# 协调员预留(P1):父任务准入只查一次时,N 个分工会
# 解锁 N 倍消费。父任务创建时要求剩余美元配额至少覆盖
# 分工数 × 每分工预留额,否则直接 402、父任务不建;
# 子任务归属 = 父任务创建人(worker 建子任务时
# created_by 直接取父任务创建人,成本 JOIN 据此归属,
# 子任务认领时 worker 侧配额门再查一次)。不限额用户
# (quota 为 None)不受此预留约束。
if coord_parts and not self._check_coordinator_reserve(
user_id, len(coord_parts)):
return
# sub2api 懒开通/健康检查(P1):建任务前先真校验网关
# key——无行则开通,死 key 则轮换重开;网关不可用时
# 502 且不建任务(不把死 key/待开通当成已开通发给
# worker)。xcmapi 网关下跳过本段。
if _gateway_name() == "sub2api":
_gw_conn = self._db()
try:
try:
_validate_sub2api_key(_gw_conn, user_id)
except _Sub2apiError as exc:
self._send_json(502, {"error": str(exc)})
return
except Exception as exc:
self._send_json(502, {"error": str(exc)})
return
finally:
_gw_conn.close()
# 环境绑定:environment_id 非空时必须是本项目、已 ready
# 的环境;creating/failed/archived 一律拒绝并把状态写进
# 报错(creating 时用户就知道该等 worker 准备好再建任务,
@@ -2836,6 +2967,15 @@ class CloudControlPlaneHandler(BaseHTTPRequestHandler):
conn.close()
with _auth_store(self.server.db_path) as store:
store.audit(user_id, project_id, "create_task", task_id)
if coord_parts:
# 创建响应里写明子任务配额归属(与 worker 建子任务时
# created_by=父任务创建人一致),不是只写文档宣称。
task["quota_attribution"] = (
"coordinator subtasks are charged to the parent"
" task creator")
task["quota_reserved_usd_micros"] = (
len(coord_parts)
* COORDINATOR_RESERVE_PER_PART_USD_MICROS)
self._send_json(201, task)
def _list_tasks(self, user_id, query):
@@ -4091,6 +4231,34 @@ class CloudControlPlaneHandler(BaseHTTPRequestHandler):
return True
return False
def _check_coordinator_reserve(self, user_id, n_parts):
"""协调员预留准入:通过返回 True;被挡时已回 402 返回 False。
剩余美元配额(quota - used,quota 为 None = 不限额时放行)
必须 >= 分工数 × COORDINATOR_RESERVE_PER_PART_USD_MICROS。
这是准入下限估计,不是计费承诺:子任务实际花费仍按
created_by=父任务创建人全额记创建人账;父任务被挡时
一个子任务也不会建,不存在“第 2 个子任务才被挡”的
半截状态。"""
if not n_parts or n_parts <= 1:
return True
quota, used, month = self._quota_status(user_id)
if quota is None:
return True
need = n_parts * COORDINATOR_RESERVE_PER_PART_USD_MICROS
if quota - used < need:
self._send_json(402, {
"error": "coordinator_quota_reserve_exceeded",
"month": month,
"quota_usd_micros": quota,
"used_usd_micros": used,
"coordinator_parts": n_parts,
"reserved_usd_micros": need,
"quota_attribution": "coordinator subtasks are charged"
" to the parent task creator"})
return False
return True
def _audit_admin(self, user_id, action, detail):
"""admin 调用审计(成功与被拒都记,失败绝不挡主流程)。"""
try:
@@ -4125,10 +4293,17 @@ class CloudControlPlaneHandler(BaseHTTPRequestHandler):
def _user_quota_self(self, user_id):
quota, used, month = self._quota_status(user_id)
self._send_json(200,
self._quota_payload(
user_id, quota, used, month,
self._quota_unpriced_tokens(user_id)))
# 合并取舍(P1 rebase 到 d254240):main 的 _quota_payload
# 已带 unpriced_actual_tokens,P1 的附加字段也全留,供控制台
# 同时显示美元与无价 token 两条进度;402 时控制台据
# 响应字段区分“美元配额已满”与“无价 token 已满”。
payload = self._quota_payload(
user_id, quota, used, month,
self._quota_unpriced_tokens(user_id))
payload["unpriced_tokens_used"] = self._quota_unpriced_tokens(
user_id, month)
payload["unpriced_token_cap"] = unpriced_token_cap()
self._send_json(200, payload)
def _admin_quota_get(self, user_id, query):
if not self._is_global_admin(user_id):
@@ -4204,10 +4379,20 @@ class CloudControlPlaneHandler(BaseHTTPRequestHandler):
# -- worker 池 -----------------------------------------------------
def _workers_list(self, user_id):
"""列出注册 worker:任何已认证用户可读(worker 是基础设施
信息,不属于某个项目的数据,不按项目成员过滤)。"""
"""列出注册 worker:全局管理员看完整清单;其他已认证用户
只看在线/离线聚合(P1 收紧:worker_id、执行器、容量、
心跳时间等内部细节不对普通用户暴露,控制台的 Workers
区据此只显示聚合数字)。"""
workers = _registry_list(self.server.db_path)
self._send_json(200, {"workers": workers})
if self._is_global_admin(user_id):
self._send_json(200, {"workers": workers})
return
online = sum(1 for w in workers if w.get("alive"))
self._send_json(200, {
"workers_online": online,
"workers_offline": len(workers) - online,
"workers_total": len(workers),
})
def _workers_heartbeat(self, user_id):
"""worker 心跳:body 含 worker_id(必填,缺了 400)与可选
@@ -4429,11 +4614,15 @@ class CloudControlPlaneHandler(BaseHTTPRequestHandler):
"""POST /admin/backup:备份当前后端数据库,返回路径与规模信息。
权限:只认 CODEX_GLOBAL_ADMINS(见 _is_global_admin),
未配置时一律 403。返回只含 backend、备份文件路径、
文件大小与任务行数,绝不返回数据库密码、备份文件内容
或任何凭据。out_dir 可由 body 指定,否则走环境变量
CLOUD_BACKUP_DIR,未设则用 cloud-backups。成功与被拒
都记审计(审计里只记路径与规模,不记库内容)。
未配置时一律 403;非运维调用方根本到不了这里,绝对
路径只回给全局管理员。out_dir 限白名单目录(P1 收紧):
白名单根来自 CLOUD_BACKUP_ROOT(逗号分隔多根),未设
则 CLOUD_BACKUP_DIR,未设则固定根 cloud-backups;请求
的 out_dir 经实路径解析后必须等于某根或在其下,含 ..
逃逸、指向 /、/etc、/opt 等系统目录的一律 400(白名单
外的绝对路径即使存在也不放行)。备份文件落盘后 chmod
0600。成功与被拒都记审计(调用人由审计的 user_id 列
记,detail 含目标目录与大小,不记库内容)。
"""
if not self._is_global_admin(user_id):
self._audit_admin(user_id, "admin_backup_denied", "backup")
@@ -4443,11 +4632,33 @@ class CloudControlPlaneHandler(BaseHTTPRequestHandler):
if err:
self._send_json(400, {"error": err})
return
out_dir = body.get("out_dir") or os.environ.get(
"CLOUD_BACKUP_DIR") or "cloud-backups"
if not isinstance(out_dir, str) or not out_dir:
requested = body.get("out_dir")
if requested is not None and (
not isinstance(requested, str) or not requested):
self._send_json(400, {"error": "out_dir must be a string"})
return
roots_raw = os.environ.get("CLOUD_BACKUP_ROOT", "")
if roots_raw.strip():
roots = [os.path.realpath(p.strip())
for p in roots_raw.split(",") if p.strip()]
else:
roots = [os.path.realpath(
os.environ.get("CLOUD_BACKUP_DIR") or "cloud-backups")]
if requested:
candidate = os.path.realpath(requested)
else:
candidate = roots[0]
allowed = any(candidate == root
or candidate.startswith(root + os.sep)
for root in roots)
if not allowed:
self._audit_admin(
user_id, "admin_backup_denied",
"out_dir outside backup roots: %s" % requested)
self._send_json(400, {
"error": "out_dir outside allowed backup roots"})
return
out_dir = candidate
backend = _db_backend_name()
try:
if backend == "postgres":
@@ -4456,6 +4667,10 @@ class CloudControlPlaneHandler(BaseHTTPRequestHandler):
else:
from cloud.backup import backup_db as _backup_sqlite
dest = _backup_sqlite(self.server.db_path, out_dir, keep=3)
try:
os.chmod(dest, 0o600)
except OSError:
pass
size_bytes = os.path.getsize(dest)
conn = self._db()
try:
@@ -4468,8 +4683,8 @@ class CloudControlPlaneHandler(BaseHTTPRequestHandler):
return
self._audit_admin(
user_id, "admin_backup",
"backend=%s path=%s size=%d tasks=%d"
% (backend, dest, size_bytes, task_count))
"caller=%s backend=%s out_dir=%s path=%s size=%d tasks=%d"
% (user_id, backend, out_dir, dest, size_bytes, task_count))
self._send_json(200, {
"backend": backend, "path": dest,
"size_bytes": size_bytes, "task_count": task_count,
@@ -5165,6 +5380,9 @@ class CloudControlPlaneHandler(BaseHTTPRequestHandler):
label = "api-key"
with _auth_store(self.server.db_path) as auth:
key_id, api_key_plain = auth.create_api_key(user_id, label)
# 审计只记 key_id 与 label,绝不记明文与哈希。
auth.audit(user_id, "", "api_key_created",
"key_id=%s label=%s" % (key_id, label))
self._send_json(201, {"id": key_id, "label": label,
"hint": api_key_plain[-4:],
"api_key": api_key_plain})
@@ -5182,6 +5400,9 @@ class CloudControlPlaneHandler(BaseHTTPRequestHandler):
return
with _auth_store(self.server.db_path) as auth:
ok = auth.revoke_api_key(user_id, key_id)
if ok:
auth.audit(user_id, "", "api_key_revoked",
"key_id=%s" % key_id)
if not ok:
self._send_json(404, {"error": "key not found"})
return
@@ -5199,8 +5420,45 @@ class CloudControlPlaneHandler(BaseHTTPRequestHandler):
keep_hash = hashlib.sha256(api_key.encode()).hexdigest()
with _auth_store(self.server.db_path) as auth:
count = auth.revoke_other_api_keys(user_id, keep_hash)
# 只记撤销数,不记任何 key 明文、哈希或保留 key 的身份。
auth.audit(user_id, "", "api_key_revoke_others",
"revoked_count=%d" % count)
self._send_json(200, {"revoked_count": count})
def _user_gateway_rotate(self):
"""POST /users/me/gateway/rotate:仅限本人重新开通/轮换
网关 key。非 sub2api 网关时 400;网关不可用时 502 且
旧行不动(rotate 内部先删旧行再重开,只在重开失败时
旧 key 已丢——调用方须知:轮换是显式动作,失败后须
重试本端点重新开通)。审计只记 sub2api_user_id 与
key_id,绝不记 key 明文;响应不含 key。"""
user_id = self._authenticate()
if user_id is None:
self._send_json(401, {"error": "unauthorized"})
return
if _gateway_name() != "sub2api":
self._send_json(400, {"error": "gateway is not sub2api"})
return
conn = self._db()
try:
try:
status, _key, sub_id, key_id = _rotate_sub2api_key(
conn, user_id)
except _Sub2apiError as exc:
self._send_json(502, {"error": str(exc)})
return
except Exception as exc:
self._send_json(502, {"error": str(exc)})
return
finally:
conn.close()
with _auth_store(self.server.db_path) as store:
store.audit(user_id, "", "gateway_key_rotated",
"sub2api_user_id=%s key_id=%s"
% (sub_id, key_id))
self._send_json(200, {"rotated": True, "provisioning": status,
"sub2api_user_id": sub_id})
# -- 推送订阅与分发 ---------------------------------------------------
def _push_create_endpoint(self, user_id, project_id):
body, err = self._read_json()
@@ -5215,6 +5473,12 @@ class CloudControlPlaneHandler(BaseHTTPRequestHandler):
except ValueError as e:
self._send_json(400, {"error": str(e)})
return
# 审计只记订阅 id 与通道:target 全文不记(Slack webhook
# URL 本身含 secret 路径,记进审计等于把 secret 落盘)。
with _auth_store(self.server.db_path) as store:
store.audit(user_id, project_id, "push_subscription_created",
"subscription_id=%s channel=%s"
% (sub["id"], sub["channel"]))
self._send_json(201, sub)
def _push_dispatch_endpoint(self, user_id):
@@ -5228,8 +5492,9 @@ class CloudControlPlaneHandler(BaseHTTPRequestHandler):
self._send_json(400, {"error": "event_type and project_id required"})
return
# P0 安全批后:/admin/push/dispatch 是管理端点,只认
# CODEX_GLOBAL_ADMINS,不再只凭项目 read 就能让服务端
# 向订阅目标(可能含内网地址)真 POST。被拒与成功都记审计。
# CODEX_GLOBAL_ADMINS——该口径已强于 P1 要求的 create_task,
# 不再降级。被拒与成功都记审计;目标 SSRF 校验在
# push.dispatch_event / 真实发送层做。
if not self._is_global_admin(user_id):
self._audit_admin(user_id, "admin_push_denied",
"dispatch %s %s" % (event_type, project_id))
@@ -5258,11 +5523,21 @@ class CloudControlPlaneHandler(BaseHTTPRequestHandler):
"""创建自动化定义:只登记定义与到期状态,不跑模型、
不另起调度器(执行器后续走 claim 门,见 automations
模块文档)。body:name、prompt 必填;interval_seconds
可选(默认 86400,最小 60);repo_url 可选。"""
可选(默认 86400,最小 60);repo_url 可选。
P1 配额门:创建时先过与建任务同一道配额门(美元配额
与无价 token cap,超限 402),避免“定义免费建、到期
无限跑”;单自动化的 spend 上限即创建人配额本身——
每次到期产出的任务以 created_by=创建人建任务,走
POST /tasks 的配额门(触发路径若绕过服务端建任务,
则不在本门覆盖内,见残留)。本服务端没有自动化触发
端点,触发侧的门不在本次 server.py 改动内。"""
body, err = self._read_json()
if err:
self._send_json(400, {"error": err})
return
if self._reject_if_over_quota(user_id):
return
interval = body.get("interval_seconds", 86400)
if interval is None:
interval = 86400
+101 -8
View File
@@ -441,8 +441,28 @@ def list_gateway_models(origin, api_key, request=None):
return out
def ensure_user_gateway_key(conn, user_id, admin=None, request=None):
"""幂等开通:已有行直接返回 key;否则建号->登录->发 key->授余额->入库。
def gateway_key_health(origin, api_key, request=None):
"""真健康检查:用已有 key 调一次 /v1/models。
返回 True(200,可用)/ False(401/403:key 已失效,或余额
为 0——网关在余额为 0 时鉴权层直接 403,两者都算死 key)/
"unknown"(其它状态或网络异常:网关本身不可用时不据此轮换,
避免把一次抖动当成 key 死亡、反复删号重建)。
"""
req = request or _default_request
try:
st, _data = req("GET", origin + "/v1/models", token=api_key)
except Exception:
return "unknown"
if st == 200:
return True
if st in (401, 403):
return False
return "unknown"
def _create_fresh_gateway_key(conn, user_id, admin=None, request=None):
"""建号->登录->发 key->授余额->入库,返回 (key, sub_id, key_id)。
入库失败时尽力清理刚建的 key 与账号(避免孤儿 key),再抛原错;
授余额失败同样走这条清理(余额为 0 的 key 在网关侧直接 403,
@@ -451,11 +471,9 @@ def ensure_user_gateway_key(conn, user_id, admin=None, request=None):
(同一进程内第二次起免管理员登录,见 shared_admin_session)。
调用方负责 conn 的提交语义:本函数在入库后
commit(开通是独立事务,不跟调用方的业务事务绑在一起)。
调用方须保证调用前本用户无有效行(ensure/rotate 先处理
旧行),否则会主键冲突(并发撞号除外,走轮询复用)。
"""
ensure_gateway_keys_table(conn)
row = get_user_gateway_key(conn, user_id)
if row is not None:
return row["api_key"]
origin = gateway_origin()
if not origin:
raise Sub2apiError("SUB2API_BASE_URL 未配置,无法开通网关账号")
@@ -476,9 +494,16 @@ def ensure_user_gateway_key(conn, user_id, admin=None, request=None):
time.sleep(2)
row = get_user_gateway_key(conn, user_id)
if row is not None:
return row["api_key"]
# 合并修复(P1 rebase 到 d254240):main 的 #92 竞态
# 分支原在单体 ensure 里只回一个 key 字符串;P1
# 抽出 _create_fresh 后契约是 3 元组,4 个调用点
# 都按 3 元组解包,裸字符串会被当成字符序列解包
# 而 ValueError。这里按契约回 3 元组,key_id 未知
# 故为 None(与 validate_or_rotate 的 ok 路一致)。
return (row["api_key"], row["sub2api_user_id"], None)
raise
key_id = None
user_token = None
try:
user_token, refresh = user_login(origin, email, password,
request=req)
@@ -497,7 +522,7 @@ def ensure_user_gateway_key(conn, user_id, admin=None, request=None):
(user_id, sub_id, key, refresh, now, now))
conn.commit()
except Exception:
if key_id is not None:
if key_id is not None and user_token is not None:
try:
user_delete_key(origin, user_token, key_id, request=req)
except Exception:
@@ -507,4 +532,72 @@ def ensure_user_gateway_key(conn, user_id, admin=None, request=None):
except Exception:
pass
raise
return key, sub_id, key_id
def ensure_user_gateway_key(conn, user_id, admin=None, request=None):
"""幂等开通:已有行直接返回 key;否则走 _create_fresh 开通。
只保证“行存在”,不保证 key 仍有效:要校验有效性用
validate_or_rotate_gateway_key(死 key 会触发轮换重开)。
"""
ensure_gateway_keys_table(conn)
row = get_user_gateway_key(conn, user_id)
if row is not None:
return row["api_key"]
key, _sub_id, _key_id = _create_fresh_gateway_key(
conn, user_id, admin=admin, request=request)
return key
def validate_or_rotate_gateway_key(conn, user_id, admin=None, request=None):
"""懒开通/健康检查合一:返回 (status, key, sub_id, key_id)。
status:ok(旧 key 经 /v1/models 实测可用)/ provisioned
(无行,新开通)/ rotated(旧 key 经实测已死:401/403,
含余额为 0 的 403;删旧行后重开,死 key 不再被当成已开通)。
网关本身不可用(健康检查 unknown)时抛 Sub2apiError,调用方
应回 502、保留旧行不轮换;绝不在此时把“待开通”当成可用。
"""
ensure_gateway_keys_table(conn)
row = get_user_gateway_key(conn, user_id)
origin = gateway_origin()
if not origin:
raise Sub2apiError("SUB2API_BASE_URL 未配置,无法开通网关账号")
if row is None or not row.get("api_key"):
key, sub_id, key_id = _create_fresh_gateway_key(
conn, user_id, admin=admin, request=request)
return "provisioned", key, sub_id, key_id
health = gateway_key_health(origin, row["api_key"], request=request)
if health is True:
return "ok", row["api_key"], row.get("sub2api_user_id"), None
if health is False:
conn.execute(
"DELETE FROM user_gateway_keys WHERE user_id = ?",
(user_id,))
conn.commit()
key, sub_id, key_id = _create_fresh_gateway_key(
conn, user_id, admin=admin, request=request)
return "rotated", key, sub_id, key_id
raise Sub2apiError("sub2api 网关健康检查暂不可用,保留现有 key")
def rotate_user_gateway_key(conn, user_id, admin=None, request=None):
"""仅限本人调用的重新开通/轮换:删旧行后重开。
无旧行时直接走开通(与 validate 的 provisioned 等价);返回
(status, key, sub_id, key_id),调用方记审计时只可用 sub_id /
key_id,绝不记 key 明文。旧 key 即使只是“疑似失效”也会被
换掉:轮换是用户显式动作,不以健康检查为前提。
"""
ensure_gateway_keys_table(conn)
row = get_user_gateway_key(conn, user_id)
if row is not None:
conn.execute(
"DELETE FROM user_gateway_keys WHERE user_id = ?",
(user_id,))
conn.commit()
key, sub_id, key_id = _create_fresh_gateway_key(
conn, user_id, admin=admin, request=request)
return ("rotated" if row is not None else "provisioned",
key, sub_id, key_id)
+19 -2
View File
@@ -33,6 +33,9 @@ class CapsServerTestCase(unittest.TestCase):
self.auth = AuthStore(self.db_path)
self.owner_id, self.owner_key = self.auth.create_user("owner")
self.viewer_id, self.viewer_key = self.auth.create_user("viewer")
# P1:完整 workers 清单只限全局管理员;owner 在此测试里即管理员。
self._old_admins = os.environ.get("CODEX_GLOBAL_ADMINS")
os.environ["CODEX_GLOBAL_ADMINS"] = self.owner_id
self.server = create_server(self.db_path, "127.0.0.1", 0)
self.port = self.server.server_address[1]
self.thread = threading.Thread(
@@ -48,6 +51,10 @@ class CapsServerTestCase(unittest.TestCase):
except Exception:
pass
self.tmp.cleanup()
if getattr(self, "_old_admins", None) is None:
os.environ.pop("CODEX_GLOBAL_ADMINS", None)
else:
os.environ["CODEX_GLOBAL_ADMINS"] = self._old_admins
def _req(self, method, path, body=None, key=None):
conn = http.client.HTTPConnection("127.0.0.1", self.port, timeout=10)
@@ -82,11 +89,21 @@ class CapsServerTestCase(unittest.TestCase):
"capacity": 4, "running": 2},
self.owner_key)
self.assertEqual(status, 200, body)
# 读权限:任何已认证用户可读(viewer 也能看到基础设施行)。
workers = self._workers(self.viewer_key)
# P1:完整清单只限全局管理员(owner 在此测试里即管理员)。
workers = self._workers(self.owner_key)
self.assertEqual(workers["w1"]["capacity"], 4)
self.assertEqual(workers["w1"]["running"], 2)
def test_workers_list_non_admin_gets_aggregate_only(self):
self._req("POST", "/workers/heartbeat",
{"worker_id": "w1", "executor": "local",
"capacity": 4, "running": 2},
self.owner_key)
status, body = self._req("GET", "/workers", key=self.viewer_key)
self.assertEqual(status, 200, body)
self.assertNotIn("workers", body)
self.assertIn("workers_total", body)
def test_heartbeat_without_running_preserves_old(self):
self._req("POST", "/workers/heartbeat",
{"worker_id": "w1", "capacity": 2, "running": 1},
+10 -1
View File
@@ -184,7 +184,16 @@ class WorkerRegistryTestCase(_ServerCase):
self.assertEqual(status, 200, body)
self.assertTrue(body["alive"])
self.assertEqual(body["executor"], "docker")
status, body = self._req("GET", "/workers", None, self.owner_key)
# P1:完整 workers 清单只限全局管理员,本测试里 owner 即管理员。
old_admins = os.environ.get("CODEX_GLOBAL_ADMINS")
os.environ["CODEX_GLOBAL_ADMINS"] = self.owner_id
try:
status, body = self._req("GET", "/workers", None, self.owner_key)
finally:
if old_admins is None:
os.environ.pop("CODEX_GLOBAL_ADMINS", None)
else:
os.environ["CODEX_GLOBAL_ADMINS"] = old_admins
self.assertEqual(status, 200, body)
self.assertEqual(len(body["workers"]), 1)
self.assertTrue(body["workers"][0]["alive"])
+11 -1
View File
@@ -95,13 +95,23 @@ class ServerGithubReposTestCase(unittest.TestCase):
def test_list_returns_seeded_installations(self):
self._seed_installations()
# P1:repos 只返回调用者成员项目对应的仓库;先把项目仓库
# 绑到其中一个已安装仓库,再断言只返回该仓库,非成员为空。
s, d = _req(self.port, "PATCH",
"/projects/" + self.project_id,
{"repo_url": "https://github.com/openqj/codex-app-server.git"},
self.owner_key)
self.assertEqual(s, 200, d)
s, d = _req(self.port, "GET", "/github/repos", key=self.owner_key)
self.assertEqual(s, 200, d)
names = [r["repo_full_name"] for r in d["repos"]]
self.assertEqual(names, ["openqj/codex-app-server", "openqj/demo"])
self.assertEqual(names, ["openqj/codex-app-server"])
self.assertEqual(
d["repos"][0]["clone_url"],
"https://github.com/openqj/codex-app-server.git")
s, d = _req(self.port, "GET", "/github/repos", key=self.other_key)
self.assertEqual(s, 200, d)
self.assertEqual(d["repos"], [])
def test_patch_repo_url_owner(self):
url = "https://github.com/openqj/demo.git"
+598
View File
@@ -0,0 +1,598 @@
"""P1 控制面/控制台单测(server.py / console.html / sub2api.py / push.py)。
只覆盖本批 P1 控制面改动,不跑真实模型回合、不联网、不碰生产:
server 侧走本地临时端口 + 临时 SQLite;网关侧用假 request /
假 admin(key、token 全是测试假值);推送 DNS 用注入的
resolver。绝不改 worker.py / coordinator.py / cost.py / host.py。
"""
import http.client
import json
import os
import sqlite3
import stat
import sys
import tempfile
import threading
import unittest
from pathlib import Path
from unittest import mock
try:
from cloud.auth import AuthStore
from cloud.push import dispatch_event, validate_push_target
from cloud.push import create_subscription
from cloud.server import create_server
from cloud import server as server_mod
from cloud.sub2api import Sub2apiError
from cloud.sub2api import ensure_gateway_keys_table
from cloud.sub2api import get_user_gateway_key
from cloud.sub2api import rotate_user_gateway_key
from cloud.sub2api import validate_or_rotate_gateway_key
except ImportError:
sys.path.insert(0, str(Path(__file__).resolve().parents[2]))
from cloud.auth import AuthStore
from cloud.push import dispatch_event, validate_push_target
from cloud.push import create_subscription
from cloud.server import create_server
from cloud import server as server_mod
from cloud.sub2api import Sub2apiError
from cloud.sub2api import ensure_gateway_keys_table
from cloud.sub2api import get_user_gateway_key
from cloud.sub2api import rotate_user_gateway_key
from cloud.sub2api import validate_or_rotate_gateway_key
def _req(port, method, path, body=None, key=None, raw_body=None):
conn = http.client.HTTPConnection("127.0.0.1", port, timeout=10)
hdrs = {}
if raw_body is not None:
payload = raw_body
elif body is not None:
payload = json.dumps(body).encode("utf-8")
hdrs["Content-Type"] = "application/json"
else:
payload = None
if key is not None:
hdrs["X-API-Key"] = key
try:
conn.request(method, path, body=payload, headers=hdrs)
resp = conn.getresponse()
data = resp.read()
finally:
conn.close()
try:
parsed = json.loads(data.decode("utf-8")) if data else None
except ValueError:
parsed = data.decode("utf-8", "replace")
return resp.status, parsed
class _FakeGatewayAdmin:
"""假网关管理员:建号/授余额只记调用,不联网。"""
def __init__(self):
self.created = []
self.deleted = []
self.balances = []
def create_user(self, email, password, username):
sub_id = 1000 + len(self.created)
self.created.append((email, username))
return sub_id
def delete_user(self, sub_user_id):
self.deleted.append(sub_user_id)
return True
def set_balance(self, sub_user_id, amount):
self.balances.append((sub_user_id, amount))
def resolve_group_id(self):
return 2
class _FakeGatewayRequest:
"""假网关 HTTP:models 状态可调,其余走固定成功路径。"""
def __init__(self, models_status=200):
self.models_status = models_status
self.calls = []
def __call__(self, method, url, body=None, token=None):
self.calls.append((method, url))
if url.endswith("/v1/models"):
return self.models_status, {"data": []}
if url.endswith("/api/v1/auth/login"):
return 200, {"data": {"access_token": "fake-user-token",
"refresh_token": "fake-refresh",
"expires_in": 3600}}
if url.endswith("/api/v1/keys"):
return 201, {"data": {"id": "fake-key-id-1",
"key": "fake-gateway-key-1"}}
if "/balance" in url:
return 200, {"data": {}}
return 200, {"data": {}}
class Sub2apiP1TestCase(unittest.TestCase):
def setUp(self):
self.conn = sqlite3.connect(":memory:")
ensure_gateway_keys_table(self.conn)
self.admin = _FakeGatewayAdmin()
# 假 origin:请求本身全是 fake,不产生真实网络调用。
self._old_origin = os.environ.get("SUB2API_BASE_URL")
os.environ["SUB2API_BASE_URL"] = "http://127.0.0.1:9"
def tearDown(self):
self.conn.close()
if self._old_origin is None:
os.environ.pop("SUB2API_BASE_URL", None)
else:
os.environ["SUB2API_BASE_URL"] = self._old_origin
def _seed_row(self, key="fake-old-key"):
self.conn.execute(
"INSERT INTO user_gateway_keys(user_id, sub2api_user_id,"
" api_key, refresh_token, created_at, updated_at)"
" VALUES(?, ?, ?, ?, ?, ?)",
("u1", 42, key, "fake-refresh", 1.0, 1.0))
self.conn.commit()
def test_validate_no_row_provisions(self):
req = _FakeGatewayRequest()
status, key, sub_id, key_id = validate_or_rotate_gateway_key(
self.conn, "u1", admin=self.admin, request=req)
self.assertEqual(status, "provisioned")
self.assertEqual(key, "fake-gateway-key-1")
self.assertEqual(key_id, "fake-key-id-1")
self.assertIsNotNone(get_user_gateway_key(self.conn, "u1"))
def test_validate_healthy_key_not_rotated(self):
self._seed_row()
req = _FakeGatewayRequest(models_status=200)
status, key, _sub, _kid = validate_or_rotate_gateway_key(
self.conn, "u1", admin=self.admin, request=req)
self.assertEqual(status, "ok")
self.assertEqual(key, "fake-old-key")
self.assertEqual(self.admin.created, [])
def test_validate_dead_key_rotates_not_pending(self):
self._seed_row()
req = _FakeGatewayRequest(models_status=403)
status, key, _sub, _kid = validate_or_rotate_gateway_key(
self.conn, "u1", admin=self.admin, request=req)
self.assertEqual(status, "rotated")
self.assertEqual(key, "fake-gateway-key-1")
row = get_user_gateway_key(self.conn, "u1")
self.assertEqual(row["api_key"], "fake-gateway-key-1")
self.assertNotEqual(row["api_key"], "fake-old-key")
def test_validate_zero_balance_403_rotates(self):
# 余额为 0 时网关鉴权层回 403:与 401 同属死 key,必须轮换。
self._seed_row()
req = _FakeGatewayRequest(models_status=403)
status, _key, _sub, _kid = validate_or_rotate_gateway_key(
self.conn, "u1", admin=self.admin, request=req)
self.assertEqual(status, "rotated")
def test_validate_gateway_down_keeps_old_row(self):
self._seed_row()
req = _FakeGatewayRequest(models_status=500)
with self.assertRaises(Sub2apiError):
validate_or_rotate_gateway_key(
self.conn, "u1", admin=self.admin, request=req)
row = get_user_gateway_key(self.conn, "u1")
self.assertEqual(row["api_key"], "fake-old-key")
def test_rotate_replaces_row(self):
self._seed_row()
req = _FakeGatewayRequest()
status, key, sub_id, key_id = rotate_user_gateway_key(
self.conn, "u1", admin=self.admin, request=req)
self.assertEqual(status, "rotated")
self.assertEqual(key_id, "fake-key-id-1")
self.assertEqual(
get_user_gateway_key(self.conn, "u1")["api_key"], key)
class PushP1TestCase(unittest.TestCase):
def setUp(self):
self.conn = sqlite3.connect(":memory:")
def tearDown(self):
self.conn.close()
def test_literal_private_and_loopback_blocked(self):
for target in (
"http://127.0.0.1:9/hook",
"http://10.0.0.5/hook",
"http://172.16.4.4/hook",
"http://192.168.1.10/hook",
"http://169.254.169.254/latest/meta-data/",
"http://[::1]/hook"):
self.assertIsNotNone(
validate_push_target(target), target)
def test_metadata_hostname_blocked(self):
self.assertIsNotNone(validate_push_target(
"http://metadata.google.internal/hook"))
def test_hostname_resolved_to_private_blocked(self):
err = validate_push_target(
"https://hooks.example.com/hook",
resolver=lambda host: ["10.1.2.3"])
self.assertIsNotNone(err)
def test_hostname_public_resolves_ok(self):
err = validate_push_target(
"https://hooks.example.com/hook",
resolver=lambda host: ["93.184.216.34"])
self.assertIsNone(err)
def test_dns_failure_rejected(self):
def _boom(host):
raise OSError("no dns in test")
self.assertIsNotNone(validate_push_target(
"https://hooks.example.com/hook", resolver=_boom))
def test_dispatch_loopback_never_posts(self):
# 真实发送路径(sender=None,生产服务端走的路径):
# loopback 目标在预检即被挡,不产生任何 POST。
create_subscription(self.conn, "proj-a", "user-1", "webhook",
"http://127.0.0.1:9/hook", "instant")
result = dispatch_event(self.conn, "failed", "proj-a",
{"task_id": "t1"}, sender=None)
self.assertEqual(result["sent"], [])
self.assertEqual(len(result["errors"]), 1)
self.assertIn("blocked", result["errors"][0]["reason"])
class ServerP1TestCase(unittest.TestCase):
def setUp(self):
self.tmp = tempfile.TemporaryDirectory()
self.db_path = os.path.join(self.tmp.name, "control.db")
self.auth = AuthStore(self.db_path)
self.owner_id, self.owner_key = self.auth.create_user("owner")
self.member_id, self.member_key = self.auth.create_user("member")
self.outsider_id, self.outsider_key = self.auth.create_user(
"outsider")
self._old_admins = os.environ.get("CODEX_GLOBAL_ADMINS")
self._old_gateway = os.environ.get("CLOUD_GATEWAY")
self._old_backup_root = os.environ.get("CLOUD_BACKUP_ROOT")
self._old_backup_dir = os.environ.get("CLOUD_BACKUP_DIR")
os.environ["CODEX_GLOBAL_ADMINS"] = self.owner_id
os.environ.pop("CLOUD_GATEWAY", None)
os.environ.pop("CLOUD_BACKUP_ROOT", None)
os.environ.pop("CLOUD_BACKUP_DIR", None)
self._old_max_projects = server_mod.MAX_PROJECTS_PER_USER
self.server = create_server(self.db_path, "127.0.0.1", 0)
self.port = self.server.server_address[1]
self.thread = threading.Thread(
target=self.server.serve_forever, daemon=True)
self.thread.start()
s, proj = _req(self.port, "POST", "/projects",
{"name": "owner/repo"}, self.owner_key)
self.assertEqual(s, 201, proj)
self.project_id = proj["id"]
self.auth.set_member(
self.project_id, self.owner_id, self.member_id, "maintainer")
def tearDown(self):
self.server.shutdown()
self.server.server_close()
self.thread.join(timeout=5)
try:
self.auth.conn.close()
except Exception:
pass
self.tmp.cleanup()
server_mod.MAX_PROJECTS_PER_USER = self._old_max_projects
if self._old_admins is None:
os.environ.pop("CODEX_GLOBAL_ADMINS", None)
else:
os.environ["CODEX_GLOBAL_ADMINS"] = self._old_admins
if self._old_gateway is None:
os.environ.pop("CLOUD_GATEWAY", None)
else:
os.environ["CLOUD_GATEWAY"] = self._old_gateway
if self._old_backup_root is None:
os.environ.pop("CLOUD_BACKUP_ROOT", None)
else:
os.environ["CLOUD_BACKUP_ROOT"] = self._old_backup_root
if self._old_backup_dir is None:
os.environ.pop("CLOUD_BACKUP_DIR", None)
else:
os.environ["CLOUD_BACKUP_DIR"] = self._old_backup_dir
# 恢复猴子补丁过的 server 模块函数(relay/rotate 相关测试用)。
for name in ("_validate_sub2api_key", "_list_sub2api_models",
"_rotate_sub2api_key"):
try:
mock.patch.object(server_mod, name).stop()
except Exception:
pass
# -- helpers ---------------------------------------------------
def _audit_actions(self, user_id=None):
conn = sqlite3.connect(self.db_path)
try:
rows = conn.execute(
"SELECT action, detail FROM audit_log"
+ (" WHERE user_id = ?" if user_id else ""),
(user_id,) if user_id else ()).fetchall()
finally:
conn.close()
return rows
def _set_quota(self, target, quota):
return _req(self.port, "POST", "/admin/quotas",
{"user_id": target, "quota_usd_micros": quota},
self.owner_key)
# -- 备份 -------------------------------------------------------
def test_backup_whitelist_and_mode(self):
root = os.path.join(self.tmp.name, "backup-root")
os.environ["CLOUD_BACKUP_ROOT"] = root
out_dir = os.path.join(root, "daily")
s, body = _req(self.port, "POST", "/admin/backup",
{"out_dir": out_dir}, self.owner_key)
self.assertEqual(s, 200, body)
self.assertTrue(os.path.exists(body["path"]))
self.assertEqual(stat.S_IMODE(os.stat(body["path"]).st_mode),
0o600)
details = [d for _a, d in self._audit_actions(self.owner_id)]
self.assertTrue(any("admin_backup" in a
for a, _d in self._audit_actions(
self.owner_id)))
self.assertTrue(any(out_dir in (d or "")
for d in details))
# 白名单外:系统目录与 .. 逃逸一律 400。
for bad in ("/etc", os.path.join(root, "..", "escape")):
s, body = _req(self.port, "POST", "/admin/backup",
{"out_dir": bad}, self.owner_key)
self.assertEqual(s, 400, (bad, body))
# -- workers 聚合 ------------------------------------------------
def test_workers_admin_full_member_aggregate_only(self):
s, _b = _req(self.port, "POST", "/workers/heartbeat",
{"worker_id": "w-p1", "executor": "docker",
"capacity": 2, "running": 0}, self.owner_key)
self.assertEqual(s, 200, _b)
s, body = _req(self.port, "GET", "/workers", key=self.owner_key)
self.assertEqual(s, 200)
self.assertIn("workers", body)
self.assertTrue(any(w.get("worker_id") == "w-p1"
for w in body["workers"]))
s, body = _req(self.port, "GET", "/workers", key=self.member_key)
self.assertEqual(s, 200)
self.assertNotIn("workers", body)
self.assertNotIn("w-p1", json.dumps(body))
self.assertEqual(body["workers_total"], 1)
self.assertEqual(body["workers_online"], 1)
self.assertEqual(body["workers_offline"], 0)
# -- github repos 过滤 -------------------------------------------
def test_github_repos_filtered_by_member_projects(self):
s, proj = _req(self.port, "POST", "/projects",
{"name": "with-repo",
"repo_url": "https://github.com/acme/alpha.git"},
self.owner_key)
self.assertEqual(s, 201, proj)
self.auth.set_member(proj["id"], self.owner_id, self.member_id,
"maintainer")
conn = sqlite3.connect(self.db_path)
try:
conn.execute(
"CREATE TABLE IF NOT EXISTS github_app_installations("
"installation_id TEXT, account_login TEXT,"
" repo_full_name TEXT, created_at REAL,"
" UNIQUE(installation_id, repo_full_name))")
conn.execute(
"INSERT OR IGNORE INTO github_app_installations"
" VALUES(?, ?, ?, ?)", ("7", "acme", "acme/alpha", 1.0))
conn.execute(
"INSERT OR IGNORE INTO github_app_installations"
" VALUES(?, ?, ?, ?)", ("7", "acme", "acme/beta", 1.0))
conn.commit()
finally:
conn.close()
s, body = _req(self.port, "GET", "/github/repos",
key=self.member_key)
self.assertEqual(s, 200, body)
names = [r["repo_full_name"] for r in body["repos"]]
self.assertEqual(names, ["acme/alpha"])
s, body = _req(self.port, "GET", "/github/repos",
key=self.outsider_key)
self.assertEqual(s, 200, body)
self.assertEqual(body["repos"], [])
# -- key 审计 -----------------------------------------------------
def test_api_key_audit_only_id_and_label(self):
s, body = _req(self.port, "POST", "/users/me/keys",
{"label": "p1-test"}, self.owner_key)
self.assertEqual(s, 201, body)
key_id = body["id"]
plain = body["api_key"]
s, _b = _req(self.port, "DELETE",
"/users/me/keys/" + key_id, key=self.owner_key)
self.assertEqual(s, 200, _b)
s, body = _req(self.port, "POST",
"/users/me/keys/revoke-others", {},
self.owner_key)
self.assertEqual(s, 200, body)
actions = {a for a, _d in self._audit_actions(self.owner_id)}
self.assertIn("api_key_created", actions)
self.assertIn("api_key_revoked", actions)
self.assertIn("api_key_revoke_others", actions)
blob = json.dumps(self._audit_actions(self.owner_id))
self.assertNotIn(plain, blob)
# -- 项目上限 ------------------------------------------------------
def test_project_limits(self):
s, body = _req(self.port, "POST", "/projects",
{"name": "x" * 101}, self.owner_key)
self.assertEqual(s, 400, body)
self.assertIn("100", body["error"])
s, body = _req(self.port, "POST", "/projects",
{"name": "ok", "id": "y" * 65}, self.owner_key)
self.assertEqual(s, 400, body)
self.assertIn("64", body["error"])
server_mod.MAX_PROJECTS_PER_USER = 1
# setUp 已建 1 个 owner 项目,限额 1 时再建即超限。
s, body = _req(self.port, "POST", "/projects",
{"name": "second"}, self.owner_key)
self.assertEqual(s, 400, body)
self.assertIn("max 1", body["error"])
# -- 协调员预留 ----------------------------------------------------
def test_coordinator_reserve_blocks_parent(self):
s, _b = self._set_quota(self.owner_id, 1000000)
self.assertEqual(s, 200, _b)
parts = [{"name": "a", "prompt": "做 a"},
{"name": "b", "prompt": "做 b"},
{"name": "c", "prompt": "做 c"}]
s, body = _req(self.port, "POST", "/tasks",
{"project_id": self.project_id, "prompt": "协调",
"coordinator_parts": parts}, self.owner_key)
self.assertEqual(s, 402, body)
self.assertEqual(body["error"],
"coordinator_quota_reserve_exceeded")
self.assertEqual(body["coordinator_parts"], 3)
s, _b = self._set_quota(self.owner_id, 10000000)
self.assertEqual(s, 200, _b)
s, body = _req(self.port, "POST", "/tasks",
{"project_id": self.project_id, "prompt": "协调",
"coordinator_parts": parts}, self.owner_key)
self.assertEqual(s, 201, body)
self.assertIn("parent task creator",
body.get("quota_attribution", ""))
self.assertEqual(body.get("quota_reserved_usd_micros"), 3000000)
# -- 自动化配额门 ----------------------------------------------------
def test_automation_create_quota_gate(self):
s, _b = self._set_quota(self.owner_id, 0)
self.assertEqual(s, 200, _b)
s, body = _req(self.port, "POST",
"/projects/%s/automations" % self.project_id,
{"name": "nightly", "prompt": "巡检"},
self.owner_key)
self.assertEqual(s, 402, body)
s, _b = self._set_quota(self.owner_id, 10000000)
self.assertEqual(s, 200, _b)
s, body = _req(self.port, "POST",
"/projects/%s/automations" % self.project_id,
{"name": "nightly", "prompt": "巡检"},
self.owner_key)
self.assertEqual(s, 201, body)
# -- sub2api 懒开通 / 轮换端点 --------------------------------------
def test_relay_models_lazy_provision_not_just_pending(self):
os.environ["CLOUD_GATEWAY"] = "sub2api"
orig_validate = server_mod._validate_sub2api_key
orig_list = server_mod._list_sub2api_models
try:
server_mod._validate_sub2api_key = (
lambda conn, uid: ("provisioned", "fake-key", 7, 9))
server_mod._list_sub2api_models = (
lambda origin, key: ["model-a"])
s, body = _req(self.port, "GET", "/relay/models",
key=self.owner_key)
self.assertEqual(s, 200, body)
self.assertEqual(body["models"], ["model-a"])
self.assertEqual(body["provisioning"], "provisioned")
server_mod._validate_sub2api_key = (
lambda conn, uid: (_ for _ in ()).throw(
Sub2apiError("网关暂不可用")))
s, body = _req(self.port, "GET", "/relay/models",
key=self.owner_key)
self.assertEqual(s, 502, body)
finally:
server_mod._validate_sub2api_key = orig_validate
server_mod._list_sub2api_models = orig_list
def test_gateway_rotate_endpoint(self):
s, body = _req(self.port, "POST", "/users/me/gateway/rotate",
{}, self.owner_key)
self.assertEqual(s, 400, body)
os.environ["CLOUD_GATEWAY"] = "sub2api"
orig_rotate = server_mod._rotate_sub2api_key
try:
server_mod._rotate_sub2api_key = (
lambda conn, uid: ("rotated", "fake-key", 7, 9))
s, body = _req(self.port, "POST", "/users/me/gateway/rotate",
{}, self.owner_key)
self.assertEqual(s, 200, body)
self.assertTrue(body["rotated"])
self.assertNotIn("api_key", body)
actions = {a for a, _d in self._audit_actions(self.owner_id)}
self.assertIn("gateway_key_rotated", actions)
finally:
server_mod._rotate_sub2api_key = orig_rotate
# -- 推送审计与分发 SSRF ----------------------------------------------
def test_push_audit_and_dispatch_ssrf(self):
s, sub = _req(self.port, "POST",
"/projects/%s/push/subscriptions" % self.project_id,
{"channel": "webhook",
"target": "https://example.com/hook",
"preference": "instant"}, self.owner_key)
self.assertEqual(s, 201, sub)
actions = {a for a, _d in self._audit_actions(self.owner_id)}
self.assertIn("push_subscription_created", actions)
s, _b = _req(self.port, "DELETE",
"/projects/%s/push/subscriptions/%s"
% (self.project_id, sub["id"]), key=self.owner_key)
self.assertEqual(s, 200, _b)
actions = {a for a, _d in self._audit_actions(self.owner_id)}
self.assertIn("push_subscription_deleted", actions)
# 内网目标:分发不 POST、sent 为空、错误含 delivery failed。
s, sub = _req(self.port, "POST",
"/projects/%s/push/subscriptions" % self.project_id,
{"channel": "webhook",
"target": "http://127.0.0.1:9/hook",
"preference": "instant"}, self.owner_key)
self.assertEqual(s, 201, sub)
s, body = _req(self.port, "POST", "/admin/push/dispatch",
{"event_type": "failed",
"project_id": self.project_id,
"payload": {"task_id": "t1"}}, self.owner_key)
self.assertEqual(s, 200, body)
self.assertEqual(body["sent"], [])
self.assertEqual(len(body["errors"]), 1)
self.assertIn("delivery failed", body["errors"][0]["reason"])
actions = {a for a, _d in self._audit_actions(self.owner_id)}
self.assertIn("admin_push_dispatch", actions)
# -- 配额双字段与控制台文案 --------------------------------------------
def test_quota_payload_has_unpriced_fields(self):
s, body = _req(self.port, "GET", "/users/me/quota",
key=self.owner_key)
self.assertEqual(s, 200, body)
self.assertIn("unpriced_tokens_used", body)
self.assertIn("unpriced_token_cap", body)
def test_console_p1_markers(self):
conn = http.client.HTTPConnection(
"127.0.0.1", self.port, timeout=10)
try:
conn.request("GET", "/console")
resp = conn.getresponse()
text = resp.read().decode("utf-8")
finally:
conn.close()
self.assertEqual(resp.status, 200)
for marker in (
"登记评审需求",
"记录允许",
"记录拒绝",
"当前版本只留回执,不拦截实时行为",
"模型回合结束、改动在 worktree 里尚未合并",
"控制面已标记 merged,不代表 git 主分支已合入",
"协调员与 goal/计划互斥",
"仅 failed/interrupted/merge-failed 可重试",
"仅 maintainer 可合并"):
self.assertIn(marker, text)
if __name__ == "__main__":
unittest.main()
+534
View File
@@ -0,0 +1,534 @@
"""P1 执行面针对性单测(只测本路改动,不跑全量三套)。
运行(在 ~/workspace/codex-app-server-p1security 下):
python3 -m unittest discover -s cloud/tests -p "test_p1_security_worker.py" -v
说明:只用注入的假 runner 与测试假值,不跑真实模型回合、不联网、
不登录任何服务;env 里的值只是占位用的假名,绝不是真实密钥。
本文件只归本路(执行面)所有,不碰 test_p1_security_server.py。
"""
import json
import os
import sqlite3
import sys
import tempfile
import threading
import time
import unittest
from pathlib import Path
from types import SimpleNamespace
from unittest import mock
try:
import cloud.sandbox as sandbox_mod
from cloud.cost import CostStore
from cloud.pr_exec import enqueue_pr_job
from cloud.sandbox import DockerExecutor, LocalExecutor
from cloud.server import init_db
from cloud.worker import Worker, _code_signature
except ImportError: # discover -s cloud/tests 时把项目根目录补进 sys.path
sys.path.insert(0, str(Path(__file__).resolve().parents[2]))
import cloud.sandbox as sandbox_mod
from cloud.cost import CostStore
from cloud.pr_exec import enqueue_pr_job
from cloud.sandbox import DockerExecutor, LocalExecutor
from cloud.server import init_db
from cloud.worker import Worker, _code_signature
FAKE_ENV = {"XCM_API_KEY": "test-placeholder"}
def _completed_runner(cmd):
if "dispatch" in cmd:
name = cmd[cmd.index("--name") + 1]
else:
name = cmd[cmd.index("--task") + 1]
return SimpleNamespace(
returncode=0,
stdout=json.dumps({"task": name, "status": "completed",
"text": "done"}),
stderr="")
def _usage_runner(usage, calls):
def fake(cmd):
calls.append(list(cmd))
return SimpleNamespace(
returncode=0,
stdout=json.dumps({"status": "completed", "text": "done",
"usage": usage}),
stderr="")
return fake
def _insert_task(db_path, task_id, created_by="tester", status="queued",
project_id="p1", **cols):
now = time.time()
conn = sqlite3.connect(db_path)
try:
conn.execute(
"INSERT INTO tasks(id, project_id, prompt, status,"
" idempotency_key, depends_on, priority, created_by,"
" created_at, updated_at, last_heartbeat)"
" VALUES(?, ?, ?, ?, NULL, '[]', 0, ?, ?, ?, NULL)",
(task_id, project_id, "do work", status, created_by, now, now),
)
for key, value in cols.items():
conn.execute(
"UPDATE tasks SET %s = ? WHERE id = ?" % key,
(value, task_id))
conn.commit()
finally:
conn.close()
def _insert_quota(db_path, user_id, micros):
conn = sqlite3.connect(db_path)
try:
conn.execute(
"CREATE TABLE IF NOT EXISTS user_quotas("
"user_id TEXT PRIMARY KEY, quota_usd_micros BIGINT,"
" updated_at DOUBLE PRECISION)")
conn.execute(
"INSERT INTO user_quotas(user_id, quota_usd_micros, updated_at)"
" VALUES(?, ?, ?)",
(user_id, micros, time.time()))
conn.commit()
finally:
conn.close()
def _insert_cost(db_path, task_id, model, input_tokens, output_tokens,
micros, source="actual"):
_insert_task(db_path, task_id, status="completed")
store = CostStore(db_path)
try:
store.record(task_id, "p1", model, input_tokens=input_tokens,
output_tokens=output_tokens, usd_micros=micros,
source=source)
finally:
try:
store._conn.close()
except Exception:
pass
def _status(db_path, task_id):
conn = sqlite3.connect(db_path)
try:
row = conn.execute(
"SELECT status FROM tasks WHERE id = ?", (task_id,)).fetchone()
finally:
conn.close()
return row[0] if row else None
def _error_text(db_path, task_id):
conn = sqlite3.connect(db_path)
try:
row = conn.execute(
"SELECT error_text FROM tasks WHERE id = ?",
(task_id,)).fetchone()
finally:
conn.close()
return row[0] if row else None
class P1WorkerTestCase(unittest.TestCase):
def setUp(self):
self.tmp = tempfile.TemporaryDirectory()
self.db_path = os.path.join(self.tmp.name, "control.db")
init_db(self.db_path)
self.calls = []
def tearDown(self):
self.tmp.cleanup()
def _worker(self, runner=None, **kwargs):
kwargs.setdefault("env", FAKE_ENV)
return Worker(self.db_path, "/fake/host.py",
runner=runner or _completed_runner, **kwargs)
# 1. 无配额行按默认 $10 额度(不再是不限额)
def test_missing_quota_row_uses_default_quota(self):
_insert_cost(self.db_path, "donor-a", "gpt-6.1-sol",
1, 0, 11000000)
_insert_cost(self.db_path, "donor-b", "gpt-6.1-sol",
1, 0, 9000000)
worker = self._worker()
over = worker._quota_exceeded(
{"created_by": "tester", "project_id": "p1"}, None)
# donor-a 归 tester:已花 $11 > 默认 $10,挡住。
self.assertIsNotNone(over)
self.assertIn("quota_exceeded", over)
# donor-b 同样归 tester 会把总额推得更高,单独测另一用户:
_insert_task(self.db_path, "donor-c", created_by="rich",
status="completed")
store = CostStore(self.db_path)
try:
store.record("donor-c", "p1", "gpt-6.1-sol", input_tokens=1,
output_tokens=0, usd_micros=9000000,
source="actual")
finally:
try:
store._conn.close()
except Exception:
pass
under = worker._quota_exceeded(
{"created_by": "rich", "project_id": "p1"}, None)
self.assertIsNone(under)
# 2. 显式 NULL 配额行仍是不限额
def test_explicit_null_quota_row_stays_unlimited(self):
_insert_quota(self.db_path, "tester", None)
_insert_cost(self.db_path, "donor", "gpt-6.1-sol",
1, 0, 999000000)
worker = self._worker()
self.assertIsNone(worker._quota_exceeded(
{"created_by": "tester", "project_id": "p1"}, None))
# 3. 属主为空拒绝新回合(不再返回 None 放行)
def test_ownerless_task_refused(self):
worker = self._worker()
for owner in (None, ""):
err = worker._quota_exceeded(
{"created_by": owner, "project_id": "p1"}, None)
self.assertIsNotNone(err)
self.assertIn("quota_exceeded", err)
# 4. 协调员子任务属主跟父任务走
def test_child_inherits_parent_owner(self):
_insert_quota(self.db_path, "tester", 100)
_insert_cost(self.db_path, "donor", "gpt-6.1-sol",
1, 0, 200)
_insert_task(self.db_path, "parent", created_by="tester")
_insert_task(self.db_path, "parent2", created_by="fresh")
worker = self._worker()
blocked = worker._quota_exceeded(
{"created_by": None, "parent_task_id": "parent",
"project_id": "p1"}, None)
self.assertIsNotNone(blocked)
self.assertIn("quota_exceeded", blocked)
# fresh 用户无配额行、未花钱:默认额度内放行。
self.assertIsNone(worker._quota_exceeded(
{"created_by": None, "parent_task_id": "parent2",
"project_id": "p1"}, None))
# 5. 查库失败 fail-closed(新回合拒绝,不自动放行)
def test_quota_db_failure_is_fail_closed(self):
worker = self._worker()
with mock.patch("cloud.worker._db_connect",
side_effect=RuntimeError("test db down")):
err = worker._quota_exceeded(
{"created_by": "tester", "project_id": "p1"}, None)
self.assertIsNotNone(err)
self.assertTrue(err.startswith("quota_check_failed:"))
# 6. Docker 配额看门狗:命中杀容器、未命中不杀
def test_docker_quota_watchdog(self):
killed = []
def fake(cmd, timeout=None):
killed.append(list(cmd))
return SimpleNamespace(returncode=0, stdout="", stderr="")
executor = DockerExecutor(runner=fake)
worker = Worker(self.db_path, "/fake/host.py",
env=FAKE_ENV, executor=executor)
# 未命中:0.2 秒后停表,kill 不应被调。
stop = threading.Event()
threading.Timer(0.2, stop.set).start()
worker._docker_quota_watchdog(
"T1", "codex-test", stop, lambda: False, interval=0.05)
self.assertEqual(killed, [])
# 命中:立即杀容器并记命中标记。
stop2 = threading.Event()
worker._docker_quota_watchdog(
"T2", "codex-test", stop2, lambda: True, interval=0.05)
self.assertTrue(any("kill" in c for c in killed))
self.assertTrue(worker._take_quota_hit("T2"))
# 命中已消费,第二次取不到(不污染下一轮)。
self.assertFalse(worker._take_quota_hit("T2"))
# 7. usage_cost_usd 按实际成本累加,max_cost_usd 不再是死参数
def test_usage_cost_accumulates_and_budget_trips(self):
_insert_task(self.db_path, "T1")
worker = self._worker(
runner=_usage_runner(
{"input_tokens": 10000, "cached_input_tokens": 0,
"output_tokens": 1000, "total_tokens": 11000},
self.calls),
model="gpt-6.1-sol", max_cost_usd=0.01)
self.assertFalse(worker._budget_exceeded())
result = worker.run_one()
self.assertEqual(result["status"], "completed")
# 10000*$2 + 1000*$10 = $0.03,超过 $0.01 上限。
self.assertAlmostEqual(worker.usage_cost_usd, 0.03, places=6)
self.assertTrue(worker._budget_exceeded())
conn = sqlite3.connect(self.db_path)
try:
row = conn.execute(
"SELECT usd_micros, source FROM cost_records"
" WHERE task_id = 'T1'").fetchone()
finally:
conn.close()
self.assertEqual(row, (30000, "actual"))
# 8. 取消分支先记成本再返回(与配额中断收尾同款记账点)
def test_cancel_branch_records_cost(self):
worker = self._worker()
task = {"id": "T1", "project_id": "p1", "prompt": "do work"}
with worker._db_lock:
cost_id = worker._record_usage_cost(
task, "gpt-6.1-sol",
{"input_tokens": 10000, "cached_input_tokens": 0,
"output_tokens": 1000, "total_tokens": 11000},
"done", 0)
self.assertIsNotNone(cost_id)
self.assertAlmostEqual(worker.usage_cost_usd, 0.03, places=6)
# 无用量时回落估算,不累加 usage_cost_usd。
with worker._db_lock:
worker._record_usage_cost(
{"id": "T2", "project_id": "p1", "prompt": "do work"},
"gpt-6.1-sol", None, "done", 0)
self.assertAlmostEqual(worker.usage_cost_usd, 0.03, places=6)
# 9. 估算行不计入无价 token 上限
def test_estimated_rows_excluded_from_unpriced_cap(self):
_insert_cost(self.db_path, "donor-est", "gpt-5.6-terra",
100000, 0, None, source="estimated")
worker = self._worker()
old = os.environ.get("CLOUD_QUOTA_UNPRICED_TOKEN_CAP")
os.environ["CLOUD_QUOTA_UNPRICED_TOKEN_CAP"] = "100"
try:
# 10 万估算 token 在上限 100 下仍放行(估算不是真花费)。
self.assertIsNone(worker._quota_exceeded(
{"created_by": "tester", "project_id": "p1"}, None))
_insert_cost(self.db_path, "donor-act", "gpt-5.6-terra",
75, 75, None, source="actual")
err = worker._quota_exceeded(
{"created_by": "tester", "project_id": "p1"}, None)
self.assertIsNotNone(err)
self.assertIn("无价模型 token", err)
finally:
if old is None:
os.environ.pop("CLOUD_QUOTA_UNPRICED_TOKEN_CAP", None)
else:
os.environ["CLOUD_QUOTA_UNPRICED_TOKEN_CAP"] = old
# 10. cache_write 不计入无价上限(只 input+output 计)
def test_cache_write_excluded_from_unpriced_cap(self):
worker = self._worker()
old = os.environ.get("CLOUD_QUOTA_UNPRICED_TOKEN_CAP")
os.environ["CLOUD_QUOTA_UNPRICED_TOKEN_CAP"] = "100"
try:
usage = {"input_tokens": 10, "output_tokens": 5,
"cache_write_input_tokens": 9999999,
"total_tokens": 15}
self.assertIsNone(worker._quota_exceeded(
{"created_by": "tester", "project_id": "p1",
"model": "gpt-5.6-terra"}, usage))
usage2 = {"input_tokens": 200, "output_tokens": 0,
"cache_write_input_tokens": 0,
"total_tokens": 200}
err = worker._quota_exceeded(
{"created_by": "tester", "project_id": "p1",
"model": "gpt-5.6-terra"}, usage2)
self.assertIsNotNone(err)
finally:
if old is None:
os.environ.pop("CLOUD_QUOTA_UNPRICED_TOKEN_CAP", None)
else:
os.environ["CLOUD_QUOTA_UNPRICED_TOKEN_CAP"] = old
# 11. 协调员子任务继承父任务模型/强度,分工显式值优先
def test_coordinator_child_inherits_model_effort(self):
worker = self._worker()
task = {"id": "coord", "project_id": "p1", "prompt": "coord",
"priority": 0, "created_by": "tester", "model": "gpt-6.1-sol",
"effort": "high", "executor": None, "environment_id": None}
children = worker._coord_sync_children(task, [
{"name": "a", "prompt": "do a", "model": None,
"effort": None, "executor": None},
{"name": "b", "prompt": "do b", "model": "gpt-5.6-terra",
"effort": "low", "executor": None},
])
self.assertEqual(len(children), 2)
conn = sqlite3.connect(self.db_path)
try:
rows = {r[0]: (r[1], r[2]) for r in conn.execute(
"SELECT coordinator_part, model, effort FROM tasks"
" WHERE parent_task_id = 'coord'").fetchall()}
finally:
conn.close()
self.assertEqual(rows["a"], ("gpt-6.1-sol", "high"))
self.assertEqual(rows["b"], ("gpt-5.6-terra", "low"))
# 12. 断点续跑:Local/Docker/popen 三路都不传 --goal-text
def test_goal_resume_breakpoint_omits_goal_text(self):
calls = []
def fake(cmd):
calls.append(list(cmd))
return SimpleNamespace(
returncode=0,
stdout="goal met after rounds\n"
"usage input=10 cached=0 cache_write=0 output=5 "
"reasoning=0 total=15 completions=1\n",
stderr="")
task = {"id": "T1", "project_id": "p1", "prompt": "orig goal",
"verify_command": "true", "goal_max_rounds": 3}
local = LocalExecutor("/fake/host.py", runner=fake)
local.goal_resume_task(task, None)
self.assertIn("goal-resume", calls[-1])
self.assertNotIn("--goal-text", calls[-1])
docker_calls = []
def dfake(cmd, timeout=None):
docker_calls.append(list(cmd))
return SimpleNamespace(
returncode=0,
stdout="goal met after rounds\n"
"usage input=10 cached=0 cache_write=0 output=5 "
"reasoning=0 total=15 completions=1\n",
stderr="")
docker = DockerExecutor(runner=dfake, env=FAKE_ENV)
with mock.patch("cloud.sandbox.shutil.which",
return_value="/usr/bin/docker"):
docker.goal_resume_task(task, None)
self.assertIn("goal-resume", docker_calls[-1])
self.assertNotIn("--goal-text", docker_calls[-1])
with mock.patch.object(sandbox_mod.subprocess, "Popen") as popen:
popen.return_value = SimpleNamespace(pid=9999)
local.popen_task(
{"id": "T1", "prompt": "resume msg",
"_resume_kind": "breakpoint"})
# popen_task 默认 kind=dispatch;显式测 goal-resume:
local.popen_task(
{"id": "T1", "prompt": "resume msg",
"_resume_kind": "breakpoint"}, kind="goal-resume")
cmd = popen.call_args[0][0]
self.assertIn("goal-resume", cmd)
self.assertNotIn("--goal-text", cmd)
# 13. 开新目标:goal-resume 带 --goal-text(旧行为保留)
def test_goal_resume_new_goal_keeps_goal_text(self):
calls = []
def fake(cmd):
calls.append(list(cmd))
return SimpleNamespace(
returncode=0,
stdout="goal met after rounds\n"
"usage input=10 cached=0 cache_write=0 output=5 "
"reasoning=0 total=15 completions=1\n",
stderr="")
task = {"id": "T1", "project_id": "p1", "prompt": "new goal"}
local = LocalExecutor("/fake/host.py", runner=fake)
local.goal_resume_task(task, "new goal")
self.assertIn("--goal-text", calls[-1])
self.assertIn("new goal", calls[-1])
# 14. goal 轮次上限:sandbox loud fail + worker 认领后 failed
def test_goal_max_rounds_cap(self):
task = {"id": "T1", "project_id": "p1", "prompt": "g",
"verify_command": "true", "goal_max_rounds": 101}
local = LocalExecutor("/fake/host.py", runner=_completed_runner)
with self.assertRaises(ValueError):
local.goal_task(task)
_insert_task(self.db_path, "T1", verify_command="true",
goal_max_rounds=101)
worker = self._worker()
result = worker.run_one()
self.assertEqual(result["status"], "failed")
self.assertIn("上限", result["error"])
self.assertEqual(_status(self.db_path, "T1"), "failed")
self.assertEqual(self.calls, [])
# 15. PR 非 local:显式 needs_local_worker,任务与排队都不动
def test_pr_non_local_needs_local_worker(self):
conn = sqlite3.connect(self.db_path)
try:
job_id = enqueue_pr_job(conn, "T1", "p1", "test-host")
finally:
conn.close()
executor = DockerExecutor(runner=lambda cmd, timeout=None:
SimpleNamespace(returncode=0, stdout="",
stderr=""))
worker = Worker(self.db_path, "/fake/host.py", env=FAKE_ENV,
executor=executor, host_name="test-host")
results = worker._run_pr_jobs()
self.assertEqual(len(results), 1)
self.assertEqual(results[0]["status"], "needs_local_worker")
self.assertEqual(results[0]["pr_job_id"], job_id)
conn = sqlite3.connect(self.db_path)
try:
row = conn.execute(
"SELECT status FROM pr_jobs WHERE id = ?",
(job_id,)).fetchone()
finally:
conn.close()
self.assertEqual(row[0], "queued")
# 16. 多文件签名 + 签名缺失/已变时认领 fail-closed
def test_code_signature_multi_file_and_claim_refusal(self):
sig = _code_signature("/fake/host.py")
self.assertIsNotNone(sig)
self.assertEqual(len(sig), 6)
for entry in sig:
self.assertEqual(len(entry), 2)
_insert_task(self.db_path, "T1")
worker = self._worker()
worker._code_signature = ("stale-signature",)
result = worker.run_one()
self.assertFalse(result["dispatched"])
self.assertEqual(result["reason"], "already_claimed")
self.assertEqual(_status(self.db_path, "T1"), "queued")
worker._code_signature = None
result = worker.run_one()
self.assertFalse(result["dispatched"])
self.assertEqual(_status(self.db_path, "T1"), "queued")
# 17. goal 首轮撞已存在 → 断点续跑一次,不直接 failed
def test_goal_already_exists_falls_back_to_breakpoint(self):
calls = []
def fake(cmd):
calls.append(list(cmd))
if "goal-resume" in cmd:
return SimpleNamespace(
returncode=0,
stdout="round 1: turn=completed verify_rc=0 "
"verify_out=\ngoal met after rounds\n"
"usage input=10 cached=0 cache_write=0 output=5 "
"reasoning=0 total=15 completions=1\n",
stderr="")
if "goal" in cmd:
return SimpleNamespace(
returncode=0,
stdout="host 任务已存在:T1 已存在",
stderr="")
return _completed_runner(cmd)
_insert_task(self.db_path, "T1", verify_command="true",
goal_max_rounds=2)
worker = self._worker(runner=fake)
result = worker.run_one()
self.assertEqual(result["status"], "completed")
# 注意:cmd 是元素列表,"goal" in cmd 只匹配恰好等于
# "goal" 的元素,goal-resume 命令里并没有 "goal" 元素,
# 所以按“任一元素含 goal 子串”过滤。
goal_cmds = [c for c in calls
if any("goal" in e for e in c)]
self.assertTrue(any("goal-resume" in c for c in goal_cmds))
resume_cmd = [c for c in goal_cmds if "goal-resume" in c][0]
self.assertNotIn("--goal-text", resume_cmd)
self.assertEqual(_status(self.db_path, "T1"), "completed")
if __name__ == "__main__":
unittest.main()
+6 -1
View File
@@ -307,7 +307,12 @@ class WorkerPrJobsTestCase(unittest.TestCase):
enqueue_pr_job(conn, "t1", "p1", worker.host_name)
finally:
conn.close()
self.assertEqual(worker._run_pr_jobs(), [])
# P1:Docker worker 不再静默返回空,改为显式 needs_local_worker,
# 任务保持 queued 等 local worker 认领。
jobs = worker._run_pr_jobs()
self.assertEqual(len(jobs), 1)
self.assertEqual(jobs[0]["status"], "needs_local_worker")
self.assertEqual(jobs[0]["task_id"], "t1")
def test_autoscaler_forces_host_name(self):
scaler = Autoscaler(self.db_path, [])
+7
View File
@@ -41,6 +41,9 @@ class WebhookExecTestCase(unittest.TestCase):
self.owner_id, self.owner_key = self.auth.create_user("owner")
self._old_admins = os.environ.get("CODEX_GLOBAL_ADMINS")
os.environ["CODEX_GLOBAL_ADMINS"] = self.owner_id
# P1:备份 out_dir 限白名单根;本测试的 tmp 目录即白名单根。
self._old_backup_root = os.environ.get("CLOUD_BACKUP_ROOT")
os.environ["CLOUD_BACKUP_ROOT"] = self.tmp.name
self.viewer_id, self.viewer_key = self.auth.create_user("viewer")
self.server = create_server(self.db_path, "127.0.0.1", 0)
self.port = self.server.server_address[1]
@@ -67,6 +70,10 @@ class WebhookExecTestCase(unittest.TestCase):
os.environ.pop("CODEX_GLOBAL_ADMINS", None)
else:
os.environ["CODEX_GLOBAL_ADMINS"] = self._old_admins
if getattr(self, "_old_backup_root", None) is None:
os.environ.pop("CLOUD_BACKUP_ROOT", None)
else:
os.environ["CLOUD_BACKUP_ROOT"] = self._old_backup_root
def _req(self, method, path, body=None, key=None, headers=None,
raw_body=None):
+10
View File
@@ -336,6 +336,11 @@ class TestRealSenders(PushTestCase):
def setUp(self):
super().setUp()
# P1 SSRF 门默认挡 loopback;本地录制服务器测试显式放行,
# 生产默认值仍关闭(不要把本开关当成生产默认)。
self._old_push_allow = os.environ.get(
"CODEX_PUSH_ALLOW_PRIVATE_TARGETS")
os.environ["CODEX_PUSH_ALLOW_PRIVATE_TARGETS"] = "1"
_RecordingHandler.requests = []
_RecordingHandler.status = 200
self.server = http.server.ThreadingHTTPServer(
@@ -349,6 +354,11 @@ class TestRealSenders(PushTestCase):
self.server.shutdown()
self.server.server_close()
self.thread.join(timeout=5)
if getattr(self, "_old_push_allow", None) is None:
os.environ.pop("CODEX_PUSH_ALLOW_PRIVATE_TARGETS", None)
else:
os.environ["CODEX_PUSH_ALLOW_PRIVATE_TARGETS"] = (
self._old_push_allow)
super().tearDown()
def _url(self, path="/hook"):
+2 -1
View File
@@ -331,7 +331,8 @@ class TestGoalExecutors(unittest.TestCase):
{"id": "t1", "prompt": "do it",
"verify_command": "true"})
self.assertEqual(result["status"], "failed")
self.assertIn("请新建任务", result["goal_error"])
# P1 新语义:goal 失败请用继续而非重试(旧“请新建任务”已废)。
self.assertIn("请用继续而非重试", result["goal_error"])
def test_local_goal_task_without_verify_is_llm_decided(self):
# 新语义:没有验收命令不再抛错——走模型自决分支:
+10
View File
@@ -114,6 +114,11 @@ class _Listener:
class GoalPrNotifyTestCase(unittest.TestCase):
def setUp(self):
# P1 SSRF 门默认挡 loopback;本地监听器测试显式放行,
# 生产默认值仍关闭。
self._old_push_allow = os.environ.get(
"CODEX_PUSH_ALLOW_PRIVATE_TARGETS")
os.environ["CODEX_PUSH_ALLOW_PRIVATE_TARGETS"] = "1"
self.tmp = tempfile.TemporaryDirectory()
self.db_path = os.path.join(self.tmp.name, "control.db")
self.auth = AuthStore(self.db_path)
@@ -133,6 +138,11 @@ class GoalPrNotifyTestCase(unittest.TestCase):
self.server.shutdown()
self.server.server_close()
self.thread.join(timeout=5)
if getattr(self, "_old_push_allow", None) is None:
os.environ.pop("CODEX_PUSH_ALLOW_PRIVATE_TARGETS", None)
else:
os.environ["CODEX_PUSH_ALLOW_PRIVATE_TARGETS"] = (
self._old_push_allow)
try:
self.auth.conn.close()
except Exception:
+10
View File
@@ -74,6 +74,11 @@ def _completed_runner(cmd):
class WorkerTestCase(unittest.TestCase):
def setUp(self):
# P1 SSRF 门默认挡 loopback;通知测试用本地监听器,显式放行,
# 生产默认值仍关闭。
self._old_push_allow = os.environ.get(
"CODEX_PUSH_ALLOW_PRIVATE_TARGETS")
os.environ["CODEX_PUSH_ALLOW_PRIVATE_TARGETS"] = "1"
self.tmp = tempfile.TemporaryDirectory()
self.db_path = os.path.join(self.tmp.name, "control.db")
init_db(self.db_path)
@@ -81,6 +86,11 @@ class WorkerTestCase(unittest.TestCase):
self.calls = []
def tearDown(self):
if getattr(self, "_old_push_allow", None) is None:
os.environ.pop("CODEX_PUSH_ALLOW_PRIVATE_TARGETS", None)
else:
os.environ["CODEX_PUSH_ALLOW_PRIVATE_TARGETS"] = (
self._old_push_allow)
self.tmp.cleanup()
def _worker(self, runner=None, **kwargs):
+604 -55
View File
@@ -139,6 +139,7 @@ from cloud.coordinator import DEFAULT_POLL_INTERVAL as _COORD_POLL_INTERVAL
from cloud.coordinator import DEFAULT_TIMEOUT_SECONDS as _COORD_TIMEOUT
from cloud.coordinator import ensure_columns as _coord_ensure_columns
from cloud.coordinator import parse_parts as _coord_parse_parts
from cloud.coordinator import nonterminal_children as _coord_nonterminal
from cloud.cost import CostStore, estimate_tokens, price_usd_micros, \
sum_usage, unpriced_token_cap
from cloud.db import connect as _db_connect
@@ -170,19 +171,68 @@ DEFAULT_ENV_KEY = "XCM_API_KEY"
# gpt-6.1-sol 路由当日实测持续 503,terra 同日实测 1 秒完成;
# 任务未指定模型时 worker 与 host.py 都落到这里。
DEFAULT_MODEL = "gpt-5.6-terra"
# P1 执行面:goal 轮次上限(有限轮次形态)。goal_mode=1 的 0 轮
# (模型宣告才结束)不在此限——它的上界是配额(配额看门狗与
# 认领/边界门),不是轮次;旧验收形态的 goal_max_rounds 必须
# 在 1..GOAL_MAX_ROUNDS_CAP 之间,超限的任务 worker 在认领后
# 直接 failed(建任务时的 400 校验在 server.py,属另一路文件,
# 本路只做执行侧防线,见 run_one 的 goal 上限门)。
GOAL_MAX_ROUNDS_CAP = 100
# P1 执行面:无配额行时的默认配额(微美元),与 server.py
# _default_quota_usd_micros() 同值($10/月,环境变量
# CLOUD_DEFAULT_QUOTA_USD_MICROS 可覆盖)。P0 前“无行 =
# 不限额”已废:worker 侧查不到行时按此默认额度算,显式
# NULL 行才是不限额(与服务端 _quota_status 同口径)。
# host.py dispatch 的原话是 "task 已存在: <name>,先换名或 discard 它"
#(英文 task + 已存在),取共有子串做判断,不依赖标点。
_ALREADY_EXISTS_MARKER = "已存在"
def _code_signature():
"""本文件签名(mtime_ns, size):部署 staging 换目录后必变;
取不到时返回 None(此时自退检查跳过,绝不误杀)。"""
def _default_quota_usd_micros():
"""无配额行时的默认配额(微美元),与 server 侧同值。
server.py 的 _default_quota_usd_micros() 在另一路文件里,
worker 不跨模块 import 它(避免循环 import),这里复刻
同款解析:环境变量 CLOUD_DEFAULT_QUOTA_USD_MICROS,非法
值回落 10_000_000。"""
try:
st = os.stat(os.path.abspath(__file__))
return (st.st_mtime_ns, st.st_size)
return max(0, int(os.environ.get(
"CLOUD_DEFAULT_QUOTA_USD_MICROS", "10000000")))
except (TypeError, ValueError):
return 10000000
def _code_signature(host_py_path=None):
"""多文件签名:worker/sandbox/coordinator/cost/server/host
各取 (mtime_ns, size) 组成的元组;任一文件取不到即 None。
P1 执行面扩面:旧版只签 worker.py,部署只换了 sandbox 或
host.py 时旧 worker 会继续用旧代码抢任务。host.py 取
host_py_path(真实文件时用它,否则用 cloud/ 上一级的
host.py);单测常传 /fake/host.py 这种不存在的路径,此时
回落到仓库 host.py,不因此判签名缺失。取不到时返回 None,
调用方必须 fail-closed(不认领、不执行),绝不按“没变”
放行——旧版“取不到当没变”是 fail-open,已废。"""
here = os.path.dirname(os.path.abspath(__file__))
candidates = [
os.path.join(here, "worker.py"),
os.path.join(here, "sandbox.py"),
os.path.join(here, "coordinator.py"),
os.path.join(here, "cost.py"),
os.path.join(here, "server.py"),
]
if host_py_path and os.path.isfile(host_py_path):
candidates.append(os.path.abspath(host_py_path))
else:
candidates.append(os.path.join(os.path.dirname(here), "host.py"))
sig = []
try:
for path in candidates:
st = os.stat(path)
sig.append((st.st_mtime_ns, st.st_size))
except OSError:
return None
return tuple(sig)
def _missing_key_error(env_key):
@@ -447,16 +497,46 @@ class Worker:
# 注入的普通 dict 同样可读环境变量(无该键时默认 1)。
self.capacity = resolve_capacity(capacity, env=self._env)
# 常驻启动时的代码签名:run_loop 每轮边界比对,变了就自退
# (见 run_loop 的代码变化自退检查)。
self._code_signature = _code_signature()
# (见 run_loop 的代码变化自退检查);_claim 每次认领前
# 也比对(见 _code_signature_valid),不要只在轮次边界查。
self._code_signature = _code_signature(host_py_path)
# 配额看门狗命中过的任务(Docker 同步路径用):看门狗杀
# 容器后,同步返回的 failed 要被 _run_attempt 改写成
# quota_exceeded,否则 run_one 会把它当普通失败重试,
# 配额中断就续跑下去了。受 _quota_hit_lock 保护。
self._quota_hit_tasks = set()
self._quota_hit_lock = threading.Lock()
def _code_changed(self):
"""启动后 worker.py 是否被换过(部署换目录/手动改代码):
签名取不到时一律当没变,不误杀。"""
now = _code_signature()
"""启动后关键代码是否被换过(部署换目录/手动改代码):
签名取不到时返回 False(是否“变了”无从判断),但这
不等于放行——放行与否看 _code_signature_valid。"""
now = _code_signature(self.host_py_path)
return (now is not None and self._code_signature is not None
and now != self._code_signature)
def _code_signature_valid(self):
"""签名是否可信且未变:启动签名与现签名都必须取得到
且相等,否则 False。False 时 _claim 拒绝认领、run_loop
自退(75)、_run_pr_jobs 不碰任务——fail-closed,绝不
让签名缺失的 worker 抢任务(旧版取不到当没变是 fail-open)。"""
now = _code_signature(self.host_py_path)
return (now is not None and self._code_signature is not None
and now == self._code_signature)
def _mark_quota_hit(self, task_id):
with self._quota_hit_lock:
self._quota_hit_tasks.add(task_id)
def _take_quota_hit(self, task_id):
"""取走一次配额看门狗命中(命中即消费,避免旧命中污染
同一任务的下一轮):返回 True 表示本轮确被看门狗杀过。"""
with self._quota_hit_lock:
if task_id in self._quota_hit_tasks:
self._quota_hit_tasks.discard(task_id)
return True
return False
def register(self):
"""向控制面注册本 worker(写 workers 表,失败只记日志)。
@@ -573,7 +653,17 @@ class Worker:
里清(本地 _execute_via_process / Docker 同步路径),
认领后提前返回的三道门(缺 key / 仓库 / 配额)在
run_one 里显式清占位;CAS 语义本身一句未改。
P1 签名门:占位之前先比对多文件签名,签名缺失或已变
时直接返回 False(fail-closed,不抢任务、不占槽),
并记日志;不要只在 run_loop 轮次边界查——并行池线程
直接调 run_one/_claim,轮次边界查不到它们。
"""
if not self._code_signature_valid():
log_event(self.logger, "worker_claim_refused_code_signature",
task_id=task_id,
message="code signature missing or changed,"
" refusing claim")
return False
with self._running_lock:
already_running = task_id in self._running
if not already_running and \
@@ -865,14 +955,51 @@ class Worker:
_check_unpriced_token_cap 同口径):USD 配额不计量无价
模型,默认模型 terra 无价,若无此上限则配额对默认模型
永远不生效;有配额行的用户当月无价 token(input +
output,已记账 + 本轮)达到上限即停。无配额行 = 不限额。
查询本身异常时按不限额处理(fail-open 并记一条警告):
配额是第二道网,绝不能因查询抖动把全部任务挡在门外;
服务端建任务/分支/重试时的准入仍是第一道。
output,已记账 + 本轮)达到上限即停。无价上限只计
source='actual' 的行:估算行(estimated)不是真实花费,
计入上限会把“没花钱”算成“花超了”,且与 cost.py
_report 的 unpriced_actual_tokens 口径一致。
P1 执行面三处收口(旧版三处都是 fail-open,已废):
- 属主:任务 created_by 为空时先查父任务(协调员子任务
行可能无 created_by,属主跟父任务走);仍查不到时
拒绝新回合,不再返回 None 放行。created_by 字面为
"system" 的系统任务按普通属主算(无行时同样走默认
额度,不另设无限额的系统预算)。
- 无配额行:不再是不限额,按默认配额
_default_quota_usd_micros()($10/月)算;只有显式
NULL 行才是不限额(与 P0 的服务端口径一致,管理员
显式设 NULL 的行仍被尊重)。
- 查库失败:返回 "quota_check_failed: ..."(fail-closed),
绝不返回 None 放行。调用方分流:新回合边界(认领门、
重试/fallback/后续消息边界)见此串即拒绝新回合并记
worker_quota_stop 审计;执行中轮询(popen 轮询、
Docker 配额看门狗)见此串只记警告、不中途杀当前轮
——查询抖动不应杀掉已在跑的回合,但下一轮边界仍会
拒绝(见 run_one 的 _quota_hit 过滤)。查库失败的
审计由边界调用方记,这里只记日志,避免轮询高频写审计。
"""
owner = (task or {}).get("created_by")
if not owner and (task or {}).get("parent_task_id"):
# 协调员子任务属主跟父任务走(与 _gateway_context
# 同款回查);查不到时不猜,直接走下面的拒绝分支。
try:
conn = _db_connect(self.db_path)
try:
row = conn.execute(
"SELECT created_by FROM tasks WHERE id = ?",
(task["parent_task_id"],)).fetchone()
finally:
conn.close()
if row is not None:
try:
owner = row[0]
except (TypeError, IndexError):
owner = None
except Exception:
owner = None
if not owner:
return None
return ("quota_exceeded: 任务缺少属主,无法查配额,"
"拒绝新回合(请归属真实用户后再排队)")
try:
month = datetime.now(timezone.utc).strftime("%Y-%m")
conn = _db_connect(self.db_path)
@@ -887,19 +1014,33 @@ class Worker:
" WHERE user_id = ?",
(owner,)).fetchone()
if row is None:
return None
quota = row[0]
# P1:无行按默认额度,不是不限额(P0 服务端
# 已在建号/登录时补行,worker 侧仍可能遇到
# 老库未补行的用户,这里按默认额度兜底,
# 不写行——写行是服务端的职责,worker 不抢)。
quota = _default_quota_usd_micros()
else:
quota = row[0]
if quota is None:
return None
# cost_records 由 CostStore 懒建:全新库上它还不存
# 在,直接 JOIN 会报错——先实例化一次建表再查(与
# 服务端 _quota_status 同款做法),否则 quota=0
#(一分钱也不能花)在全新库上会被误放行。
# P1 补:建表必须落在本连接上,不只 CostStore 的
# 连接上——文件库两者指向同一文件所以旧代码碰巧
# 能用,:memory: 库两者是不同空库,本连接查 JOIN
# 会直接报错并(P1 fail-closed 后)误拒全部任务。
_cost_store = CostStore(self.db_path)
try:
_cost_store._conn.close()
except Exception:
pass
try:
conn.execute(CostStore._COST_DDL)
conn.commit()
except Exception:
pass
used_row = conn.execute(
"SELECT COALESCE(SUM(c.usd_micros), 0)"
" FROM cost_records c JOIN tasks t"
@@ -912,7 +1053,7 @@ class Worker:
" FROM cost_records c JOIN tasks t"
" ON c.task_id = t.id"
" WHERE t.created_by = ? AND c.day LIKE ?"
" AND c.usd_micros IS NULL",
" AND c.usd_micros IS NULL AND c.source = 'actual'",
(owner, month + "%")).fetchone()
unpriced_used = unpriced_row[0] if unpriced_row else 0
finally:
@@ -940,11 +1081,15 @@ class Worker:
% (unpriced_used, cap, month))
return None
except Exception as exc:
# P1 fail-closed:查库失败不再返回 None。调用方按
# 前缀分流(见本函数 docstring):quota_check_failed
# 在新回合边界 = 拒绝,在执行中轮询 = 只警告不杀轮。
try:
self.logger.warning("quota check failed: %s", exc)
except Exception:
pass
return None
return ("quota_check_failed: 配额查询失败,拒绝新回合"
"(查库异常,非配额用完)")
def _ready_tasks(self):
"""当前就绪任务列表(pick_next 的批量版)。
@@ -1123,6 +1268,41 @@ class Worker:
pass
return
def _docker_quota_watchdog(self, task_id, container_name,
stop_event, quota_check, interval=30.0):
"""Docker 同步路径的配额看门狗(P1,复用取消看门狗模式)。
同步路径阻塞在 docker run 里,run_one 的边界检查要等
本轮返回才跑得到——配额在别处被花完时,本轮会继续烧
到自然结束。本线程每 interval 秒(默认 30 秒,不超过
30 秒)用 quota_check 查一次已记账配额:命中时按容器
名真杀容器并记 _quota_hit_tasks,_run_attempt 在同步
返回后把结果改写成 quota_exceeded,run_one 走统一的
配额中断收尾(interrupted + 配额原因 + 已花成本照记,
下一轮不再续跑)。查库失败时 quota_check 返回 False
(run_one 的 _quota_hit 会把 quota_check_failed 前缀
过滤掉,见 _quota_exceeded docstring),本线程只继续
轮询、不杀当前轮;下一轮边界仍会拒绝。诚实边界:看
门狗只能看到已记账用量 + 往轮累计用量,本轮容器内尚
未回传的 token 它看不到——那部分在结果返回后由边界
门拦住,不会悄悄续跑下一轮。
"""
while not stop_event.wait(interval):
try:
over = bool(quota_check()) if quota_check else False
except Exception:
over = False
if not over:
continue
self._mark_quota_hit(task_id)
try:
kill = getattr(self.executor, "kill_container", None)
if kill is not None and container_name:
kill(container_name)
except Exception:
pass
return
def cleanup_stale_containers(self):
"""启动清理兜底:清掉上一轮残留的 codex- 容器。
@@ -2069,6 +2249,7 @@ class Worker:
# 按容器名真杀;非 Docker 路径不启动,避免空线程)。
_watchdog_stop = None
_watchdog_thread = None
_quota_watchdog_thread = None
if is_docker and hasattr(self.executor, "kill_container"):
try:
_container_name = self.executor.container_name(task)
@@ -2081,6 +2262,17 @@ class Worker:
args=(task_id, _container_name, _watchdog_stop),
daemon=True)
_watchdog_thread.start()
# P1 配额看门狗:只在同步 Docker 路径起(进程
# 模式的 _execute_via_process 已每 30 秒轮询
# quota_check,不重复起)。quota_check 为 None
# 时不起(无配额检查可做,不空转线程)。
if quota_check is not None and not self._use_popen():
_quota_watchdog_thread = threading.Thread(
target=self._docker_quota_watchdog,
args=(task_id, _container_name,
_watchdog_stop, quota_check),
daemon=True)
_quota_watchdog_thread.start()
try:
gateway_provider, gateway_env = self._ensure_gateway(task)
if self._use_popen():
@@ -2123,8 +2315,14 @@ class Worker:
prep_err)
if goal_mode:
if task.get("_resume"):
# P1 断点续跑传 None(不传 --goal-text,
# host 沿用原 goal+verify);开新目标
# 才传本轮 prompt 当新 goal_text。
_resume_text = None if task.get(
"_resume_kind") == "breakpoint" \
else task["prompt"]
result = self.executor.goal_resume_task(
task, task["prompt"],
task, _resume_text,
timeout=effective_docker_timeout,
extra_mounts=tenant_mounts,
network=tenant_network,
@@ -2165,8 +2363,11 @@ class Worker:
if task.get("_resume") and isinstance(
self.executor, LocalExecutor):
_resume_text = None if task.get(
"_resume_kind") == "breakpoint" \
else task["prompt"]
result = self.executor.goal_resume_task(
task, task["prompt"],
task, _resume_text,
extra_env=gateway_env,
model_provider=gateway_provider)
elif isinstance(self.executor, LocalExecutor):
@@ -2220,6 +2421,8 @@ class Worker:
_watchdog_stop.set()
if _watchdog_thread is not None:
_watchdog_thread.join(timeout=2)
if _quota_watchdog_thread is not None:
_quota_watchdog_thread.join(timeout=2)
# 同步路径执行结束:清掉取消用的容器标记(取消路径里
# 若已清过,这里再清一次无害)。进程模式的 proc 由
# _execute_via_process 自己的 finally 清,这里只清
@@ -2236,6 +2439,18 @@ class Worker:
if result is not None and result.get("status") == "failed" and \
"docker run timeout" in (result.get("stdout", "") or ""):
result = dict(result, status="timeout")
# P1 配额看门狗命中改写:同步路径被看门狗杀容器后,
# 执行器只返回普通 failed,run_one 无法区分“模型失败”
# 与“配额杀容器”——命中时改写成 quota_exceeded,run_one
# 走统一配额中断收尾(interrupted + 配额原因 + 成本照记,
# 不重试、不续跑下一轮)。改写前先取走命中标记,避免
# 旧命中污染同一任务的后续尝试;超时标记优先于配额
# 改写的情况不存在(看门狗杀容器时执行器不会同时报超时)。
if result is not None and self._take_quota_hit(task_id):
result = dict(result, status="quota_exceeded")
if error is None:
error = ("quota_exceeded: Docker 配额看门狗已杀容器"
"(配额中断,下一轮不再续跑)")
return result, error
def _attempt_error(self, result, error, token):
@@ -2384,7 +2599,12 @@ class Worker:
task.get("priority", 0) or 0, task["created_by"],
now, now,
part.get("executor") or task.get("executor"),
part.get("model"), part.get("effort"),
# P1:分工未指定模型/强度时继承父任务(旧版
# 直接写 part 的 None,子任务会落到 worker
# 默认模型/强度,与父任务的显式选择脱节,
# 既改成本也改行为,且排查时看不出继承关系)。
part.get("model") or task.get("model"),
part.get("effort") or task.get("effort"),
task.get("environment_id"), task["id"], name))
self._audit(
task["project_id"], "coordinator_child_created",
@@ -2422,6 +2642,109 @@ class Worker:
conn.close()
return rows
def _coord_mark_orphans(self, task, child_ids):
"""把仍未终态的子任务标 interrupted 并记审计(P1 孤儿收口)。
只对 CAS 改到行的子任务记审计(rowcount==1):已被别的
worker 终态化的子任务不动,避免覆盖它的结果。返回被标
记的子任务 id 列表。正在跑的子任务被标 interrupted 后,
跑它的 worker 会在轮询里看到库状态并杀进程/容器(popen
每 1 秒查库、Docker 取消看门狗每 1 秒查库),不会无声
继续烧钱;若那个 worker 也已死,子任务至少已是终态,
不会再被认领烧钱。
"""
marked = []
rows = self._coord_child_rows(child_ids)
pending = _coord_nonterminal([
{"id": cid, "status": (rows.get(cid) or {}).get("status")}
for cid in child_ids])
if not pending:
return marked
conn = _db_connect(self.db_path)
try:
for ch in pending:
cur = conn.execute(
"UPDATE tasks SET status = 'interrupted',"
" updated_at = ? WHERE id = ? AND status NOT IN"
" ('completed', 'failed', 'interrupted', 'merged',"
" 'merge-failed')",
(time.time(), ch["id"]))
if cur.rowcount == 1:
marked.append(ch["id"])
self._audit(
task["project_id"], "coordinator_child_orphaned",
"%s -> %s (parent %s)"
% (ch["id"], "interrupted", task["id"]))
conn.commit()
finally:
conn.close()
return marked
def sweep_orphaned_coordinator_children(self):
"""常驻循环每轮的孤儿扫尾(P1):父已终态或父行失踪、子
仍未终态时,把子标 interrupted 并记审计。
父仍在 running/queued 时绝不动子(协调员等齐期间子任务
未终态是正常状态)。父行失踪(被删)同样算孤儿:没有
父会再来合流,子任务继续跑只是烧钱。返回被标记的
(parent_id, child_id) 对列表;查库失败时返回 [] 且只记
日志,不炸循环。
"""
marked = []
try:
conn = _db_connect(self.db_path)
try:
rows = conn.execute(
"SELECT id, parent_task_id, project_id FROM tasks"
" WHERE parent_task_id IS NOT NULL AND status NOT IN"
" ('completed', 'failed', 'interrupted', 'merged',"
" 'merge-failed')").fetchall()
parents = {}
for row in rows:
try:
pid = row["parent_task_id"]
except (KeyError, IndexError, TypeError):
continue
if pid not in parents:
prow = conn.execute(
"SELECT status FROM tasks WHERE id = ?",
(pid,)).fetchone()
try:
parents[pid] = prow["status"] \
if prow is not None else None
except (KeyError, IndexError, TypeError):
parents[pid] = None
for row in rows:
try:
cid = row["id"]
pid = row["parent_task_id"]
proj = row["project_id"]
except (KeyError, IndexError, TypeError):
continue
pstatus = parents.get(pid)
if pstatus is not None and \
pstatus not in TERMINAL_STATUSES:
continue
cur = conn.execute(
"UPDATE tasks SET status = 'interrupted',"
" updated_at = ? WHERE id = ? AND status NOT IN"
" ('completed', 'failed', 'interrupted', 'merged',"
" 'merge-failed')",
(time.time(), cid))
if cur.rowcount == 1:
marked.append((pid, cid))
self._audit(
proj, "coordinator_child_orphaned",
"%s -> interrupted (parent %s, parent"
" status %s)" % (cid, pid, pstatus))
conn.commit()
finally:
conn.close()
except Exception as exc:
log_event(self.logger, "worker_orphan_sweep_failed",
message="orphan sweep failed: %s" % exc)
return marked
def _run_coordinator(self, task, started_at, poll_interval=None,
timeout=None):
"""协调员执行:建子任务 → 等齐 → 合流汇总写回。
@@ -2460,6 +2783,7 @@ class Worker:
timed_out = True
break
time.sleep(poll_interval)
orphaned = []
if cancelled or timed_out:
rows = self._coord_child_rows(child_ids)
for cid in child_ids:
@@ -2469,6 +2793,15 @@ class Worker:
self.cancel(cid)
except Exception:
pass
# P1:cancel() 对已终态/不存在的子任务返回 False,
# 跨 worker 在跑的子任务也可能因竞态仍未终态——
# cancel 之后仍未终态的一律直接标 interrupted 并记
# 审计(_coord_mark_orphans),绝不留孤儿无声烧钱;
# 标记结果写进汇总报告,调用方在详情里看得到。
try:
orphaned = self._coord_mark_orphans(task, child_ids)
except Exception:
orphaned = []
rows = self._coord_child_rows(child_ids)
merged = []
for ch in children:
@@ -2479,6 +2812,9 @@ class Worker:
"result_text": row.get("result_text"),
"error_text": row.get("error_text")})
report, overall = _coord_build_report(merged)
if orphaned:
report += ("\n\n孤儿子任务已标 interrupted:%s"
% "、".join(orphaned))
if cancelled:
final = "interrupted"
error = "cancelled"
@@ -2531,6 +2867,47 @@ class Worker:
"executor": self._executor_name(), "stdout": report,
"error": error, "events": [], "cost_record_id": None}
def _record_usage_cost(self, task, actual_model, usage_total,
stdout, fallback_count):
"""记一条任务成本并累加 worker 级用量(调用方须持 _db_lock)。
P1 成本修齐:旧版 max_cost_usd 参数存在但 usage_cost_usd
从未累加——_budget_exceeded 的成本门永远是 0,等于没设。
本函数是唯一记账点(终局块与取消分支共用):actual 行
按价目表算 usd_micros(无价模型为 None),有价行才累加
usage_cost_usd(无价行不折算成 0 美元,仍受无价上限管);
无用量时回落字符估算记 estimated,绝不冒充 actual。
cache_write_input_tokens 不进这里:它只在 usage dict 里
做观测(见 cost.py 的不计费声明),不加进 total、不用
于定价。返回 cost record id。
"""
task_id = task["id"]
if usage_total and (usage_total.get("total_tokens") or 0) > 0:
micros = price_usd_micros(
actual_model, usage_total.get("input_tokens", 0),
usage_total.get("cached_input_tokens", 0),
usage_total.get("output_tokens", 0))
cost_id = self.costs.record(
task_id, task["project_id"], actual_model,
input_tokens=usage_total.get("input_tokens", 0),
output_tokens=usage_total.get("output_tokens", 0),
cached_input_tokens=usage_total.get(
"cached_input_tokens", 0),
usd_micros=micros,
source="actual", fallback_count=fallback_count)
else:
micros = None
total = estimate_tokens(task.get("prompt") or "", stdout)
cost_id = self.costs.record(
task_id, task["project_id"], actual_model,
input_tokens=total, output_tokens=0, usd_micros=None,
source="estimated", fallback_count=fallback_count)
if micros is not None:
# max_cost_usd 不再是死参数:只有有价花费才累加,
# _budget_exceeded 据此在下一认领前挡住超支 worker。
self.usage_cost_usd += micros / 1000000.0
return cost_id
def run_one(self, task=None):
"""认领并执行一个任务,返回结果 dict(见模块 docstring)。
@@ -2552,11 +2929,32 @@ class Worker:
# 成它(只换内存里的副本,原 prompt 不动);_resume 标记
# 让 _run_attempt 在 goal 模式下走 goal-resume 而不是 goal
# (goal 会撞已存在,只有 goal-resume 能在旧线程上开新循环)。
# P1 goal-resume 分流(断点 vs 开新目标,三处一致的 worker 侧):
# - 旧验收形态(verify_command 非空):一律断点续跑——
# 绝不能把续跑消息当新 goal_text 传给 host.py,否则
# host 的 goal-resume --goal-text 分支会用 verify=None
# 开新循环,把原 verify_command 覆盖掉;断点调用不传
# --goal-text,host 沿用原 goal + 原 verify + 原轮次,
# 从 goal_rounds_done 的下一轮续跑。续跑消息本轮不
# 当 prompt 用(断点语义就是续原目标),跑完照常清掉。
# - goal_mode=1(无验收):开新目标——续跑消息就是新
# goal_text(服务端 /continue 带 goal_mode=1 时已把
# task.goal 写成该消息,这里与之对齐)。
# - 非 goal 任务:沿用旧行为(prompt 换成续跑消息,走
# 已存在续跑 run 路径)。
resume_message = task.get("resume_message")
if resume_message:
task = dict(task)
task["prompt"] = resume_message
task["_resume"] = True
if task.get("verify_command"):
task["_resume"] = True
task["_resume_kind"] = "breakpoint"
elif task.get("goal_mode"):
task["prompt"] = resume_message
task["_resume"] = True
task["_resume_kind"] = "new-goal"
else:
task["prompt"] = resume_message
task["_resume"] = True
self._audit(task["project_id"], "worker_claim",
"%s executor=%s" % (task_id, self._executor_name()))
# log_event 只接受 task_id/project_id/status/message 四个
@@ -2647,6 +3045,56 @@ class Worker:
"stdout": "", "error": quota_error, "events": [],
"cost_record_id": None}
# P1 goal 轮次上限门(执行侧防线):旧验收形态的
# goal_max_rounds 超过 GOAL_MAX_ROUNDS_CAP 时直接 failed,
# 不进 host.py goal 循环烧轮次;goal_mode=1 的 0(模型宣告
# 才结束)豁免轮次上限(上界是配额),但 0 配上旧验收命令
# 时同样 failed(服务端建任务时 0 本就 400,老行若有 0
# 验收形态属于脏数据,不按不限轮次跑)。建任务时的 400
# 校验在 server.py(另一路文件),这里是执行侧第二道门,
# 错误文本写明上限与“请新建合规任务”,不静默截断轮次
# (截断会改任务语义,调用方看不出被改过)。
_goal_rounds = task.get("goal_max_rounds")
if (task.get("verify_command") or task.get("goal_mode")) \
and _goal_rounds is not None:
_goal_cap_error = None
try:
_goal_rounds_int = int(_goal_rounds)
except (TypeError, ValueError):
_goal_rounds_int = None
if _goal_rounds_int is not None \
and _goal_rounds_int > GOAL_MAX_ROUNDS_CAP:
_goal_cap_error = (
"goal_max_rounds 超过上限 %d 轮(当前 %d),"
"请新建轮次合规的任务" % (
GOAL_MAX_ROUNDS_CAP, _goal_rounds_int))
elif _goal_rounds_int == 0 and task.get("verify_command") \
and not task.get("goal_mode"):
_goal_cap_error = (
"旧验收形态 goal_max_rounds=0 表示不限轮次,"
"已禁用(上限 %d 轮),请新建轮次合规的任务"
% GOAL_MAX_ROUNDS_CAP)
if _goal_cap_error is not None:
self._set_status(task_id, "failed")
self._write_execution_result(
task_id, error_text=_goal_cap_error,
actual_model=self.model,
started_at=started_at, finished_at=time.time(),
only_if_status="failed")
self._audit(task["project_id"], "worker_goal_cap_rejected",
"%s: %s" % (task_id, _goal_cap_error))
log_event(self.logger, "worker_goal_cap_rejected",
task_id=task_id, project_id=task["project_id"],
status="failed")
self._running_clear(task_id)
return {"dispatched": True, "task_id": task_id,
"host_name": self.executor._host_name(task)
if hasattr(self.executor, "_host_name") else task_id,
"status": "failed",
"executor": self._executor_name(),
"stdout": "", "error": _goal_cap_error, "events": [],
"cost_record_id": None}
# 协调员分支:不跑模型回合,改为按分工建并行子任务、等齐、
# 合流汇总。已过 key 门与配额门;仓库门已对协调员任务本身
# 跑过(项目仓库解析失败会在前面直接 failed,不会建一堆
@@ -2721,7 +3169,14 @@ class Worker:
def _quota_hit():
# 闭包读当前值:usage_total 与 task 在重试/fallback
# 时会被重绑,轮询掐断要按最新值判断。
return self._quota_exceeded(task, usage_total) is not None
# P1 分流:执行中轮询只认真正的 quota_exceeded,
# quota_check_failed(查库失败)不杀当前轮——查询
# 抖动不应杀掉已在跑的回合;但它在重试/fallback/
# 后续消息等新回合边界仍是拒绝(那些边界直接调
# _quota_exceeded,不走这个过滤闭包)。
err = self._quota_exceeded(task, usage_total)
return err is not None and \
not err.startswith("quota_check_failed:")
# 分支任务:先 fork 再跑首轮(见 _run_fork);首轮失败
# 不自动重试/fallback,失败后走重试入口重跑首轮(已存在
@@ -2763,6 +3218,39 @@ class Worker:
# 喂回,或新形态的模型自决多轮),worker 级重试/fallback
# 不再叠加——同任务重跑会撞 host 任务名已存在,第二
# 次尝试必失败且污染错误文本。
# P1 唯一例外:goal 首轮撞“已存在”时(重试入口
# 重排同一 goal 任务的典型死法),不再直接 failed,
# 而是走一次断点续跑(goal-resume 不传 --goal-text,
# 沿用原 goal+verify,从断点续跑);断点续跑的
# 结果即本轮结果(成功则完成,仍失败则以续跑的
# 错误为准,不再拼“已存在”)。已是续跑的任务
# (_resume)不再二次回落,避免断点循环。
# 分支任务(forked_from)不走这里:fork 首轮的
# 已存在语义是 dispatch 续跑,已在 _run_attempt
# 内处理,不叠加 goal 断点。
if not task.get("_resume") and not task.get("forked_from"):
_combined = ((result or {}).get("stdout", "") or "") \
+ ((result or {}).get("stderr", "") or "")
_goal_err = (result or {}).get("goal_error") or ""
if _ALREADY_EXISTS_MARKER in _combined or \
_ALREADY_EXISTS_MARKER in _goal_err or \
_ALREADY_EXISTS_MARKER in (error or ""):
self._audit(task["project_id"],
"worker_goal_resume_fallback",
"%s: goal 已存在,走断点续跑" % task_id)
_resume_task = dict(task)
_resume_task["_resume"] = True
_resume_task["_resume_kind"] = "breakpoint"
result, error = self._run_attempt(
_resume_task, quota_check=_quota_hit)
usage_total = sum_usage(
usage_total, (result or {}).get("usage"))
self._deliver_peer_messages(task, result)
if (result or {}).get("status") == "quota_exceeded":
quota_error = self._quota_exceeded(
task, usage_total) or \
"quota_exceeded: 本月配额已用完"
quota_stop = True
break
if result.get("status") == "completed":
break
@@ -2900,6 +3388,18 @@ class Worker:
turn_id=(result or {}).get("turn_id"),
root_turn_id=(result or {}).get("root_turn_id"),
only_if_status="interrupted")
# P1:取消分支先记一条成本再返回(与配额中断收尾
# 同款记账点)。旧版这里直接返回 cost_record_id=None,
# 被取消前已花的模型费用就从账上丢了;取消不等于没
# 花钱。记账失败不挡取消返回(costs.record 异常时
# cost_id 置 None,任务状态仍是 interrupted)。
try:
with self._db_lock:
_cancel_cost_id = self._record_usage_cost(
task, actual_model, usage_total,
(result or {}).get("stdout", ""), fallback_count)
except Exception:
_cancel_cost_id = None
log_event(self.logger, "worker_cancelled", task_id=task_id,
project_id=task["project_id"], status="interrupted")
return {"dispatched": True, "task_id": task_id,
@@ -2908,7 +3408,7 @@ class Worker:
"status": "interrupted", "executor": self._executor_name(),
"stdout": (result or {}).get("stdout", ""),
"error": "cancelled", "events": [],
"cost_record_id": None}
"cost_record_id": _cancel_cost_id}
stdout = result.get("stdout", "") or ""
host_status = result.get("host_status")
@@ -2979,32 +3479,12 @@ class Worker:
self._cleanup_task_seed(task)
with self._db_lock:
self.usage_turns += 1
# 成本:执行器带回上游真实用量时记 actual(input/output/
# cached 全是实测值,usd 按价目表算、无价模型记 None,
# 报表里进 unpriced_actual_tokens 而不是 0 美元);
# 没带回用量时回落字符估算记 estimated,绝不冒充 actual。
# fallback_count 复用同一记录;模型记最后真正跑的那个
# (fallback 成功时就是 fallback 模型)。cost 记录共用
# 一条连接,并发下必须持锁写,否则两个线程共用同一
# sqlite 连接会直接报错。
if usage_total and (usage_total.get("total_tokens") or 0) > 0:
cost_id = self.costs.record(
task_id, task["project_id"], actual_model,
input_tokens=usage_total.get("input_tokens", 0),
output_tokens=usage_total.get("output_tokens", 0),
cached_input_tokens=usage_total.get(
"cached_input_tokens", 0),
usd_micros=price_usd_micros(
actual_model, usage_total.get("input_tokens", 0),
usage_total.get("cached_input_tokens", 0),
usage_total.get("output_tokens", 0)),
source="actual", fallback_count=fallback_count)
else:
total = estimate_tokens(task.get("prompt") or "", stdout)
cost_id = self.costs.record(
task_id, task["project_id"], actual_model,
input_tokens=total, output_tokens=0, usd_micros=None,
source="estimated", fallback_count=fallback_count)
# 成本记账走 _record_usage_cost(终局与取消分支共用
# 的唯一记账点):actual/estimated 分流、usage_cost_usd
# 累加都在里面;cost 记录共用一条连接,并发下必须
# 持锁写,否则两个线程共用同一 sqlite 连接会直接报错。
cost_id = self._record_usage_cost(
task, actual_model, usage_total, stdout, fallback_count)
if final_written:
self._audit(task["project_id"],
"worker_quota_stop" if quota_stop else
@@ -3064,9 +3544,65 @@ class Worker:
发 review-requested/merge-failed,这里只写回任务终态。
跳过(PrSkipped,如任务已不是 merged)与落盘失败都记
failed + error,任务本身保持 merged、可重调 /pr 再排队。
P1 非 local 通道:旧版非 local worker 直接返回 [],
排队的 pr 任务看起来像“没人处理”且无任何痕迹——
静默未完成。现在改为显式 needs_local_worker:只读
peek 本主机 queued 的 pr 任务,逐条记审计 + 日志并
返回 needs_local_worker 条目,任务本身保持 merged、
pr 任务保持 queued(留给 local worker 认领),绝不
把它标 failed(failed 会让调用方以为 PR 已失败,且
任务仍可重调 /pr 的语义会被混淆)。为什么不直接把
pr_jobs 行写成 needs_local_worker 状态:pr_jobs 的认
领只认 queued(claim_pr_job 在 pr_exec.py,本路不改
该文件),写成新状态后 local worker 也认领不到,任务
会被永久搁置;真要落库状态,需 pr_exec.py 的认领同
步改成含 needs_local_worker 或 local 重排队——该改动
留给协调员(见汇报残留),本路不伪造一个“写了状态
却没人能接”的通道。签名无效时同样不碰任务(返回 [],
只记日志),fail-closed。
"""
if self._executor_name() != "local":
if not self._code_signature_valid():
log_event(self.logger, "worker_pr_refused_code_signature",
message="code signature missing or changed,"
" refusing pr jobs")
return []
if self._executor_name() != "local":
results = []
try:
from cloud.pr_exec import ensure_pr_jobs_table
conn = _db_connect(self.db_path)
try:
ensure_pr_jobs_table(conn)
rows = conn.execute(
"SELECT id, task_id, project_id FROM pr_jobs"
" WHERE host = ? AND status = 'queued'",
(self.host_name,)).fetchall()
finally:
conn.close()
for row in rows:
try:
job_id = row["id"]
task_id = row["task_id"]
project_id = row["project_id"]
except (KeyError, IndexError, TypeError):
continue
reason = ("PR 执行需要 local worker(host-worktrees"
" 只在主机上可见),本 worker 为 %s,"
"任务保持 queued 等 local worker 认领"
% self._executor_name())
self._audit(project_id, "pr_job_needs_local_worker",
"%s: %s" % (job_id, reason))
log_event(self.logger, "pr_job_needs_local_worker",
task_id=task_id, status="needs_local_worker",
message=reason)
results.append({"pr_job_id": job_id,
"task_id": task_id,
"status": "needs_local_worker",
"error": reason})
except Exception as exc:
log_event(self.logger, "pr_job_peek_failed",
message="pr job peek failed: %s" % exc)
return results
results = []
while True:
conn = _db_connect(self.db_path)
@@ -3312,8 +3848,12 @@ class Worker:
# 任务;退出码 75(EX_TEMPFAIL,失败类):unit 托管
# 的由 systemd on-failure 重拉新版,autoscaler 托管
# 的由 reconcile 重拉,孤儿直接消失。单次 run_all
# 不走这里(进程本来就退出)。
if self._code_changed():
# 不走这里(进程本来就退出)。P1 fail-closed:
# 签名缺失同样自退(旧版只在“变了”时退,取不到
# 签名时继续跑是 fail-open);每次认领前的签名门
# 在 _claim 里(池线程不走轮次边界也拦得住)。
if self._code_changed() or \
not self._code_signature_valid():
log_event(self.logger, "worker_code_changed_exit",
message="worker code changed, exiting "
"for restart")
@@ -3336,6 +3876,15 @@ class Worker:
except Exception as exc:
log_event(self.logger, "worker_sweep_failed",
message="worker sweep failed: %s" % exc)
# P1 孤儿扫尾(接在陈旧回收之后):协调员本身
# 被回收标 interrupted 后,它的未终态子任务
# 在这里被统一标 interrupted;协调员仍在跑时
# 本函数不动子任务,见 sweep 方法 docstring。
try:
self.sweep_orphaned_coordinator_children()
except Exception as exc:
log_event(self.logger, "worker_orphan_sweep_failed",
message="orphan sweep failed: %s" % exc)
# 每轮结束写心跳:哪怕本轮一个任务都没派,worker 活着
# 这件事本身也要被 server 看到(文件心跳 + 注册表心跳)。
self._write_heartbeat()