P1 security batch: quota execution checks, cost accounting, goal/coordinator semantics, PR channel, orphan signature, control-plane and console hardening
- Docker quota watchdog + fail-closed quota checks, system tasks need real owner - max_cost_usd actually accumulated, cancel records cost, estimated rows excluded from unpriced cap, cache_write declared non-billed - goal-resume keeps original verify on breakpoint resume, goal retry falls back to resume, goal_max_rounds capped at 100 at create and execution - coordinator children inherit parent model/effort, orphans marked interrupted with audit - PR jobs on docker workers return explicit needs_local_worker instead of silent empty - worker code signature covers 6 files and is checked before claim, fail-closed - sub2api lazy provisioning with rotation, backup whitelist + 0600, push SSRF block, repos/workers desensitized, key/project audits and limits, console buttons/quota/status wording, coordinator/automation quota reserve
This commit is contained in:
1 parent
d254240d84
commit
1402dc67fd
20 files changed
+2616
-179
No files matched your search
+79
-12
@@ -160,7 +160,7 @@
|
||||
</div>
|
||||
<textarea id="prompt" placeholder="任务 prompt(必填)"></textarea>
|
||||
<textarea id="coordParts" placeholder="协调员子任务 JSON(可选),留空为普通任务"></textarea>
|
||||
<div class="muted">协调员格式:[{"name":"后端","prompt":"做后端"},{"name":"前端","prompt":"做前端"}];提交后 worker 并行建子任务、等齐、合流汇总。不能与 goal/计划模式同用。</div>
|
||||
<div class="muted">协调员格式:[{"name":"后端","prompt":"做后端"},{"name":"前端","prompt":"做前端"}];提交后 worker 并行建子任务、等齐、合流汇总。不能与 goal/计划模式同用。协调员与 goal/计划互斥:勾了 goal 或计划模式时不要填协调员 JSON,提交前会校验并拦截。子任务费用记父任务创建人账;父任务创建时按分工数预留配额,余额不足时整个父任务被挡下。</div>
|
||||
<div class="bar">
|
||||
<select id="modelSelect">
|
||||
<option value="">模型:默认(跟随配置)</option>
|
||||
@@ -265,10 +265,11 @@
|
||||
<input id="reviewSha" placeholder="指定提交用:提交 sha" style="width:100%">
|
||||
<textarea id="reviewInstructions" placeholder="自定义评审指令用:评审指令正文"></textarea>
|
||||
<div class="bar">
|
||||
<button id="btnCreateReview" class="primary" onclick="postReview()">发起评审</button>
|
||||
<button id="btnCreateReview" class="primary" onclick="postReview()">登记评审需求</button>
|
||||
<button onclick="if (currentTask) loadReviews(currentTask.id)">刷新队列</button>
|
||||
<span id="reviewMsg" class="muted"></span>
|
||||
</div>
|
||||
<div class="muted">本基线只登记评审需求(requested),不自动跑评审;评审是否执行、结果如何,以评审队列里的记录为准。</div>
|
||||
</div>
|
||||
<div id="reviewList" class="muted">点任务行查看详情后,点“刷新队列”查看评审记录</div>
|
||||
<h3>实时链路</h3>
|
||||
@@ -286,7 +287,7 @@
|
||||
<div id="followupList" class="muted"></div>
|
||||
<h3>运行时审批队列</h3>
|
||||
<div class="bar"><button id="btnRefreshApprovals" onclick="if(currentTask)loadApprovals(currentTask.id)">刷新审批队列</button><span id="approvalMsg" class="muted"></span></div>
|
||||
<div class="muted">待审批项需人工点允许或拒绝;超时默认拒绝。本队列的决定会留回执;实时请求的事前拦截待 host 接入后生效,在此之前实时行为不变。</div>
|
||||
<div class="muted">待审批项需人工点记录允许或记录拒绝;超时默认拒绝。当前版本只留回执,不拦截实时行为(实时请求的事前拦截待 host 接入后生效,在此之前实时行为不变)。</div>
|
||||
<div id="approvalList" class="muted"></div>
|
||||
<h3>给其他任务发消息</h3>
|
||||
<div id="peerMessageForm">
|
||||
@@ -354,7 +355,12 @@ async function api(path, method, body) {
|
||||
var msg = (data && data.error) ? data.error : ("HTTP " + r.status);
|
||||
if (r.status === 401) msg += "(key 未填或无效)";
|
||||
if (r.status === 429) msg += "(触发限流,稍后再试)";
|
||||
if (r.status === 402) msg += "(本月配额已用完,联系管理员调整)";
|
||||
if (r.status === 402) {
|
||||
// 402 要区分两种满:服务端在响应里带哪个上限字段就是哪个满。
|
||||
if (data && data.unpriced_token_cap !== undefined) msg += "(无价 token 已满,联系管理员调整)";
|
||||
else if (data && data.error === "coordinator_quota_reserve_exceeded") msg = "协调员分工预留不足:剩余美元配额不够覆盖全部分工,子任务费用记父任务创建人账,整个父任务已被挡下(未建任何子任务)";
|
||||
else msg += "(美元配额已满,联系管理员调整)";
|
||||
}
|
||||
throw new Error(msg);
|
||||
}
|
||||
return data;
|
||||
@@ -394,13 +400,22 @@ async function loadQuota() {
|
||||
try {
|
||||
var d = await api("/users/me/quota", "GET");
|
||||
var used = (d.used_usd_micros / 1000000).toFixed(2);
|
||||
var usdLine;
|
||||
if (d.quota_usd_micros === null || d.quota_usd_micros === undefined) {
|
||||
el.textContent = "本月用量 $" + used + "(不限额)";
|
||||
usdLine = "美元:本月用量 $" + used + "(不限额)";
|
||||
} else {
|
||||
var q = (d.quota_usd_micros / 1000000).toFixed(2);
|
||||
var rem = (d.remaining_usd_micros / 1000000).toFixed(2);
|
||||
el.textContent = "本月用量 $" + used + " / $" + q + "(剩余 $" + rem + ")";
|
||||
usdLine = "美元:本月用量 $" + used + " / $" + q + "(剩余 $" + rem + ")";
|
||||
}
|
||||
// 两条进度并列:美元一条,无价 token 一条;402 时按哪条满区分报错。
|
||||
var tokLine = "无价 token:";
|
||||
if (d.unpriced_token_cap === undefined || d.unpriced_tokens_used === undefined) {
|
||||
tokLine += "未返回(旧服务端)";
|
||||
} else {
|
||||
tokLine += d.unpriced_tokens_used + " / " + d.unpriced_token_cap + " tokens";
|
||||
}
|
||||
el.textContent = usdLine + " | " + tokLine;
|
||||
} catch (e) { el.textContent = "配额加载失败:" + e.message; }
|
||||
}
|
||||
|
||||
@@ -453,8 +468,8 @@ async function loadApprovals(taskId) {
|
||||
}
|
||||
var btns = "";
|
||||
if (a.status === "pending") {
|
||||
btns = " <button onclick=\"decideApproval('" + esc(a.id) + "', 'approved')\">允许</button>" +
|
||||
" <button onclick=\"decideApproval('" + esc(a.id) + "', 'denied')\">拒绝</button>";
|
||||
btns = " <button onclick=\"decideApproval('" + esc(a.id) + "', 'approved')\">记录允许</button>" +
|
||||
" <button onclick=\"decideApproval('" + esc(a.id) + "', 'denied')\">记录拒绝</button>";
|
||||
}
|
||||
return "[" + esc(a.kind) + "|" + esc(a.status) + "] " + esc(a.summary) + receipt + btns;
|
||||
}).join("<br>");
|
||||
@@ -465,13 +480,20 @@ async function decideApproval(approvalId, decision) {
|
||||
var verb = decision === "approved" ? "approve" : "deny";
|
||||
try {
|
||||
await api("/tasks/" + encodeURIComponent(currentTask.id) + "/approvals/" + encodeURIComponent(approvalId) + "/" + verb, "POST", {});
|
||||
$("#approvalMsg").innerHTML = "<span class='ok'>已" + (decision === "approved" ? "允许" : "拒绝") + "</span>";
|
||||
$("#approvalMsg").innerHTML = "<span class='ok'>已记录" + (decision === "approved" ? "允许" : "拒绝") + "(当前版本只留回执,不拦截实时行为)</span>";
|
||||
loadApprovals(currentTask.id);
|
||||
} catch (e) { $("#approvalMsg").innerHTML = "<span class='err'>决定失败:" + esc(e.message) + "</span>"; }
|
||||
}
|
||||
async function loadWorkers() {
|
||||
try {
|
||||
var d = await api("/workers", "GET");
|
||||
// 非全局管理员只拿到聚合数字(服务端 P1 收紧):不展示 worker 清单。
|
||||
if (d.workers === undefined && d.workers_total !== undefined) {
|
||||
if (!d.workers_total) { $("#workers").textContent = "无 worker 注册"; return; }
|
||||
$("#workers").textContent = "在线 " + d.workers_online + " / 离线 " + d.workers_offline +
|
||||
"(共 " + d.workers_total + ";完整 worker 清单仅全局管理员可见)";
|
||||
return;
|
||||
}
|
||||
var list = d.workers || d || [];
|
||||
if (!list.length) { $("#workers").textContent = "无 worker 注册"; return; }
|
||||
$("#workers").innerHTML = list.map(function (w) {
|
||||
@@ -759,7 +781,7 @@ async function postReview() {
|
||||
await api("/tasks/" + encodeURIComponent(currentTask.id) + "/reviews", "POST", body);
|
||||
$("#reviewMsg").innerHTML = "<span class='ok'>评审已登记(requested)</span>";
|
||||
loadReviews(currentTask.id);
|
||||
} catch (e) { $("#reviewMsg").innerHTML = "<span class='err'>发起评审失败:" + esc(e.message) + "</span>"; }
|
||||
} catch (e) { $("#reviewMsg").innerHTML = "<span class='err'>登记评审需求失败:" + esc(e.message) + "</span>"; }
|
||||
}
|
||||
async function submitReviewFindings(reviewId) {
|
||||
if (!currentTask) return;
|
||||
@@ -778,8 +800,19 @@ async function showDetail(taskId) {
|
||||
document.querySelectorAll("#taskRows tr").forEach(function (tr) {
|
||||
tr.classList.toggle("selected", tr.getAttribute("data-id") === taskId);
|
||||
});
|
||||
// 每个任务状态配一句中文解释(只解释状态语义,不改后端状态机)。
|
||||
var STATUS_EXPLAIN = {
|
||||
queued: "排队中,尚未被 worker 认领",
|
||||
running: "运行中,模型回合进行中",
|
||||
completed: "模型回合结束、改动在 worktree 里尚未合并",
|
||||
failed: "模型回合失败,可重试",
|
||||
interrupted: "已被取消/中断,可重试",
|
||||
merged: "控制面已标记 merged,不代表 git 主分支已合入",
|
||||
"merge-failed": "合并落盘失败,可重试"
|
||||
};
|
||||
$("#detail").innerHTML =
|
||||
"状态 <b class='st-" + esc(t.status) + "'>" + esc(t.status) + "</b> | " +
|
||||
"状态 <b class='st-" + esc(t.status) + "'>" + esc(t.status) + "</b>(" +
|
||||
esc(STATUS_EXPLAIN[t.status] || "未知状态") + ") | " +
|
||||
"executor " + esc(t.executor || "") + " | 主机 " + esc(t.worktree_host || "") +
|
||||
" | 模型 " + esc(t.model || "默认") + " | 强度 " + esc(t.effort || "默认") +
|
||||
" | 权限 " + esc(t.approval || "默认") +
|
||||
@@ -800,7 +833,35 @@ async function showDetail(taskId) {
|
||||
pre.style.display = "block";
|
||||
$("#detailActions").style.display = "flex";
|
||||
$("#btnFork").style.display = TRACE_TERMINAL.indexOf(t.status) >= 0 ? "" : "none";
|
||||
$("#actionMsg").textContent = "";
|
||||
// 按钮按状态 + 当前用户项目角色禁用并写明原因(前端禁用只是
|
||||
// 防误点,真正鉴权仍在服务端每次请求时做;角色来自项目列表
|
||||
// 的 my_role 字段,未知角色时按最严(仅取消可用)处理)。
|
||||
(function () {
|
||||
var proj = null;
|
||||
(projectsCache || []).forEach(function (p) { if (p.id === t.project_id) proj = p; });
|
||||
var role = proj ? proj.my_role : null;
|
||||
var isMaintainer = (role === "maintainer" || role === "owner");
|
||||
var canCreate = (role === "contributor" || isMaintainer);
|
||||
var terminal = TRACE_TERMINAL.indexOf(t.status) >= 0;
|
||||
var rules = [
|
||||
{ btn: "#btnRetry", ok: ["failed", "interrupted", "merge-failed"].indexOf(t.status) >= 0 && isMaintainer,
|
||||
why: "仅 failed/interrupted/merge-failed 可重试" + (isMaintainer ? "" : ",且仅 maintainer 可重试") },
|
||||
{ btn: "#btnCancel", ok: !terminal && canCreate,
|
||||
why: terminal ? "终态任务不可取消" : (canCreate ? "" : "仅项目成员(contributor 及以上)可取消") },
|
||||
{ btn: "#btnMerge", ok: t.status === "completed" && isMaintainer,
|
||||
why: "仅 completed 可合并" + (isMaintainer ? "" : ",且仅 maintainer 可合并") },
|
||||
{ btn: "#btnPr", ok: t.status === "merged" && isMaintainer,
|
||||
why: "仅 merged 可提 PR" + (isMaintainer ? "" : ",且仅 maintainer 可提 PR") }
|
||||
];
|
||||
var reasons = [];
|
||||
rules.forEach(function (r) {
|
||||
var b = $(r.btn);
|
||||
b.disabled = !r.ok;
|
||||
b.title = r.ok ? "" : r.why;
|
||||
if (!r.ok && r.why) reasons.push(b.textContent.replace(/(.*$/, "") + ":不可用——" + r.why);
|
||||
});
|
||||
$("#actionMsg").textContent = reasons.length ? reasons.join(";") : "";
|
||||
})();
|
||||
$("#costBox").innerHTML = "";
|
||||
$("#forkForm").style.display = "none";
|
||||
$("#forkMsgSpan").textContent = "";
|
||||
@@ -1062,6 +1123,12 @@ $("#createTask").onclick = async function () {
|
||||
try { body.coordinator_parts = JSON.parse(coordRaw); }
|
||||
catch (e) { $("#taskMsg").textContent = "协调员子任务 JSON 解析失败"; return; }
|
||||
if (!Array.isArray(body.coordinator_parts)) { $("#taskMsg").textContent = "协调员子任务必须是数组"; return; }
|
||||
// 协调员与 goal/计划互斥:提交前校验并在表单处写明冲突
|
||||
// (服务端同样 400,但不要等到服务端才发现)。
|
||||
if ($("#goalMode").checked || $("#planMode").checked) {
|
||||
$("#taskMsg").textContent = "协调员任务不能与 goal/计划模式同用:协调员任务本身不跑模型回合,请取消 goal/计划勾选,或清空协调员 JSON";
|
||||
return;
|
||||
}
|
||||
}
|
||||
var atts = gatherAttachments();
|
||||
if (atts.length) {
|
||||
|
||||
@@ -16,6 +16,15 @@
|
||||
超时时同样取消挂起的子任务并标 failed,报告里写清哪几项
|
||||
没完成。取消路径的协调员标 interrupted,不发完成/失败推送
|
||||
(与普通任务的取消口径一致)。
|
||||
- 孤儿收口(P1):协调员本身死亡(worker 进程死、VM 回收)
|
||||
时没有“取消者”替它取消子任务,正在跑的子任务会无声继续
|
||||
烧钱。worker 侧两道收口:① 协调员取消/超时终局时,cancel
|
||||
之后仍未终态的子任务由 _coord_mark_orphans 直接标
|
||||
interrupted 并记审计;② 常驻循环每轮的
|
||||
sweep_orphaned_coordinator_children 把“父已终态(或父行
|
||||
已失踪)但子仍未终态”的子任务统一标 interrupted 并记审计
|
||||
(父仍在跑时绝不动子任务)。本模块只提供纯函数
|
||||
nonterminal_children 供 worker 判断,DB 写回在 worker 侧。
|
||||
|
||||
本模块只放纯逻辑(校验、解析、汇总、建表),不碰 DB 之外的
|
||||
任何东西,单测不需要 worker/server。
|
||||
@@ -128,6 +137,28 @@ def ensure_columns(conn):
|
||||
pass
|
||||
|
||||
|
||||
TERMINAL_STATUSES = frozenset(
|
||||
{"completed", "failed", "interrupted", "merged", "merge-failed"})
|
||||
|
||||
|
||||
def nonterminal_children(children):
|
||||
"""未终态的子任务(P1 孤儿判断用,纯函数)。
|
||||
|
||||
children 为 [{name, id, status, ...}](worker 的 _coord_child_rows
|
||||
拼出来的 merged 形态,或 _coord_sync_children 的 [{name, id}]
|
||||
补上 status 后的形态);返回其中 status 不在终态集里的项
|
||||
(缺 status 按未终态算,绝不把“查不到”当成“已完成”)。
|
||||
worker 的孤儿标记与取消/超时终局都以它为准。
|
||||
"""
|
||||
out = []
|
||||
for ch in children or []:
|
||||
if not isinstance(ch, dict):
|
||||
continue
|
||||
if ch.get("status") not in TERMINAL_STATUSES:
|
||||
out.append(ch)
|
||||
return out
|
||||
|
||||
|
||||
def build_report(children):
|
||||
"""合流汇总:children 为 [{name, id, status, result_text,
|
||||
error_text}],返回 (report_text, overall_status)。
|
||||
|
||||
@@ -18,6 +18,28 @@ may charge differently. Override per model with
|
||||
with no price record ``usd_micros=None`` and reports count their
|
||||
tokens in ``unpriced_actual_tokens`` instead of pricing them at zero.
|
||||
|
||||
cache_write_input_tokens: EXPLICITLY NOT BILLED HERE (P1 decision,
|
||||
not an oversight). host.py collects ``cacheWriteInputTokens`` from
|
||||
upstream tokenUsage events and sandbox.py carries it in usage dicts
|
||||
for observability, but it is NOT added to ``total_tokens`` (upstream
|
||||
total is input + output), NOT stored in ``cost_records`` (``record``
|
||||
takes input/output/cached only), NOT priced by ``price_usd_micros``,
|
||||
and NOT counted toward the unpriced-token cap (worker counts input +
|
||||
output from ``source="actual"`` rows only). Rationale: no price-table
|
||||
entry and no relay bill evidence treats cache creation as a
|
||||
separately billed token stream for the OpenAI-style relay models in
|
||||
use; cache_write here is prompt-cache bookkeeping detail, not a
|
||||
second charge on top of input. If a future provider bills cache
|
||||
creation separately, the price table must gain an explicit
|
||||
cache_write rate FIRST -- only then may ``record``/pricing count it.
|
||||
Until then, counting it would double-charge input tokens that are
|
||||
already priced (or inflate the unpriced cap with non-spend).
|
||||
|
||||
Unpriced cap scope (P1): ``unpriced_actual_tokens`` and the worker's
|
||||
cap query count ``source="actual"`` rows only. Estimated rows
|
||||
(``source="estimated"``, character-count fallback when no usage
|
||||
events arrive) are never real spend and must not trip the cap.
|
||||
|
||||
Dead end, documented so nobody retries it: the app-server's
|
||||
account/usage/read method requires ChatGPT account authentication and
|
||||
fails on a relay API key (2026-10-26 probe: ``codex account
|
||||
@@ -98,6 +120,9 @@ def price_usd_micros(model, input_tokens, cached_input_tokens,
|
||||
output rate. Reasoning tokens are already inside output_tokens
|
||||
(provider total = input + output), never added twice. Returns None
|
||||
when the model has no price entry (caller records usd as unknown).
|
||||
cache_write_input_tokens is deliberately NOT a parameter here:
|
||||
it is not billed separately (see module docstring); callers must
|
||||
not add it to input_tokens to “count it anyway”.
|
||||
"""
|
||||
rates = model_prices().get(model)
|
||||
if rates is None:
|
||||
|
||||
+125
-5
@@ -37,15 +37,22 @@
|
||||
发送端是后续待接线的工作,见模块末尾的缺口说明。
|
||||
- 本模块不跑真实模型回合。webhook/slack 的真实发送只 POST
|
||||
到订阅者自己填的 target URL(创建订阅已要求项目
|
||||
create_task 权限;能建订阅的成员本来就能经任务执行代码,
|
||||
因此不对 target 做内网屏蔽——屏蔽既拦不住提权,也会误伤
|
||||
本地接收器)。密钥、token、证书一律不由本模块读取、写入
|
||||
或返回。
|
||||
create_task 权限)。P1 收紧后:真实发送层对 target 做 SSRF
|
||||
校验——loopback、私有网段、链路本地、云元数据地址一律不
|
||||
POST;主机名先 DNS 解析再按解析出的 IP 判,解析失败拒绝
|
||||
(见 validate_push_target)。本地接收器如需放行,须运维显式
|
||||
置 CODEX_PUSH_ALLOW_PRIVATE_TARGETS=1(默认关闭,测试用
|
||||
本地接收器也须经此开关,不在代码里默认放行 loopback)。
|
||||
密钥、token、证书一律不由本模块读取、写入或返回。
|
||||
"""
|
||||
|
||||
import ipaddress
|
||||
import json
|
||||
import os
|
||||
import socket
|
||||
import sqlite3
|
||||
import urllib.error
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
from cloud.db import connect as _db_connect
|
||||
from cloud.db import PgConnection as _PgConnection
|
||||
@@ -67,6 +74,7 @@ __all__ = [
|
||||
"send_webhook",
|
||||
"send_slack",
|
||||
"dispatch_event",
|
||||
"validate_push_target",
|
||||
]
|
||||
|
||||
CHANNELS = ("webhook", "slack", "webpush", "apns", "fcm")
|
||||
@@ -311,6 +319,91 @@ def _require_http_url(target, channel):
|
||||
return target
|
||||
|
||||
|
||||
# 云元数据主机名:即使解析出的 IP 恰好不在私有段,也直接挡。
|
||||
_BLOCKED_HOSTNAMES = frozenset({"metadata.google.internal"})
|
||||
|
||||
|
||||
def _allow_private_targets():
|
||||
"""运维显式开关:仅 CODEX_PUSH_ALLOW_PRIVATE_TARGETS=1 时放行
|
||||
内网目标(本地接收器场景)。默认关闭;开关只影响 IP 段判定,
|
||||
不影响 scheme 校验。"""
|
||||
return os.environ.get(
|
||||
"CODEX_PUSH_ALLOW_PRIVATE_TARGETS", "").strip() == "1"
|
||||
|
||||
|
||||
def _default_resolver(hostname):
|
||||
"""默认 DNS 解析:返回 IP 字符串列表;解析失败抛异常,
|
||||
由 validate_push_target 转成“拒绝”。"""
|
||||
infos = socket.getaddrinfo(hostname, None)
|
||||
out = []
|
||||
for info in infos:
|
||||
sockaddr = info[4] if len(info) > 4 else None
|
||||
if sockaddr and sockaddr[0]:
|
||||
out.append(str(sockaddr[0]))
|
||||
return out
|
||||
|
||||
|
||||
def _ip_is_blocked(ip_text):
|
||||
"""非全局单播 IP 一律算内网/特殊地址:loopback、私有网段、
|
||||
链路本地(169.254/16,含云元数据 169.254.169.254)、保留段、
|
||||
组播、未指定地址都在 is_global=False 里。"""
|
||||
try:
|
||||
return not ipaddress.ip_address(ip_text).is_global
|
||||
except ValueError:
|
||||
return True
|
||||
|
||||
|
||||
def validate_push_target(target, resolver=None, check_dns=True):
|
||||
"""SSRF 校验:通过返回 None,否则返回原因字符串(不抛)。
|
||||
|
||||
顺序:scheme(http/https)-> 主机名存在 -> 元数据主机名 ->
|
||||
IP 字面量直接判段 -> (check_dns 时)DNS 解析后逐 IP 判段,
|
||||
解析失败或解析出空结果都拒绝(DNS 重绑定下解析时与发送时
|
||||
仍可能不一致,这是本层能做的上限,见残留说明)。
|
||||
check_dns=False 时只做字面量判定:供直接调 send_* 但不想
|
||||
做 DNS 的调用方用;dispatch 的真实发送路径一律
|
||||
check_dns=True,显式 sender 路径不走本预检(见 dispatch)。
|
||||
"""
|
||||
if not isinstance(target, str) or not (
|
||||
target.startswith("http://") or target.startswith("https://")):
|
||||
return "target must be an http(s) URL"
|
||||
try:
|
||||
host = urllib.parse.urlparse(target).hostname
|
||||
except ValueError:
|
||||
return "target host is invalid"
|
||||
if not host:
|
||||
return "target host required"
|
||||
lowered = host.lower()
|
||||
if lowered in _BLOCKED_HOSTNAMES or lowered.endswith(
|
||||
".metadata.google.internal"):
|
||||
return "target host is a cloud metadata address"
|
||||
if _allow_private_targets():
|
||||
return None
|
||||
try:
|
||||
ipaddress.ip_address(host)
|
||||
is_literal = True
|
||||
except ValueError:
|
||||
is_literal = False
|
||||
if is_literal:
|
||||
if _ip_is_blocked(host):
|
||||
return "target resolves to a non-public address: %s" % host
|
||||
return None
|
||||
if not check_dns:
|
||||
return None
|
||||
resolve = resolver or _default_resolver
|
||||
try:
|
||||
ips = resolve(host)
|
||||
except Exception:
|
||||
return "DNS resolution failed for target host"
|
||||
if not ips:
|
||||
return "DNS resolution failed for target host"
|
||||
for ip_text in ips:
|
||||
if _ip_is_blocked(ip_text):
|
||||
return ("target host resolves to a non-public address: %s"
|
||||
% ip_text)
|
||||
return None
|
||||
|
||||
|
||||
def send_webhook(message):
|
||||
"""webhook 通道真实发送:POST JSON 到订阅的 target URL。
|
||||
|
||||
@@ -319,6 +412,11 @@ def send_webhook(message):
|
||||
RuntimeError(dispatch 记入 errors,不算 sent)。
|
||||
"""
|
||||
target = _require_http_url(message["target"], "webhook")
|
||||
blocked = validate_push_target(
|
||||
target, resolver=message.get("_resolver"))
|
||||
if blocked is not None:
|
||||
raise RuntimeError(
|
||||
"webhook delivery failed: target blocked: %s" % blocked)
|
||||
payload = message["payload"] if isinstance(
|
||||
message["payload"], dict) else {}
|
||||
ok, detail = _post_json(target, {
|
||||
@@ -340,6 +438,11 @@ def send_slack(message):
|
||||
事件附 error 摘要(截断 500 字)。非 2xx 抛 RuntimeError。
|
||||
"""
|
||||
target = _require_http_url(message["target"], "slack")
|
||||
blocked = validate_push_target(
|
||||
target, resolver=message.get("_resolver"))
|
||||
if blocked is not None:
|
||||
raise RuntimeError(
|
||||
"slack delivery failed: target blocked: %s" % blocked)
|
||||
payload = message["payload"] if isinstance(
|
||||
message["payload"], dict) else {}
|
||||
text = "[codex] task %s %s (project %s)" % (
|
||||
@@ -357,7 +460,8 @@ def send_slack(message):
|
||||
DEFAULT_SENDERS = {"webhook": send_webhook, "slack": send_slack}
|
||||
|
||||
|
||||
def dispatch_event(db, event_type, project_id, payload, sender=None):
|
||||
def dispatch_event(db, event_type, project_id, payload, sender=None,
|
||||
resolver=None):
|
||||
"""分发一次状态变化事件,只发本项目的订阅。
|
||||
|
||||
参数:
|
||||
@@ -418,6 +522,7 @@ def dispatch_event(db, event_type, project_id, payload, sender=None):
|
||||
}]
|
||||
return result
|
||||
|
||||
real_send = sender is None
|
||||
if sender is None:
|
||||
# 内置真实发送端:webhook/slack 真 POST;显式 sender
|
||||
# 传入时不走这里(注入优先),webpush/apns/fcm 永远
|
||||
@@ -445,6 +550,21 @@ def dispatch_event(db, event_type, project_id, payload, sender=None):
|
||||
result["errors"].append(
|
||||
dict(base, reason=MISSING_CREDENTIAL_REASONS[sub["channel"]]))
|
||||
continue
|
||||
# SSRF 预检(真实发送路径,发送前挡,不 POST):含 DNS
|
||||
# 的全量校验。显式 sender 路径不预检:注入的 sender 是
|
||||
# 调用方自控的投递(测试 seam),不是服务端发起的 HTTP,
|
||||
# 挡它不增加生产安全性;生产服务端一律走 sender=None
|
||||
# 的真实发送,必经本预检与 send_webhook/send_slack 的
|
||||
# 二次校验。原因文案含 "delivery failed",与旧调用方
|
||||
# 对投递失败的断言兼容。
|
||||
if real_send:
|
||||
blocked = validate_push_target(
|
||||
sub["target"], resolver=resolver, check_dns=True)
|
||||
if blocked is not None:
|
||||
result["errors"].append(
|
||||
dict(base, reason="%s delivery failed: target blocked:"
|
||||
" %s" % (sub["channel"], blocked)))
|
||||
continue
|
||||
# webhook/slack:sender=None 时已换成内置真实发送端;
|
||||
# 显式 sender 里没有该通道时明确报错,不静默成功。
|
||||
channel_sender = _resolve_channel_sender(sender, sub["channel"])
|
||||
|
||||
+83
-36
@@ -423,6 +423,15 @@ class LocalExecutor(Executor):
|
||||
rounds = task.get("goal_max_rounds")
|
||||
if rounds is None:
|
||||
rounds = 3
|
||||
# P1 轮次上限(与 worker 的 GOAL_MAX_ROUNDS_CAP 同值 100):
|
||||
# 有限轮次超限直接 loud fail,不静默截断;goal_mode 的 0
|
||||
#(不限轮次)放行,上界是配额;旧验收形态的 0 同样拒绝
|
||||
#(0 在旧形态里表示不限轮次,P1 已禁用无上限验收循环)。
|
||||
if rounds is not None and rounds > 100:
|
||||
raise ValueError("goal_max_rounds 超过上限 100 轮")
|
||||
if rounds == 0 and verify and not task.get("goal_mode"):
|
||||
raise ValueError(
|
||||
"旧验收形态 goal_max_rounds=0 表示不限轮次,已禁用")
|
||||
cmd = self._base_command(model=task.get("model"),
|
||||
effort=task.get("effort"),
|
||||
approval=task.get("approval"),
|
||||
@@ -440,26 +449,35 @@ class LocalExecutor(Executor):
|
||||
return self._goal_result_from_runner(
|
||||
task["id"], self._run_local(cmd, extra_env))
|
||||
|
||||
def goal_resume_task(self, task, goal_text, extra_env=None,
|
||||
model_provider=None) -> dict:
|
||||
"""调 host.py goal-resume --goal-text:在旧线程上开新目标
|
||||
循环(旧任务接续 goal 模式,云端 /continue 带 goal_mode=1
|
||||
时走这里)。命令:host.py goal-resume --task <host 名>
|
||||
--goal-text <新目标> --max-rounds 0(新语义不限轮次)。
|
||||
def goal_resume_task(self, task, goal_text=None, extra_env=None,
|
||||
model_provider=None, max_rounds=None) -> dict:
|
||||
"""调 host.py goal-resume,分两种调用(P1,与 worker、
|
||||
host 三处一致,绝不能混用):
|
||||
- 断点续跑:goal_text=None(worker 的 _resume_kind=
|
||||
"breakpoint" 时传 None)。命令不带 --goal-text、默认
|
||||
也不带 --max-rounds:host 沿用原 goal + 原 verify_command
|
||||
+ 原 goal_max_rounds,从断点轮次续跑。传了 --goal-text
|
||||
就会走 host 的开新目标分支、把原 verify 覆盖成 None,
|
||||
所以断点调用绝不能顺手把续跑消息塞进 goal_text。
|
||||
- 开新目标:goal_text 为新目标文本(云端 /continue 带
|
||||
goal_mode=1 时走这里)。命令带 --goal-text 与
|
||||
--max-rounds(默认 0,新语义不限轮次,上界是配额)。
|
||||
不传 --repo:goal-resume 操作已有 host 任务,不新建
|
||||
worktree。结论解析与 goal_task 同款。
|
||||
"""
|
||||
if not isinstance(task, dict) or task.get("id") is None:
|
||||
raise ValueError("任务缺少 id")
|
||||
if not goal_text:
|
||||
raise ValueError("goal-resume 缺少目标文本")
|
||||
cmd = self._base_command(model=task.get("model"),
|
||||
effort=task.get("effort"),
|
||||
approval=task.get("approval"),
|
||||
model_provider=model_provider)
|
||||
cmd += ["goal-resume", "--task", self._host_name(task),
|
||||
"--goal-text", goal_text,
|
||||
"--max-rounds", "0"]
|
||||
cmd += ["goal-resume", "--task", self._host_name(task)]
|
||||
if goal_text:
|
||||
cmd += ["--goal-text", goal_text,
|
||||
"--max-rounds",
|
||||
str(max_rounds if max_rounds is not None else 0)]
|
||||
elif max_rounds is not None:
|
||||
cmd += ["--max-rounds", str(max_rounds)]
|
||||
return self._goal_result_from_runner(
|
||||
task["id"], self._run_local(cmd, extra_env))
|
||||
|
||||
@@ -469,7 +487,9 @@ class LocalExecutor(Executor):
|
||||
解析用完整 stdout(结论在末尾),存库的 stdout 才截尾;
|
||||
returncode 非 0 一律 failed;为 0 时无结论也判 failed,
|
||||
绝不把“没打印结论”当成达标。host 任务已存在时(同任务
|
||||
/retry 撞名,goal 没有 run 续跑语义)错误写明请新建任务。
|
||||
/retry 撞名,goal 没有 run 续跑语义)错误写明“goal 失败
|
||||
请用继续而非重试”(P1:重试在已存在 host-state 上必失败,
|
||||
断点续跑才是正确路径,见 worker 的回落逻辑)。
|
||||
"""
|
||||
returncode = getattr(result, "returncode", 1)
|
||||
stdout_full = getattr(result, "stdout", "") or ""
|
||||
@@ -478,8 +498,10 @@ class LocalExecutor(Executor):
|
||||
combined = stdout_full + stderr
|
||||
if "已存在" in combined:
|
||||
met, goal_error = None, (
|
||||
"host 任务已存在:goal 任务不支持同任务重试,"
|
||||
"请新建任务")
|
||||
"host 任务已存在:goal 失败请用继续而非重试——"
|
||||
"同任务重试会撞已存在 host-state,worker 会先"
|
||||
"尝试断点续跑,断点续跑仍失败时请用继续(不要"
|
||||
"重排同一任务)")
|
||||
status = "failed"
|
||||
elif returncode != 0:
|
||||
status = "failed"
|
||||
@@ -516,9 +538,11 @@ class LocalExecutor(Executor):
|
||||
killpg 才能连同子孙一起杀掉。注意:只有默认 runner(真实
|
||||
subprocess)时才用本方法;单测注入的假 runner 走同步路径,
|
||||
不走这里。kind=goal 时建 goal 命令、kind=goal-resume 时建
|
||||
goal-resume 命令(旧任务接续 goal 模式,目标文本取本轮
|
||||
prompt;结论解析由 worker 的 _execute_via_process 按 kind
|
||||
分发到 _goal_result_from_runner)。goal 语义与同步 goal_task
|
||||
goal-resume 命令(结论解析由 worker 的 _execute_via_process
|
||||
按 kind 分发到 _goal_result_from_runner)。goal-resume 分流
|
||||
(P1):_resume_kind="breakpoint" 时不传 --goal-text(断点
|
||||
续跑,沿用原 goal+verify),其它情况才把本轮 prompt 当
|
||||
--goal-text(开新目标)。goal 语义与同步 goal_task
|
||||
一致:新形态(goal_mode)无验收命令时 --verify 不传、
|
||||
--max-rounds 传 0(0 不能用 or 兜底成 3);旧形态仍传验收
|
||||
命令与轮次。seed_dir 只对 dispatch/goal 追加(run 续跑与
|
||||
@@ -545,6 +569,11 @@ class LocalExecutor(Executor):
|
||||
rounds = task.get("goal_max_rounds")
|
||||
if rounds is None:
|
||||
rounds = 3
|
||||
if rounds is not None and rounds > 100:
|
||||
raise ValueError("goal_max_rounds 超过上限 100 轮")
|
||||
if rounds == 0 and verify and not task.get("goal_mode"):
|
||||
raise ValueError(
|
||||
"旧验收形态 goal_max_rounds=0 表示不限轮次,已禁用")
|
||||
cmd += ["goal", "--name", self._host_name(task),
|
||||
"--task", goal_text]
|
||||
if verify:
|
||||
@@ -556,12 +585,15 @@ class LocalExecutor(Executor):
|
||||
if seed_dir:
|
||||
cmd += ["--seed-dir", seed_dir]
|
||||
elif kind == "goal-resume":
|
||||
# 旧任务接续 goal:host 侧用 --goal-text 在旧线程上开
|
||||
# 新目标循环(新语义,不限轮次);task["prompt"] 在续跑
|
||||
# 时已被 worker 换成续跑消息。
|
||||
cmd += ["goal-resume", "--task", self._host_name(task),
|
||||
"--goal-text", task["prompt"],
|
||||
"--max-rounds", "0"]
|
||||
# P1 分流(与 goal_resume_task 同款契约):
|
||||
# _resume_kind="breakpoint" 时不传 --goal-text(host
|
||||
# 沿用原 goal+verify 断点续跑,绝不能用续跑消息覆盖
|
||||
# 原 verify);其它情况(开新目标)才用 task["prompt"]
|
||||
#(worker 已换成续跑消息)当 --goal-text,--max-rounds 0。
|
||||
cmd += ["goal-resume", "--task", self._host_name(task)]
|
||||
if task.get("_resume_kind") != "breakpoint":
|
||||
cmd += ["--goal-text", task["prompt"],
|
||||
"--max-rounds", "0"]
|
||||
else:
|
||||
cmd += ["dispatch", "--name", self._host_name(task),
|
||||
"--task", task["prompt"]]
|
||||
@@ -1101,6 +1133,12 @@ class DockerExecutor(Executor):
|
||||
rounds = task.get("goal_max_rounds")
|
||||
if rounds is None:
|
||||
rounds = 3
|
||||
# P1 轮次上限(与 Local goal_task 同款,不静默截断)。
|
||||
if rounds is not None and rounds > 100:
|
||||
raise ValueError("goal_max_rounds 超过上限 100 轮")
|
||||
if rounds == 0 and verify and not task.get("goal_mode"):
|
||||
raise ValueError(
|
||||
"旧验收形态 goal_max_rounds=0 表示不限轮次,已禁用")
|
||||
inner_args = ["goal", "--name", self._host_name(task),
|
||||
"--task", goal_text]
|
||||
if verify:
|
||||
@@ -1150,8 +1188,10 @@ class DockerExecutor(Executor):
|
||||
combined = stdout_full + stderr
|
||||
if "已存在" in combined:
|
||||
met, goal_error = None, (
|
||||
"host 任务已存在:goal 任务不支持同任务重试,"
|
||||
"请新建任务")
|
||||
"host 任务已存在:goal 失败请用继续而非重试——"
|
||||
"同任务重试会撞已存在 host-state,worker 会先"
|
||||
"尝试断点续跑,断点续跑仍失败时请用继续(不要"
|
||||
"重排同一任务)")
|
||||
status = "failed"
|
||||
elif returncode != 0:
|
||||
status = "failed"
|
||||
@@ -1181,22 +1221,29 @@ class DockerExecutor(Executor):
|
||||
"usage": _parse_goal_usage(stdout_full),
|
||||
}
|
||||
|
||||
def goal_resume_task(self, task, goal_text, timeout=None,
|
||||
def goal_resume_task(self, task, goal_text=None, timeout=None,
|
||||
extra_mounts=None, network=None,
|
||||
extra_env=None, model_provider=None) -> dict:
|
||||
"""容器内调 host.py goal-resume --goal-text:在旧线程上开
|
||||
新目标循环(旧任务接续 goal 模式)。解析与 goal_task 同款
|
||||
(完整 stdout 结论、存库截尾),但没有“已存在”分支:
|
||||
goal-resume 本来就是操作已有 host 任务,不存在 dispatch
|
||||
撞名问题;returncode 非 0 或无结论一律 failed。
|
||||
extra_env=None, model_provider=None,
|
||||
max_rounds=None) -> dict:
|
||||
"""容器内调 host.py goal-resume,分两种调用(P1,与 Local
|
||||
goal_resume_task 同款契约):断点续跑传 goal_text=None
|
||||
(不带 --goal-text,host 沿用原 goal+verify+轮次),开新
|
||||
目标才传 goal_text(带 --goal-text 与 --max-rounds,默认 0)。
|
||||
解析与 goal_task 同款(完整 stdout 结论、存库截尾),但
|
||||
没有“已存在”分支:goal-resume 本来就是操作已有 host 任务,
|
||||
不存在 dispatch 撞名问题;returncode 非 0 或无结论一律 failed。
|
||||
"""
|
||||
if not isinstance(task, dict) or task.get("id") is None:
|
||||
raise ValueError("任务缺少 id")
|
||||
if not goal_text:
|
||||
raise ValueError("goal-resume 缺少目标文本")
|
||||
_resume_args = ["goal-resume", "--task", self._host_name(task)]
|
||||
if goal_text:
|
||||
_resume_args += ["--goal-text", goal_text, "--max-rounds",
|
||||
str(max_rounds
|
||||
if max_rounds is not None else 0)]
|
||||
elif max_rounds is not None:
|
||||
_resume_args += ["--max-rounds", str(max_rounds)]
|
||||
inner = self._inner_host_command(
|
||||
"goal-resume", "--task", self._host_name(task),
|
||||
"--goal-text", goal_text, "--max-rounds", "0",
|
||||
*_resume_args,
|
||||
model=task.get("model"), effort=task.get("effort"),
|
||||
approval=task.get("approval"),
|
||||
model_provider=model_provider)
|
||||
|
||||
+329
-54
@@ -113,12 +113,15 @@ from cloud.oauth import (
|
||||
)
|
||||
from cloud.relay import RelayError as _RelayError
|
||||
from cloud.relay import get_relay_models as _get_relay_models
|
||||
from cloud.sub2api import Sub2apiError as _Sub2apiError
|
||||
from cloud.sub2api import ensure_gateway_keys_table as _ensure_gateway_keys
|
||||
from cloud.sub2api import ensure_user_gateway_key as _ensure_sub2api_key
|
||||
from cloud.sub2api import gateway_name as _gateway_name
|
||||
from cloud.sub2api import gateway_origin as _gateway_origin
|
||||
from cloud.sub2api import get_user_gateway_key as _get_sub2api_key
|
||||
from cloud.sub2api import list_gateway_models as _list_sub2api_models
|
||||
from cloud.sub2api import rotate_user_gateway_key as _rotate_sub2api_key
|
||||
from cloud.sub2api import validate_or_rotate_gateway_key as _validate_sub2api_key
|
||||
from cloud.repos import validate_repo_url
|
||||
from cloud.push import (
|
||||
create_subscription as _push_create,
|
||||
@@ -167,6 +170,31 @@ TRACE_STREAM_MAX_SECONDS = 3600
|
||||
TRACE_STREAM_POLL_SECONDS = 1.0
|
||||
TRACE_DEFAULT_DIR = "/opt/codex-app-server/work/host-traces"
|
||||
|
||||
# P1 控制面限额(建项目时校验,超限 400 并在错误里写明数值):
|
||||
# 单用户项目数上限、项目名长度上限、自定义项目 id 长度上限。
|
||||
# 均可经环境变量调(运维侧),缺省值见下方;改动只影响新项目,
|
||||
# 已有超限项目不受影响(不删、不改名)。
|
||||
def _env_int(name, default):
|
||||
try:
|
||||
value = int(os.environ.get(name, "") or default)
|
||||
except (TypeError, ValueError):
|
||||
return default
|
||||
return value if value > 0 else default
|
||||
|
||||
|
||||
MAX_PROJECTS_PER_USER = _env_int("CODEX_MAX_PROJECTS_PER_USER", 50)
|
||||
MAX_PROJECT_NAME_LEN = _env_int("CODEX_MAX_PROJECT_NAME_LEN", 100)
|
||||
MAX_PROJECT_ID_LEN = _env_int("CODEX_MAX_PROJECT_ID_LEN", 64)
|
||||
# 协调员父任务预留:每分工至少预留的美元额度(微美元)。
|
||||
# 父任务准入只查一次时,N 个分工可解锁 N 倍消费;本预留要求
|
||||
# 剩余配额 >= 分工数 × 本值,否则父任务直接 402(见
|
||||
# _check_coordinator_reserve)。缺省 $1/分工,可经环境变量调;
|
||||
# 它是准入下限估计,不是实际计费——实际花费仍按子任务
|
||||
# created_by=父任务创建人记到创建人账上(worker 建子任务时
|
||||
# created_by 直接取父任务创建人,成本 JOIN 据此归属)。
|
||||
COORDINATOR_RESERVE_PER_PART_USD_MICROS = _env_int(
|
||||
"CODEX_COORDINATOR_RESERVE_PER_PART_USD_MICROS", 1000000)
|
||||
|
||||
|
||||
def _trace_file_name(task_id):
|
||||
"""云任务 id -> 链路文件名:与 cloud.sandbox 的 _host_name
|
||||
@@ -1451,6 +1479,9 @@ class CloudControlPlaneHandler(BaseHTTPRequestHandler):
|
||||
else:
|
||||
self._send_json(404, {"error": "not found"})
|
||||
return
|
||||
if path == "/users/me/gateway/rotate" and method == "POST":
|
||||
self._user_gateway_rotate()
|
||||
return
|
||||
if path.startswith("/users/me/keys"):
|
||||
# API keys 管理:全部只操作认证用户自己的 key。
|
||||
if method == "GET" and path == "/users/me/keys":
|
||||
@@ -1830,10 +1861,35 @@ class CloudControlPlaneHandler(BaseHTTPRequestHandler):
|
||||
if not name or not isinstance(name, str):
|
||||
self._send_json(400, {"error": "name required"})
|
||||
return
|
||||
if len(name) > MAX_PROJECT_NAME_LEN:
|
||||
self._send_json(400, {
|
||||
"error": "project name too long: max %d characters"
|
||||
% MAX_PROJECT_NAME_LEN})
|
||||
return
|
||||
project_id = body.get("id") or uuid.uuid4().hex
|
||||
if not isinstance(project_id, str):
|
||||
self._send_json(400, {"error": "id must be a string"})
|
||||
return
|
||||
if len(project_id) > MAX_PROJECT_ID_LEN:
|
||||
self._send_json(400, {
|
||||
"error": "project id too long: max %d characters"
|
||||
% MAX_PROJECT_ID_LEN})
|
||||
return
|
||||
# 单用户项目数上限:按 projects.owner_id 计(成员身份加入
|
||||
# 的项目不计入自己的上限,避免“被拉进很多项目就建不了
|
||||
# 项目”);超限 400 且不建项目。
|
||||
_count_conn = self._db()
|
||||
try:
|
||||
_owned = _count_conn.execute(
|
||||
"SELECT COUNT(*) FROM projects WHERE owner_id = ?",
|
||||
(user_id,)).fetchone()[0]
|
||||
finally:
|
||||
_count_conn.close()
|
||||
if _owned >= MAX_PROJECTS_PER_USER:
|
||||
self._send_json(400, {
|
||||
"error": "project limit exceeded: max %d projects"
|
||||
" per user" % MAX_PROJECTS_PER_USER})
|
||||
return
|
||||
# repo_url 可选:不传时为 None(老行为不变);传了则与 worker
|
||||
# 共用 cloud.repos.validate_repo_url 的同一套校验,不合法时
|
||||
# 400 且不建项目,绝不把带凭证的 URL 存进库。
|
||||
@@ -1947,43 +2003,78 @@ class CloudControlPlaneHandler(BaseHTTPRequestHandler):
|
||||
self._send_json(200, {"project": _project_to_dict(row)})
|
||||
|
||||
def _github_repos_list(self, user_id):
|
||||
"""GET /github/repos:列出本服务 GitHub App 安装记录里的仓
|
||||
库(installation_repositories 事件同步进 github_app_
|
||||
installations 表),给控制台建项目/补仓库时的下拉菜单用。
|
||||
任何已认证用户可列(只含仓库名,不含密钥);选了哪个仓库
|
||||
只影响调用者自己有管理权的项目(PATCH 仍按 owner 校验)。
|
||||
"""GET /github/repos:只返回调用者成员项目对应的仓库。
|
||||
|
||||
P1 收紧前返回全部安装记录(任一已认证用户可看到全部
|
||||
安装仓库名,属跨租户信息泄露)。现按 project_members
|
||||
过滤:只保留调用者是成员的项目的 repo_url 对应的仓库
|
||||
(clone_url 归一化后比对:去 .git 后缀、比对小写);
|
||||
调用者无成员项目或项目未填 repo_url 时返回空列表。
|
||||
手填 URL 的能力不变——没进 App 安装的公开仓库仍可手动填。"""
|
||||
conn = self._db()
|
||||
try:
|
||||
rows = list_installations(conn)
|
||||
member_ids = {
|
||||
row["project_id"]
|
||||
for row in conn.execute(
|
||||
"SELECT project_id FROM project_members"
|
||||
" WHERE user_id = ?",
|
||||
(user_id,)).fetchall()
|
||||
}
|
||||
allowed = set()
|
||||
for row in conn.execute(
|
||||
"SELECT id, repo_url FROM projects").fetchall():
|
||||
if row["id"] not in member_ids or not row["repo_url"]:
|
||||
continue
|
||||
url = str(row["repo_url"]).strip().lower()
|
||||
if url.endswith(".git"):
|
||||
url = url[:-4]
|
||||
allowed.add(url)
|
||||
finally:
|
||||
conn.close()
|
||||
repos = [{
|
||||
"installation_id": r["installation_id"],
|
||||
"account_login": r["account_login"],
|
||||
"repo_full_name": r["repo_full_name"],
|
||||
"clone_url": "https://github.com/%s.git" % r["repo_full_name"],
|
||||
} for r in rows]
|
||||
repos = []
|
||||
for r in rows:
|
||||
clone_url = "https://github.com/%s.git" % r["repo_full_name"]
|
||||
normalized = clone_url.lower()
|
||||
if normalized.endswith(".git"):
|
||||
normalized = normalized[:-4]
|
||||
if normalized not in allowed:
|
||||
continue
|
||||
repos.append({
|
||||
"installation_id": r["installation_id"],
|
||||
"account_login": r["account_login"],
|
||||
"repo_full_name": r["repo_full_name"],
|
||||
"clone_url": clone_url,
|
||||
})
|
||||
self._send_json(200, {"repos": repos})
|
||||
|
||||
def _list_projects(self, user_id):
|
||||
conn = self._db()
|
||||
try:
|
||||
member_ids = {
|
||||
row["project_id"]
|
||||
for row in conn.execute(
|
||||
"SELECT project_id FROM project_members WHERE user_id = ?",
|
||||
(user_id,),
|
||||
).fetchall()
|
||||
}
|
||||
member_rows = conn.execute(
|
||||
"SELECT project_id, role FROM project_members"
|
||||
" WHERE user_id = ?",
|
||||
(user_id,)).fetchall()
|
||||
member_ids = {row["project_id"] for row in member_rows}
|
||||
roles = {row["project_id"]: row["role"]
|
||||
for row in member_rows}
|
||||
rows = conn.execute(
|
||||
"SELECT id, name, owner_id, created_at, repo_url, auto_pr"
|
||||
" FROM projects ORDER BY created_at, id"
|
||||
).fetchall()
|
||||
finally:
|
||||
conn.close()
|
||||
projects = [_project_to_dict(r) for r in rows
|
||||
if r["id"] in member_ids]
|
||||
projects = []
|
||||
for r in rows:
|
||||
if r["id"] not in member_ids:
|
||||
continue
|
||||
# 附加字段 my_role(只增字段,不改已有成功语义):
|
||||
# 控制台据此按角色禁用重试/合并/提 PR 等按钮;服务端
|
||||
# 权限仍以每次请求的 check_permission 为准,前端禁用
|
||||
# 只是防误点,不是鉴权。
|
||||
item = _project_to_dict(r)
|
||||
item["my_role"] = roles.get(r["id"])
|
||||
projects.append(item)
|
||||
self._send_json(200, {"projects": projects})
|
||||
|
||||
# -- 项目子资源:记忆 / 知识库 / 成本 -------------------------------
|
||||
@@ -2119,6 +2210,11 @@ class CloudControlPlaneHandler(BaseHTTPRequestHandler):
|
||||
return
|
||||
deleted = _push_delete(self.server.db_path,
|
||||
subscription_id=parts[3])
|
||||
if deleted:
|
||||
with _auth_store(self.server.db_path) as store:
|
||||
store.audit(
|
||||
user_id, project_id, "push_subscription_deleted",
|
||||
"subscription_id=%s" % parts[3])
|
||||
self._send_json(200, {"deleted": bool(deleted)})
|
||||
return
|
||||
# /projects/{id}/automations
|
||||
@@ -2389,31 +2485,33 @@ class CloudControlPlaneHandler(BaseHTTPRequestHandler):
|
||||
# 缓存(见 cloud/relay.py)。单测可经 server.relay_fetcher
|
||||
# 注入假取数函数;中转挂了或未配 key 时 502,不编造列表。
|
||||
# 网关为 sub2api 时:模型目录因用户而异(key 绑定的组
|
||||
# 决定),取调用者自己的网关 key 现查;无 key 时返回空
|
||||
# 列表 + provisioning 标记(控制台据此提示开通中),不
|
||||
# 在页面加载时悄悄开通(开通在建号与任务时发生)。
|
||||
# 决定)。P1 收紧:首次调模型目录即做真正的懒开通/
|
||||
# 健康检查,不再只回 provisioning: pending——无行则开通,
|
||||
# 旧 key 经 /v1/models 实测已死(401/403,含余额为 0 的
|
||||
# 403)则轮换重开;网关本身不可用时回 502、保留旧行。
|
||||
# key 明文绝不进响应、日志、缓存。
|
||||
if _gateway_name() == "sub2api":
|
||||
conn = self._db()
|
||||
try:
|
||||
row = _get_sub2api_key(conn, user_id)
|
||||
try:
|
||||
conn.commit()
|
||||
except Exception:
|
||||
pass
|
||||
status, api_key, _sub, _kid = _validate_sub2api_key(
|
||||
conn, user_id)
|
||||
except _Sub2apiError as exc:
|
||||
self._send_json(502, {"error": str(exc)})
|
||||
return
|
||||
except Exception as exc:
|
||||
self._send_json(502, {"error": str(exc)})
|
||||
return
|
||||
finally:
|
||||
conn.close()
|
||||
if row is None or not row.get("api_key"):
|
||||
self._send_json(200, {"models": [], "gateway": "sub2api",
|
||||
"provisioning": "pending"})
|
||||
return
|
||||
try:
|
||||
models = _list_sub2api_models(_gateway_origin(),
|
||||
row["api_key"])
|
||||
models = _list_sub2api_models(_gateway_origin(), api_key)
|
||||
except Exception as exc:
|
||||
self._send_json(502, {"error": str(exc)})
|
||||
return
|
||||
self._send_json(200, {"models": models, "gateway": "sub2api",
|
||||
"cached": False, "updated_at": None})
|
||||
"cached": False, "updated_at": None,
|
||||
"provisioning": status})
|
||||
return
|
||||
fetcher = getattr(self.server, "relay_fetcher", None)
|
||||
try:
|
||||
@@ -2647,6 +2745,12 @@ class CloudControlPlaneHandler(BaseHTTPRequestHandler):
|
||||
self._send_json(400, {
|
||||
"error": "goal_max_rounds must be >= 1"})
|
||||
return
|
||||
# P1:旧验收形态轮次设上限 100(与 worker/sandbox 两道门
|
||||
# 一致;新 goal_mode=1 不走本分支,仍由配额兜底)。
|
||||
if goal_max_rounds > 100:
|
||||
self._send_json(400, {
|
||||
"error": "goal_max_rounds 超过上限 100 轮"})
|
||||
return
|
||||
# 协调员:coordinator_parts 非空时任务本身不跑模型回合,
|
||||
# worker 认领后按分工建并行子任务、等齐、合流;与 goal/
|
||||
# plan/验收命令语义冲突(那些都是“本任务跑模型”的意思),
|
||||
@@ -2673,6 +2777,33 @@ class CloudControlPlaneHandler(BaseHTTPRequestHandler):
|
||||
return
|
||||
if self._reject_if_over_quota(user_id):
|
||||
return
|
||||
# 协调员预留(P1):父任务准入只查一次时,N 个分工会
|
||||
# 解锁 N 倍消费。父任务创建时要求剩余美元配额至少覆盖
|
||||
# 分工数 × 每分工预留额,否则直接 402、父任务不建;
|
||||
# 子任务归属 = 父任务创建人(worker 建子任务时
|
||||
# created_by 直接取父任务创建人,成本 JOIN 据此归属,
|
||||
# 子任务认领时 worker 侧配额门再查一次)。不限额用户
|
||||
# (quota 为 None)不受此预留约束。
|
||||
if coord_parts and not self._check_coordinator_reserve(
|
||||
user_id, len(coord_parts)):
|
||||
return
|
||||
# sub2api 懒开通/健康检查(P1):建任务前先真校验网关
|
||||
# key——无行则开通,死 key 则轮换重开;网关不可用时
|
||||
# 502 且不建任务(不把死 key/待开通当成已开通发给
|
||||
# worker)。xcmapi 网关下跳过本段。
|
||||
if _gateway_name() == "sub2api":
|
||||
_gw_conn = self._db()
|
||||
try:
|
||||
try:
|
||||
_validate_sub2api_key(_gw_conn, user_id)
|
||||
except _Sub2apiError as exc:
|
||||
self._send_json(502, {"error": str(exc)})
|
||||
return
|
||||
except Exception as exc:
|
||||
self._send_json(502, {"error": str(exc)})
|
||||
return
|
||||
finally:
|
||||
_gw_conn.close()
|
||||
# 环境绑定:environment_id 非空时必须是本项目、已 ready
|
||||
# 的环境;creating/failed/archived 一律拒绝并把状态写进
|
||||
# 报错(creating 时用户就知道该等 worker 准备好再建任务,
|
||||
@@ -2836,6 +2967,15 @@ class CloudControlPlaneHandler(BaseHTTPRequestHandler):
|
||||
conn.close()
|
||||
with _auth_store(self.server.db_path) as store:
|
||||
store.audit(user_id, project_id, "create_task", task_id)
|
||||
if coord_parts:
|
||||
# 创建响应里写明子任务配额归属(与 worker 建子任务时
|
||||
# created_by=父任务创建人一致),不是只写文档宣称。
|
||||
task["quota_attribution"] = (
|
||||
"coordinator subtasks are charged to the parent"
|
||||
" task creator")
|
||||
task["quota_reserved_usd_micros"] = (
|
||||
len(coord_parts)
|
||||
* COORDINATOR_RESERVE_PER_PART_USD_MICROS)
|
||||
self._send_json(201, task)
|
||||
|
||||
def _list_tasks(self, user_id, query):
|
||||
@@ -4091,6 +4231,34 @@ class CloudControlPlaneHandler(BaseHTTPRequestHandler):
|
||||
return True
|
||||
return False
|
||||
|
||||
def _check_coordinator_reserve(self, user_id, n_parts):
|
||||
"""协调员预留准入:通过返回 True;被挡时已回 402 返回 False。
|
||||
|
||||
剩余美元配额(quota - used,quota 为 None = 不限额时放行)
|
||||
必须 >= 分工数 × COORDINATOR_RESERVE_PER_PART_USD_MICROS。
|
||||
这是准入下限估计,不是计费承诺:子任务实际花费仍按
|
||||
created_by=父任务创建人全额记创建人账;父任务被挡时
|
||||
一个子任务也不会建,不存在“第 2 个子任务才被挡”的
|
||||
半截状态。"""
|
||||
if not n_parts or n_parts <= 1:
|
||||
return True
|
||||
quota, used, month = self._quota_status(user_id)
|
||||
if quota is None:
|
||||
return True
|
||||
need = n_parts * COORDINATOR_RESERVE_PER_PART_USD_MICROS
|
||||
if quota - used < need:
|
||||
self._send_json(402, {
|
||||
"error": "coordinator_quota_reserve_exceeded",
|
||||
"month": month,
|
||||
"quota_usd_micros": quota,
|
||||
"used_usd_micros": used,
|
||||
"coordinator_parts": n_parts,
|
||||
"reserved_usd_micros": need,
|
||||
"quota_attribution": "coordinator subtasks are charged"
|
||||
" to the parent task creator"})
|
||||
return False
|
||||
return True
|
||||
|
||||
def _audit_admin(self, user_id, action, detail):
|
||||
"""admin 调用审计(成功与被拒都记,失败绝不挡主流程)。"""
|
||||
try:
|
||||
@@ -4125,10 +4293,17 @@ class CloudControlPlaneHandler(BaseHTTPRequestHandler):
|
||||
|
||||
def _user_quota_self(self, user_id):
|
||||
quota, used, month = self._quota_status(user_id)
|
||||
self._send_json(200,
|
||||
self._quota_payload(
|
||||
user_id, quota, used, month,
|
||||
self._quota_unpriced_tokens(user_id)))
|
||||
# 合并取舍(P1 rebase 到 d254240):main 的 _quota_payload
|
||||
# 已带 unpriced_actual_tokens,P1 的附加字段也全留,供控制台
|
||||
# 同时显示美元与无价 token 两条进度;402 时控制台据
|
||||
# 响应字段区分“美元配额已满”与“无价 token 已满”。
|
||||
payload = self._quota_payload(
|
||||
user_id, quota, used, month,
|
||||
self._quota_unpriced_tokens(user_id))
|
||||
payload["unpriced_tokens_used"] = self._quota_unpriced_tokens(
|
||||
user_id, month)
|
||||
payload["unpriced_token_cap"] = unpriced_token_cap()
|
||||
self._send_json(200, payload)
|
||||
|
||||
def _admin_quota_get(self, user_id, query):
|
||||
if not self._is_global_admin(user_id):
|
||||
@@ -4204,10 +4379,20 @@ class CloudControlPlaneHandler(BaseHTTPRequestHandler):
|
||||
|
||||
# -- worker 池 -----------------------------------------------------
|
||||
def _workers_list(self, user_id):
|
||||
"""列出注册 worker:任何已认证用户可读(worker 是基础设施
|
||||
信息,不属于某个项目的数据,不按项目成员过滤)。"""
|
||||
"""列出注册 worker:全局管理员看完整清单;其他已认证用户
|
||||
只看在线/离线聚合(P1 收紧:worker_id、执行器、容量、
|
||||
心跳时间等内部细节不对普通用户暴露,控制台的 Workers
|
||||
区据此只显示聚合数字)。"""
|
||||
workers = _registry_list(self.server.db_path)
|
||||
self._send_json(200, {"workers": workers})
|
||||
if self._is_global_admin(user_id):
|
||||
self._send_json(200, {"workers": workers})
|
||||
return
|
||||
online = sum(1 for w in workers if w.get("alive"))
|
||||
self._send_json(200, {
|
||||
"workers_online": online,
|
||||
"workers_offline": len(workers) - online,
|
||||
"workers_total": len(workers),
|
||||
})
|
||||
|
||||
def _workers_heartbeat(self, user_id):
|
||||
"""worker 心跳:body 含 worker_id(必填,缺了 400)与可选
|
||||
@@ -4429,11 +4614,15 @@ class CloudControlPlaneHandler(BaseHTTPRequestHandler):
|
||||
"""POST /admin/backup:备份当前后端数据库,返回路径与规模信息。
|
||||
|
||||
权限:只认 CODEX_GLOBAL_ADMINS(见 _is_global_admin),
|
||||
未配置时一律 403。返回只含 backend、备份文件路径、
|
||||
文件大小与任务行数,绝不返回数据库密码、备份文件内容
|
||||
或任何凭据。out_dir 可由 body 指定,否则走环境变量
|
||||
CLOUD_BACKUP_DIR,未设则用 cloud-backups。成功与被拒
|
||||
都记审计(审计里只记路径与规模,不记库内容)。
|
||||
未配置时一律 403;非运维调用方根本到不了这里,绝对
|
||||
路径只回给全局管理员。out_dir 限白名单目录(P1 收紧):
|
||||
白名单根来自 CLOUD_BACKUP_ROOT(逗号分隔多根),未设
|
||||
则 CLOUD_BACKUP_DIR,未设则固定根 cloud-backups;请求
|
||||
的 out_dir 经实路径解析后必须等于某根或在其下,含 ..
|
||||
逃逸、指向 /、/etc、/opt 等系统目录的一律 400(白名单
|
||||
外的绝对路径即使存在也不放行)。备份文件落盘后 chmod
|
||||
0600。成功与被拒都记审计(调用人由审计的 user_id 列
|
||||
记,detail 含目标目录与大小,不记库内容)。
|
||||
"""
|
||||
if not self._is_global_admin(user_id):
|
||||
self._audit_admin(user_id, "admin_backup_denied", "backup")
|
||||
@@ -4443,11 +4632,33 @@ class CloudControlPlaneHandler(BaseHTTPRequestHandler):
|
||||
if err:
|
||||
self._send_json(400, {"error": err})
|
||||
return
|
||||
out_dir = body.get("out_dir") or os.environ.get(
|
||||
"CLOUD_BACKUP_DIR") or "cloud-backups"
|
||||
if not isinstance(out_dir, str) or not out_dir:
|
||||
requested = body.get("out_dir")
|
||||
if requested is not None and (
|
||||
not isinstance(requested, str) or not requested):
|
||||
self._send_json(400, {"error": "out_dir must be a string"})
|
||||
return
|
||||
roots_raw = os.environ.get("CLOUD_BACKUP_ROOT", "")
|
||||
if roots_raw.strip():
|
||||
roots = [os.path.realpath(p.strip())
|
||||
for p in roots_raw.split(",") if p.strip()]
|
||||
else:
|
||||
roots = [os.path.realpath(
|
||||
os.environ.get("CLOUD_BACKUP_DIR") or "cloud-backups")]
|
||||
if requested:
|
||||
candidate = os.path.realpath(requested)
|
||||
else:
|
||||
candidate = roots[0]
|
||||
allowed = any(candidate == root
|
||||
or candidate.startswith(root + os.sep)
|
||||
for root in roots)
|
||||
if not allowed:
|
||||
self._audit_admin(
|
||||
user_id, "admin_backup_denied",
|
||||
"out_dir outside backup roots: %s" % requested)
|
||||
self._send_json(400, {
|
||||
"error": "out_dir outside allowed backup roots"})
|
||||
return
|
||||
out_dir = candidate
|
||||
backend = _db_backend_name()
|
||||
try:
|
||||
if backend == "postgres":
|
||||
@@ -4456,6 +4667,10 @@ class CloudControlPlaneHandler(BaseHTTPRequestHandler):
|
||||
else:
|
||||
from cloud.backup import backup_db as _backup_sqlite
|
||||
dest = _backup_sqlite(self.server.db_path, out_dir, keep=3)
|
||||
try:
|
||||
os.chmod(dest, 0o600)
|
||||
except OSError:
|
||||
pass
|
||||
size_bytes = os.path.getsize(dest)
|
||||
conn = self._db()
|
||||
try:
|
||||
@@ -4468,8 +4683,8 @@ class CloudControlPlaneHandler(BaseHTTPRequestHandler):
|
||||
return
|
||||
self._audit_admin(
|
||||
user_id, "admin_backup",
|
||||
"backend=%s path=%s size=%d tasks=%d"
|
||||
% (backend, dest, size_bytes, task_count))
|
||||
"caller=%s backend=%s out_dir=%s path=%s size=%d tasks=%d"
|
||||
% (user_id, backend, out_dir, dest, size_bytes, task_count))
|
||||
self._send_json(200, {
|
||||
"backend": backend, "path": dest,
|
||||
"size_bytes": size_bytes, "task_count": task_count,
|
||||
@@ -5165,6 +5380,9 @@ class CloudControlPlaneHandler(BaseHTTPRequestHandler):
|
||||
label = "api-key"
|
||||
with _auth_store(self.server.db_path) as auth:
|
||||
key_id, api_key_plain = auth.create_api_key(user_id, label)
|
||||
# 审计只记 key_id 与 label,绝不记明文与哈希。
|
||||
auth.audit(user_id, "", "api_key_created",
|
||||
"key_id=%s label=%s" % (key_id, label))
|
||||
self._send_json(201, {"id": key_id, "label": label,
|
||||
"hint": api_key_plain[-4:],
|
||||
"api_key": api_key_plain})
|
||||
@@ -5182,6 +5400,9 @@ class CloudControlPlaneHandler(BaseHTTPRequestHandler):
|
||||
return
|
||||
with _auth_store(self.server.db_path) as auth:
|
||||
ok = auth.revoke_api_key(user_id, key_id)
|
||||
if ok:
|
||||
auth.audit(user_id, "", "api_key_revoked",
|
||||
"key_id=%s" % key_id)
|
||||
if not ok:
|
||||
self._send_json(404, {"error": "key not found"})
|
||||
return
|
||||
@@ -5199,8 +5420,45 @@ class CloudControlPlaneHandler(BaseHTTPRequestHandler):
|
||||
keep_hash = hashlib.sha256(api_key.encode()).hexdigest()
|
||||
with _auth_store(self.server.db_path) as auth:
|
||||
count = auth.revoke_other_api_keys(user_id, keep_hash)
|
||||
# 只记撤销数,不记任何 key 明文、哈希或保留 key 的身份。
|
||||
auth.audit(user_id, "", "api_key_revoke_others",
|
||||
"revoked_count=%d" % count)
|
||||
self._send_json(200, {"revoked_count": count})
|
||||
|
||||
def _user_gateway_rotate(self):
|
||||
"""POST /users/me/gateway/rotate:仅限本人重新开通/轮换
|
||||
网关 key。非 sub2api 网关时 400;网关不可用时 502 且
|
||||
旧行不动(rotate 内部先删旧行再重开,只在重开失败时
|
||||
旧 key 已丢——调用方须知:轮换是显式动作,失败后须
|
||||
重试本端点重新开通)。审计只记 sub2api_user_id 与
|
||||
key_id,绝不记 key 明文;响应不含 key。"""
|
||||
user_id = self._authenticate()
|
||||
if user_id is None:
|
||||
self._send_json(401, {"error": "unauthorized"})
|
||||
return
|
||||
if _gateway_name() != "sub2api":
|
||||
self._send_json(400, {"error": "gateway is not sub2api"})
|
||||
return
|
||||
conn = self._db()
|
||||
try:
|
||||
try:
|
||||
status, _key, sub_id, key_id = _rotate_sub2api_key(
|
||||
conn, user_id)
|
||||
except _Sub2apiError as exc:
|
||||
self._send_json(502, {"error": str(exc)})
|
||||
return
|
||||
except Exception as exc:
|
||||
self._send_json(502, {"error": str(exc)})
|
||||
return
|
||||
finally:
|
||||
conn.close()
|
||||
with _auth_store(self.server.db_path) as store:
|
||||
store.audit(user_id, "", "gateway_key_rotated",
|
||||
"sub2api_user_id=%s key_id=%s"
|
||||
% (sub_id, key_id))
|
||||
self._send_json(200, {"rotated": True, "provisioning": status,
|
||||
"sub2api_user_id": sub_id})
|
||||
|
||||
# -- 推送订阅与分发 ---------------------------------------------------
|
||||
def _push_create_endpoint(self, user_id, project_id):
|
||||
body, err = self._read_json()
|
||||
@@ -5215,6 +5473,12 @@ class CloudControlPlaneHandler(BaseHTTPRequestHandler):
|
||||
except ValueError as e:
|
||||
self._send_json(400, {"error": str(e)})
|
||||
return
|
||||
# 审计只记订阅 id 与通道:target 全文不记(Slack webhook
|
||||
# URL 本身含 secret 路径,记进审计等于把 secret 落盘)。
|
||||
with _auth_store(self.server.db_path) as store:
|
||||
store.audit(user_id, project_id, "push_subscription_created",
|
||||
"subscription_id=%s channel=%s"
|
||||
% (sub["id"], sub["channel"]))
|
||||
self._send_json(201, sub)
|
||||
|
||||
def _push_dispatch_endpoint(self, user_id):
|
||||
@@ -5228,8 +5492,9 @@ class CloudControlPlaneHandler(BaseHTTPRequestHandler):
|
||||
self._send_json(400, {"error": "event_type and project_id required"})
|
||||
return
|
||||
# P0 安全批后:/admin/push/dispatch 是管理端点,只认
|
||||
# CODEX_GLOBAL_ADMINS,不再只凭项目 read 就能让服务端
|
||||
# 向订阅目标(可能含内网地址)真 POST。被拒与成功都记审计。
|
||||
# CODEX_GLOBAL_ADMINS——该口径已强于 P1 要求的 create_task,
|
||||
# 不再降级。被拒与成功都记审计;目标 SSRF 校验在
|
||||
# push.dispatch_event / 真实发送层做。
|
||||
if not self._is_global_admin(user_id):
|
||||
self._audit_admin(user_id, "admin_push_denied",
|
||||
"dispatch %s %s" % (event_type, project_id))
|
||||
@@ -5258,11 +5523,21 @@ class CloudControlPlaneHandler(BaseHTTPRequestHandler):
|
||||
"""创建自动化定义:只登记定义与到期状态,不跑模型、
|
||||
不另起调度器(执行器后续走 claim 门,见 automations
|
||||
模块文档)。body:name、prompt 必填;interval_seconds
|
||||
可选(默认 86400,最小 60);repo_url 可选。"""
|
||||
可选(默认 86400,最小 60);repo_url 可选。
|
||||
|
||||
P1 配额门:创建时先过与建任务同一道配额门(美元配额
|
||||
与无价 token cap,超限 402),避免“定义免费建、到期
|
||||
无限跑”;单自动化的 spend 上限即创建人配额本身——
|
||||
每次到期产出的任务以 created_by=创建人建任务,走
|
||||
POST /tasks 的配额门(触发路径若绕过服务端建任务,
|
||||
则不在本门覆盖内,见残留)。本服务端没有自动化触发
|
||||
端点,触发侧的门不在本次 server.py 改动内。"""
|
||||
body, err = self._read_json()
|
||||
if err:
|
||||
self._send_json(400, {"error": err})
|
||||
return
|
||||
if self._reject_if_over_quota(user_id):
|
||||
return
|
||||
interval = body.get("interval_seconds", 86400)
|
||||
if interval is None:
|
||||
interval = 86400
|
||||
|
||||
+101
-8
@@ -441,8 +441,28 @@ def list_gateway_models(origin, api_key, request=None):
|
||||
return out
|
||||
|
||||
|
||||
def ensure_user_gateway_key(conn, user_id, admin=None, request=None):
|
||||
"""幂等开通:已有行直接返回 key;否则建号->登录->发 key->授余额->入库。
|
||||
def gateway_key_health(origin, api_key, request=None):
|
||||
"""真健康检查:用已有 key 调一次 /v1/models。
|
||||
|
||||
返回 True(200,可用)/ False(401/403:key 已失效,或余额
|
||||
为 0——网关在余额为 0 时鉴权层直接 403,两者都算死 key)/
|
||||
"unknown"(其它状态或网络异常:网关本身不可用时不据此轮换,
|
||||
避免把一次抖动当成 key 死亡、反复删号重建)。
|
||||
"""
|
||||
req = request or _default_request
|
||||
try:
|
||||
st, _data = req("GET", origin + "/v1/models", token=api_key)
|
||||
except Exception:
|
||||
return "unknown"
|
||||
if st == 200:
|
||||
return True
|
||||
if st in (401, 403):
|
||||
return False
|
||||
return "unknown"
|
||||
|
||||
|
||||
def _create_fresh_gateway_key(conn, user_id, admin=None, request=None):
|
||||
"""建号->登录->发 key->授余额->入库,返回 (key, sub_id, key_id)。
|
||||
|
||||
入库失败时尽力清理刚建的 key 与账号(避免孤儿 key),再抛原错;
|
||||
授余额失败同样走这条清理(余额为 0 的 key 在网关侧直接 403,
|
||||
@@ -451,11 +471,9 @@ def ensure_user_gateway_key(conn, user_id, admin=None, request=None):
|
||||
(同一进程内第二次起免管理员登录,见 shared_admin_session)。
|
||||
调用方负责 conn 的提交语义:本函数在入库后
|
||||
commit(开通是独立事务,不跟调用方的业务事务绑在一起)。
|
||||
调用方须保证调用前本用户无有效行(ensure/rotate 先处理
|
||||
旧行),否则会主键冲突(并发撞号除外,走轮询复用)。
|
||||
"""
|
||||
ensure_gateway_keys_table(conn)
|
||||
row = get_user_gateway_key(conn, user_id)
|
||||
if row is not None:
|
||||
return row["api_key"]
|
||||
origin = gateway_origin()
|
||||
if not origin:
|
||||
raise Sub2apiError("SUB2API_BASE_URL 未配置,无法开通网关账号")
|
||||
@@ -476,9 +494,16 @@ def ensure_user_gateway_key(conn, user_id, admin=None, request=None):
|
||||
time.sleep(2)
|
||||
row = get_user_gateway_key(conn, user_id)
|
||||
if row is not None:
|
||||
return row["api_key"]
|
||||
# 合并修复(P1 rebase 到 d254240):main 的 #92 竞态
|
||||
# 分支原在单体 ensure 里只回一个 key 字符串;P1
|
||||
# 抽出 _create_fresh 后契约是 3 元组,4 个调用点
|
||||
# 都按 3 元组解包,裸字符串会被当成字符序列解包
|
||||
# 而 ValueError。这里按契约回 3 元组,key_id 未知
|
||||
# 故为 None(与 validate_or_rotate 的 ok 路一致)。
|
||||
return (row["api_key"], row["sub2api_user_id"], None)
|
||||
raise
|
||||
key_id = None
|
||||
user_token = None
|
||||
try:
|
||||
user_token, refresh = user_login(origin, email, password,
|
||||
request=req)
|
||||
@@ -497,7 +522,7 @@ def ensure_user_gateway_key(conn, user_id, admin=None, request=None):
|
||||
(user_id, sub_id, key, refresh, now, now))
|
||||
conn.commit()
|
||||
except Exception:
|
||||
if key_id is not None:
|
||||
if key_id is not None and user_token is not None:
|
||||
try:
|
||||
user_delete_key(origin, user_token, key_id, request=req)
|
||||
except Exception:
|
||||
@@ -507,4 +532,72 @@ def ensure_user_gateway_key(conn, user_id, admin=None, request=None):
|
||||
except Exception:
|
||||
pass
|
||||
raise
|
||||
return key, sub_id, key_id
|
||||
|
||||
|
||||
def ensure_user_gateway_key(conn, user_id, admin=None, request=None):
|
||||
"""幂等开通:已有行直接返回 key;否则走 _create_fresh 开通。
|
||||
|
||||
只保证“行存在”,不保证 key 仍有效:要校验有效性用
|
||||
validate_or_rotate_gateway_key(死 key 会触发轮换重开)。
|
||||
"""
|
||||
ensure_gateway_keys_table(conn)
|
||||
row = get_user_gateway_key(conn, user_id)
|
||||
if row is not None:
|
||||
return row["api_key"]
|
||||
key, _sub_id, _key_id = _create_fresh_gateway_key(
|
||||
conn, user_id, admin=admin, request=request)
|
||||
return key
|
||||
|
||||
|
||||
def validate_or_rotate_gateway_key(conn, user_id, admin=None, request=None):
|
||||
"""懒开通/健康检查合一:返回 (status, key, sub_id, key_id)。
|
||||
|
||||
status:ok(旧 key 经 /v1/models 实测可用)/ provisioned
|
||||
(无行,新开通)/ rotated(旧 key 经实测已死:401/403,
|
||||
含余额为 0 的 403;删旧行后重开,死 key 不再被当成已开通)。
|
||||
网关本身不可用(健康检查 unknown)时抛 Sub2apiError,调用方
|
||||
应回 502、保留旧行不轮换;绝不在此时把“待开通”当成可用。
|
||||
"""
|
||||
ensure_gateway_keys_table(conn)
|
||||
row = get_user_gateway_key(conn, user_id)
|
||||
origin = gateway_origin()
|
||||
if not origin:
|
||||
raise Sub2apiError("SUB2API_BASE_URL 未配置,无法开通网关账号")
|
||||
if row is None or not row.get("api_key"):
|
||||
key, sub_id, key_id = _create_fresh_gateway_key(
|
||||
conn, user_id, admin=admin, request=request)
|
||||
return "provisioned", key, sub_id, key_id
|
||||
health = gateway_key_health(origin, row["api_key"], request=request)
|
||||
if health is True:
|
||||
return "ok", row["api_key"], row.get("sub2api_user_id"), None
|
||||
if health is False:
|
||||
conn.execute(
|
||||
"DELETE FROM user_gateway_keys WHERE user_id = ?",
|
||||
(user_id,))
|
||||
conn.commit()
|
||||
key, sub_id, key_id = _create_fresh_gateway_key(
|
||||
conn, user_id, admin=admin, request=request)
|
||||
return "rotated", key, sub_id, key_id
|
||||
raise Sub2apiError("sub2api 网关健康检查暂不可用,保留现有 key")
|
||||
|
||||
|
||||
def rotate_user_gateway_key(conn, user_id, admin=None, request=None):
|
||||
"""仅限本人调用的重新开通/轮换:删旧行后重开。
|
||||
|
||||
无旧行时直接走开通(与 validate 的 provisioned 等价);返回
|
||||
(status, key, sub_id, key_id),调用方记审计时只可用 sub_id /
|
||||
key_id,绝不记 key 明文。旧 key 即使只是“疑似失效”也会被
|
||||
换掉:轮换是用户显式动作,不以健康检查为前提。
|
||||
"""
|
||||
ensure_gateway_keys_table(conn)
|
||||
row = get_user_gateway_key(conn, user_id)
|
||||
if row is not None:
|
||||
conn.execute(
|
||||
"DELETE FROM user_gateway_keys WHERE user_id = ?",
|
||||
(user_id,))
|
||||
conn.commit()
|
||||
key, sub_id, key_id = _create_fresh_gateway_key(
|
||||
conn, user_id, admin=admin, request=request)
|
||||
return ("rotated" if row is not None else "provisioned",
|
||||
key, sub_id, key_id)
|
||||
@@ -33,6 +33,9 @@ class CapsServerTestCase(unittest.TestCase):
|
||||
self.auth = AuthStore(self.db_path)
|
||||
self.owner_id, self.owner_key = self.auth.create_user("owner")
|
||||
self.viewer_id, self.viewer_key = self.auth.create_user("viewer")
|
||||
# P1:完整 workers 清单只限全局管理员;owner 在此测试里即管理员。
|
||||
self._old_admins = os.environ.get("CODEX_GLOBAL_ADMINS")
|
||||
os.environ["CODEX_GLOBAL_ADMINS"] = self.owner_id
|
||||
self.server = create_server(self.db_path, "127.0.0.1", 0)
|
||||
self.port = self.server.server_address[1]
|
||||
self.thread = threading.Thread(
|
||||
@@ -48,6 +51,10 @@ class CapsServerTestCase(unittest.TestCase):
|
||||
except Exception:
|
||||
pass
|
||||
self.tmp.cleanup()
|
||||
if getattr(self, "_old_admins", None) is None:
|
||||
os.environ.pop("CODEX_GLOBAL_ADMINS", None)
|
||||
else:
|
||||
os.environ["CODEX_GLOBAL_ADMINS"] = self._old_admins
|
||||
|
||||
def _req(self, method, path, body=None, key=None):
|
||||
conn = http.client.HTTPConnection("127.0.0.1", self.port, timeout=10)
|
||||
@@ -82,11 +89,21 @@ class CapsServerTestCase(unittest.TestCase):
|
||||
"capacity": 4, "running": 2},
|
||||
self.owner_key)
|
||||
self.assertEqual(status, 200, body)
|
||||
# 读权限:任何已认证用户可读(viewer 也能看到基础设施行)。
|
||||
workers = self._workers(self.viewer_key)
|
||||
# P1:完整清单只限全局管理员(owner 在此测试里即管理员)。
|
||||
workers = self._workers(self.owner_key)
|
||||
self.assertEqual(workers["w1"]["capacity"], 4)
|
||||
self.assertEqual(workers["w1"]["running"], 2)
|
||||
|
||||
def test_workers_list_non_admin_gets_aggregate_only(self):
|
||||
self._req("POST", "/workers/heartbeat",
|
||||
{"worker_id": "w1", "executor": "local",
|
||||
"capacity": 4, "running": 2},
|
||||
self.owner_key)
|
||||
status, body = self._req("GET", "/workers", key=self.viewer_key)
|
||||
self.assertEqual(status, 200, body)
|
||||
self.assertNotIn("workers", body)
|
||||
self.assertIn("workers_total", body)
|
||||
|
||||
def test_heartbeat_without_running_preserves_old(self):
|
||||
self._req("POST", "/workers/heartbeat",
|
||||
{"worker_id": "w1", "capacity": 2, "running": 1},
|
||||
|
||||
@@ -184,7 +184,16 @@ class WorkerRegistryTestCase(_ServerCase):
|
||||
self.assertEqual(status, 200, body)
|
||||
self.assertTrue(body["alive"])
|
||||
self.assertEqual(body["executor"], "docker")
|
||||
status, body = self._req("GET", "/workers", None, self.owner_key)
|
||||
# P1:完整 workers 清单只限全局管理员,本测试里 owner 即管理员。
|
||||
old_admins = os.environ.get("CODEX_GLOBAL_ADMINS")
|
||||
os.environ["CODEX_GLOBAL_ADMINS"] = self.owner_id
|
||||
try:
|
||||
status, body = self._req("GET", "/workers", None, self.owner_key)
|
||||
finally:
|
||||
if old_admins is None:
|
||||
os.environ.pop("CODEX_GLOBAL_ADMINS", None)
|
||||
else:
|
||||
os.environ["CODEX_GLOBAL_ADMINS"] = old_admins
|
||||
self.assertEqual(status, 200, body)
|
||||
self.assertEqual(len(body["workers"]), 1)
|
||||
self.assertTrue(body["workers"][0]["alive"])
|
||||
|
||||
@@ -95,13 +95,23 @@ class ServerGithubReposTestCase(unittest.TestCase):
|
||||
|
||||
def test_list_returns_seeded_installations(self):
|
||||
self._seed_installations()
|
||||
# P1:repos 只返回调用者成员项目对应的仓库;先把项目仓库
|
||||
# 绑到其中一个已安装仓库,再断言只返回该仓库,非成员为空。
|
||||
s, d = _req(self.port, "PATCH",
|
||||
"/projects/" + self.project_id,
|
||||
{"repo_url": "https://github.com/openqj/codex-app-server.git"},
|
||||
self.owner_key)
|
||||
self.assertEqual(s, 200, d)
|
||||
s, d = _req(self.port, "GET", "/github/repos", key=self.owner_key)
|
||||
self.assertEqual(s, 200, d)
|
||||
names = [r["repo_full_name"] for r in d["repos"]]
|
||||
self.assertEqual(names, ["openqj/codex-app-server", "openqj/demo"])
|
||||
self.assertEqual(names, ["openqj/codex-app-server"])
|
||||
self.assertEqual(
|
||||
d["repos"][0]["clone_url"],
|
||||
"https://github.com/openqj/codex-app-server.git")
|
||||
s, d = _req(self.port, "GET", "/github/repos", key=self.other_key)
|
||||
self.assertEqual(s, 200, d)
|
||||
self.assertEqual(d["repos"], [])
|
||||
|
||||
def test_patch_repo_url_owner(self):
|
||||
url = "https://github.com/openqj/demo.git"
|
||||
|
||||
@@ -0,0 +1,598 @@
|
||||
"""P1 控制面/控制台单测(server.py / console.html / sub2api.py / push.py)。
|
||||
|
||||
只覆盖本批 P1 控制面改动,不跑真实模型回合、不联网、不碰生产:
|
||||
server 侧走本地临时端口 + 临时 SQLite;网关侧用假 request /
|
||||
假 admin(key、token 全是测试假值);推送 DNS 用注入的
|
||||
resolver。绝不改 worker.py / coordinator.py / cost.py / host.py。
|
||||
"""
|
||||
|
||||
import http.client
|
||||
import json
|
||||
import os
|
||||
import sqlite3
|
||||
import stat
|
||||
import sys
|
||||
import tempfile
|
||||
import threading
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
from unittest import mock
|
||||
|
||||
try:
|
||||
from cloud.auth import AuthStore
|
||||
from cloud.push import dispatch_event, validate_push_target
|
||||
from cloud.push import create_subscription
|
||||
from cloud.server import create_server
|
||||
from cloud import server as server_mod
|
||||
from cloud.sub2api import Sub2apiError
|
||||
from cloud.sub2api import ensure_gateway_keys_table
|
||||
from cloud.sub2api import get_user_gateway_key
|
||||
from cloud.sub2api import rotate_user_gateway_key
|
||||
from cloud.sub2api import validate_or_rotate_gateway_key
|
||||
except ImportError:
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[2]))
|
||||
from cloud.auth import AuthStore
|
||||
from cloud.push import dispatch_event, validate_push_target
|
||||
from cloud.push import create_subscription
|
||||
from cloud.server import create_server
|
||||
from cloud import server as server_mod
|
||||
from cloud.sub2api import Sub2apiError
|
||||
from cloud.sub2api import ensure_gateway_keys_table
|
||||
from cloud.sub2api import get_user_gateway_key
|
||||
from cloud.sub2api import rotate_user_gateway_key
|
||||
from cloud.sub2api import validate_or_rotate_gateway_key
|
||||
|
||||
|
||||
def _req(port, method, path, body=None, key=None, raw_body=None):
|
||||
conn = http.client.HTTPConnection("127.0.0.1", port, timeout=10)
|
||||
hdrs = {}
|
||||
if raw_body is not None:
|
||||
payload = raw_body
|
||||
elif body is not None:
|
||||
payload = json.dumps(body).encode("utf-8")
|
||||
hdrs["Content-Type"] = "application/json"
|
||||
else:
|
||||
payload = None
|
||||
if key is not None:
|
||||
hdrs["X-API-Key"] = key
|
||||
try:
|
||||
conn.request(method, path, body=payload, headers=hdrs)
|
||||
resp = conn.getresponse()
|
||||
data = resp.read()
|
||||
finally:
|
||||
conn.close()
|
||||
try:
|
||||
parsed = json.loads(data.decode("utf-8")) if data else None
|
||||
except ValueError:
|
||||
parsed = data.decode("utf-8", "replace")
|
||||
return resp.status, parsed
|
||||
|
||||
|
||||
class _FakeGatewayAdmin:
|
||||
"""假网关管理员:建号/授余额只记调用,不联网。"""
|
||||
|
||||
def __init__(self):
|
||||
self.created = []
|
||||
self.deleted = []
|
||||
self.balances = []
|
||||
|
||||
def create_user(self, email, password, username):
|
||||
sub_id = 1000 + len(self.created)
|
||||
self.created.append((email, username))
|
||||
return sub_id
|
||||
|
||||
def delete_user(self, sub_user_id):
|
||||
self.deleted.append(sub_user_id)
|
||||
return True
|
||||
|
||||
def set_balance(self, sub_user_id, amount):
|
||||
self.balances.append((sub_user_id, amount))
|
||||
|
||||
def resolve_group_id(self):
|
||||
return 2
|
||||
|
||||
|
||||
class _FakeGatewayRequest:
|
||||
"""假网关 HTTP:models 状态可调,其余走固定成功路径。"""
|
||||
|
||||
def __init__(self, models_status=200):
|
||||
self.models_status = models_status
|
||||
self.calls = []
|
||||
|
||||
def __call__(self, method, url, body=None, token=None):
|
||||
self.calls.append((method, url))
|
||||
if url.endswith("/v1/models"):
|
||||
return self.models_status, {"data": []}
|
||||
if url.endswith("/api/v1/auth/login"):
|
||||
return 200, {"data": {"access_token": "fake-user-token",
|
||||
"refresh_token": "fake-refresh",
|
||||
"expires_in": 3600}}
|
||||
if url.endswith("/api/v1/keys"):
|
||||
return 201, {"data": {"id": "fake-key-id-1",
|
||||
"key": "fake-gateway-key-1"}}
|
||||
if "/balance" in url:
|
||||
return 200, {"data": {}}
|
||||
return 200, {"data": {}}
|
||||
|
||||
|
||||
class Sub2apiP1TestCase(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.conn = sqlite3.connect(":memory:")
|
||||
ensure_gateway_keys_table(self.conn)
|
||||
self.admin = _FakeGatewayAdmin()
|
||||
# 假 origin:请求本身全是 fake,不产生真实网络调用。
|
||||
self._old_origin = os.environ.get("SUB2API_BASE_URL")
|
||||
os.environ["SUB2API_BASE_URL"] = "http://127.0.0.1:9"
|
||||
|
||||
def tearDown(self):
|
||||
self.conn.close()
|
||||
if self._old_origin is None:
|
||||
os.environ.pop("SUB2API_BASE_URL", None)
|
||||
else:
|
||||
os.environ["SUB2API_BASE_URL"] = self._old_origin
|
||||
|
||||
def _seed_row(self, key="fake-old-key"):
|
||||
self.conn.execute(
|
||||
"INSERT INTO user_gateway_keys(user_id, sub2api_user_id,"
|
||||
" api_key, refresh_token, created_at, updated_at)"
|
||||
" VALUES(?, ?, ?, ?, ?, ?)",
|
||||
("u1", 42, key, "fake-refresh", 1.0, 1.0))
|
||||
self.conn.commit()
|
||||
|
||||
def test_validate_no_row_provisions(self):
|
||||
req = _FakeGatewayRequest()
|
||||
status, key, sub_id, key_id = validate_or_rotate_gateway_key(
|
||||
self.conn, "u1", admin=self.admin, request=req)
|
||||
self.assertEqual(status, "provisioned")
|
||||
self.assertEqual(key, "fake-gateway-key-1")
|
||||
self.assertEqual(key_id, "fake-key-id-1")
|
||||
self.assertIsNotNone(get_user_gateway_key(self.conn, "u1"))
|
||||
|
||||
def test_validate_healthy_key_not_rotated(self):
|
||||
self._seed_row()
|
||||
req = _FakeGatewayRequest(models_status=200)
|
||||
status, key, _sub, _kid = validate_or_rotate_gateway_key(
|
||||
self.conn, "u1", admin=self.admin, request=req)
|
||||
self.assertEqual(status, "ok")
|
||||
self.assertEqual(key, "fake-old-key")
|
||||
self.assertEqual(self.admin.created, [])
|
||||
|
||||
def test_validate_dead_key_rotates_not_pending(self):
|
||||
self._seed_row()
|
||||
req = _FakeGatewayRequest(models_status=403)
|
||||
status, key, _sub, _kid = validate_or_rotate_gateway_key(
|
||||
self.conn, "u1", admin=self.admin, request=req)
|
||||
self.assertEqual(status, "rotated")
|
||||
self.assertEqual(key, "fake-gateway-key-1")
|
||||
row = get_user_gateway_key(self.conn, "u1")
|
||||
self.assertEqual(row["api_key"], "fake-gateway-key-1")
|
||||
self.assertNotEqual(row["api_key"], "fake-old-key")
|
||||
|
||||
def test_validate_zero_balance_403_rotates(self):
|
||||
# 余额为 0 时网关鉴权层回 403:与 401 同属死 key,必须轮换。
|
||||
self._seed_row()
|
||||
req = _FakeGatewayRequest(models_status=403)
|
||||
status, _key, _sub, _kid = validate_or_rotate_gateway_key(
|
||||
self.conn, "u1", admin=self.admin, request=req)
|
||||
self.assertEqual(status, "rotated")
|
||||
|
||||
def test_validate_gateway_down_keeps_old_row(self):
|
||||
self._seed_row()
|
||||
req = _FakeGatewayRequest(models_status=500)
|
||||
with self.assertRaises(Sub2apiError):
|
||||
validate_or_rotate_gateway_key(
|
||||
self.conn, "u1", admin=self.admin, request=req)
|
||||
row = get_user_gateway_key(self.conn, "u1")
|
||||
self.assertEqual(row["api_key"], "fake-old-key")
|
||||
|
||||
def test_rotate_replaces_row(self):
|
||||
self._seed_row()
|
||||
req = _FakeGatewayRequest()
|
||||
status, key, sub_id, key_id = rotate_user_gateway_key(
|
||||
self.conn, "u1", admin=self.admin, request=req)
|
||||
self.assertEqual(status, "rotated")
|
||||
self.assertEqual(key_id, "fake-key-id-1")
|
||||
self.assertEqual(
|
||||
get_user_gateway_key(self.conn, "u1")["api_key"], key)
|
||||
|
||||
|
||||
class PushP1TestCase(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.conn = sqlite3.connect(":memory:")
|
||||
|
||||
def tearDown(self):
|
||||
self.conn.close()
|
||||
|
||||
def test_literal_private_and_loopback_blocked(self):
|
||||
for target in (
|
||||
"http://127.0.0.1:9/hook",
|
||||
"http://10.0.0.5/hook",
|
||||
"http://172.16.4.4/hook",
|
||||
"http://192.168.1.10/hook",
|
||||
"http://169.254.169.254/latest/meta-data/",
|
||||
"http://[::1]/hook"):
|
||||
self.assertIsNotNone(
|
||||
validate_push_target(target), target)
|
||||
|
||||
def test_metadata_hostname_blocked(self):
|
||||
self.assertIsNotNone(validate_push_target(
|
||||
"http://metadata.google.internal/hook"))
|
||||
|
||||
def test_hostname_resolved_to_private_blocked(self):
|
||||
err = validate_push_target(
|
||||
"https://hooks.example.com/hook",
|
||||
resolver=lambda host: ["10.1.2.3"])
|
||||
self.assertIsNotNone(err)
|
||||
|
||||
def test_hostname_public_resolves_ok(self):
|
||||
err = validate_push_target(
|
||||
"https://hooks.example.com/hook",
|
||||
resolver=lambda host: ["93.184.216.34"])
|
||||
self.assertIsNone(err)
|
||||
|
||||
def test_dns_failure_rejected(self):
|
||||
def _boom(host):
|
||||
raise OSError("no dns in test")
|
||||
self.assertIsNotNone(validate_push_target(
|
||||
"https://hooks.example.com/hook", resolver=_boom))
|
||||
|
||||
def test_dispatch_loopback_never_posts(self):
|
||||
# 真实发送路径(sender=None,生产服务端走的路径):
|
||||
# loopback 目标在预检即被挡,不产生任何 POST。
|
||||
create_subscription(self.conn, "proj-a", "user-1", "webhook",
|
||||
"http://127.0.0.1:9/hook", "instant")
|
||||
result = dispatch_event(self.conn, "failed", "proj-a",
|
||||
{"task_id": "t1"}, sender=None)
|
||||
self.assertEqual(result["sent"], [])
|
||||
self.assertEqual(len(result["errors"]), 1)
|
||||
self.assertIn("blocked", result["errors"][0]["reason"])
|
||||
|
||||
|
||||
class ServerP1TestCase(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.tmp = tempfile.TemporaryDirectory()
|
||||
self.db_path = os.path.join(self.tmp.name, "control.db")
|
||||
self.auth = AuthStore(self.db_path)
|
||||
self.owner_id, self.owner_key = self.auth.create_user("owner")
|
||||
self.member_id, self.member_key = self.auth.create_user("member")
|
||||
self.outsider_id, self.outsider_key = self.auth.create_user(
|
||||
"outsider")
|
||||
self._old_admins = os.environ.get("CODEX_GLOBAL_ADMINS")
|
||||
self._old_gateway = os.environ.get("CLOUD_GATEWAY")
|
||||
self._old_backup_root = os.environ.get("CLOUD_BACKUP_ROOT")
|
||||
self._old_backup_dir = os.environ.get("CLOUD_BACKUP_DIR")
|
||||
os.environ["CODEX_GLOBAL_ADMINS"] = self.owner_id
|
||||
os.environ.pop("CLOUD_GATEWAY", None)
|
||||
os.environ.pop("CLOUD_BACKUP_ROOT", None)
|
||||
os.environ.pop("CLOUD_BACKUP_DIR", None)
|
||||
self._old_max_projects = server_mod.MAX_PROJECTS_PER_USER
|
||||
self.server = create_server(self.db_path, "127.0.0.1", 0)
|
||||
self.port = self.server.server_address[1]
|
||||
self.thread = threading.Thread(
|
||||
target=self.server.serve_forever, daemon=True)
|
||||
self.thread.start()
|
||||
s, proj = _req(self.port, "POST", "/projects",
|
||||
{"name": "owner/repo"}, self.owner_key)
|
||||
self.assertEqual(s, 201, proj)
|
||||
self.project_id = proj["id"]
|
||||
self.auth.set_member(
|
||||
self.project_id, self.owner_id, self.member_id, "maintainer")
|
||||
|
||||
def tearDown(self):
|
||||
self.server.shutdown()
|
||||
self.server.server_close()
|
||||
self.thread.join(timeout=5)
|
||||
try:
|
||||
self.auth.conn.close()
|
||||
except Exception:
|
||||
pass
|
||||
self.tmp.cleanup()
|
||||
server_mod.MAX_PROJECTS_PER_USER = self._old_max_projects
|
||||
if self._old_admins is None:
|
||||
os.environ.pop("CODEX_GLOBAL_ADMINS", None)
|
||||
else:
|
||||
os.environ["CODEX_GLOBAL_ADMINS"] = self._old_admins
|
||||
if self._old_gateway is None:
|
||||
os.environ.pop("CLOUD_GATEWAY", None)
|
||||
else:
|
||||
os.environ["CLOUD_GATEWAY"] = self._old_gateway
|
||||
if self._old_backup_root is None:
|
||||
os.environ.pop("CLOUD_BACKUP_ROOT", None)
|
||||
else:
|
||||
os.environ["CLOUD_BACKUP_ROOT"] = self._old_backup_root
|
||||
if self._old_backup_dir is None:
|
||||
os.environ.pop("CLOUD_BACKUP_DIR", None)
|
||||
else:
|
||||
os.environ["CLOUD_BACKUP_DIR"] = self._old_backup_dir
|
||||
# 恢复猴子补丁过的 server 模块函数(relay/rotate 相关测试用)。
|
||||
for name in ("_validate_sub2api_key", "_list_sub2api_models",
|
||||
"_rotate_sub2api_key"):
|
||||
try:
|
||||
mock.patch.object(server_mod, name).stop()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# -- helpers ---------------------------------------------------
|
||||
def _audit_actions(self, user_id=None):
|
||||
conn = sqlite3.connect(self.db_path)
|
||||
try:
|
||||
rows = conn.execute(
|
||||
"SELECT action, detail FROM audit_log"
|
||||
+ (" WHERE user_id = ?" if user_id else ""),
|
||||
(user_id,) if user_id else ()).fetchall()
|
||||
finally:
|
||||
conn.close()
|
||||
return rows
|
||||
|
||||
def _set_quota(self, target, quota):
|
||||
return _req(self.port, "POST", "/admin/quotas",
|
||||
{"user_id": target, "quota_usd_micros": quota},
|
||||
self.owner_key)
|
||||
|
||||
# -- 备份 -------------------------------------------------------
|
||||
def test_backup_whitelist_and_mode(self):
|
||||
root = os.path.join(self.tmp.name, "backup-root")
|
||||
os.environ["CLOUD_BACKUP_ROOT"] = root
|
||||
out_dir = os.path.join(root, "daily")
|
||||
s, body = _req(self.port, "POST", "/admin/backup",
|
||||
{"out_dir": out_dir}, self.owner_key)
|
||||
self.assertEqual(s, 200, body)
|
||||
self.assertTrue(os.path.exists(body["path"]))
|
||||
self.assertEqual(stat.S_IMODE(os.stat(body["path"]).st_mode),
|
||||
0o600)
|
||||
details = [d for _a, d in self._audit_actions(self.owner_id)]
|
||||
self.assertTrue(any("admin_backup" in a
|
||||
for a, _d in self._audit_actions(
|
||||
self.owner_id)))
|
||||
self.assertTrue(any(out_dir in (d or "")
|
||||
for d in details))
|
||||
# 白名单外:系统目录与 .. 逃逸一律 400。
|
||||
for bad in ("/etc", os.path.join(root, "..", "escape")):
|
||||
s, body = _req(self.port, "POST", "/admin/backup",
|
||||
{"out_dir": bad}, self.owner_key)
|
||||
self.assertEqual(s, 400, (bad, body))
|
||||
|
||||
# -- workers 聚合 ------------------------------------------------
|
||||
def test_workers_admin_full_member_aggregate_only(self):
|
||||
s, _b = _req(self.port, "POST", "/workers/heartbeat",
|
||||
{"worker_id": "w-p1", "executor": "docker",
|
||||
"capacity": 2, "running": 0}, self.owner_key)
|
||||
self.assertEqual(s, 200, _b)
|
||||
s, body = _req(self.port, "GET", "/workers", key=self.owner_key)
|
||||
self.assertEqual(s, 200)
|
||||
self.assertIn("workers", body)
|
||||
self.assertTrue(any(w.get("worker_id") == "w-p1"
|
||||
for w in body["workers"]))
|
||||
s, body = _req(self.port, "GET", "/workers", key=self.member_key)
|
||||
self.assertEqual(s, 200)
|
||||
self.assertNotIn("workers", body)
|
||||
self.assertNotIn("w-p1", json.dumps(body))
|
||||
self.assertEqual(body["workers_total"], 1)
|
||||
self.assertEqual(body["workers_online"], 1)
|
||||
self.assertEqual(body["workers_offline"], 0)
|
||||
|
||||
# -- github repos 过滤 -------------------------------------------
|
||||
def test_github_repos_filtered_by_member_projects(self):
|
||||
s, proj = _req(self.port, "POST", "/projects",
|
||||
{"name": "with-repo",
|
||||
"repo_url": "https://github.com/acme/alpha.git"},
|
||||
self.owner_key)
|
||||
self.assertEqual(s, 201, proj)
|
||||
self.auth.set_member(proj["id"], self.owner_id, self.member_id,
|
||||
"maintainer")
|
||||
conn = sqlite3.connect(self.db_path)
|
||||
try:
|
||||
conn.execute(
|
||||
"CREATE TABLE IF NOT EXISTS github_app_installations("
|
||||
"installation_id TEXT, account_login TEXT,"
|
||||
" repo_full_name TEXT, created_at REAL,"
|
||||
" UNIQUE(installation_id, repo_full_name))")
|
||||
conn.execute(
|
||||
"INSERT OR IGNORE INTO github_app_installations"
|
||||
" VALUES(?, ?, ?, ?)", ("7", "acme", "acme/alpha", 1.0))
|
||||
conn.execute(
|
||||
"INSERT OR IGNORE INTO github_app_installations"
|
||||
" VALUES(?, ?, ?, ?)", ("7", "acme", "acme/beta", 1.0))
|
||||
conn.commit()
|
||||
finally:
|
||||
conn.close()
|
||||
s, body = _req(self.port, "GET", "/github/repos",
|
||||
key=self.member_key)
|
||||
self.assertEqual(s, 200, body)
|
||||
names = [r["repo_full_name"] for r in body["repos"]]
|
||||
self.assertEqual(names, ["acme/alpha"])
|
||||
s, body = _req(self.port, "GET", "/github/repos",
|
||||
key=self.outsider_key)
|
||||
self.assertEqual(s, 200, body)
|
||||
self.assertEqual(body["repos"], [])
|
||||
|
||||
# -- key 审计 -----------------------------------------------------
|
||||
def test_api_key_audit_only_id_and_label(self):
|
||||
s, body = _req(self.port, "POST", "/users/me/keys",
|
||||
{"label": "p1-test"}, self.owner_key)
|
||||
self.assertEqual(s, 201, body)
|
||||
key_id = body["id"]
|
||||
plain = body["api_key"]
|
||||
s, _b = _req(self.port, "DELETE",
|
||||
"/users/me/keys/" + key_id, key=self.owner_key)
|
||||
self.assertEqual(s, 200, _b)
|
||||
s, body = _req(self.port, "POST",
|
||||
"/users/me/keys/revoke-others", {},
|
||||
self.owner_key)
|
||||
self.assertEqual(s, 200, body)
|
||||
actions = {a for a, _d in self._audit_actions(self.owner_id)}
|
||||
self.assertIn("api_key_created", actions)
|
||||
self.assertIn("api_key_revoked", actions)
|
||||
self.assertIn("api_key_revoke_others", actions)
|
||||
blob = json.dumps(self._audit_actions(self.owner_id))
|
||||
self.assertNotIn(plain, blob)
|
||||
|
||||
# -- 项目上限 ------------------------------------------------------
|
||||
def test_project_limits(self):
|
||||
s, body = _req(self.port, "POST", "/projects",
|
||||
{"name": "x" * 101}, self.owner_key)
|
||||
self.assertEqual(s, 400, body)
|
||||
self.assertIn("100", body["error"])
|
||||
s, body = _req(self.port, "POST", "/projects",
|
||||
{"name": "ok", "id": "y" * 65}, self.owner_key)
|
||||
self.assertEqual(s, 400, body)
|
||||
self.assertIn("64", body["error"])
|
||||
server_mod.MAX_PROJECTS_PER_USER = 1
|
||||
# setUp 已建 1 个 owner 项目,限额 1 时再建即超限。
|
||||
s, body = _req(self.port, "POST", "/projects",
|
||||
{"name": "second"}, self.owner_key)
|
||||
self.assertEqual(s, 400, body)
|
||||
self.assertIn("max 1", body["error"])
|
||||
|
||||
# -- 协调员预留 ----------------------------------------------------
|
||||
def test_coordinator_reserve_blocks_parent(self):
|
||||
s, _b = self._set_quota(self.owner_id, 1000000)
|
||||
self.assertEqual(s, 200, _b)
|
||||
parts = [{"name": "a", "prompt": "做 a"},
|
||||
{"name": "b", "prompt": "做 b"},
|
||||
{"name": "c", "prompt": "做 c"}]
|
||||
s, body = _req(self.port, "POST", "/tasks",
|
||||
{"project_id": self.project_id, "prompt": "协调",
|
||||
"coordinator_parts": parts}, self.owner_key)
|
||||
self.assertEqual(s, 402, body)
|
||||
self.assertEqual(body["error"],
|
||||
"coordinator_quota_reserve_exceeded")
|
||||
self.assertEqual(body["coordinator_parts"], 3)
|
||||
s, _b = self._set_quota(self.owner_id, 10000000)
|
||||
self.assertEqual(s, 200, _b)
|
||||
s, body = _req(self.port, "POST", "/tasks",
|
||||
{"project_id": self.project_id, "prompt": "协调",
|
||||
"coordinator_parts": parts}, self.owner_key)
|
||||
self.assertEqual(s, 201, body)
|
||||
self.assertIn("parent task creator",
|
||||
body.get("quota_attribution", ""))
|
||||
self.assertEqual(body.get("quota_reserved_usd_micros"), 3000000)
|
||||
|
||||
# -- 自动化配额门 ----------------------------------------------------
|
||||
def test_automation_create_quota_gate(self):
|
||||
s, _b = self._set_quota(self.owner_id, 0)
|
||||
self.assertEqual(s, 200, _b)
|
||||
s, body = _req(self.port, "POST",
|
||||
"/projects/%s/automations" % self.project_id,
|
||||
{"name": "nightly", "prompt": "巡检"},
|
||||
self.owner_key)
|
||||
self.assertEqual(s, 402, body)
|
||||
s, _b = self._set_quota(self.owner_id, 10000000)
|
||||
self.assertEqual(s, 200, _b)
|
||||
s, body = _req(self.port, "POST",
|
||||
"/projects/%s/automations" % self.project_id,
|
||||
{"name": "nightly", "prompt": "巡检"},
|
||||
self.owner_key)
|
||||
self.assertEqual(s, 201, body)
|
||||
|
||||
# -- sub2api 懒开通 / 轮换端点 --------------------------------------
|
||||
def test_relay_models_lazy_provision_not_just_pending(self):
|
||||
os.environ["CLOUD_GATEWAY"] = "sub2api"
|
||||
orig_validate = server_mod._validate_sub2api_key
|
||||
orig_list = server_mod._list_sub2api_models
|
||||
try:
|
||||
server_mod._validate_sub2api_key = (
|
||||
lambda conn, uid: ("provisioned", "fake-key", 7, 9))
|
||||
server_mod._list_sub2api_models = (
|
||||
lambda origin, key: ["model-a"])
|
||||
s, body = _req(self.port, "GET", "/relay/models",
|
||||
key=self.owner_key)
|
||||
self.assertEqual(s, 200, body)
|
||||
self.assertEqual(body["models"], ["model-a"])
|
||||
self.assertEqual(body["provisioning"], "provisioned")
|
||||
server_mod._validate_sub2api_key = (
|
||||
lambda conn, uid: (_ for _ in ()).throw(
|
||||
Sub2apiError("网关暂不可用")))
|
||||
s, body = _req(self.port, "GET", "/relay/models",
|
||||
key=self.owner_key)
|
||||
self.assertEqual(s, 502, body)
|
||||
finally:
|
||||
server_mod._validate_sub2api_key = orig_validate
|
||||
server_mod._list_sub2api_models = orig_list
|
||||
|
||||
def test_gateway_rotate_endpoint(self):
|
||||
s, body = _req(self.port, "POST", "/users/me/gateway/rotate",
|
||||
{}, self.owner_key)
|
||||
self.assertEqual(s, 400, body)
|
||||
os.environ["CLOUD_GATEWAY"] = "sub2api"
|
||||
orig_rotate = server_mod._rotate_sub2api_key
|
||||
try:
|
||||
server_mod._rotate_sub2api_key = (
|
||||
lambda conn, uid: ("rotated", "fake-key", 7, 9))
|
||||
s, body = _req(self.port, "POST", "/users/me/gateway/rotate",
|
||||
{}, self.owner_key)
|
||||
self.assertEqual(s, 200, body)
|
||||
self.assertTrue(body["rotated"])
|
||||
self.assertNotIn("api_key", body)
|
||||
actions = {a for a, _d in self._audit_actions(self.owner_id)}
|
||||
self.assertIn("gateway_key_rotated", actions)
|
||||
finally:
|
||||
server_mod._rotate_sub2api_key = orig_rotate
|
||||
|
||||
# -- 推送审计与分发 SSRF ----------------------------------------------
|
||||
def test_push_audit_and_dispatch_ssrf(self):
|
||||
s, sub = _req(self.port, "POST",
|
||||
"/projects/%s/push/subscriptions" % self.project_id,
|
||||
{"channel": "webhook",
|
||||
"target": "https://example.com/hook",
|
||||
"preference": "instant"}, self.owner_key)
|
||||
self.assertEqual(s, 201, sub)
|
||||
actions = {a for a, _d in self._audit_actions(self.owner_id)}
|
||||
self.assertIn("push_subscription_created", actions)
|
||||
s, _b = _req(self.port, "DELETE",
|
||||
"/projects/%s/push/subscriptions/%s"
|
||||
% (self.project_id, sub["id"]), key=self.owner_key)
|
||||
self.assertEqual(s, 200, _b)
|
||||
actions = {a for a, _d in self._audit_actions(self.owner_id)}
|
||||
self.assertIn("push_subscription_deleted", actions)
|
||||
# 内网目标:分发不 POST、sent 为空、错误含 delivery failed。
|
||||
s, sub = _req(self.port, "POST",
|
||||
"/projects/%s/push/subscriptions" % self.project_id,
|
||||
{"channel": "webhook",
|
||||
"target": "http://127.0.0.1:9/hook",
|
||||
"preference": "instant"}, self.owner_key)
|
||||
self.assertEqual(s, 201, sub)
|
||||
s, body = _req(self.port, "POST", "/admin/push/dispatch",
|
||||
{"event_type": "failed",
|
||||
"project_id": self.project_id,
|
||||
"payload": {"task_id": "t1"}}, self.owner_key)
|
||||
self.assertEqual(s, 200, body)
|
||||
self.assertEqual(body["sent"], [])
|
||||
self.assertEqual(len(body["errors"]), 1)
|
||||
self.assertIn("delivery failed", body["errors"][0]["reason"])
|
||||
actions = {a for a, _d in self._audit_actions(self.owner_id)}
|
||||
self.assertIn("admin_push_dispatch", actions)
|
||||
|
||||
# -- 配额双字段与控制台文案 --------------------------------------------
|
||||
def test_quota_payload_has_unpriced_fields(self):
|
||||
s, body = _req(self.port, "GET", "/users/me/quota",
|
||||
key=self.owner_key)
|
||||
self.assertEqual(s, 200, body)
|
||||
self.assertIn("unpriced_tokens_used", body)
|
||||
self.assertIn("unpriced_token_cap", body)
|
||||
|
||||
def test_console_p1_markers(self):
|
||||
conn = http.client.HTTPConnection(
|
||||
"127.0.0.1", self.port, timeout=10)
|
||||
try:
|
||||
conn.request("GET", "/console")
|
||||
resp = conn.getresponse()
|
||||
text = resp.read().decode("utf-8")
|
||||
finally:
|
||||
conn.close()
|
||||
self.assertEqual(resp.status, 200)
|
||||
for marker in (
|
||||
"登记评审需求",
|
||||
"记录允许",
|
||||
"记录拒绝",
|
||||
"当前版本只留回执,不拦截实时行为",
|
||||
"模型回合结束、改动在 worktree 里尚未合并",
|
||||
"控制面已标记 merged,不代表 git 主分支已合入",
|
||||
"协调员与 goal/计划互斥",
|
||||
"仅 failed/interrupted/merge-failed 可重试",
|
||||
"仅 maintainer 可合并"):
|
||||
self.assertIn(marker, text)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,534 @@
|
||||
"""P1 执行面针对性单测(只测本路改动,不跑全量三套)。
|
||||
|
||||
运行(在 ~/workspace/codex-app-server-p1security 下):
|
||||
python3 -m unittest discover -s cloud/tests -p "test_p1_security_worker.py" -v
|
||||
|
||||
说明:只用注入的假 runner 与测试假值,不跑真实模型回合、不联网、
|
||||
不登录任何服务;env 里的值只是占位用的假名,绝不是真实密钥。
|
||||
本文件只归本路(执行面)所有,不碰 test_p1_security_server.py。
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
import sqlite3
|
||||
import sys
|
||||
import tempfile
|
||||
import threading
|
||||
import time
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
from unittest import mock
|
||||
|
||||
try:
|
||||
import cloud.sandbox as sandbox_mod
|
||||
from cloud.cost import CostStore
|
||||
from cloud.pr_exec import enqueue_pr_job
|
||||
from cloud.sandbox import DockerExecutor, LocalExecutor
|
||||
from cloud.server import init_db
|
||||
from cloud.worker import Worker, _code_signature
|
||||
except ImportError: # discover -s cloud/tests 时把项目根目录补进 sys.path
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[2]))
|
||||
import cloud.sandbox as sandbox_mod
|
||||
from cloud.cost import CostStore
|
||||
from cloud.pr_exec import enqueue_pr_job
|
||||
from cloud.sandbox import DockerExecutor, LocalExecutor
|
||||
from cloud.server import init_db
|
||||
from cloud.worker import Worker, _code_signature
|
||||
|
||||
FAKE_ENV = {"XCM_API_KEY": "test-placeholder"}
|
||||
|
||||
|
||||
def _completed_runner(cmd):
|
||||
if "dispatch" in cmd:
|
||||
name = cmd[cmd.index("--name") + 1]
|
||||
else:
|
||||
name = cmd[cmd.index("--task") + 1]
|
||||
return SimpleNamespace(
|
||||
returncode=0,
|
||||
stdout=json.dumps({"task": name, "status": "completed",
|
||||
"text": "done"}),
|
||||
stderr="")
|
||||
|
||||
|
||||
def _usage_runner(usage, calls):
|
||||
def fake(cmd):
|
||||
calls.append(list(cmd))
|
||||
return SimpleNamespace(
|
||||
returncode=0,
|
||||
stdout=json.dumps({"status": "completed", "text": "done",
|
||||
"usage": usage}),
|
||||
stderr="")
|
||||
return fake
|
||||
|
||||
|
||||
def _insert_task(db_path, task_id, created_by="tester", status="queued",
|
||||
project_id="p1", **cols):
|
||||
now = time.time()
|
||||
conn = sqlite3.connect(db_path)
|
||||
try:
|
||||
conn.execute(
|
||||
"INSERT INTO tasks(id, project_id, prompt, status,"
|
||||
" idempotency_key, depends_on, priority, created_by,"
|
||||
" created_at, updated_at, last_heartbeat)"
|
||||
" VALUES(?, ?, ?, ?, NULL, '[]', 0, ?, ?, ?, NULL)",
|
||||
(task_id, project_id, "do work", status, created_by, now, now),
|
||||
)
|
||||
for key, value in cols.items():
|
||||
conn.execute(
|
||||
"UPDATE tasks SET %s = ? WHERE id = ?" % key,
|
||||
(value, task_id))
|
||||
conn.commit()
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
def _insert_quota(db_path, user_id, micros):
|
||||
conn = sqlite3.connect(db_path)
|
||||
try:
|
||||
conn.execute(
|
||||
"CREATE TABLE IF NOT EXISTS user_quotas("
|
||||
"user_id TEXT PRIMARY KEY, quota_usd_micros BIGINT,"
|
||||
" updated_at DOUBLE PRECISION)")
|
||||
conn.execute(
|
||||
"INSERT INTO user_quotas(user_id, quota_usd_micros, updated_at)"
|
||||
" VALUES(?, ?, ?)",
|
||||
(user_id, micros, time.time()))
|
||||
conn.commit()
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
def _insert_cost(db_path, task_id, model, input_tokens, output_tokens,
|
||||
micros, source="actual"):
|
||||
_insert_task(db_path, task_id, status="completed")
|
||||
store = CostStore(db_path)
|
||||
try:
|
||||
store.record(task_id, "p1", model, input_tokens=input_tokens,
|
||||
output_tokens=output_tokens, usd_micros=micros,
|
||||
source=source)
|
||||
finally:
|
||||
try:
|
||||
store._conn.close()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
def _status(db_path, task_id):
|
||||
conn = sqlite3.connect(db_path)
|
||||
try:
|
||||
row = conn.execute(
|
||||
"SELECT status FROM tasks WHERE id = ?", (task_id,)).fetchone()
|
||||
finally:
|
||||
conn.close()
|
||||
return row[0] if row else None
|
||||
|
||||
|
||||
def _error_text(db_path, task_id):
|
||||
conn = sqlite3.connect(db_path)
|
||||
try:
|
||||
row = conn.execute(
|
||||
"SELECT error_text FROM tasks WHERE id = ?",
|
||||
(task_id,)).fetchone()
|
||||
finally:
|
||||
conn.close()
|
||||
return row[0] if row else None
|
||||
|
||||
|
||||
class P1WorkerTestCase(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.tmp = tempfile.TemporaryDirectory()
|
||||
self.db_path = os.path.join(self.tmp.name, "control.db")
|
||||
init_db(self.db_path)
|
||||
self.calls = []
|
||||
|
||||
def tearDown(self):
|
||||
self.tmp.cleanup()
|
||||
|
||||
def _worker(self, runner=None, **kwargs):
|
||||
kwargs.setdefault("env", FAKE_ENV)
|
||||
return Worker(self.db_path, "/fake/host.py",
|
||||
runner=runner or _completed_runner, **kwargs)
|
||||
|
||||
# 1. 无配额行按默认 $10 额度(不再是不限额)
|
||||
def test_missing_quota_row_uses_default_quota(self):
|
||||
_insert_cost(self.db_path, "donor-a", "gpt-6.1-sol",
|
||||
1, 0, 11000000)
|
||||
_insert_cost(self.db_path, "donor-b", "gpt-6.1-sol",
|
||||
1, 0, 9000000)
|
||||
worker = self._worker()
|
||||
over = worker._quota_exceeded(
|
||||
{"created_by": "tester", "project_id": "p1"}, None)
|
||||
# donor-a 归 tester:已花 $11 > 默认 $10,挡住。
|
||||
self.assertIsNotNone(over)
|
||||
self.assertIn("quota_exceeded", over)
|
||||
# donor-b 同样归 tester 会把总额推得更高,单独测另一用户:
|
||||
_insert_task(self.db_path, "donor-c", created_by="rich",
|
||||
status="completed")
|
||||
store = CostStore(self.db_path)
|
||||
try:
|
||||
store.record("donor-c", "p1", "gpt-6.1-sol", input_tokens=1,
|
||||
output_tokens=0, usd_micros=9000000,
|
||||
source="actual")
|
||||
finally:
|
||||
try:
|
||||
store._conn.close()
|
||||
except Exception:
|
||||
pass
|
||||
under = worker._quota_exceeded(
|
||||
{"created_by": "rich", "project_id": "p1"}, None)
|
||||
self.assertIsNone(under)
|
||||
|
||||
# 2. 显式 NULL 配额行仍是不限额
|
||||
def test_explicit_null_quota_row_stays_unlimited(self):
|
||||
_insert_quota(self.db_path, "tester", None)
|
||||
_insert_cost(self.db_path, "donor", "gpt-6.1-sol",
|
||||
1, 0, 999000000)
|
||||
worker = self._worker()
|
||||
self.assertIsNone(worker._quota_exceeded(
|
||||
{"created_by": "tester", "project_id": "p1"}, None))
|
||||
|
||||
# 3. 属主为空拒绝新回合(不再返回 None 放行)
|
||||
def test_ownerless_task_refused(self):
|
||||
worker = self._worker()
|
||||
for owner in (None, ""):
|
||||
err = worker._quota_exceeded(
|
||||
{"created_by": owner, "project_id": "p1"}, None)
|
||||
self.assertIsNotNone(err)
|
||||
self.assertIn("quota_exceeded", err)
|
||||
|
||||
# 4. 协调员子任务属主跟父任务走
|
||||
def test_child_inherits_parent_owner(self):
|
||||
_insert_quota(self.db_path, "tester", 100)
|
||||
_insert_cost(self.db_path, "donor", "gpt-6.1-sol",
|
||||
1, 0, 200)
|
||||
_insert_task(self.db_path, "parent", created_by="tester")
|
||||
_insert_task(self.db_path, "parent2", created_by="fresh")
|
||||
worker = self._worker()
|
||||
blocked = worker._quota_exceeded(
|
||||
{"created_by": None, "parent_task_id": "parent",
|
||||
"project_id": "p1"}, None)
|
||||
self.assertIsNotNone(blocked)
|
||||
self.assertIn("quota_exceeded", blocked)
|
||||
# fresh 用户无配额行、未花钱:默认额度内放行。
|
||||
self.assertIsNone(worker._quota_exceeded(
|
||||
{"created_by": None, "parent_task_id": "parent2",
|
||||
"project_id": "p1"}, None))
|
||||
|
||||
# 5. 查库失败 fail-closed(新回合拒绝,不自动放行)
|
||||
def test_quota_db_failure_is_fail_closed(self):
|
||||
worker = self._worker()
|
||||
with mock.patch("cloud.worker._db_connect",
|
||||
side_effect=RuntimeError("test db down")):
|
||||
err = worker._quota_exceeded(
|
||||
{"created_by": "tester", "project_id": "p1"}, None)
|
||||
self.assertIsNotNone(err)
|
||||
self.assertTrue(err.startswith("quota_check_failed:"))
|
||||
|
||||
# 6. Docker 配额看门狗:命中杀容器、未命中不杀
|
||||
def test_docker_quota_watchdog(self):
|
||||
killed = []
|
||||
|
||||
def fake(cmd, timeout=None):
|
||||
killed.append(list(cmd))
|
||||
return SimpleNamespace(returncode=0, stdout="", stderr="")
|
||||
|
||||
executor = DockerExecutor(runner=fake)
|
||||
worker = Worker(self.db_path, "/fake/host.py",
|
||||
env=FAKE_ENV, executor=executor)
|
||||
# 未命中:0.2 秒后停表,kill 不应被调。
|
||||
stop = threading.Event()
|
||||
threading.Timer(0.2, stop.set).start()
|
||||
worker._docker_quota_watchdog(
|
||||
"T1", "codex-test", stop, lambda: False, interval=0.05)
|
||||
self.assertEqual(killed, [])
|
||||
# 命中:立即杀容器并记命中标记。
|
||||
stop2 = threading.Event()
|
||||
worker._docker_quota_watchdog(
|
||||
"T2", "codex-test", stop2, lambda: True, interval=0.05)
|
||||
self.assertTrue(any("kill" in c for c in killed))
|
||||
self.assertTrue(worker._take_quota_hit("T2"))
|
||||
# 命中已消费,第二次取不到(不污染下一轮)。
|
||||
self.assertFalse(worker._take_quota_hit("T2"))
|
||||
|
||||
# 7. usage_cost_usd 按实际成本累加,max_cost_usd 不再是死参数
|
||||
def test_usage_cost_accumulates_and_budget_trips(self):
|
||||
_insert_task(self.db_path, "T1")
|
||||
worker = self._worker(
|
||||
runner=_usage_runner(
|
||||
{"input_tokens": 10000, "cached_input_tokens": 0,
|
||||
"output_tokens": 1000, "total_tokens": 11000},
|
||||
self.calls),
|
||||
model="gpt-6.1-sol", max_cost_usd=0.01)
|
||||
self.assertFalse(worker._budget_exceeded())
|
||||
result = worker.run_one()
|
||||
self.assertEqual(result["status"], "completed")
|
||||
# 10000*$2 + 1000*$10 = $0.03,超过 $0.01 上限。
|
||||
self.assertAlmostEqual(worker.usage_cost_usd, 0.03, places=6)
|
||||
self.assertTrue(worker._budget_exceeded())
|
||||
conn = sqlite3.connect(self.db_path)
|
||||
try:
|
||||
row = conn.execute(
|
||||
"SELECT usd_micros, source FROM cost_records"
|
||||
" WHERE task_id = 'T1'").fetchone()
|
||||
finally:
|
||||
conn.close()
|
||||
self.assertEqual(row, (30000, "actual"))
|
||||
|
||||
# 8. 取消分支先记成本再返回(与配额中断收尾同款记账点)
|
||||
def test_cancel_branch_records_cost(self):
|
||||
worker = self._worker()
|
||||
task = {"id": "T1", "project_id": "p1", "prompt": "do work"}
|
||||
with worker._db_lock:
|
||||
cost_id = worker._record_usage_cost(
|
||||
task, "gpt-6.1-sol",
|
||||
{"input_tokens": 10000, "cached_input_tokens": 0,
|
||||
"output_tokens": 1000, "total_tokens": 11000},
|
||||
"done", 0)
|
||||
self.assertIsNotNone(cost_id)
|
||||
self.assertAlmostEqual(worker.usage_cost_usd, 0.03, places=6)
|
||||
# 无用量时回落估算,不累加 usage_cost_usd。
|
||||
with worker._db_lock:
|
||||
worker._record_usage_cost(
|
||||
{"id": "T2", "project_id": "p1", "prompt": "do work"},
|
||||
"gpt-6.1-sol", None, "done", 0)
|
||||
self.assertAlmostEqual(worker.usage_cost_usd, 0.03, places=6)
|
||||
|
||||
# 9. 估算行不计入无价 token 上限
|
||||
def test_estimated_rows_excluded_from_unpriced_cap(self):
|
||||
_insert_cost(self.db_path, "donor-est", "gpt-5.6-terra",
|
||||
100000, 0, None, source="estimated")
|
||||
worker = self._worker()
|
||||
old = os.environ.get("CLOUD_QUOTA_UNPRICED_TOKEN_CAP")
|
||||
os.environ["CLOUD_QUOTA_UNPRICED_TOKEN_CAP"] = "100"
|
||||
try:
|
||||
# 10 万估算 token 在上限 100 下仍放行(估算不是真花费)。
|
||||
self.assertIsNone(worker._quota_exceeded(
|
||||
{"created_by": "tester", "project_id": "p1"}, None))
|
||||
_insert_cost(self.db_path, "donor-act", "gpt-5.6-terra",
|
||||
75, 75, None, source="actual")
|
||||
err = worker._quota_exceeded(
|
||||
{"created_by": "tester", "project_id": "p1"}, None)
|
||||
self.assertIsNotNone(err)
|
||||
self.assertIn("无价模型 token", err)
|
||||
finally:
|
||||
if old is None:
|
||||
os.environ.pop("CLOUD_QUOTA_UNPRICED_TOKEN_CAP", None)
|
||||
else:
|
||||
os.environ["CLOUD_QUOTA_UNPRICED_TOKEN_CAP"] = old
|
||||
|
||||
# 10. cache_write 不计入无价上限(只 input+output 计)
|
||||
def test_cache_write_excluded_from_unpriced_cap(self):
|
||||
worker = self._worker()
|
||||
old = os.environ.get("CLOUD_QUOTA_UNPRICED_TOKEN_CAP")
|
||||
os.environ["CLOUD_QUOTA_UNPRICED_TOKEN_CAP"] = "100"
|
||||
try:
|
||||
usage = {"input_tokens": 10, "output_tokens": 5,
|
||||
"cache_write_input_tokens": 9999999,
|
||||
"total_tokens": 15}
|
||||
self.assertIsNone(worker._quota_exceeded(
|
||||
{"created_by": "tester", "project_id": "p1",
|
||||
"model": "gpt-5.6-terra"}, usage))
|
||||
usage2 = {"input_tokens": 200, "output_tokens": 0,
|
||||
"cache_write_input_tokens": 0,
|
||||
"total_tokens": 200}
|
||||
err = worker._quota_exceeded(
|
||||
{"created_by": "tester", "project_id": "p1",
|
||||
"model": "gpt-5.6-terra"}, usage2)
|
||||
self.assertIsNotNone(err)
|
||||
finally:
|
||||
if old is None:
|
||||
os.environ.pop("CLOUD_QUOTA_UNPRICED_TOKEN_CAP", None)
|
||||
else:
|
||||
os.environ["CLOUD_QUOTA_UNPRICED_TOKEN_CAP"] = old
|
||||
|
||||
# 11. 协调员子任务继承父任务模型/强度,分工显式值优先
|
||||
def test_coordinator_child_inherits_model_effort(self):
|
||||
worker = self._worker()
|
||||
task = {"id": "coord", "project_id": "p1", "prompt": "coord",
|
||||
"priority": 0, "created_by": "tester", "model": "gpt-6.1-sol",
|
||||
"effort": "high", "executor": None, "environment_id": None}
|
||||
children = worker._coord_sync_children(task, [
|
||||
{"name": "a", "prompt": "do a", "model": None,
|
||||
"effort": None, "executor": None},
|
||||
{"name": "b", "prompt": "do b", "model": "gpt-5.6-terra",
|
||||
"effort": "low", "executor": None},
|
||||
])
|
||||
self.assertEqual(len(children), 2)
|
||||
conn = sqlite3.connect(self.db_path)
|
||||
try:
|
||||
rows = {r[0]: (r[1], r[2]) for r in conn.execute(
|
||||
"SELECT coordinator_part, model, effort FROM tasks"
|
||||
" WHERE parent_task_id = 'coord'").fetchall()}
|
||||
finally:
|
||||
conn.close()
|
||||
self.assertEqual(rows["a"], ("gpt-6.1-sol", "high"))
|
||||
self.assertEqual(rows["b"], ("gpt-5.6-terra", "low"))
|
||||
|
||||
# 12. 断点续跑:Local/Docker/popen 三路都不传 --goal-text
|
||||
def test_goal_resume_breakpoint_omits_goal_text(self):
|
||||
calls = []
|
||||
|
||||
def fake(cmd):
|
||||
calls.append(list(cmd))
|
||||
return SimpleNamespace(
|
||||
returncode=0,
|
||||
stdout="goal met after rounds\n"
|
||||
"usage input=10 cached=0 cache_write=0 output=5 "
|
||||
"reasoning=0 total=15 completions=1\n",
|
||||
stderr="")
|
||||
|
||||
task = {"id": "T1", "project_id": "p1", "prompt": "orig goal",
|
||||
"verify_command": "true", "goal_max_rounds": 3}
|
||||
local = LocalExecutor("/fake/host.py", runner=fake)
|
||||
local.goal_resume_task(task, None)
|
||||
self.assertIn("goal-resume", calls[-1])
|
||||
self.assertNotIn("--goal-text", calls[-1])
|
||||
docker_calls = []
|
||||
|
||||
def dfake(cmd, timeout=None):
|
||||
docker_calls.append(list(cmd))
|
||||
return SimpleNamespace(
|
||||
returncode=0,
|
||||
stdout="goal met after rounds\n"
|
||||
"usage input=10 cached=0 cache_write=0 output=5 "
|
||||
"reasoning=0 total=15 completions=1\n",
|
||||
stderr="")
|
||||
|
||||
docker = DockerExecutor(runner=dfake, env=FAKE_ENV)
|
||||
with mock.patch("cloud.sandbox.shutil.which",
|
||||
return_value="/usr/bin/docker"):
|
||||
docker.goal_resume_task(task, None)
|
||||
self.assertIn("goal-resume", docker_calls[-1])
|
||||
self.assertNotIn("--goal-text", docker_calls[-1])
|
||||
with mock.patch.object(sandbox_mod.subprocess, "Popen") as popen:
|
||||
popen.return_value = SimpleNamespace(pid=9999)
|
||||
local.popen_task(
|
||||
{"id": "T1", "prompt": "resume msg",
|
||||
"_resume_kind": "breakpoint"})
|
||||
# popen_task 默认 kind=dispatch;显式测 goal-resume:
|
||||
local.popen_task(
|
||||
{"id": "T1", "prompt": "resume msg",
|
||||
"_resume_kind": "breakpoint"}, kind="goal-resume")
|
||||
cmd = popen.call_args[0][0]
|
||||
self.assertIn("goal-resume", cmd)
|
||||
self.assertNotIn("--goal-text", cmd)
|
||||
|
||||
# 13. 开新目标:goal-resume 带 --goal-text(旧行为保留)
|
||||
def test_goal_resume_new_goal_keeps_goal_text(self):
|
||||
calls = []
|
||||
|
||||
def fake(cmd):
|
||||
calls.append(list(cmd))
|
||||
return SimpleNamespace(
|
||||
returncode=0,
|
||||
stdout="goal met after rounds\n"
|
||||
"usage input=10 cached=0 cache_write=0 output=5 "
|
||||
"reasoning=0 total=15 completions=1\n",
|
||||
stderr="")
|
||||
|
||||
task = {"id": "T1", "project_id": "p1", "prompt": "new goal"}
|
||||
local = LocalExecutor("/fake/host.py", runner=fake)
|
||||
local.goal_resume_task(task, "new goal")
|
||||
self.assertIn("--goal-text", calls[-1])
|
||||
self.assertIn("new goal", calls[-1])
|
||||
|
||||
# 14. goal 轮次上限:sandbox loud fail + worker 认领后 failed
|
||||
def test_goal_max_rounds_cap(self):
|
||||
task = {"id": "T1", "project_id": "p1", "prompt": "g",
|
||||
"verify_command": "true", "goal_max_rounds": 101}
|
||||
local = LocalExecutor("/fake/host.py", runner=_completed_runner)
|
||||
with self.assertRaises(ValueError):
|
||||
local.goal_task(task)
|
||||
_insert_task(self.db_path, "T1", verify_command="true",
|
||||
goal_max_rounds=101)
|
||||
worker = self._worker()
|
||||
result = worker.run_one()
|
||||
self.assertEqual(result["status"], "failed")
|
||||
self.assertIn("上限", result["error"])
|
||||
self.assertEqual(_status(self.db_path, "T1"), "failed")
|
||||
self.assertEqual(self.calls, [])
|
||||
|
||||
# 15. PR 非 local:显式 needs_local_worker,任务与排队都不动
|
||||
def test_pr_non_local_needs_local_worker(self):
|
||||
conn = sqlite3.connect(self.db_path)
|
||||
try:
|
||||
job_id = enqueue_pr_job(conn, "T1", "p1", "test-host")
|
||||
finally:
|
||||
conn.close()
|
||||
executor = DockerExecutor(runner=lambda cmd, timeout=None:
|
||||
SimpleNamespace(returncode=0, stdout="",
|
||||
stderr=""))
|
||||
worker = Worker(self.db_path, "/fake/host.py", env=FAKE_ENV,
|
||||
executor=executor, host_name="test-host")
|
||||
results = worker._run_pr_jobs()
|
||||
self.assertEqual(len(results), 1)
|
||||
self.assertEqual(results[0]["status"], "needs_local_worker")
|
||||
self.assertEqual(results[0]["pr_job_id"], job_id)
|
||||
conn = sqlite3.connect(self.db_path)
|
||||
try:
|
||||
row = conn.execute(
|
||||
"SELECT status FROM pr_jobs WHERE id = ?",
|
||||
(job_id,)).fetchone()
|
||||
finally:
|
||||
conn.close()
|
||||
self.assertEqual(row[0], "queued")
|
||||
|
||||
# 16. 多文件签名 + 签名缺失/已变时认领 fail-closed
|
||||
def test_code_signature_multi_file_and_claim_refusal(self):
|
||||
sig = _code_signature("/fake/host.py")
|
||||
self.assertIsNotNone(sig)
|
||||
self.assertEqual(len(sig), 6)
|
||||
for entry in sig:
|
||||
self.assertEqual(len(entry), 2)
|
||||
_insert_task(self.db_path, "T1")
|
||||
worker = self._worker()
|
||||
worker._code_signature = ("stale-signature",)
|
||||
result = worker.run_one()
|
||||
self.assertFalse(result["dispatched"])
|
||||
self.assertEqual(result["reason"], "already_claimed")
|
||||
self.assertEqual(_status(self.db_path, "T1"), "queued")
|
||||
worker._code_signature = None
|
||||
result = worker.run_one()
|
||||
self.assertFalse(result["dispatched"])
|
||||
self.assertEqual(_status(self.db_path, "T1"), "queued")
|
||||
|
||||
# 17. goal 首轮撞已存在 → 断点续跑一次,不直接 failed
|
||||
def test_goal_already_exists_falls_back_to_breakpoint(self):
|
||||
calls = []
|
||||
|
||||
def fake(cmd):
|
||||
calls.append(list(cmd))
|
||||
if "goal-resume" in cmd:
|
||||
return SimpleNamespace(
|
||||
returncode=0,
|
||||
stdout="round 1: turn=completed verify_rc=0 "
|
||||
"verify_out=\ngoal met after rounds\n"
|
||||
"usage input=10 cached=0 cache_write=0 output=5 "
|
||||
"reasoning=0 total=15 completions=1\n",
|
||||
stderr="")
|
||||
if "goal" in cmd:
|
||||
return SimpleNamespace(
|
||||
returncode=0,
|
||||
stdout="host 任务已存在:T1 已存在",
|
||||
stderr="")
|
||||
return _completed_runner(cmd)
|
||||
|
||||
_insert_task(self.db_path, "T1", verify_command="true",
|
||||
goal_max_rounds=2)
|
||||
worker = self._worker(runner=fake)
|
||||
result = worker.run_one()
|
||||
self.assertEqual(result["status"], "completed")
|
||||
# 注意:cmd 是元素列表,"goal" in cmd 只匹配恰好等于
|
||||
# "goal" 的元素,goal-resume 命令里并没有 "goal" 元素,
|
||||
# 所以按“任一元素含 goal 子串”过滤。
|
||||
goal_cmds = [c for c in calls
|
||||
if any("goal" in e for e in c)]
|
||||
self.assertTrue(any("goal-resume" in c for c in goal_cmds))
|
||||
resume_cmd = [c for c in goal_cmds if "goal-resume" in c][0]
|
||||
self.assertNotIn("--goal-text", resume_cmd)
|
||||
self.assertEqual(_status(self.db_path, "T1"), "completed")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -307,7 +307,12 @@ class WorkerPrJobsTestCase(unittest.TestCase):
|
||||
enqueue_pr_job(conn, "t1", "p1", worker.host_name)
|
||||
finally:
|
||||
conn.close()
|
||||
self.assertEqual(worker._run_pr_jobs(), [])
|
||||
# P1:Docker worker 不再静默返回空,改为显式 needs_local_worker,
|
||||
# 任务保持 queued 等 local worker 认领。
|
||||
jobs = worker._run_pr_jobs()
|
||||
self.assertEqual(len(jobs), 1)
|
||||
self.assertEqual(jobs[0]["status"], "needs_local_worker")
|
||||
self.assertEqual(jobs[0]["task_id"], "t1")
|
||||
|
||||
def test_autoscaler_forces_host_name(self):
|
||||
scaler = Autoscaler(self.db_path, [])
|
||||
|
||||
@@ -41,6 +41,9 @@ class WebhookExecTestCase(unittest.TestCase):
|
||||
self.owner_id, self.owner_key = self.auth.create_user("owner")
|
||||
self._old_admins = os.environ.get("CODEX_GLOBAL_ADMINS")
|
||||
os.environ["CODEX_GLOBAL_ADMINS"] = self.owner_id
|
||||
# P1:备份 out_dir 限白名单根;本测试的 tmp 目录即白名单根。
|
||||
self._old_backup_root = os.environ.get("CLOUD_BACKUP_ROOT")
|
||||
os.environ["CLOUD_BACKUP_ROOT"] = self.tmp.name
|
||||
self.viewer_id, self.viewer_key = self.auth.create_user("viewer")
|
||||
self.server = create_server(self.db_path, "127.0.0.1", 0)
|
||||
self.port = self.server.server_address[1]
|
||||
@@ -67,6 +70,10 @@ class WebhookExecTestCase(unittest.TestCase):
|
||||
os.environ.pop("CODEX_GLOBAL_ADMINS", None)
|
||||
else:
|
||||
os.environ["CODEX_GLOBAL_ADMINS"] = self._old_admins
|
||||
if getattr(self, "_old_backup_root", None) is None:
|
||||
os.environ.pop("CLOUD_BACKUP_ROOT", None)
|
||||
else:
|
||||
os.environ["CLOUD_BACKUP_ROOT"] = self._old_backup_root
|
||||
|
||||
def _req(self, method, path, body=None, key=None, headers=None,
|
||||
raw_body=None):
|
||||
|
||||
@@ -336,6 +336,11 @@ class TestRealSenders(PushTestCase):
|
||||
|
||||
def setUp(self):
|
||||
super().setUp()
|
||||
# P1 SSRF 门默认挡 loopback;本地录制服务器测试显式放行,
|
||||
# 生产默认值仍关闭(不要把本开关当成生产默认)。
|
||||
self._old_push_allow = os.environ.get(
|
||||
"CODEX_PUSH_ALLOW_PRIVATE_TARGETS")
|
||||
os.environ["CODEX_PUSH_ALLOW_PRIVATE_TARGETS"] = "1"
|
||||
_RecordingHandler.requests = []
|
||||
_RecordingHandler.status = 200
|
||||
self.server = http.server.ThreadingHTTPServer(
|
||||
@@ -349,6 +354,11 @@ class TestRealSenders(PushTestCase):
|
||||
self.server.shutdown()
|
||||
self.server.server_close()
|
||||
self.thread.join(timeout=5)
|
||||
if getattr(self, "_old_push_allow", None) is None:
|
||||
os.environ.pop("CODEX_PUSH_ALLOW_PRIVATE_TARGETS", None)
|
||||
else:
|
||||
os.environ["CODEX_PUSH_ALLOW_PRIVATE_TARGETS"] = (
|
||||
self._old_push_allow)
|
||||
super().tearDown()
|
||||
|
||||
def _url(self, path="/hook"):
|
||||
|
||||
@@ -331,7 +331,8 @@ class TestGoalExecutors(unittest.TestCase):
|
||||
{"id": "t1", "prompt": "do it",
|
||||
"verify_command": "true"})
|
||||
self.assertEqual(result["status"], "failed")
|
||||
self.assertIn("请新建任务", result["goal_error"])
|
||||
# P1 新语义:goal 失败请用继续而非重试(旧“请新建任务”已废)。
|
||||
self.assertIn("请用继续而非重试", result["goal_error"])
|
||||
|
||||
def test_local_goal_task_without_verify_is_llm_decided(self):
|
||||
# 新语义:没有验收命令不再抛错——走模型自决分支:
|
||||
|
||||
@@ -114,6 +114,11 @@ class _Listener:
|
||||
|
||||
class GoalPrNotifyTestCase(unittest.TestCase):
|
||||
def setUp(self):
|
||||
# P1 SSRF 门默认挡 loopback;本地监听器测试显式放行,
|
||||
# 生产默认值仍关闭。
|
||||
self._old_push_allow = os.environ.get(
|
||||
"CODEX_PUSH_ALLOW_PRIVATE_TARGETS")
|
||||
os.environ["CODEX_PUSH_ALLOW_PRIVATE_TARGETS"] = "1"
|
||||
self.tmp = tempfile.TemporaryDirectory()
|
||||
self.db_path = os.path.join(self.tmp.name, "control.db")
|
||||
self.auth = AuthStore(self.db_path)
|
||||
@@ -133,6 +138,11 @@ class GoalPrNotifyTestCase(unittest.TestCase):
|
||||
self.server.shutdown()
|
||||
self.server.server_close()
|
||||
self.thread.join(timeout=5)
|
||||
if getattr(self, "_old_push_allow", None) is None:
|
||||
os.environ.pop("CODEX_PUSH_ALLOW_PRIVATE_TARGETS", None)
|
||||
else:
|
||||
os.environ["CODEX_PUSH_ALLOW_PRIVATE_TARGETS"] = (
|
||||
self._old_push_allow)
|
||||
try:
|
||||
self.auth.conn.close()
|
||||
except Exception:
|
||||
|
||||
@@ -74,6 +74,11 @@ def _completed_runner(cmd):
|
||||
|
||||
class WorkerTestCase(unittest.TestCase):
|
||||
def setUp(self):
|
||||
# P1 SSRF 门默认挡 loopback;通知测试用本地监听器,显式放行,
|
||||
# 生产默认值仍关闭。
|
||||
self._old_push_allow = os.environ.get(
|
||||
"CODEX_PUSH_ALLOW_PRIVATE_TARGETS")
|
||||
os.environ["CODEX_PUSH_ALLOW_PRIVATE_TARGETS"] = "1"
|
||||
self.tmp = tempfile.TemporaryDirectory()
|
||||
self.db_path = os.path.join(self.tmp.name, "control.db")
|
||||
init_db(self.db_path)
|
||||
@@ -81,6 +86,11 @@ class WorkerTestCase(unittest.TestCase):
|
||||
self.calls = []
|
||||
|
||||
def tearDown(self):
|
||||
if getattr(self, "_old_push_allow", None) is None:
|
||||
os.environ.pop("CODEX_PUSH_ALLOW_PRIVATE_TARGETS", None)
|
||||
else:
|
||||
os.environ["CODEX_PUSH_ALLOW_PRIVATE_TARGETS"] = (
|
||||
self._old_push_allow)
|
||||
self.tmp.cleanup()
|
||||
|
||||
def _worker(self, runner=None, **kwargs):
|
||||
|
||||
+604
-55
@@ -139,6 +139,7 @@ from cloud.coordinator import DEFAULT_POLL_INTERVAL as _COORD_POLL_INTERVAL
|
||||
from cloud.coordinator import DEFAULT_TIMEOUT_SECONDS as _COORD_TIMEOUT
|
||||
from cloud.coordinator import ensure_columns as _coord_ensure_columns
|
||||
from cloud.coordinator import parse_parts as _coord_parse_parts
|
||||
from cloud.coordinator import nonterminal_children as _coord_nonterminal
|
||||
from cloud.cost import CostStore, estimate_tokens, price_usd_micros, \
|
||||
sum_usage, unpriced_token_cap
|
||||
from cloud.db import connect as _db_connect
|
||||
@@ -170,19 +171,68 @@ DEFAULT_ENV_KEY = "XCM_API_KEY"
|
||||
# gpt-6.1-sol 路由当日实测持续 503,terra 同日实测 1 秒完成;
|
||||
# 任务未指定模型时 worker 与 host.py 都落到这里。
|
||||
DEFAULT_MODEL = "gpt-5.6-terra"
|
||||
# P1 执行面:goal 轮次上限(有限轮次形态)。goal_mode=1 的 0 轮
|
||||
# (模型宣告才结束)不在此限——它的上界是配额(配额看门狗与
|
||||
# 认领/边界门),不是轮次;旧验收形态的 goal_max_rounds 必须
|
||||
# 在 1..GOAL_MAX_ROUNDS_CAP 之间,超限的任务 worker 在认领后
|
||||
# 直接 failed(建任务时的 400 校验在 server.py,属另一路文件,
|
||||
# 本路只做执行侧防线,见 run_one 的 goal 上限门)。
|
||||
GOAL_MAX_ROUNDS_CAP = 100
|
||||
# P1 执行面:无配额行时的默认配额(微美元),与 server.py
|
||||
# _default_quota_usd_micros() 同值($10/月,环境变量
|
||||
# CLOUD_DEFAULT_QUOTA_USD_MICROS 可覆盖)。P0 前“无行 =
|
||||
# 不限额”已废:worker 侧查不到行时按此默认额度算,显式
|
||||
# NULL 行才是不限额(与服务端 _quota_status 同口径)。
|
||||
# host.py dispatch 的原话是 "task 已存在: <name>,先换名或 discard 它"
|
||||
#(英文 task + 已存在),取共有子串做判断,不依赖标点。
|
||||
_ALREADY_EXISTS_MARKER = "已存在"
|
||||
|
||||
|
||||
def _code_signature():
|
||||
"""本文件签名(mtime_ns, size):部署 staging 换目录后必变;
|
||||
取不到时返回 None(此时自退检查跳过,绝不误杀)。"""
|
||||
def _default_quota_usd_micros():
|
||||
"""无配额行时的默认配额(微美元),与 server 侧同值。
|
||||
|
||||
server.py 的 _default_quota_usd_micros() 在另一路文件里,
|
||||
worker 不跨模块 import 它(避免循环 import),这里复刻
|
||||
同款解析:环境变量 CLOUD_DEFAULT_QUOTA_USD_MICROS,非法
|
||||
值回落 10_000_000。"""
|
||||
try:
|
||||
st = os.stat(os.path.abspath(__file__))
|
||||
return (st.st_mtime_ns, st.st_size)
|
||||
return max(0, int(os.environ.get(
|
||||
"CLOUD_DEFAULT_QUOTA_USD_MICROS", "10000000")))
|
||||
except (TypeError, ValueError):
|
||||
return 10000000
|
||||
|
||||
|
||||
def _code_signature(host_py_path=None):
|
||||
"""多文件签名:worker/sandbox/coordinator/cost/server/host
|
||||
各取 (mtime_ns, size) 组成的元组;任一文件取不到即 None。
|
||||
|
||||
P1 执行面扩面:旧版只签 worker.py,部署只换了 sandbox 或
|
||||
host.py 时旧 worker 会继续用旧代码抢任务。host.py 取
|
||||
host_py_path(真实文件时用它,否则用 cloud/ 上一级的
|
||||
host.py);单测常传 /fake/host.py 这种不存在的路径,此时
|
||||
回落到仓库 host.py,不因此判签名缺失。取不到时返回 None,
|
||||
调用方必须 fail-closed(不认领、不执行),绝不按“没变”
|
||||
放行——旧版“取不到当没变”是 fail-open,已废。"""
|
||||
here = os.path.dirname(os.path.abspath(__file__))
|
||||
candidates = [
|
||||
os.path.join(here, "worker.py"),
|
||||
os.path.join(here, "sandbox.py"),
|
||||
os.path.join(here, "coordinator.py"),
|
||||
os.path.join(here, "cost.py"),
|
||||
os.path.join(here, "server.py"),
|
||||
]
|
||||
if host_py_path and os.path.isfile(host_py_path):
|
||||
candidates.append(os.path.abspath(host_py_path))
|
||||
else:
|
||||
candidates.append(os.path.join(os.path.dirname(here), "host.py"))
|
||||
sig = []
|
||||
try:
|
||||
for path in candidates:
|
||||
st = os.stat(path)
|
||||
sig.append((st.st_mtime_ns, st.st_size))
|
||||
except OSError:
|
||||
return None
|
||||
return tuple(sig)
|
||||
|
||||
|
||||
def _missing_key_error(env_key):
|
||||
@@ -447,16 +497,46 @@ class Worker:
|
||||
# 注入的普通 dict 同样可读环境变量(无该键时默认 1)。
|
||||
self.capacity = resolve_capacity(capacity, env=self._env)
|
||||
# 常驻启动时的代码签名:run_loop 每轮边界比对,变了就自退
|
||||
# (见 run_loop 的代码变化自退检查)。
|
||||
self._code_signature = _code_signature()
|
||||
# (见 run_loop 的代码变化自退检查);_claim 每次认领前
|
||||
# 也比对(见 _code_signature_valid),不要只在轮次边界查。
|
||||
self._code_signature = _code_signature(host_py_path)
|
||||
# 配额看门狗命中过的任务(Docker 同步路径用):看门狗杀
|
||||
# 容器后,同步返回的 failed 要被 _run_attempt 改写成
|
||||
# quota_exceeded,否则 run_one 会把它当普通失败重试,
|
||||
# 配额中断就续跑下去了。受 _quota_hit_lock 保护。
|
||||
self._quota_hit_tasks = set()
|
||||
self._quota_hit_lock = threading.Lock()
|
||||
|
||||
def _code_changed(self):
|
||||
"""启动后 worker.py 是否被换过(部署换目录/手动改代码):
|
||||
签名取不到时一律当没变,不误杀。"""
|
||||
now = _code_signature()
|
||||
"""启动后关键代码是否被换过(部署换目录/手动改代码):
|
||||
签名取不到时返回 False(是否“变了”无从判断),但这
|
||||
不等于放行——放行与否看 _code_signature_valid。"""
|
||||
now = _code_signature(self.host_py_path)
|
||||
return (now is not None and self._code_signature is not None
|
||||
and now != self._code_signature)
|
||||
|
||||
def _code_signature_valid(self):
|
||||
"""签名是否可信且未变:启动签名与现签名都必须取得到
|
||||
且相等,否则 False。False 时 _claim 拒绝认领、run_loop
|
||||
自退(75)、_run_pr_jobs 不碰任务——fail-closed,绝不
|
||||
让签名缺失的 worker 抢任务(旧版取不到当没变是 fail-open)。"""
|
||||
now = _code_signature(self.host_py_path)
|
||||
return (now is not None and self._code_signature is not None
|
||||
and now == self._code_signature)
|
||||
|
||||
def _mark_quota_hit(self, task_id):
|
||||
with self._quota_hit_lock:
|
||||
self._quota_hit_tasks.add(task_id)
|
||||
|
||||
def _take_quota_hit(self, task_id):
|
||||
"""取走一次配额看门狗命中(命中即消费,避免旧命中污染
|
||||
同一任务的下一轮):返回 True 表示本轮确被看门狗杀过。"""
|
||||
with self._quota_hit_lock:
|
||||
if task_id in self._quota_hit_tasks:
|
||||
self._quota_hit_tasks.discard(task_id)
|
||||
return True
|
||||
return False
|
||||
|
||||
def register(self):
|
||||
"""向控制面注册本 worker(写 workers 表,失败只记日志)。
|
||||
|
||||
@@ -573,7 +653,17 @@ class Worker:
|
||||
里清(本地 _execute_via_process / Docker 同步路径),
|
||||
认领后提前返回的三道门(缺 key / 仓库 / 配额)在
|
||||
run_one 里显式清占位;CAS 语义本身一句未改。
|
||||
P1 签名门:占位之前先比对多文件签名,签名缺失或已变
|
||||
时直接返回 False(fail-closed,不抢任务、不占槽),
|
||||
并记日志;不要只在 run_loop 轮次边界查——并行池线程
|
||||
直接调 run_one/_claim,轮次边界查不到它们。
|
||||
"""
|
||||
if not self._code_signature_valid():
|
||||
log_event(self.logger, "worker_claim_refused_code_signature",
|
||||
task_id=task_id,
|
||||
message="code signature missing or changed,"
|
||||
" refusing claim")
|
||||
return False
|
||||
with self._running_lock:
|
||||
already_running = task_id in self._running
|
||||
if not already_running and \
|
||||
@@ -865,14 +955,51 @@ class Worker:
|
||||
_check_unpriced_token_cap 同口径):USD 配额不计量无价
|
||||
模型,默认模型 terra 无价,若无此上限则配额对默认模型
|
||||
永远不生效;有配额行的用户当月无价 token(input +
|
||||
output,已记账 + 本轮)达到上限即停。无配额行 = 不限额。
|
||||
查询本身异常时按不限额处理(fail-open 并记一条警告):
|
||||
配额是第二道网,绝不能因查询抖动把全部任务挡在门外;
|
||||
服务端建任务/分支/重试时的准入仍是第一道。
|
||||
output,已记账 + 本轮)达到上限即停。无价上限只计
|
||||
source='actual' 的行:估算行(estimated)不是真实花费,
|
||||
计入上限会把“没花钱”算成“花超了”,且与 cost.py
|
||||
_report 的 unpriced_actual_tokens 口径一致。
|
||||
P1 执行面三处收口(旧版三处都是 fail-open,已废):
|
||||
- 属主:任务 created_by 为空时先查父任务(协调员子任务
|
||||
行可能无 created_by,属主跟父任务走);仍查不到时
|
||||
拒绝新回合,不再返回 None 放行。created_by 字面为
|
||||
"system" 的系统任务按普通属主算(无行时同样走默认
|
||||
额度,不另设无限额的系统预算)。
|
||||
- 无配额行:不再是不限额,按默认配额
|
||||
_default_quota_usd_micros()($10/月)算;只有显式
|
||||
NULL 行才是不限额(与 P0 的服务端口径一致,管理员
|
||||
显式设 NULL 的行仍被尊重)。
|
||||
- 查库失败:返回 "quota_check_failed: ..."(fail-closed),
|
||||
绝不返回 None 放行。调用方分流:新回合边界(认领门、
|
||||
重试/fallback/后续消息边界)见此串即拒绝新回合并记
|
||||
worker_quota_stop 审计;执行中轮询(popen 轮询、
|
||||
Docker 配额看门狗)见此串只记警告、不中途杀当前轮
|
||||
——查询抖动不应杀掉已在跑的回合,但下一轮边界仍会
|
||||
拒绝(见 run_one 的 _quota_hit 过滤)。查库失败的
|
||||
审计由边界调用方记,这里只记日志,避免轮询高频写审计。
|
||||
"""
|
||||
owner = (task or {}).get("created_by")
|
||||
if not owner and (task or {}).get("parent_task_id"):
|
||||
# 协调员子任务属主跟父任务走(与 _gateway_context
|
||||
# 同款回查);查不到时不猜,直接走下面的拒绝分支。
|
||||
try:
|
||||
conn = _db_connect(self.db_path)
|
||||
try:
|
||||
row = conn.execute(
|
||||
"SELECT created_by FROM tasks WHERE id = ?",
|
||||
(task["parent_task_id"],)).fetchone()
|
||||
finally:
|
||||
conn.close()
|
||||
if row is not None:
|
||||
try:
|
||||
owner = row[0]
|
||||
except (TypeError, IndexError):
|
||||
owner = None
|
||||
except Exception:
|
||||
owner = None
|
||||
if not owner:
|
||||
return None
|
||||
return ("quota_exceeded: 任务缺少属主,无法查配额,"
|
||||
"拒绝新回合(请归属真实用户后再排队)")
|
||||
try:
|
||||
month = datetime.now(timezone.utc).strftime("%Y-%m")
|
||||
conn = _db_connect(self.db_path)
|
||||
@@ -887,19 +1014,33 @@ class Worker:
|
||||
" WHERE user_id = ?",
|
||||
(owner,)).fetchone()
|
||||
if row is None:
|
||||
return None
|
||||
quota = row[0]
|
||||
# P1:无行按默认额度,不是不限额(P0 服务端
|
||||
# 已在建号/登录时补行,worker 侧仍可能遇到
|
||||
# 老库未补行的用户,这里按默认额度兜底,
|
||||
# 不写行——写行是服务端的职责,worker 不抢)。
|
||||
quota = _default_quota_usd_micros()
|
||||
else:
|
||||
quota = row[0]
|
||||
if quota is None:
|
||||
return None
|
||||
# cost_records 由 CostStore 懒建:全新库上它还不存
|
||||
# 在,直接 JOIN 会报错——先实例化一次建表再查(与
|
||||
# 服务端 _quota_status 同款做法),否则 quota=0
|
||||
#(一分钱也不能花)在全新库上会被误放行。
|
||||
# P1 补:建表必须落在本连接上,不只 CostStore 的
|
||||
# 连接上——文件库两者指向同一文件所以旧代码碰巧
|
||||
# 能用,:memory: 库两者是不同空库,本连接查 JOIN
|
||||
# 会直接报错并(P1 fail-closed 后)误拒全部任务。
|
||||
_cost_store = CostStore(self.db_path)
|
||||
try:
|
||||
_cost_store._conn.close()
|
||||
except Exception:
|
||||
pass
|
||||
try:
|
||||
conn.execute(CostStore._COST_DDL)
|
||||
conn.commit()
|
||||
except Exception:
|
||||
pass
|
||||
used_row = conn.execute(
|
||||
"SELECT COALESCE(SUM(c.usd_micros), 0)"
|
||||
" FROM cost_records c JOIN tasks t"
|
||||
@@ -912,7 +1053,7 @@ class Worker:
|
||||
" FROM cost_records c JOIN tasks t"
|
||||
" ON c.task_id = t.id"
|
||||
" WHERE t.created_by = ? AND c.day LIKE ?"
|
||||
" AND c.usd_micros IS NULL",
|
||||
" AND c.usd_micros IS NULL AND c.source = 'actual'",
|
||||
(owner, month + "%")).fetchone()
|
||||
unpriced_used = unpriced_row[0] if unpriced_row else 0
|
||||
finally:
|
||||
@@ -940,11 +1081,15 @@ class Worker:
|
||||
% (unpriced_used, cap, month))
|
||||
return None
|
||||
except Exception as exc:
|
||||
# P1 fail-closed:查库失败不再返回 None。调用方按
|
||||
# 前缀分流(见本函数 docstring):quota_check_failed
|
||||
# 在新回合边界 = 拒绝,在执行中轮询 = 只警告不杀轮。
|
||||
try:
|
||||
self.logger.warning("quota check failed: %s", exc)
|
||||
except Exception:
|
||||
pass
|
||||
return None
|
||||
return ("quota_check_failed: 配额查询失败,拒绝新回合"
|
||||
"(查库异常,非配额用完)")
|
||||
|
||||
def _ready_tasks(self):
|
||||
"""当前就绪任务列表(pick_next 的批量版)。
|
||||
@@ -1123,6 +1268,41 @@ class Worker:
|
||||
pass
|
||||
return
|
||||
|
||||
def _docker_quota_watchdog(self, task_id, container_name,
|
||||
stop_event, quota_check, interval=30.0):
|
||||
"""Docker 同步路径的配额看门狗(P1,复用取消看门狗模式)。
|
||||
|
||||
同步路径阻塞在 docker run 里,run_one 的边界检查要等
|
||||
本轮返回才跑得到——配额在别处被花完时,本轮会继续烧
|
||||
到自然结束。本线程每 interval 秒(默认 30 秒,不超过
|
||||
30 秒)用 quota_check 查一次已记账配额:命中时按容器
|
||||
名真杀容器并记 _quota_hit_tasks,_run_attempt 在同步
|
||||
返回后把结果改写成 quota_exceeded,run_one 走统一的
|
||||
配额中断收尾(interrupted + 配额原因 + 已花成本照记,
|
||||
下一轮不再续跑)。查库失败时 quota_check 返回 False
|
||||
(run_one 的 _quota_hit 会把 quota_check_failed 前缀
|
||||
过滤掉,见 _quota_exceeded docstring),本线程只继续
|
||||
轮询、不杀当前轮;下一轮边界仍会拒绝。诚实边界:看
|
||||
门狗只能看到已记账用量 + 往轮累计用量,本轮容器内尚
|
||||
未回传的 token 它看不到——那部分在结果返回后由边界
|
||||
门拦住,不会悄悄续跑下一轮。
|
||||
"""
|
||||
while not stop_event.wait(interval):
|
||||
try:
|
||||
over = bool(quota_check()) if quota_check else False
|
||||
except Exception:
|
||||
over = False
|
||||
if not over:
|
||||
continue
|
||||
self._mark_quota_hit(task_id)
|
||||
try:
|
||||
kill = getattr(self.executor, "kill_container", None)
|
||||
if kill is not None and container_name:
|
||||
kill(container_name)
|
||||
except Exception:
|
||||
pass
|
||||
return
|
||||
|
||||
def cleanup_stale_containers(self):
|
||||
"""启动清理兜底:清掉上一轮残留的 codex- 容器。
|
||||
|
||||
@@ -2069,6 +2249,7 @@ class Worker:
|
||||
# 按容器名真杀;非 Docker 路径不启动,避免空线程)。
|
||||
_watchdog_stop = None
|
||||
_watchdog_thread = None
|
||||
_quota_watchdog_thread = None
|
||||
if is_docker and hasattr(self.executor, "kill_container"):
|
||||
try:
|
||||
_container_name = self.executor.container_name(task)
|
||||
@@ -2081,6 +2262,17 @@ class Worker:
|
||||
args=(task_id, _container_name, _watchdog_stop),
|
||||
daemon=True)
|
||||
_watchdog_thread.start()
|
||||
# P1 配额看门狗:只在同步 Docker 路径起(进程
|
||||
# 模式的 _execute_via_process 已每 30 秒轮询
|
||||
# quota_check,不重复起)。quota_check 为 None
|
||||
# 时不起(无配额检查可做,不空转线程)。
|
||||
if quota_check is not None and not self._use_popen():
|
||||
_quota_watchdog_thread = threading.Thread(
|
||||
target=self._docker_quota_watchdog,
|
||||
args=(task_id, _container_name,
|
||||
_watchdog_stop, quota_check),
|
||||
daemon=True)
|
||||
_quota_watchdog_thread.start()
|
||||
try:
|
||||
gateway_provider, gateway_env = self._ensure_gateway(task)
|
||||
if self._use_popen():
|
||||
@@ -2123,8 +2315,14 @@ class Worker:
|
||||
prep_err)
|
||||
if goal_mode:
|
||||
if task.get("_resume"):
|
||||
# P1 断点续跑传 None(不传 --goal-text,
|
||||
# host 沿用原 goal+verify);开新目标
|
||||
# 才传本轮 prompt 当新 goal_text。
|
||||
_resume_text = None if task.get(
|
||||
"_resume_kind") == "breakpoint" \
|
||||
else task["prompt"]
|
||||
result = self.executor.goal_resume_task(
|
||||
task, task["prompt"],
|
||||
task, _resume_text,
|
||||
timeout=effective_docker_timeout,
|
||||
extra_mounts=tenant_mounts,
|
||||
network=tenant_network,
|
||||
@@ -2165,8 +2363,11 @@ class Worker:
|
||||
|
||||
if task.get("_resume") and isinstance(
|
||||
self.executor, LocalExecutor):
|
||||
_resume_text = None if task.get(
|
||||
"_resume_kind") == "breakpoint" \
|
||||
else task["prompt"]
|
||||
result = self.executor.goal_resume_task(
|
||||
task, task["prompt"],
|
||||
task, _resume_text,
|
||||
extra_env=gateway_env,
|
||||
model_provider=gateway_provider)
|
||||
elif isinstance(self.executor, LocalExecutor):
|
||||
@@ -2220,6 +2421,8 @@ class Worker:
|
||||
_watchdog_stop.set()
|
||||
if _watchdog_thread is not None:
|
||||
_watchdog_thread.join(timeout=2)
|
||||
if _quota_watchdog_thread is not None:
|
||||
_quota_watchdog_thread.join(timeout=2)
|
||||
# 同步路径执行结束:清掉取消用的容器标记(取消路径里
|
||||
# 若已清过,这里再清一次无害)。进程模式的 proc 由
|
||||
# _execute_via_process 自己的 finally 清,这里只清
|
||||
@@ -2236,6 +2439,18 @@ class Worker:
|
||||
if result is not None and result.get("status") == "failed" and \
|
||||
"docker run timeout" in (result.get("stdout", "") or ""):
|
||||
result = dict(result, status="timeout")
|
||||
# P1 配额看门狗命中改写:同步路径被看门狗杀容器后,
|
||||
# 执行器只返回普通 failed,run_one 无法区分“模型失败”
|
||||
# 与“配额杀容器”——命中时改写成 quota_exceeded,run_one
|
||||
# 走统一配额中断收尾(interrupted + 配额原因 + 成本照记,
|
||||
# 不重试、不续跑下一轮)。改写前先取走命中标记,避免
|
||||
# 旧命中污染同一任务的后续尝试;超时标记优先于配额
|
||||
# 改写的情况不存在(看门狗杀容器时执行器不会同时报超时)。
|
||||
if result is not None and self._take_quota_hit(task_id):
|
||||
result = dict(result, status="quota_exceeded")
|
||||
if error is None:
|
||||
error = ("quota_exceeded: Docker 配额看门狗已杀容器"
|
||||
"(配额中断,下一轮不再续跑)")
|
||||
return result, error
|
||||
|
||||
def _attempt_error(self, result, error, token):
|
||||
@@ -2384,7 +2599,12 @@ class Worker:
|
||||
task.get("priority", 0) or 0, task["created_by"],
|
||||
now, now,
|
||||
part.get("executor") or task.get("executor"),
|
||||
part.get("model"), part.get("effort"),
|
||||
# P1:分工未指定模型/强度时继承父任务(旧版
|
||||
# 直接写 part 的 None,子任务会落到 worker
|
||||
# 默认模型/强度,与父任务的显式选择脱节,
|
||||
# 既改成本也改行为,且排查时看不出继承关系)。
|
||||
part.get("model") or task.get("model"),
|
||||
part.get("effort") or task.get("effort"),
|
||||
task.get("environment_id"), task["id"], name))
|
||||
self._audit(
|
||||
task["project_id"], "coordinator_child_created",
|
||||
@@ -2422,6 +2642,109 @@ class Worker:
|
||||
conn.close()
|
||||
return rows
|
||||
|
||||
def _coord_mark_orphans(self, task, child_ids):
|
||||
"""把仍未终态的子任务标 interrupted 并记审计(P1 孤儿收口)。
|
||||
|
||||
只对 CAS 改到行的子任务记审计(rowcount==1):已被别的
|
||||
worker 终态化的子任务不动,避免覆盖它的结果。返回被标
|
||||
记的子任务 id 列表。正在跑的子任务被标 interrupted 后,
|
||||
跑它的 worker 会在轮询里看到库状态并杀进程/容器(popen
|
||||
每 1 秒查库、Docker 取消看门狗每 1 秒查库),不会无声
|
||||
继续烧钱;若那个 worker 也已死,子任务至少已是终态,
|
||||
不会再被认领烧钱。
|
||||
"""
|
||||
marked = []
|
||||
rows = self._coord_child_rows(child_ids)
|
||||
pending = _coord_nonterminal([
|
||||
{"id": cid, "status": (rows.get(cid) or {}).get("status")}
|
||||
for cid in child_ids])
|
||||
if not pending:
|
||||
return marked
|
||||
conn = _db_connect(self.db_path)
|
||||
try:
|
||||
for ch in pending:
|
||||
cur = conn.execute(
|
||||
"UPDATE tasks SET status = 'interrupted',"
|
||||
" updated_at = ? WHERE id = ? AND status NOT IN"
|
||||
" ('completed', 'failed', 'interrupted', 'merged',"
|
||||
" 'merge-failed')",
|
||||
(time.time(), ch["id"]))
|
||||
if cur.rowcount == 1:
|
||||
marked.append(ch["id"])
|
||||
self._audit(
|
||||
task["project_id"], "coordinator_child_orphaned",
|
||||
"%s -> %s (parent %s)"
|
||||
% (ch["id"], "interrupted", task["id"]))
|
||||
conn.commit()
|
||||
finally:
|
||||
conn.close()
|
||||
return marked
|
||||
|
||||
def sweep_orphaned_coordinator_children(self):
|
||||
"""常驻循环每轮的孤儿扫尾(P1):父已终态或父行失踪、子
|
||||
仍未终态时,把子标 interrupted 并记审计。
|
||||
|
||||
父仍在 running/queued 时绝不动子(协调员等齐期间子任务
|
||||
未终态是正常状态)。父行失踪(被删)同样算孤儿:没有
|
||||
父会再来合流,子任务继续跑只是烧钱。返回被标记的
|
||||
(parent_id, child_id) 对列表;查库失败时返回 [] 且只记
|
||||
日志,不炸循环。
|
||||
"""
|
||||
marked = []
|
||||
try:
|
||||
conn = _db_connect(self.db_path)
|
||||
try:
|
||||
rows = conn.execute(
|
||||
"SELECT id, parent_task_id, project_id FROM tasks"
|
||||
" WHERE parent_task_id IS NOT NULL AND status NOT IN"
|
||||
" ('completed', 'failed', 'interrupted', 'merged',"
|
||||
" 'merge-failed')").fetchall()
|
||||
parents = {}
|
||||
for row in rows:
|
||||
try:
|
||||
pid = row["parent_task_id"]
|
||||
except (KeyError, IndexError, TypeError):
|
||||
continue
|
||||
if pid not in parents:
|
||||
prow = conn.execute(
|
||||
"SELECT status FROM tasks WHERE id = ?",
|
||||
(pid,)).fetchone()
|
||||
try:
|
||||
parents[pid] = prow["status"] \
|
||||
if prow is not None else None
|
||||
except (KeyError, IndexError, TypeError):
|
||||
parents[pid] = None
|
||||
for row in rows:
|
||||
try:
|
||||
cid = row["id"]
|
||||
pid = row["parent_task_id"]
|
||||
proj = row["project_id"]
|
||||
except (KeyError, IndexError, TypeError):
|
||||
continue
|
||||
pstatus = parents.get(pid)
|
||||
if pstatus is not None and \
|
||||
pstatus not in TERMINAL_STATUSES:
|
||||
continue
|
||||
cur = conn.execute(
|
||||
"UPDATE tasks SET status = 'interrupted',"
|
||||
" updated_at = ? WHERE id = ? AND status NOT IN"
|
||||
" ('completed', 'failed', 'interrupted', 'merged',"
|
||||
" 'merge-failed')",
|
||||
(time.time(), cid))
|
||||
if cur.rowcount == 1:
|
||||
marked.append((pid, cid))
|
||||
self._audit(
|
||||
proj, "coordinator_child_orphaned",
|
||||
"%s -> interrupted (parent %s, parent"
|
||||
" status %s)" % (cid, pid, pstatus))
|
||||
conn.commit()
|
||||
finally:
|
||||
conn.close()
|
||||
except Exception as exc:
|
||||
log_event(self.logger, "worker_orphan_sweep_failed",
|
||||
message="orphan sweep failed: %s" % exc)
|
||||
return marked
|
||||
|
||||
def _run_coordinator(self, task, started_at, poll_interval=None,
|
||||
timeout=None):
|
||||
"""协调员执行:建子任务 → 等齐 → 合流汇总写回。
|
||||
@@ -2460,6 +2783,7 @@ class Worker:
|
||||
timed_out = True
|
||||
break
|
||||
time.sleep(poll_interval)
|
||||
orphaned = []
|
||||
if cancelled or timed_out:
|
||||
rows = self._coord_child_rows(child_ids)
|
||||
for cid in child_ids:
|
||||
@@ -2469,6 +2793,15 @@ class Worker:
|
||||
self.cancel(cid)
|
||||
except Exception:
|
||||
pass
|
||||
# P1:cancel() 对已终态/不存在的子任务返回 False,
|
||||
# 跨 worker 在跑的子任务也可能因竞态仍未终态——
|
||||
# cancel 之后仍未终态的一律直接标 interrupted 并记
|
||||
# 审计(_coord_mark_orphans),绝不留孤儿无声烧钱;
|
||||
# 标记结果写进汇总报告,调用方在详情里看得到。
|
||||
try:
|
||||
orphaned = self._coord_mark_orphans(task, child_ids)
|
||||
except Exception:
|
||||
orphaned = []
|
||||
rows = self._coord_child_rows(child_ids)
|
||||
merged = []
|
||||
for ch in children:
|
||||
@@ -2479,6 +2812,9 @@ class Worker:
|
||||
"result_text": row.get("result_text"),
|
||||
"error_text": row.get("error_text")})
|
||||
report, overall = _coord_build_report(merged)
|
||||
if orphaned:
|
||||
report += ("\n\n孤儿子任务已标 interrupted:%s"
|
||||
% "、".join(orphaned))
|
||||
if cancelled:
|
||||
final = "interrupted"
|
||||
error = "cancelled"
|
||||
@@ -2531,6 +2867,47 @@ class Worker:
|
||||
"executor": self._executor_name(), "stdout": report,
|
||||
"error": error, "events": [], "cost_record_id": None}
|
||||
|
||||
def _record_usage_cost(self, task, actual_model, usage_total,
|
||||
stdout, fallback_count):
|
||||
"""记一条任务成本并累加 worker 级用量(调用方须持 _db_lock)。
|
||||
|
||||
P1 成本修齐:旧版 max_cost_usd 参数存在但 usage_cost_usd
|
||||
从未累加——_budget_exceeded 的成本门永远是 0,等于没设。
|
||||
本函数是唯一记账点(终局块与取消分支共用):actual 行
|
||||
按价目表算 usd_micros(无价模型为 None),有价行才累加
|
||||
usage_cost_usd(无价行不折算成 0 美元,仍受无价上限管);
|
||||
无用量时回落字符估算记 estimated,绝不冒充 actual。
|
||||
cache_write_input_tokens 不进这里:它只在 usage dict 里
|
||||
做观测(见 cost.py 的不计费声明),不加进 total、不用
|
||||
于定价。返回 cost record id。
|
||||
"""
|
||||
task_id = task["id"]
|
||||
if usage_total and (usage_total.get("total_tokens") or 0) > 0:
|
||||
micros = price_usd_micros(
|
||||
actual_model, usage_total.get("input_tokens", 0),
|
||||
usage_total.get("cached_input_tokens", 0),
|
||||
usage_total.get("output_tokens", 0))
|
||||
cost_id = self.costs.record(
|
||||
task_id, task["project_id"], actual_model,
|
||||
input_tokens=usage_total.get("input_tokens", 0),
|
||||
output_tokens=usage_total.get("output_tokens", 0),
|
||||
cached_input_tokens=usage_total.get(
|
||||
"cached_input_tokens", 0),
|
||||
usd_micros=micros,
|
||||
source="actual", fallback_count=fallback_count)
|
||||
else:
|
||||
micros = None
|
||||
total = estimate_tokens(task.get("prompt") or "", stdout)
|
||||
cost_id = self.costs.record(
|
||||
task_id, task["project_id"], actual_model,
|
||||
input_tokens=total, output_tokens=0, usd_micros=None,
|
||||
source="estimated", fallback_count=fallback_count)
|
||||
if micros is not None:
|
||||
# max_cost_usd 不再是死参数:只有有价花费才累加,
|
||||
# _budget_exceeded 据此在下一认领前挡住超支 worker。
|
||||
self.usage_cost_usd += micros / 1000000.0
|
||||
return cost_id
|
||||
|
||||
def run_one(self, task=None):
|
||||
"""认领并执行一个任务,返回结果 dict(见模块 docstring)。
|
||||
|
||||
@@ -2552,11 +2929,32 @@ class Worker:
|
||||
# 成它(只换内存里的副本,原 prompt 不动);_resume 标记
|
||||
# 让 _run_attempt 在 goal 模式下走 goal-resume 而不是 goal
|
||||
# (goal 会撞已存在,只有 goal-resume 能在旧线程上开新循环)。
|
||||
# P1 goal-resume 分流(断点 vs 开新目标,三处一致的 worker 侧):
|
||||
# - 旧验收形态(verify_command 非空):一律断点续跑——
|
||||
# 绝不能把续跑消息当新 goal_text 传给 host.py,否则
|
||||
# host 的 goal-resume --goal-text 分支会用 verify=None
|
||||
# 开新循环,把原 verify_command 覆盖掉;断点调用不传
|
||||
# --goal-text,host 沿用原 goal + 原 verify + 原轮次,
|
||||
# 从 goal_rounds_done 的下一轮续跑。续跑消息本轮不
|
||||
# 当 prompt 用(断点语义就是续原目标),跑完照常清掉。
|
||||
# - goal_mode=1(无验收):开新目标——续跑消息就是新
|
||||
# goal_text(服务端 /continue 带 goal_mode=1 时已把
|
||||
# task.goal 写成该消息,这里与之对齐)。
|
||||
# - 非 goal 任务:沿用旧行为(prompt 换成续跑消息,走
|
||||
# 已存在续跑 run 路径)。
|
||||
resume_message = task.get("resume_message")
|
||||
if resume_message:
|
||||
task = dict(task)
|
||||
task["prompt"] = resume_message
|
||||
task["_resume"] = True
|
||||
if task.get("verify_command"):
|
||||
task["_resume"] = True
|
||||
task["_resume_kind"] = "breakpoint"
|
||||
elif task.get("goal_mode"):
|
||||
task["prompt"] = resume_message
|
||||
task["_resume"] = True
|
||||
task["_resume_kind"] = "new-goal"
|
||||
else:
|
||||
task["prompt"] = resume_message
|
||||
task["_resume"] = True
|
||||
self._audit(task["project_id"], "worker_claim",
|
||||
"%s executor=%s" % (task_id, self._executor_name()))
|
||||
# log_event 只接受 task_id/project_id/status/message 四个
|
||||
@@ -2647,6 +3045,56 @@ class Worker:
|
||||
"stdout": "", "error": quota_error, "events": [],
|
||||
"cost_record_id": None}
|
||||
|
||||
# P1 goal 轮次上限门(执行侧防线):旧验收形态的
|
||||
# goal_max_rounds 超过 GOAL_MAX_ROUNDS_CAP 时直接 failed,
|
||||
# 不进 host.py goal 循环烧轮次;goal_mode=1 的 0(模型宣告
|
||||
# 才结束)豁免轮次上限(上界是配额),但 0 配上旧验收命令
|
||||
# 时同样 failed(服务端建任务时 0 本就 400,老行若有 0
|
||||
# 验收形态属于脏数据,不按不限轮次跑)。建任务时的 400
|
||||
# 校验在 server.py(另一路文件),这里是执行侧第二道门,
|
||||
# 错误文本写明上限与“请新建合规任务”,不静默截断轮次
|
||||
# (截断会改任务语义,调用方看不出被改过)。
|
||||
_goal_rounds = task.get("goal_max_rounds")
|
||||
if (task.get("verify_command") or task.get("goal_mode")) \
|
||||
and _goal_rounds is not None:
|
||||
_goal_cap_error = None
|
||||
try:
|
||||
_goal_rounds_int = int(_goal_rounds)
|
||||
except (TypeError, ValueError):
|
||||
_goal_rounds_int = None
|
||||
if _goal_rounds_int is not None \
|
||||
and _goal_rounds_int > GOAL_MAX_ROUNDS_CAP:
|
||||
_goal_cap_error = (
|
||||
"goal_max_rounds 超过上限 %d 轮(当前 %d),"
|
||||
"请新建轮次合规的任务" % (
|
||||
GOAL_MAX_ROUNDS_CAP, _goal_rounds_int))
|
||||
elif _goal_rounds_int == 0 and task.get("verify_command") \
|
||||
and not task.get("goal_mode"):
|
||||
_goal_cap_error = (
|
||||
"旧验收形态 goal_max_rounds=0 表示不限轮次,"
|
||||
"已禁用(上限 %d 轮),请新建轮次合规的任务"
|
||||
% GOAL_MAX_ROUNDS_CAP)
|
||||
if _goal_cap_error is not None:
|
||||
self._set_status(task_id, "failed")
|
||||
self._write_execution_result(
|
||||
task_id, error_text=_goal_cap_error,
|
||||
actual_model=self.model,
|
||||
started_at=started_at, finished_at=time.time(),
|
||||
only_if_status="failed")
|
||||
self._audit(task["project_id"], "worker_goal_cap_rejected",
|
||||
"%s: %s" % (task_id, _goal_cap_error))
|
||||
log_event(self.logger, "worker_goal_cap_rejected",
|
||||
task_id=task_id, project_id=task["project_id"],
|
||||
status="failed")
|
||||
self._running_clear(task_id)
|
||||
return {"dispatched": True, "task_id": task_id,
|
||||
"host_name": self.executor._host_name(task)
|
||||
if hasattr(self.executor, "_host_name") else task_id,
|
||||
"status": "failed",
|
||||
"executor": self._executor_name(),
|
||||
"stdout": "", "error": _goal_cap_error, "events": [],
|
||||
"cost_record_id": None}
|
||||
|
||||
# 协调员分支:不跑模型回合,改为按分工建并行子任务、等齐、
|
||||
# 合流汇总。已过 key 门与配额门;仓库门已对协调员任务本身
|
||||
# 跑过(项目仓库解析失败会在前面直接 failed,不会建一堆
|
||||
@@ -2721,7 +3169,14 @@ class Worker:
|
||||
def _quota_hit():
|
||||
# 闭包读当前值:usage_total 与 task 在重试/fallback
|
||||
# 时会被重绑,轮询掐断要按最新值判断。
|
||||
return self._quota_exceeded(task, usage_total) is not None
|
||||
# P1 分流:执行中轮询只认真正的 quota_exceeded,
|
||||
# quota_check_failed(查库失败)不杀当前轮——查询
|
||||
# 抖动不应杀掉已在跑的回合;但它在重试/fallback/
|
||||
# 后续消息等新回合边界仍是拒绝(那些边界直接调
|
||||
# _quota_exceeded,不走这个过滤闭包)。
|
||||
err = self._quota_exceeded(task, usage_total)
|
||||
return err is not None and \
|
||||
not err.startswith("quota_check_failed:")
|
||||
|
||||
# 分支任务:先 fork 再跑首轮(见 _run_fork);首轮失败
|
||||
# 不自动重试/fallback,失败后走重试入口重跑首轮(已存在
|
||||
@@ -2763,6 +3218,39 @@ class Worker:
|
||||
# 喂回,或新形态的模型自决多轮),worker 级重试/fallback
|
||||
# 不再叠加——同任务重跑会撞 host 任务名已存在,第二
|
||||
# 次尝试必失败且污染错误文本。
|
||||
# P1 唯一例外:goal 首轮撞“已存在”时(重试入口
|
||||
# 重排同一 goal 任务的典型死法),不再直接 failed,
|
||||
# 而是走一次断点续跑(goal-resume 不传 --goal-text,
|
||||
# 沿用原 goal+verify,从断点续跑);断点续跑的
|
||||
# 结果即本轮结果(成功则完成,仍失败则以续跑的
|
||||
# 错误为准,不再拼“已存在”)。已是续跑的任务
|
||||
# (_resume)不再二次回落,避免断点循环。
|
||||
# 分支任务(forked_from)不走这里:fork 首轮的
|
||||
# 已存在语义是 dispatch 续跑,已在 _run_attempt
|
||||
# 内处理,不叠加 goal 断点。
|
||||
if not task.get("_resume") and not task.get("forked_from"):
|
||||
_combined = ((result or {}).get("stdout", "") or "") \
|
||||
+ ((result or {}).get("stderr", "") or "")
|
||||
_goal_err = (result or {}).get("goal_error") or ""
|
||||
if _ALREADY_EXISTS_MARKER in _combined or \
|
||||
_ALREADY_EXISTS_MARKER in _goal_err or \
|
||||
_ALREADY_EXISTS_MARKER in (error or ""):
|
||||
self._audit(task["project_id"],
|
||||
"worker_goal_resume_fallback",
|
||||
"%s: goal 已存在,走断点续跑" % task_id)
|
||||
_resume_task = dict(task)
|
||||
_resume_task["_resume"] = True
|
||||
_resume_task["_resume_kind"] = "breakpoint"
|
||||
result, error = self._run_attempt(
|
||||
_resume_task, quota_check=_quota_hit)
|
||||
usage_total = sum_usage(
|
||||
usage_total, (result or {}).get("usage"))
|
||||
self._deliver_peer_messages(task, result)
|
||||
if (result or {}).get("status") == "quota_exceeded":
|
||||
quota_error = self._quota_exceeded(
|
||||
task, usage_total) or \
|
||||
"quota_exceeded: 本月配额已用完"
|
||||
quota_stop = True
|
||||
break
|
||||
if result.get("status") == "completed":
|
||||
break
|
||||
@@ -2900,6 +3388,18 @@ class Worker:
|
||||
turn_id=(result or {}).get("turn_id"),
|
||||
root_turn_id=(result or {}).get("root_turn_id"),
|
||||
only_if_status="interrupted")
|
||||
# P1:取消分支先记一条成本再返回(与配额中断收尾
|
||||
# 同款记账点)。旧版这里直接返回 cost_record_id=None,
|
||||
# 被取消前已花的模型费用就从账上丢了;取消不等于没
|
||||
# 花钱。记账失败不挡取消返回(costs.record 异常时
|
||||
# cost_id 置 None,任务状态仍是 interrupted)。
|
||||
try:
|
||||
with self._db_lock:
|
||||
_cancel_cost_id = self._record_usage_cost(
|
||||
task, actual_model, usage_total,
|
||||
(result or {}).get("stdout", ""), fallback_count)
|
||||
except Exception:
|
||||
_cancel_cost_id = None
|
||||
log_event(self.logger, "worker_cancelled", task_id=task_id,
|
||||
project_id=task["project_id"], status="interrupted")
|
||||
return {"dispatched": True, "task_id": task_id,
|
||||
@@ -2908,7 +3408,7 @@ class Worker:
|
||||
"status": "interrupted", "executor": self._executor_name(),
|
||||
"stdout": (result or {}).get("stdout", ""),
|
||||
"error": "cancelled", "events": [],
|
||||
"cost_record_id": None}
|
||||
"cost_record_id": _cancel_cost_id}
|
||||
|
||||
stdout = result.get("stdout", "") or ""
|
||||
host_status = result.get("host_status")
|
||||
@@ -2979,32 +3479,12 @@ class Worker:
|
||||
self._cleanup_task_seed(task)
|
||||
with self._db_lock:
|
||||
self.usage_turns += 1
|
||||
# 成本:执行器带回上游真实用量时记 actual(input/output/
|
||||
# cached 全是实测值,usd 按价目表算、无价模型记 None,
|
||||
# 报表里进 unpriced_actual_tokens 而不是 0 美元);
|
||||
# 没带回用量时回落字符估算记 estimated,绝不冒充 actual。
|
||||
# fallback_count 复用同一记录;模型记最后真正跑的那个
|
||||
# (fallback 成功时就是 fallback 模型)。cost 记录共用
|
||||
# 一条连接,并发下必须持锁写,否则两个线程共用同一
|
||||
# sqlite 连接会直接报错。
|
||||
if usage_total and (usage_total.get("total_tokens") or 0) > 0:
|
||||
cost_id = self.costs.record(
|
||||
task_id, task["project_id"], actual_model,
|
||||
input_tokens=usage_total.get("input_tokens", 0),
|
||||
output_tokens=usage_total.get("output_tokens", 0),
|
||||
cached_input_tokens=usage_total.get(
|
||||
"cached_input_tokens", 0),
|
||||
usd_micros=price_usd_micros(
|
||||
actual_model, usage_total.get("input_tokens", 0),
|
||||
usage_total.get("cached_input_tokens", 0),
|
||||
usage_total.get("output_tokens", 0)),
|
||||
source="actual", fallback_count=fallback_count)
|
||||
else:
|
||||
total = estimate_tokens(task.get("prompt") or "", stdout)
|
||||
cost_id = self.costs.record(
|
||||
task_id, task["project_id"], actual_model,
|
||||
input_tokens=total, output_tokens=0, usd_micros=None,
|
||||
source="estimated", fallback_count=fallback_count)
|
||||
# 成本记账走 _record_usage_cost(终局与取消分支共用
|
||||
# 的唯一记账点):actual/estimated 分流、usage_cost_usd
|
||||
# 累加都在里面;cost 记录共用一条连接,并发下必须
|
||||
# 持锁写,否则两个线程共用同一 sqlite 连接会直接报错。
|
||||
cost_id = self._record_usage_cost(
|
||||
task, actual_model, usage_total, stdout, fallback_count)
|
||||
if final_written:
|
||||
self._audit(task["project_id"],
|
||||
"worker_quota_stop" if quota_stop else
|
||||
@@ -3064,9 +3544,65 @@ class Worker:
|
||||
发 review-requested/merge-failed,这里只写回任务终态。
|
||||
跳过(PrSkipped,如任务已不是 merged)与落盘失败都记
|
||||
failed + error,任务本身保持 merged、可重调 /pr 再排队。
|
||||
P1 非 local 通道:旧版非 local worker 直接返回 [],
|
||||
排队的 pr 任务看起来像“没人处理”且无任何痕迹——
|
||||
静默未完成。现在改为显式 needs_local_worker:只读
|
||||
peek 本主机 queued 的 pr 任务,逐条记审计 + 日志并
|
||||
返回 needs_local_worker 条目,任务本身保持 merged、
|
||||
pr 任务保持 queued(留给 local worker 认领),绝不
|
||||
把它标 failed(failed 会让调用方以为 PR 已失败,且
|
||||
任务仍可重调 /pr 的语义会被混淆)。为什么不直接把
|
||||
pr_jobs 行写成 needs_local_worker 状态:pr_jobs 的认
|
||||
领只认 queued(claim_pr_job 在 pr_exec.py,本路不改
|
||||
该文件),写成新状态后 local worker 也认领不到,任务
|
||||
会被永久搁置;真要落库状态,需 pr_exec.py 的认领同
|
||||
步改成含 needs_local_worker 或 local 重排队——该改动
|
||||
留给协调员(见汇报残留),本路不伪造一个“写了状态
|
||||
却没人能接”的通道。签名无效时同样不碰任务(返回 [],
|
||||
只记日志),fail-closed。
|
||||
"""
|
||||
if self._executor_name() != "local":
|
||||
if not self._code_signature_valid():
|
||||
log_event(self.logger, "worker_pr_refused_code_signature",
|
||||
message="code signature missing or changed,"
|
||||
" refusing pr jobs")
|
||||
return []
|
||||
if self._executor_name() != "local":
|
||||
results = []
|
||||
try:
|
||||
from cloud.pr_exec import ensure_pr_jobs_table
|
||||
conn = _db_connect(self.db_path)
|
||||
try:
|
||||
ensure_pr_jobs_table(conn)
|
||||
rows = conn.execute(
|
||||
"SELECT id, task_id, project_id FROM pr_jobs"
|
||||
" WHERE host = ? AND status = 'queued'",
|
||||
(self.host_name,)).fetchall()
|
||||
finally:
|
||||
conn.close()
|
||||
for row in rows:
|
||||
try:
|
||||
job_id = row["id"]
|
||||
task_id = row["task_id"]
|
||||
project_id = row["project_id"]
|
||||
except (KeyError, IndexError, TypeError):
|
||||
continue
|
||||
reason = ("PR 执行需要 local worker(host-worktrees"
|
||||
" 只在主机上可见),本 worker 为 %s,"
|
||||
"任务保持 queued 等 local worker 认领"
|
||||
% self._executor_name())
|
||||
self._audit(project_id, "pr_job_needs_local_worker",
|
||||
"%s: %s" % (job_id, reason))
|
||||
log_event(self.logger, "pr_job_needs_local_worker",
|
||||
task_id=task_id, status="needs_local_worker",
|
||||
message=reason)
|
||||
results.append({"pr_job_id": job_id,
|
||||
"task_id": task_id,
|
||||
"status": "needs_local_worker",
|
||||
"error": reason})
|
||||
except Exception as exc:
|
||||
log_event(self.logger, "pr_job_peek_failed",
|
||||
message="pr job peek failed: %s" % exc)
|
||||
return results
|
||||
results = []
|
||||
while True:
|
||||
conn = _db_connect(self.db_path)
|
||||
@@ -3312,8 +3848,12 @@ class Worker:
|
||||
# 任务;退出码 75(EX_TEMPFAIL,失败类):unit 托管
|
||||
# 的由 systemd on-failure 重拉新版,autoscaler 托管
|
||||
# 的由 reconcile 重拉,孤儿直接消失。单次 run_all
|
||||
# 不走这里(进程本来就退出)。
|
||||
if self._code_changed():
|
||||
# 不走这里(进程本来就退出)。P1 fail-closed:
|
||||
# 签名缺失同样自退(旧版只在“变了”时退,取不到
|
||||
# 签名时继续跑是 fail-open);每次认领前的签名门
|
||||
# 在 _claim 里(池线程不走轮次边界也拦得住)。
|
||||
if self._code_changed() or \
|
||||
not self._code_signature_valid():
|
||||
log_event(self.logger, "worker_code_changed_exit",
|
||||
message="worker code changed, exiting "
|
||||
"for restart")
|
||||
@@ -3336,6 +3876,15 @@ class Worker:
|
||||
except Exception as exc:
|
||||
log_event(self.logger, "worker_sweep_failed",
|
||||
message="worker sweep failed: %s" % exc)
|
||||
# P1 孤儿扫尾(接在陈旧回收之后):协调员本身
|
||||
# 被回收标 interrupted 后,它的未终态子任务
|
||||
# 在这里被统一标 interrupted;协调员仍在跑时
|
||||
# 本函数不动子任务,见 sweep 方法 docstring。
|
||||
try:
|
||||
self.sweep_orphaned_coordinator_children()
|
||||
except Exception as exc:
|
||||
log_event(self.logger, "worker_orphan_sweep_failed",
|
||||
message="orphan sweep failed: %s" % exc)
|
||||
# 每轮结束写心跳:哪怕本轮一个任务都没派,worker 活着
|
||||
# 这件事本身也要被 server 看到(文件心跳 + 注册表心跳)。
|
||||
self._write_heartbeat()
|
||||
|
||||
Reference in new issue
Block a user