Merge pull request #98 from openqj/feat-p1-security
feat: P1 安全批(执行中配额检查、成本实账、goal/协调员语义、PR 通道、孤儿签名、控制面与控制台加固)
This commit is contained in:
20 files changed
+2616
-179
No files matched your search
+79
-12
@@ -160,7 +160,7 @@
|
|||||||
</div>
|
</div>
|
||||||
<textarea id="prompt" placeholder="任务 prompt(必填)"></textarea>
|
<textarea id="prompt" placeholder="任务 prompt(必填)"></textarea>
|
||||||
<textarea id="coordParts" placeholder="协调员子任务 JSON(可选),留空为普通任务"></textarea>
|
<textarea id="coordParts" placeholder="协调员子任务 JSON(可选),留空为普通任务"></textarea>
|
||||||
<div class="muted">协调员格式:[{"name":"后端","prompt":"做后端"},{"name":"前端","prompt":"做前端"}];提交后 worker 并行建子任务、等齐、合流汇总。不能与 goal/计划模式同用。</div>
|
<div class="muted">协调员格式:[{"name":"后端","prompt":"做后端"},{"name":"前端","prompt":"做前端"}];提交后 worker 并行建子任务、等齐、合流汇总。不能与 goal/计划模式同用。协调员与 goal/计划互斥:勾了 goal 或计划模式时不要填协调员 JSON,提交前会校验并拦截。子任务费用记父任务创建人账;父任务创建时按分工数预留配额,余额不足时整个父任务被挡下。</div>
|
||||||
<div class="bar">
|
<div class="bar">
|
||||||
<select id="modelSelect">
|
<select id="modelSelect">
|
||||||
<option value="">模型:默认(跟随配置)</option>
|
<option value="">模型:默认(跟随配置)</option>
|
||||||
@@ -265,10 +265,11 @@
|
|||||||
<input id="reviewSha" placeholder="指定提交用:提交 sha" style="width:100%">
|
<input id="reviewSha" placeholder="指定提交用:提交 sha" style="width:100%">
|
||||||
<textarea id="reviewInstructions" placeholder="自定义评审指令用:评审指令正文"></textarea>
|
<textarea id="reviewInstructions" placeholder="自定义评审指令用:评审指令正文"></textarea>
|
||||||
<div class="bar">
|
<div class="bar">
|
||||||
<button id="btnCreateReview" class="primary" onclick="postReview()">发起评审</button>
|
<button id="btnCreateReview" class="primary" onclick="postReview()">登记评审需求</button>
|
||||||
<button onclick="if (currentTask) loadReviews(currentTask.id)">刷新队列</button>
|
<button onclick="if (currentTask) loadReviews(currentTask.id)">刷新队列</button>
|
||||||
<span id="reviewMsg" class="muted"></span>
|
<span id="reviewMsg" class="muted"></span>
|
||||||
</div>
|
</div>
|
||||||
|
<div class="muted">本基线只登记评审需求(requested),不自动跑评审;评审是否执行、结果如何,以评审队列里的记录为准。</div>
|
||||||
</div>
|
</div>
|
||||||
<div id="reviewList" class="muted">点任务行查看详情后,点“刷新队列”查看评审记录</div>
|
<div id="reviewList" class="muted">点任务行查看详情后,点“刷新队列”查看评审记录</div>
|
||||||
<h3>实时链路</h3>
|
<h3>实时链路</h3>
|
||||||
@@ -286,7 +287,7 @@
|
|||||||
<div id="followupList" class="muted"></div>
|
<div id="followupList" class="muted"></div>
|
||||||
<h3>运行时审批队列</h3>
|
<h3>运行时审批队列</h3>
|
||||||
<div class="bar"><button id="btnRefreshApprovals" onclick="if(currentTask)loadApprovals(currentTask.id)">刷新审批队列</button><span id="approvalMsg" class="muted"></span></div>
|
<div class="bar"><button id="btnRefreshApprovals" onclick="if(currentTask)loadApprovals(currentTask.id)">刷新审批队列</button><span id="approvalMsg" class="muted"></span></div>
|
||||||
<div class="muted">待审批项需人工点允许或拒绝;超时默认拒绝。本队列的决定会留回执;实时请求的事前拦截待 host 接入后生效,在此之前实时行为不变。</div>
|
<div class="muted">待审批项需人工点记录允许或记录拒绝;超时默认拒绝。当前版本只留回执,不拦截实时行为(实时请求的事前拦截待 host 接入后生效,在此之前实时行为不变)。</div>
|
||||||
<div id="approvalList" class="muted"></div>
|
<div id="approvalList" class="muted"></div>
|
||||||
<h3>给其他任务发消息</h3>
|
<h3>给其他任务发消息</h3>
|
||||||
<div id="peerMessageForm">
|
<div id="peerMessageForm">
|
||||||
@@ -354,7 +355,12 @@ async function api(path, method, body) {
|
|||||||
var msg = (data && data.error) ? data.error : ("HTTP " + r.status);
|
var msg = (data && data.error) ? data.error : ("HTTP " + r.status);
|
||||||
if (r.status === 401) msg += "(key 未填或无效)";
|
if (r.status === 401) msg += "(key 未填或无效)";
|
||||||
if (r.status === 429) msg += "(触发限流,稍后再试)";
|
if (r.status === 429) msg += "(触发限流,稍后再试)";
|
||||||
if (r.status === 402) msg += "(本月配额已用完,联系管理员调整)";
|
if (r.status === 402) {
|
||||||
|
// 402 要区分两种满:服务端在响应里带哪个上限字段就是哪个满。
|
||||||
|
if (data && data.unpriced_token_cap !== undefined) msg += "(无价 token 已满,联系管理员调整)";
|
||||||
|
else if (data && data.error === "coordinator_quota_reserve_exceeded") msg = "协调员分工预留不足:剩余美元配额不够覆盖全部分工,子任务费用记父任务创建人账,整个父任务已被挡下(未建任何子任务)";
|
||||||
|
else msg += "(美元配额已满,联系管理员调整)";
|
||||||
|
}
|
||||||
throw new Error(msg);
|
throw new Error(msg);
|
||||||
}
|
}
|
||||||
return data;
|
return data;
|
||||||
@@ -394,13 +400,22 @@ async function loadQuota() {
|
|||||||
try {
|
try {
|
||||||
var d = await api("/users/me/quota", "GET");
|
var d = await api("/users/me/quota", "GET");
|
||||||
var used = (d.used_usd_micros / 1000000).toFixed(2);
|
var used = (d.used_usd_micros / 1000000).toFixed(2);
|
||||||
|
var usdLine;
|
||||||
if (d.quota_usd_micros === null || d.quota_usd_micros === undefined) {
|
if (d.quota_usd_micros === null || d.quota_usd_micros === undefined) {
|
||||||
el.textContent = "本月用量 $" + used + "(不限额)";
|
usdLine = "美元:本月用量 $" + used + "(不限额)";
|
||||||
} else {
|
} else {
|
||||||
var q = (d.quota_usd_micros / 1000000).toFixed(2);
|
var q = (d.quota_usd_micros / 1000000).toFixed(2);
|
||||||
var rem = (d.remaining_usd_micros / 1000000).toFixed(2);
|
var rem = (d.remaining_usd_micros / 1000000).toFixed(2);
|
||||||
el.textContent = "本月用量 $" + used + " / $" + q + "(剩余 $" + rem + ")";
|
usdLine = "美元:本月用量 $" + used + " / $" + q + "(剩余 $" + rem + ")";
|
||||||
}
|
}
|
||||||
|
// 两条进度并列:美元一条,无价 token 一条;402 时按哪条满区分报错。
|
||||||
|
var tokLine = "无价 token:";
|
||||||
|
if (d.unpriced_token_cap === undefined || d.unpriced_tokens_used === undefined) {
|
||||||
|
tokLine += "未返回(旧服务端)";
|
||||||
|
} else {
|
||||||
|
tokLine += d.unpriced_tokens_used + " / " + d.unpriced_token_cap + " tokens";
|
||||||
|
}
|
||||||
|
el.textContent = usdLine + " | " + tokLine;
|
||||||
} catch (e) { el.textContent = "配额加载失败:" + e.message; }
|
} catch (e) { el.textContent = "配额加载失败:" + e.message; }
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -453,8 +468,8 @@ async function loadApprovals(taskId) {
|
|||||||
}
|
}
|
||||||
var btns = "";
|
var btns = "";
|
||||||
if (a.status === "pending") {
|
if (a.status === "pending") {
|
||||||
btns = " <button onclick=\"decideApproval('" + esc(a.id) + "', 'approved')\">允许</button>" +
|
btns = " <button onclick=\"decideApproval('" + esc(a.id) + "', 'approved')\">记录允许</button>" +
|
||||||
" <button onclick=\"decideApproval('" + esc(a.id) + "', 'denied')\">拒绝</button>";
|
" <button onclick=\"decideApproval('" + esc(a.id) + "', 'denied')\">记录拒绝</button>";
|
||||||
}
|
}
|
||||||
return "[" + esc(a.kind) + "|" + esc(a.status) + "] " + esc(a.summary) + receipt + btns;
|
return "[" + esc(a.kind) + "|" + esc(a.status) + "] " + esc(a.summary) + receipt + btns;
|
||||||
}).join("<br>");
|
}).join("<br>");
|
||||||
@@ -465,13 +480,20 @@ async function decideApproval(approvalId, decision) {
|
|||||||
var verb = decision === "approved" ? "approve" : "deny";
|
var verb = decision === "approved" ? "approve" : "deny";
|
||||||
try {
|
try {
|
||||||
await api("/tasks/" + encodeURIComponent(currentTask.id) + "/approvals/" + encodeURIComponent(approvalId) + "/" + verb, "POST", {});
|
await api("/tasks/" + encodeURIComponent(currentTask.id) + "/approvals/" + encodeURIComponent(approvalId) + "/" + verb, "POST", {});
|
||||||
$("#approvalMsg").innerHTML = "<span class='ok'>已" + (decision === "approved" ? "允许" : "拒绝") + "</span>";
|
$("#approvalMsg").innerHTML = "<span class='ok'>已记录" + (decision === "approved" ? "允许" : "拒绝") + "(当前版本只留回执,不拦截实时行为)</span>";
|
||||||
loadApprovals(currentTask.id);
|
loadApprovals(currentTask.id);
|
||||||
} catch (e) { $("#approvalMsg").innerHTML = "<span class='err'>决定失败:" + esc(e.message) + "</span>"; }
|
} catch (e) { $("#approvalMsg").innerHTML = "<span class='err'>决定失败:" + esc(e.message) + "</span>"; }
|
||||||
}
|
}
|
||||||
async function loadWorkers() {
|
async function loadWorkers() {
|
||||||
try {
|
try {
|
||||||
var d = await api("/workers", "GET");
|
var d = await api("/workers", "GET");
|
||||||
|
// 非全局管理员只拿到聚合数字(服务端 P1 收紧):不展示 worker 清单。
|
||||||
|
if (d.workers === undefined && d.workers_total !== undefined) {
|
||||||
|
if (!d.workers_total) { $("#workers").textContent = "无 worker 注册"; return; }
|
||||||
|
$("#workers").textContent = "在线 " + d.workers_online + " / 离线 " + d.workers_offline +
|
||||||
|
"(共 " + d.workers_total + ";完整 worker 清单仅全局管理员可见)";
|
||||||
|
return;
|
||||||
|
}
|
||||||
var list = d.workers || d || [];
|
var list = d.workers || d || [];
|
||||||
if (!list.length) { $("#workers").textContent = "无 worker 注册"; return; }
|
if (!list.length) { $("#workers").textContent = "无 worker 注册"; return; }
|
||||||
$("#workers").innerHTML = list.map(function (w) {
|
$("#workers").innerHTML = list.map(function (w) {
|
||||||
@@ -759,7 +781,7 @@ async function postReview() {
|
|||||||
await api("/tasks/" + encodeURIComponent(currentTask.id) + "/reviews", "POST", body);
|
await api("/tasks/" + encodeURIComponent(currentTask.id) + "/reviews", "POST", body);
|
||||||
$("#reviewMsg").innerHTML = "<span class='ok'>评审已登记(requested)</span>";
|
$("#reviewMsg").innerHTML = "<span class='ok'>评审已登记(requested)</span>";
|
||||||
loadReviews(currentTask.id);
|
loadReviews(currentTask.id);
|
||||||
} catch (e) { $("#reviewMsg").innerHTML = "<span class='err'>发起评审失败:" + esc(e.message) + "</span>"; }
|
} catch (e) { $("#reviewMsg").innerHTML = "<span class='err'>登记评审需求失败:" + esc(e.message) + "</span>"; }
|
||||||
}
|
}
|
||||||
async function submitReviewFindings(reviewId) {
|
async function submitReviewFindings(reviewId) {
|
||||||
if (!currentTask) return;
|
if (!currentTask) return;
|
||||||
@@ -778,8 +800,19 @@ async function showDetail(taskId) {
|
|||||||
document.querySelectorAll("#taskRows tr").forEach(function (tr) {
|
document.querySelectorAll("#taskRows tr").forEach(function (tr) {
|
||||||
tr.classList.toggle("selected", tr.getAttribute("data-id") === taskId);
|
tr.classList.toggle("selected", tr.getAttribute("data-id") === taskId);
|
||||||
});
|
});
|
||||||
|
// 每个任务状态配一句中文解释(只解释状态语义,不改后端状态机)。
|
||||||
|
var STATUS_EXPLAIN = {
|
||||||
|
queued: "排队中,尚未被 worker 认领",
|
||||||
|
running: "运行中,模型回合进行中",
|
||||||
|
completed: "模型回合结束、改动在 worktree 里尚未合并",
|
||||||
|
failed: "模型回合失败,可重试",
|
||||||
|
interrupted: "已被取消/中断,可重试",
|
||||||
|
merged: "控制面已标记 merged,不代表 git 主分支已合入",
|
||||||
|
"merge-failed": "合并落盘失败,可重试"
|
||||||
|
};
|
||||||
$("#detail").innerHTML =
|
$("#detail").innerHTML =
|
||||||
"状态 <b class='st-" + esc(t.status) + "'>" + esc(t.status) + "</b> | " +
|
"状态 <b class='st-" + esc(t.status) + "'>" + esc(t.status) + "</b>(" +
|
||||||
|
esc(STATUS_EXPLAIN[t.status] || "未知状态") + ") | " +
|
||||||
"executor " + esc(t.executor || "") + " | 主机 " + esc(t.worktree_host || "") +
|
"executor " + esc(t.executor || "") + " | 主机 " + esc(t.worktree_host || "") +
|
||||||
" | 模型 " + esc(t.model || "默认") + " | 强度 " + esc(t.effort || "默认") +
|
" | 模型 " + esc(t.model || "默认") + " | 强度 " + esc(t.effort || "默认") +
|
||||||
" | 权限 " + esc(t.approval || "默认") +
|
" | 权限 " + esc(t.approval || "默认") +
|
||||||
@@ -800,7 +833,35 @@ async function showDetail(taskId) {
|
|||||||
pre.style.display = "block";
|
pre.style.display = "block";
|
||||||
$("#detailActions").style.display = "flex";
|
$("#detailActions").style.display = "flex";
|
||||||
$("#btnFork").style.display = TRACE_TERMINAL.indexOf(t.status) >= 0 ? "" : "none";
|
$("#btnFork").style.display = TRACE_TERMINAL.indexOf(t.status) >= 0 ? "" : "none";
|
||||||
$("#actionMsg").textContent = "";
|
// 按钮按状态 + 当前用户项目角色禁用并写明原因(前端禁用只是
|
||||||
|
// 防误点,真正鉴权仍在服务端每次请求时做;角色来自项目列表
|
||||||
|
// 的 my_role 字段,未知角色时按最严(仅取消可用)处理)。
|
||||||
|
(function () {
|
||||||
|
var proj = null;
|
||||||
|
(projectsCache || []).forEach(function (p) { if (p.id === t.project_id) proj = p; });
|
||||||
|
var role = proj ? proj.my_role : null;
|
||||||
|
var isMaintainer = (role === "maintainer" || role === "owner");
|
||||||
|
var canCreate = (role === "contributor" || isMaintainer);
|
||||||
|
var terminal = TRACE_TERMINAL.indexOf(t.status) >= 0;
|
||||||
|
var rules = [
|
||||||
|
{ btn: "#btnRetry", ok: ["failed", "interrupted", "merge-failed"].indexOf(t.status) >= 0 && isMaintainer,
|
||||||
|
why: "仅 failed/interrupted/merge-failed 可重试" + (isMaintainer ? "" : ",且仅 maintainer 可重试") },
|
||||||
|
{ btn: "#btnCancel", ok: !terminal && canCreate,
|
||||||
|
why: terminal ? "终态任务不可取消" : (canCreate ? "" : "仅项目成员(contributor 及以上)可取消") },
|
||||||
|
{ btn: "#btnMerge", ok: t.status === "completed" && isMaintainer,
|
||||||
|
why: "仅 completed 可合并" + (isMaintainer ? "" : ",且仅 maintainer 可合并") },
|
||||||
|
{ btn: "#btnPr", ok: t.status === "merged" && isMaintainer,
|
||||||
|
why: "仅 merged 可提 PR" + (isMaintainer ? "" : ",且仅 maintainer 可提 PR") }
|
||||||
|
];
|
||||||
|
var reasons = [];
|
||||||
|
rules.forEach(function (r) {
|
||||||
|
var b = $(r.btn);
|
||||||
|
b.disabled = !r.ok;
|
||||||
|
b.title = r.ok ? "" : r.why;
|
||||||
|
if (!r.ok && r.why) reasons.push(b.textContent.replace(/(.*$/, "") + ":不可用——" + r.why);
|
||||||
|
});
|
||||||
|
$("#actionMsg").textContent = reasons.length ? reasons.join(";") : "";
|
||||||
|
})();
|
||||||
$("#costBox").innerHTML = "";
|
$("#costBox").innerHTML = "";
|
||||||
$("#forkForm").style.display = "none";
|
$("#forkForm").style.display = "none";
|
||||||
$("#forkMsgSpan").textContent = "";
|
$("#forkMsgSpan").textContent = "";
|
||||||
@@ -1062,6 +1123,12 @@ $("#createTask").onclick = async function () {
|
|||||||
try { body.coordinator_parts = JSON.parse(coordRaw); }
|
try { body.coordinator_parts = JSON.parse(coordRaw); }
|
||||||
catch (e) { $("#taskMsg").textContent = "协调员子任务 JSON 解析失败"; return; }
|
catch (e) { $("#taskMsg").textContent = "协调员子任务 JSON 解析失败"; return; }
|
||||||
if (!Array.isArray(body.coordinator_parts)) { $("#taskMsg").textContent = "协调员子任务必须是数组"; return; }
|
if (!Array.isArray(body.coordinator_parts)) { $("#taskMsg").textContent = "协调员子任务必须是数组"; return; }
|
||||||
|
// 协调员与 goal/计划互斥:提交前校验并在表单处写明冲突
|
||||||
|
// (服务端同样 400,但不要等到服务端才发现)。
|
||||||
|
if ($("#goalMode").checked || $("#planMode").checked) {
|
||||||
|
$("#taskMsg").textContent = "协调员任务不能与 goal/计划模式同用:协调员任务本身不跑模型回合,请取消 goal/计划勾选,或清空协调员 JSON";
|
||||||
|
return;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
var atts = gatherAttachments();
|
var atts = gatherAttachments();
|
||||||
if (atts.length) {
|
if (atts.length) {
|
||||||
|
|||||||
@@ -16,6 +16,15 @@
|
|||||||
超时时同样取消挂起的子任务并标 failed,报告里写清哪几项
|
超时时同样取消挂起的子任务并标 failed,报告里写清哪几项
|
||||||
没完成。取消路径的协调员标 interrupted,不发完成/失败推送
|
没完成。取消路径的协调员标 interrupted,不发完成/失败推送
|
||||||
(与普通任务的取消口径一致)。
|
(与普通任务的取消口径一致)。
|
||||||
|
- 孤儿收口(P1):协调员本身死亡(worker 进程死、VM 回收)
|
||||||
|
时没有“取消者”替它取消子任务,正在跑的子任务会无声继续
|
||||||
|
烧钱。worker 侧两道收口:① 协调员取消/超时终局时,cancel
|
||||||
|
之后仍未终态的子任务由 _coord_mark_orphans 直接标
|
||||||
|
interrupted 并记审计;② 常驻循环每轮的
|
||||||
|
sweep_orphaned_coordinator_children 把“父已终态(或父行
|
||||||
|
已失踪)但子仍未终态”的子任务统一标 interrupted 并记审计
|
||||||
|
(父仍在跑时绝不动子任务)。本模块只提供纯函数
|
||||||
|
nonterminal_children 供 worker 判断,DB 写回在 worker 侧。
|
||||||
|
|
||||||
本模块只放纯逻辑(校验、解析、汇总、建表),不碰 DB 之外的
|
本模块只放纯逻辑(校验、解析、汇总、建表),不碰 DB 之外的
|
||||||
任何东西,单测不需要 worker/server。
|
任何东西,单测不需要 worker/server。
|
||||||
@@ -128,6 +137,28 @@ def ensure_columns(conn):
|
|||||||
pass
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
TERMINAL_STATUSES = frozenset(
|
||||||
|
{"completed", "failed", "interrupted", "merged", "merge-failed"})
|
||||||
|
|
||||||
|
|
||||||
|
def nonterminal_children(children):
|
||||||
|
"""未终态的子任务(P1 孤儿判断用,纯函数)。
|
||||||
|
|
||||||
|
children 为 [{name, id, status, ...}](worker 的 _coord_child_rows
|
||||||
|
拼出来的 merged 形态,或 _coord_sync_children 的 [{name, id}]
|
||||||
|
补上 status 后的形态);返回其中 status 不在终态集里的项
|
||||||
|
(缺 status 按未终态算,绝不把“查不到”当成“已完成”)。
|
||||||
|
worker 的孤儿标记与取消/超时终局都以它为准。
|
||||||
|
"""
|
||||||
|
out = []
|
||||||
|
for ch in children or []:
|
||||||
|
if not isinstance(ch, dict):
|
||||||
|
continue
|
||||||
|
if ch.get("status") not in TERMINAL_STATUSES:
|
||||||
|
out.append(ch)
|
||||||
|
return out
|
||||||
|
|
||||||
|
|
||||||
def build_report(children):
|
def build_report(children):
|
||||||
"""合流汇总:children 为 [{name, id, status, result_text,
|
"""合流汇总:children 为 [{name, id, status, result_text,
|
||||||
error_text}],返回 (report_text, overall_status)。
|
error_text}],返回 (report_text, overall_status)。
|
||||||
|
|||||||
@@ -18,6 +18,28 @@ may charge differently. Override per model with
|
|||||||
with no price record ``usd_micros=None`` and reports count their
|
with no price record ``usd_micros=None`` and reports count their
|
||||||
tokens in ``unpriced_actual_tokens`` instead of pricing them at zero.
|
tokens in ``unpriced_actual_tokens`` instead of pricing them at zero.
|
||||||
|
|
||||||
|
cache_write_input_tokens: EXPLICITLY NOT BILLED HERE (P1 decision,
|
||||||
|
not an oversight). host.py collects ``cacheWriteInputTokens`` from
|
||||||
|
upstream tokenUsage events and sandbox.py carries it in usage dicts
|
||||||
|
for observability, but it is NOT added to ``total_tokens`` (upstream
|
||||||
|
total is input + output), NOT stored in ``cost_records`` (``record``
|
||||||
|
takes input/output/cached only), NOT priced by ``price_usd_micros``,
|
||||||
|
and NOT counted toward the unpriced-token cap (worker counts input +
|
||||||
|
output from ``source="actual"`` rows only). Rationale: no price-table
|
||||||
|
entry and no relay bill evidence treats cache creation as a
|
||||||
|
separately billed token stream for the OpenAI-style relay models in
|
||||||
|
use; cache_write here is prompt-cache bookkeeping detail, not a
|
||||||
|
second charge on top of input. If a future provider bills cache
|
||||||
|
creation separately, the price table must gain an explicit
|
||||||
|
cache_write rate FIRST -- only then may ``record``/pricing count it.
|
||||||
|
Until then, counting it would double-charge input tokens that are
|
||||||
|
already priced (or inflate the unpriced cap with non-spend).
|
||||||
|
|
||||||
|
Unpriced cap scope (P1): ``unpriced_actual_tokens`` and the worker's
|
||||||
|
cap query count ``source="actual"`` rows only. Estimated rows
|
||||||
|
(``source="estimated"``, character-count fallback when no usage
|
||||||
|
events arrive) are never real spend and must not trip the cap.
|
||||||
|
|
||||||
Dead end, documented so nobody retries it: the app-server's
|
Dead end, documented so nobody retries it: the app-server's
|
||||||
account/usage/read method requires ChatGPT account authentication and
|
account/usage/read method requires ChatGPT account authentication and
|
||||||
fails on a relay API key (2026-10-26 probe: ``codex account
|
fails on a relay API key (2026-10-26 probe: ``codex account
|
||||||
@@ -98,6 +120,9 @@ def price_usd_micros(model, input_tokens, cached_input_tokens,
|
|||||||
output rate. Reasoning tokens are already inside output_tokens
|
output rate. Reasoning tokens are already inside output_tokens
|
||||||
(provider total = input + output), never added twice. Returns None
|
(provider total = input + output), never added twice. Returns None
|
||||||
when the model has no price entry (caller records usd as unknown).
|
when the model has no price entry (caller records usd as unknown).
|
||||||
|
cache_write_input_tokens is deliberately NOT a parameter here:
|
||||||
|
it is not billed separately (see module docstring); callers must
|
||||||
|
not add it to input_tokens to “count it anyway”.
|
||||||
"""
|
"""
|
||||||
rates = model_prices().get(model)
|
rates = model_prices().get(model)
|
||||||
if rates is None:
|
if rates is None:
|
||||||
|
|||||||
+125
-5
@@ -37,15 +37,22 @@
|
|||||||
发送端是后续待接线的工作,见模块末尾的缺口说明。
|
发送端是后续待接线的工作,见模块末尾的缺口说明。
|
||||||
- 本模块不跑真实模型回合。webhook/slack 的真实发送只 POST
|
- 本模块不跑真实模型回合。webhook/slack 的真实发送只 POST
|
||||||
到订阅者自己填的 target URL(创建订阅已要求项目
|
到订阅者自己填的 target URL(创建订阅已要求项目
|
||||||
create_task 权限;能建订阅的成员本来就能经任务执行代码,
|
create_task 权限)。P1 收紧后:真实发送层对 target 做 SSRF
|
||||||
因此不对 target 做内网屏蔽——屏蔽既拦不住提权,也会误伤
|
校验——loopback、私有网段、链路本地、云元数据地址一律不
|
||||||
本地接收器)。密钥、token、证书一律不由本模块读取、写入
|
POST;主机名先 DNS 解析再按解析出的 IP 判,解析失败拒绝
|
||||||
或返回。
|
(见 validate_push_target)。本地接收器如需放行,须运维显式
|
||||||
|
置 CODEX_PUSH_ALLOW_PRIVATE_TARGETS=1(默认关闭,测试用
|
||||||
|
本地接收器也须经此开关,不在代码里默认放行 loopback)。
|
||||||
|
密钥、token、证书一律不由本模块读取、写入或返回。
|
||||||
"""
|
"""
|
||||||
|
|
||||||
|
import ipaddress
|
||||||
import json
|
import json
|
||||||
|
import os
|
||||||
|
import socket
|
||||||
import sqlite3
|
import sqlite3
|
||||||
import urllib.error
|
import urllib.error
|
||||||
|
import urllib.parse
|
||||||
import urllib.request
|
import urllib.request
|
||||||
from cloud.db import connect as _db_connect
|
from cloud.db import connect as _db_connect
|
||||||
from cloud.db import PgConnection as _PgConnection
|
from cloud.db import PgConnection as _PgConnection
|
||||||
@@ -67,6 +74,7 @@ __all__ = [
|
|||||||
"send_webhook",
|
"send_webhook",
|
||||||
"send_slack",
|
"send_slack",
|
||||||
"dispatch_event",
|
"dispatch_event",
|
||||||
|
"validate_push_target",
|
||||||
]
|
]
|
||||||
|
|
||||||
CHANNELS = ("webhook", "slack", "webpush", "apns", "fcm")
|
CHANNELS = ("webhook", "slack", "webpush", "apns", "fcm")
|
||||||
@@ -311,6 +319,91 @@ def _require_http_url(target, channel):
|
|||||||
return target
|
return target
|
||||||
|
|
||||||
|
|
||||||
|
# 云元数据主机名:即使解析出的 IP 恰好不在私有段,也直接挡。
|
||||||
|
_BLOCKED_HOSTNAMES = frozenset({"metadata.google.internal"})
|
||||||
|
|
||||||
|
|
||||||
|
def _allow_private_targets():
|
||||||
|
"""运维显式开关:仅 CODEX_PUSH_ALLOW_PRIVATE_TARGETS=1 时放行
|
||||||
|
内网目标(本地接收器场景)。默认关闭;开关只影响 IP 段判定,
|
||||||
|
不影响 scheme 校验。"""
|
||||||
|
return os.environ.get(
|
||||||
|
"CODEX_PUSH_ALLOW_PRIVATE_TARGETS", "").strip() == "1"
|
||||||
|
|
||||||
|
|
||||||
|
def _default_resolver(hostname):
|
||||||
|
"""默认 DNS 解析:返回 IP 字符串列表;解析失败抛异常,
|
||||||
|
由 validate_push_target 转成“拒绝”。"""
|
||||||
|
infos = socket.getaddrinfo(hostname, None)
|
||||||
|
out = []
|
||||||
|
for info in infos:
|
||||||
|
sockaddr = info[4] if len(info) > 4 else None
|
||||||
|
if sockaddr and sockaddr[0]:
|
||||||
|
out.append(str(sockaddr[0]))
|
||||||
|
return out
|
||||||
|
|
||||||
|
|
||||||
|
def _ip_is_blocked(ip_text):
|
||||||
|
"""非全局单播 IP 一律算内网/特殊地址:loopback、私有网段、
|
||||||
|
链路本地(169.254/16,含云元数据 169.254.169.254)、保留段、
|
||||||
|
组播、未指定地址都在 is_global=False 里。"""
|
||||||
|
try:
|
||||||
|
return not ipaddress.ip_address(ip_text).is_global
|
||||||
|
except ValueError:
|
||||||
|
return True
|
||||||
|
|
||||||
|
|
||||||
|
def validate_push_target(target, resolver=None, check_dns=True):
|
||||||
|
"""SSRF 校验:通过返回 None,否则返回原因字符串(不抛)。
|
||||||
|
|
||||||
|
顺序:scheme(http/https)-> 主机名存在 -> 元数据主机名 ->
|
||||||
|
IP 字面量直接判段 -> (check_dns 时)DNS 解析后逐 IP 判段,
|
||||||
|
解析失败或解析出空结果都拒绝(DNS 重绑定下解析时与发送时
|
||||||
|
仍可能不一致,这是本层能做的上限,见残留说明)。
|
||||||
|
check_dns=False 时只做字面量判定:供直接调 send_* 但不想
|
||||||
|
做 DNS 的调用方用;dispatch 的真实发送路径一律
|
||||||
|
check_dns=True,显式 sender 路径不走本预检(见 dispatch)。
|
||||||
|
"""
|
||||||
|
if not isinstance(target, str) or not (
|
||||||
|
target.startswith("http://") or target.startswith("https://")):
|
||||||
|
return "target must be an http(s) URL"
|
||||||
|
try:
|
||||||
|
host = urllib.parse.urlparse(target).hostname
|
||||||
|
except ValueError:
|
||||||
|
return "target host is invalid"
|
||||||
|
if not host:
|
||||||
|
return "target host required"
|
||||||
|
lowered = host.lower()
|
||||||
|
if lowered in _BLOCKED_HOSTNAMES or lowered.endswith(
|
||||||
|
".metadata.google.internal"):
|
||||||
|
return "target host is a cloud metadata address"
|
||||||
|
if _allow_private_targets():
|
||||||
|
return None
|
||||||
|
try:
|
||||||
|
ipaddress.ip_address(host)
|
||||||
|
is_literal = True
|
||||||
|
except ValueError:
|
||||||
|
is_literal = False
|
||||||
|
if is_literal:
|
||||||
|
if _ip_is_blocked(host):
|
||||||
|
return "target resolves to a non-public address: %s" % host
|
||||||
|
return None
|
||||||
|
if not check_dns:
|
||||||
|
return None
|
||||||
|
resolve = resolver or _default_resolver
|
||||||
|
try:
|
||||||
|
ips = resolve(host)
|
||||||
|
except Exception:
|
||||||
|
return "DNS resolution failed for target host"
|
||||||
|
if not ips:
|
||||||
|
return "DNS resolution failed for target host"
|
||||||
|
for ip_text in ips:
|
||||||
|
if _ip_is_blocked(ip_text):
|
||||||
|
return ("target host resolves to a non-public address: %s"
|
||||||
|
% ip_text)
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
def send_webhook(message):
|
def send_webhook(message):
|
||||||
"""webhook 通道真实发送:POST JSON 到订阅的 target URL。
|
"""webhook 通道真实发送:POST JSON 到订阅的 target URL。
|
||||||
|
|
||||||
@@ -319,6 +412,11 @@ def send_webhook(message):
|
|||||||
RuntimeError(dispatch 记入 errors,不算 sent)。
|
RuntimeError(dispatch 记入 errors,不算 sent)。
|
||||||
"""
|
"""
|
||||||
target = _require_http_url(message["target"], "webhook")
|
target = _require_http_url(message["target"], "webhook")
|
||||||
|
blocked = validate_push_target(
|
||||||
|
target, resolver=message.get("_resolver"))
|
||||||
|
if blocked is not None:
|
||||||
|
raise RuntimeError(
|
||||||
|
"webhook delivery failed: target blocked: %s" % blocked)
|
||||||
payload = message["payload"] if isinstance(
|
payload = message["payload"] if isinstance(
|
||||||
message["payload"], dict) else {}
|
message["payload"], dict) else {}
|
||||||
ok, detail = _post_json(target, {
|
ok, detail = _post_json(target, {
|
||||||
@@ -340,6 +438,11 @@ def send_slack(message):
|
|||||||
事件附 error 摘要(截断 500 字)。非 2xx 抛 RuntimeError。
|
事件附 error 摘要(截断 500 字)。非 2xx 抛 RuntimeError。
|
||||||
"""
|
"""
|
||||||
target = _require_http_url(message["target"], "slack")
|
target = _require_http_url(message["target"], "slack")
|
||||||
|
blocked = validate_push_target(
|
||||||
|
target, resolver=message.get("_resolver"))
|
||||||
|
if blocked is not None:
|
||||||
|
raise RuntimeError(
|
||||||
|
"slack delivery failed: target blocked: %s" % blocked)
|
||||||
payload = message["payload"] if isinstance(
|
payload = message["payload"] if isinstance(
|
||||||
message["payload"], dict) else {}
|
message["payload"], dict) else {}
|
||||||
text = "[codex] task %s %s (project %s)" % (
|
text = "[codex] task %s %s (project %s)" % (
|
||||||
@@ -357,7 +460,8 @@ def send_slack(message):
|
|||||||
DEFAULT_SENDERS = {"webhook": send_webhook, "slack": send_slack}
|
DEFAULT_SENDERS = {"webhook": send_webhook, "slack": send_slack}
|
||||||
|
|
||||||
|
|
||||||
def dispatch_event(db, event_type, project_id, payload, sender=None):
|
def dispatch_event(db, event_type, project_id, payload, sender=None,
|
||||||
|
resolver=None):
|
||||||
"""分发一次状态变化事件,只发本项目的订阅。
|
"""分发一次状态变化事件,只发本项目的订阅。
|
||||||
|
|
||||||
参数:
|
参数:
|
||||||
@@ -418,6 +522,7 @@ def dispatch_event(db, event_type, project_id, payload, sender=None):
|
|||||||
}]
|
}]
|
||||||
return result
|
return result
|
||||||
|
|
||||||
|
real_send = sender is None
|
||||||
if sender is None:
|
if sender is None:
|
||||||
# 内置真实发送端:webhook/slack 真 POST;显式 sender
|
# 内置真实发送端:webhook/slack 真 POST;显式 sender
|
||||||
# 传入时不走这里(注入优先),webpush/apns/fcm 永远
|
# 传入时不走这里(注入优先),webpush/apns/fcm 永远
|
||||||
@@ -445,6 +550,21 @@ def dispatch_event(db, event_type, project_id, payload, sender=None):
|
|||||||
result["errors"].append(
|
result["errors"].append(
|
||||||
dict(base, reason=MISSING_CREDENTIAL_REASONS[sub["channel"]]))
|
dict(base, reason=MISSING_CREDENTIAL_REASONS[sub["channel"]]))
|
||||||
continue
|
continue
|
||||||
|
# SSRF 预检(真实发送路径,发送前挡,不 POST):含 DNS
|
||||||
|
# 的全量校验。显式 sender 路径不预检:注入的 sender 是
|
||||||
|
# 调用方自控的投递(测试 seam),不是服务端发起的 HTTP,
|
||||||
|
# 挡它不增加生产安全性;生产服务端一律走 sender=None
|
||||||
|
# 的真实发送,必经本预检与 send_webhook/send_slack 的
|
||||||
|
# 二次校验。原因文案含 "delivery failed",与旧调用方
|
||||||
|
# 对投递失败的断言兼容。
|
||||||
|
if real_send:
|
||||||
|
blocked = validate_push_target(
|
||||||
|
sub["target"], resolver=resolver, check_dns=True)
|
||||||
|
if blocked is not None:
|
||||||
|
result["errors"].append(
|
||||||
|
dict(base, reason="%s delivery failed: target blocked:"
|
||||||
|
" %s" % (sub["channel"], blocked)))
|
||||||
|
continue
|
||||||
# webhook/slack:sender=None 时已换成内置真实发送端;
|
# webhook/slack:sender=None 时已换成内置真实发送端;
|
||||||
# 显式 sender 里没有该通道时明确报错,不静默成功。
|
# 显式 sender 里没有该通道时明确报错,不静默成功。
|
||||||
channel_sender = _resolve_channel_sender(sender, sub["channel"])
|
channel_sender = _resolve_channel_sender(sender, sub["channel"])
|
||||||
|
|||||||
+83
-36
@@ -423,6 +423,15 @@ class LocalExecutor(Executor):
|
|||||||
rounds = task.get("goal_max_rounds")
|
rounds = task.get("goal_max_rounds")
|
||||||
if rounds is None:
|
if rounds is None:
|
||||||
rounds = 3
|
rounds = 3
|
||||||
|
# P1 轮次上限(与 worker 的 GOAL_MAX_ROUNDS_CAP 同值 100):
|
||||||
|
# 有限轮次超限直接 loud fail,不静默截断;goal_mode 的 0
|
||||||
|
#(不限轮次)放行,上界是配额;旧验收形态的 0 同样拒绝
|
||||||
|
#(0 在旧形态里表示不限轮次,P1 已禁用无上限验收循环)。
|
||||||
|
if rounds is not None and rounds > 100:
|
||||||
|
raise ValueError("goal_max_rounds 超过上限 100 轮")
|
||||||
|
if rounds == 0 and verify and not task.get("goal_mode"):
|
||||||
|
raise ValueError(
|
||||||
|
"旧验收形态 goal_max_rounds=0 表示不限轮次,已禁用")
|
||||||
cmd = self._base_command(model=task.get("model"),
|
cmd = self._base_command(model=task.get("model"),
|
||||||
effort=task.get("effort"),
|
effort=task.get("effort"),
|
||||||
approval=task.get("approval"),
|
approval=task.get("approval"),
|
||||||
@@ -440,26 +449,35 @@ class LocalExecutor(Executor):
|
|||||||
return self._goal_result_from_runner(
|
return self._goal_result_from_runner(
|
||||||
task["id"], self._run_local(cmd, extra_env))
|
task["id"], self._run_local(cmd, extra_env))
|
||||||
|
|
||||||
def goal_resume_task(self, task, goal_text, extra_env=None,
|
def goal_resume_task(self, task, goal_text=None, extra_env=None,
|
||||||
model_provider=None) -> dict:
|
model_provider=None, max_rounds=None) -> dict:
|
||||||
"""调 host.py goal-resume --goal-text:在旧线程上开新目标
|
"""调 host.py goal-resume,分两种调用(P1,与 worker、
|
||||||
循环(旧任务接续 goal 模式,云端 /continue 带 goal_mode=1
|
host 三处一致,绝不能混用):
|
||||||
时走这里)。命令:host.py goal-resume --task <host 名>
|
- 断点续跑:goal_text=None(worker 的 _resume_kind=
|
||||||
--goal-text <新目标> --max-rounds 0(新语义不限轮次)。
|
"breakpoint" 时传 None)。命令不带 --goal-text、默认
|
||||||
|
也不带 --max-rounds:host 沿用原 goal + 原 verify_command
|
||||||
|
+ 原 goal_max_rounds,从断点轮次续跑。传了 --goal-text
|
||||||
|
就会走 host 的开新目标分支、把原 verify 覆盖成 None,
|
||||||
|
所以断点调用绝不能顺手把续跑消息塞进 goal_text。
|
||||||
|
- 开新目标:goal_text 为新目标文本(云端 /continue 带
|
||||||
|
goal_mode=1 时走这里)。命令带 --goal-text 与
|
||||||
|
--max-rounds(默认 0,新语义不限轮次,上界是配额)。
|
||||||
不传 --repo:goal-resume 操作已有 host 任务,不新建
|
不传 --repo:goal-resume 操作已有 host 任务,不新建
|
||||||
worktree。结论解析与 goal_task 同款。
|
worktree。结论解析与 goal_task 同款。
|
||||||
"""
|
"""
|
||||||
if not isinstance(task, dict) or task.get("id") is None:
|
if not isinstance(task, dict) or task.get("id") is None:
|
||||||
raise ValueError("任务缺少 id")
|
raise ValueError("任务缺少 id")
|
||||||
if not goal_text:
|
|
||||||
raise ValueError("goal-resume 缺少目标文本")
|
|
||||||
cmd = self._base_command(model=task.get("model"),
|
cmd = self._base_command(model=task.get("model"),
|
||||||
effort=task.get("effort"),
|
effort=task.get("effort"),
|
||||||
approval=task.get("approval"),
|
approval=task.get("approval"),
|
||||||
model_provider=model_provider)
|
model_provider=model_provider)
|
||||||
cmd += ["goal-resume", "--task", self._host_name(task),
|
cmd += ["goal-resume", "--task", self._host_name(task)]
|
||||||
"--goal-text", goal_text,
|
if goal_text:
|
||||||
"--max-rounds", "0"]
|
cmd += ["--goal-text", goal_text,
|
||||||
|
"--max-rounds",
|
||||||
|
str(max_rounds if max_rounds is not None else 0)]
|
||||||
|
elif max_rounds is not None:
|
||||||
|
cmd += ["--max-rounds", str(max_rounds)]
|
||||||
return self._goal_result_from_runner(
|
return self._goal_result_from_runner(
|
||||||
task["id"], self._run_local(cmd, extra_env))
|
task["id"], self._run_local(cmd, extra_env))
|
||||||
|
|
||||||
@@ -469,7 +487,9 @@ class LocalExecutor(Executor):
|
|||||||
解析用完整 stdout(结论在末尾),存库的 stdout 才截尾;
|
解析用完整 stdout(结论在末尾),存库的 stdout 才截尾;
|
||||||
returncode 非 0 一律 failed;为 0 时无结论也判 failed,
|
returncode 非 0 一律 failed;为 0 时无结论也判 failed,
|
||||||
绝不把“没打印结论”当成达标。host 任务已存在时(同任务
|
绝不把“没打印结论”当成达标。host 任务已存在时(同任务
|
||||||
/retry 撞名,goal 没有 run 续跑语义)错误写明请新建任务。
|
/retry 撞名,goal 没有 run 续跑语义)错误写明“goal 失败
|
||||||
|
请用继续而非重试”(P1:重试在已存在 host-state 上必失败,
|
||||||
|
断点续跑才是正确路径,见 worker 的回落逻辑)。
|
||||||
"""
|
"""
|
||||||
returncode = getattr(result, "returncode", 1)
|
returncode = getattr(result, "returncode", 1)
|
||||||
stdout_full = getattr(result, "stdout", "") or ""
|
stdout_full = getattr(result, "stdout", "") or ""
|
||||||
@@ -478,8 +498,10 @@ class LocalExecutor(Executor):
|
|||||||
combined = stdout_full + stderr
|
combined = stdout_full + stderr
|
||||||
if "已存在" in combined:
|
if "已存在" in combined:
|
||||||
met, goal_error = None, (
|
met, goal_error = None, (
|
||||||
"host 任务已存在:goal 任务不支持同任务重试,"
|
"host 任务已存在:goal 失败请用继续而非重试——"
|
||||||
"请新建任务")
|
"同任务重试会撞已存在 host-state,worker 会先"
|
||||||
|
"尝试断点续跑,断点续跑仍失败时请用继续(不要"
|
||||||
|
"重排同一任务)")
|
||||||
status = "failed"
|
status = "failed"
|
||||||
elif returncode != 0:
|
elif returncode != 0:
|
||||||
status = "failed"
|
status = "failed"
|
||||||
@@ -516,9 +538,11 @@ class LocalExecutor(Executor):
|
|||||||
killpg 才能连同子孙一起杀掉。注意:只有默认 runner(真实
|
killpg 才能连同子孙一起杀掉。注意:只有默认 runner(真实
|
||||||
subprocess)时才用本方法;单测注入的假 runner 走同步路径,
|
subprocess)时才用本方法;单测注入的假 runner 走同步路径,
|
||||||
不走这里。kind=goal 时建 goal 命令、kind=goal-resume 时建
|
不走这里。kind=goal 时建 goal 命令、kind=goal-resume 时建
|
||||||
goal-resume 命令(旧任务接续 goal 模式,目标文本取本轮
|
goal-resume 命令(结论解析由 worker 的 _execute_via_process
|
||||||
prompt;结论解析由 worker 的 _execute_via_process 按 kind
|
按 kind 分发到 _goal_result_from_runner)。goal-resume 分流
|
||||||
分发到 _goal_result_from_runner)。goal 语义与同步 goal_task
|
(P1):_resume_kind="breakpoint" 时不传 --goal-text(断点
|
||||||
|
续跑,沿用原 goal+verify),其它情况才把本轮 prompt 当
|
||||||
|
--goal-text(开新目标)。goal 语义与同步 goal_task
|
||||||
一致:新形态(goal_mode)无验收命令时 --verify 不传、
|
一致:新形态(goal_mode)无验收命令时 --verify 不传、
|
||||||
--max-rounds 传 0(0 不能用 or 兜底成 3);旧形态仍传验收
|
--max-rounds 传 0(0 不能用 or 兜底成 3);旧形态仍传验收
|
||||||
命令与轮次。seed_dir 只对 dispatch/goal 追加(run 续跑与
|
命令与轮次。seed_dir 只对 dispatch/goal 追加(run 续跑与
|
||||||
@@ -545,6 +569,11 @@ class LocalExecutor(Executor):
|
|||||||
rounds = task.get("goal_max_rounds")
|
rounds = task.get("goal_max_rounds")
|
||||||
if rounds is None:
|
if rounds is None:
|
||||||
rounds = 3
|
rounds = 3
|
||||||
|
if rounds is not None and rounds > 100:
|
||||||
|
raise ValueError("goal_max_rounds 超过上限 100 轮")
|
||||||
|
if rounds == 0 and verify and not task.get("goal_mode"):
|
||||||
|
raise ValueError(
|
||||||
|
"旧验收形态 goal_max_rounds=0 表示不限轮次,已禁用")
|
||||||
cmd += ["goal", "--name", self._host_name(task),
|
cmd += ["goal", "--name", self._host_name(task),
|
||||||
"--task", goal_text]
|
"--task", goal_text]
|
||||||
if verify:
|
if verify:
|
||||||
@@ -556,12 +585,15 @@ class LocalExecutor(Executor):
|
|||||||
if seed_dir:
|
if seed_dir:
|
||||||
cmd += ["--seed-dir", seed_dir]
|
cmd += ["--seed-dir", seed_dir]
|
||||||
elif kind == "goal-resume":
|
elif kind == "goal-resume":
|
||||||
# 旧任务接续 goal:host 侧用 --goal-text 在旧线程上开
|
# P1 分流(与 goal_resume_task 同款契约):
|
||||||
# 新目标循环(新语义,不限轮次);task["prompt"] 在续跑
|
# _resume_kind="breakpoint" 时不传 --goal-text(host
|
||||||
# 时已被 worker 换成续跑消息。
|
# 沿用原 goal+verify 断点续跑,绝不能用续跑消息覆盖
|
||||||
cmd += ["goal-resume", "--task", self._host_name(task),
|
# 原 verify);其它情况(开新目标)才用 task["prompt"]
|
||||||
"--goal-text", task["prompt"],
|
#(worker 已换成续跑消息)当 --goal-text,--max-rounds 0。
|
||||||
"--max-rounds", "0"]
|
cmd += ["goal-resume", "--task", self._host_name(task)]
|
||||||
|
if task.get("_resume_kind") != "breakpoint":
|
||||||
|
cmd += ["--goal-text", task["prompt"],
|
||||||
|
"--max-rounds", "0"]
|
||||||
else:
|
else:
|
||||||
cmd += ["dispatch", "--name", self._host_name(task),
|
cmd += ["dispatch", "--name", self._host_name(task),
|
||||||
"--task", task["prompt"]]
|
"--task", task["prompt"]]
|
||||||
@@ -1101,6 +1133,12 @@ class DockerExecutor(Executor):
|
|||||||
rounds = task.get("goal_max_rounds")
|
rounds = task.get("goal_max_rounds")
|
||||||
if rounds is None:
|
if rounds is None:
|
||||||
rounds = 3
|
rounds = 3
|
||||||
|
# P1 轮次上限(与 Local goal_task 同款,不静默截断)。
|
||||||
|
if rounds is not None and rounds > 100:
|
||||||
|
raise ValueError("goal_max_rounds 超过上限 100 轮")
|
||||||
|
if rounds == 0 and verify and not task.get("goal_mode"):
|
||||||
|
raise ValueError(
|
||||||
|
"旧验收形态 goal_max_rounds=0 表示不限轮次,已禁用")
|
||||||
inner_args = ["goal", "--name", self._host_name(task),
|
inner_args = ["goal", "--name", self._host_name(task),
|
||||||
"--task", goal_text]
|
"--task", goal_text]
|
||||||
if verify:
|
if verify:
|
||||||
@@ -1150,8 +1188,10 @@ class DockerExecutor(Executor):
|
|||||||
combined = stdout_full + stderr
|
combined = stdout_full + stderr
|
||||||
if "已存在" in combined:
|
if "已存在" in combined:
|
||||||
met, goal_error = None, (
|
met, goal_error = None, (
|
||||||
"host 任务已存在:goal 任务不支持同任务重试,"
|
"host 任务已存在:goal 失败请用继续而非重试——"
|
||||||
"请新建任务")
|
"同任务重试会撞已存在 host-state,worker 会先"
|
||||||
|
"尝试断点续跑,断点续跑仍失败时请用继续(不要"
|
||||||
|
"重排同一任务)")
|
||||||
status = "failed"
|
status = "failed"
|
||||||
elif returncode != 0:
|
elif returncode != 0:
|
||||||
status = "failed"
|
status = "failed"
|
||||||
@@ -1181,22 +1221,29 @@ class DockerExecutor(Executor):
|
|||||||
"usage": _parse_goal_usage(stdout_full),
|
"usage": _parse_goal_usage(stdout_full),
|
||||||
}
|
}
|
||||||
|
|
||||||
def goal_resume_task(self, task, goal_text, timeout=None,
|
def goal_resume_task(self, task, goal_text=None, timeout=None,
|
||||||
extra_mounts=None, network=None,
|
extra_mounts=None, network=None,
|
||||||
extra_env=None, model_provider=None) -> dict:
|
extra_env=None, model_provider=None,
|
||||||
"""容器内调 host.py goal-resume --goal-text:在旧线程上开
|
max_rounds=None) -> dict:
|
||||||
新目标循环(旧任务接续 goal 模式)。解析与 goal_task 同款
|
"""容器内调 host.py goal-resume,分两种调用(P1,与 Local
|
||||||
(完整 stdout 结论、存库截尾),但没有“已存在”分支:
|
goal_resume_task 同款契约):断点续跑传 goal_text=None
|
||||||
goal-resume 本来就是操作已有 host 任务,不存在 dispatch
|
(不带 --goal-text,host 沿用原 goal+verify+轮次),开新
|
||||||
撞名问题;returncode 非 0 或无结论一律 failed。
|
目标才传 goal_text(带 --goal-text 与 --max-rounds,默认 0)。
|
||||||
|
解析与 goal_task 同款(完整 stdout 结论、存库截尾),但
|
||||||
|
没有“已存在”分支:goal-resume 本来就是操作已有 host 任务,
|
||||||
|
不存在 dispatch 撞名问题;returncode 非 0 或无结论一律 failed。
|
||||||
"""
|
"""
|
||||||
if not isinstance(task, dict) or task.get("id") is None:
|
if not isinstance(task, dict) or task.get("id") is None:
|
||||||
raise ValueError("任务缺少 id")
|
raise ValueError("任务缺少 id")
|
||||||
if not goal_text:
|
_resume_args = ["goal-resume", "--task", self._host_name(task)]
|
||||||
raise ValueError("goal-resume 缺少目标文本")
|
if goal_text:
|
||||||
|
_resume_args += ["--goal-text", goal_text, "--max-rounds",
|
||||||
|
str(max_rounds
|
||||||
|
if max_rounds is not None else 0)]
|
||||||
|
elif max_rounds is not None:
|
||||||
|
_resume_args += ["--max-rounds", str(max_rounds)]
|
||||||
inner = self._inner_host_command(
|
inner = self._inner_host_command(
|
||||||
"goal-resume", "--task", self._host_name(task),
|
*_resume_args,
|
||||||
"--goal-text", goal_text, "--max-rounds", "0",
|
|
||||||
model=task.get("model"), effort=task.get("effort"),
|
model=task.get("model"), effort=task.get("effort"),
|
||||||
approval=task.get("approval"),
|
approval=task.get("approval"),
|
||||||
model_provider=model_provider)
|
model_provider=model_provider)
|
||||||
|
|||||||
+329
-54
@@ -113,12 +113,15 @@ from cloud.oauth import (
|
|||||||
)
|
)
|
||||||
from cloud.relay import RelayError as _RelayError
|
from cloud.relay import RelayError as _RelayError
|
||||||
from cloud.relay import get_relay_models as _get_relay_models
|
from cloud.relay import get_relay_models as _get_relay_models
|
||||||
|
from cloud.sub2api import Sub2apiError as _Sub2apiError
|
||||||
from cloud.sub2api import ensure_gateway_keys_table as _ensure_gateway_keys
|
from cloud.sub2api import ensure_gateway_keys_table as _ensure_gateway_keys
|
||||||
from cloud.sub2api import ensure_user_gateway_key as _ensure_sub2api_key
|
from cloud.sub2api import ensure_user_gateway_key as _ensure_sub2api_key
|
||||||
from cloud.sub2api import gateway_name as _gateway_name
|
from cloud.sub2api import gateway_name as _gateway_name
|
||||||
from cloud.sub2api import gateway_origin as _gateway_origin
|
from cloud.sub2api import gateway_origin as _gateway_origin
|
||||||
from cloud.sub2api import get_user_gateway_key as _get_sub2api_key
|
from cloud.sub2api import get_user_gateway_key as _get_sub2api_key
|
||||||
from cloud.sub2api import list_gateway_models as _list_sub2api_models
|
from cloud.sub2api import list_gateway_models as _list_sub2api_models
|
||||||
|
from cloud.sub2api import rotate_user_gateway_key as _rotate_sub2api_key
|
||||||
|
from cloud.sub2api import validate_or_rotate_gateway_key as _validate_sub2api_key
|
||||||
from cloud.repos import validate_repo_url
|
from cloud.repos import validate_repo_url
|
||||||
from cloud.push import (
|
from cloud.push import (
|
||||||
create_subscription as _push_create,
|
create_subscription as _push_create,
|
||||||
@@ -167,6 +170,31 @@ TRACE_STREAM_MAX_SECONDS = 3600
|
|||||||
TRACE_STREAM_POLL_SECONDS = 1.0
|
TRACE_STREAM_POLL_SECONDS = 1.0
|
||||||
TRACE_DEFAULT_DIR = "/opt/codex-app-server/work/host-traces"
|
TRACE_DEFAULT_DIR = "/opt/codex-app-server/work/host-traces"
|
||||||
|
|
||||||
|
# P1 控制面限额(建项目时校验,超限 400 并在错误里写明数值):
|
||||||
|
# 单用户项目数上限、项目名长度上限、自定义项目 id 长度上限。
|
||||||
|
# 均可经环境变量调(运维侧),缺省值见下方;改动只影响新项目,
|
||||||
|
# 已有超限项目不受影响(不删、不改名)。
|
||||||
|
def _env_int(name, default):
|
||||||
|
try:
|
||||||
|
value = int(os.environ.get(name, "") or default)
|
||||||
|
except (TypeError, ValueError):
|
||||||
|
return default
|
||||||
|
return value if value > 0 else default
|
||||||
|
|
||||||
|
|
||||||
|
MAX_PROJECTS_PER_USER = _env_int("CODEX_MAX_PROJECTS_PER_USER", 50)
|
||||||
|
MAX_PROJECT_NAME_LEN = _env_int("CODEX_MAX_PROJECT_NAME_LEN", 100)
|
||||||
|
MAX_PROJECT_ID_LEN = _env_int("CODEX_MAX_PROJECT_ID_LEN", 64)
|
||||||
|
# 协调员父任务预留:每分工至少预留的美元额度(微美元)。
|
||||||
|
# 父任务准入只查一次时,N 个分工可解锁 N 倍消费;本预留要求
|
||||||
|
# 剩余配额 >= 分工数 × 本值,否则父任务直接 402(见
|
||||||
|
# _check_coordinator_reserve)。缺省 $1/分工,可经环境变量调;
|
||||||
|
# 它是准入下限估计,不是实际计费——实际花费仍按子任务
|
||||||
|
# created_by=父任务创建人记到创建人账上(worker 建子任务时
|
||||||
|
# created_by 直接取父任务创建人,成本 JOIN 据此归属)。
|
||||||
|
COORDINATOR_RESERVE_PER_PART_USD_MICROS = _env_int(
|
||||||
|
"CODEX_COORDINATOR_RESERVE_PER_PART_USD_MICROS", 1000000)
|
||||||
|
|
||||||
|
|
||||||
def _trace_file_name(task_id):
|
def _trace_file_name(task_id):
|
||||||
"""云任务 id -> 链路文件名:与 cloud.sandbox 的 _host_name
|
"""云任务 id -> 链路文件名:与 cloud.sandbox 的 _host_name
|
||||||
@@ -1451,6 +1479,9 @@ class CloudControlPlaneHandler(BaseHTTPRequestHandler):
|
|||||||
else:
|
else:
|
||||||
self._send_json(404, {"error": "not found"})
|
self._send_json(404, {"error": "not found"})
|
||||||
return
|
return
|
||||||
|
if path == "/users/me/gateway/rotate" and method == "POST":
|
||||||
|
self._user_gateway_rotate()
|
||||||
|
return
|
||||||
if path.startswith("/users/me/keys"):
|
if path.startswith("/users/me/keys"):
|
||||||
# API keys 管理:全部只操作认证用户自己的 key。
|
# API keys 管理:全部只操作认证用户自己的 key。
|
||||||
if method == "GET" and path == "/users/me/keys":
|
if method == "GET" and path == "/users/me/keys":
|
||||||
@@ -1830,10 +1861,35 @@ class CloudControlPlaneHandler(BaseHTTPRequestHandler):
|
|||||||
if not name or not isinstance(name, str):
|
if not name or not isinstance(name, str):
|
||||||
self._send_json(400, {"error": "name required"})
|
self._send_json(400, {"error": "name required"})
|
||||||
return
|
return
|
||||||
|
if len(name) > MAX_PROJECT_NAME_LEN:
|
||||||
|
self._send_json(400, {
|
||||||
|
"error": "project name too long: max %d characters"
|
||||||
|
% MAX_PROJECT_NAME_LEN})
|
||||||
|
return
|
||||||
project_id = body.get("id") or uuid.uuid4().hex
|
project_id = body.get("id") or uuid.uuid4().hex
|
||||||
if not isinstance(project_id, str):
|
if not isinstance(project_id, str):
|
||||||
self._send_json(400, {"error": "id must be a string"})
|
self._send_json(400, {"error": "id must be a string"})
|
||||||
return
|
return
|
||||||
|
if len(project_id) > MAX_PROJECT_ID_LEN:
|
||||||
|
self._send_json(400, {
|
||||||
|
"error": "project id too long: max %d characters"
|
||||||
|
% MAX_PROJECT_ID_LEN})
|
||||||
|
return
|
||||||
|
# 单用户项目数上限:按 projects.owner_id 计(成员身份加入
|
||||||
|
# 的项目不计入自己的上限,避免“被拉进很多项目就建不了
|
||||||
|
# 项目”);超限 400 且不建项目。
|
||||||
|
_count_conn = self._db()
|
||||||
|
try:
|
||||||
|
_owned = _count_conn.execute(
|
||||||
|
"SELECT COUNT(*) FROM projects WHERE owner_id = ?",
|
||||||
|
(user_id,)).fetchone()[0]
|
||||||
|
finally:
|
||||||
|
_count_conn.close()
|
||||||
|
if _owned >= MAX_PROJECTS_PER_USER:
|
||||||
|
self._send_json(400, {
|
||||||
|
"error": "project limit exceeded: max %d projects"
|
||||||
|
" per user" % MAX_PROJECTS_PER_USER})
|
||||||
|
return
|
||||||
# repo_url 可选:不传时为 None(老行为不变);传了则与 worker
|
# repo_url 可选:不传时为 None(老行为不变);传了则与 worker
|
||||||
# 共用 cloud.repos.validate_repo_url 的同一套校验,不合法时
|
# 共用 cloud.repos.validate_repo_url 的同一套校验,不合法时
|
||||||
# 400 且不建项目,绝不把带凭证的 URL 存进库。
|
# 400 且不建项目,绝不把带凭证的 URL 存进库。
|
||||||
@@ -1947,43 +2003,78 @@ class CloudControlPlaneHandler(BaseHTTPRequestHandler):
|
|||||||
self._send_json(200, {"project": _project_to_dict(row)})
|
self._send_json(200, {"project": _project_to_dict(row)})
|
||||||
|
|
||||||
def _github_repos_list(self, user_id):
|
def _github_repos_list(self, user_id):
|
||||||
"""GET /github/repos:列出本服务 GitHub App 安装记录里的仓
|
"""GET /github/repos:只返回调用者成员项目对应的仓库。
|
||||||
库(installation_repositories 事件同步进 github_app_
|
|
||||||
installations 表),给控制台建项目/补仓库时的下拉菜单用。
|
P1 收紧前返回全部安装记录(任一已认证用户可看到全部
|
||||||
任何已认证用户可列(只含仓库名,不含密钥);选了哪个仓库
|
安装仓库名,属跨租户信息泄露)。现按 project_members
|
||||||
只影响调用者自己有管理权的项目(PATCH 仍按 owner 校验)。
|
过滤:只保留调用者是成员的项目的 repo_url 对应的仓库
|
||||||
|
(clone_url 归一化后比对:去 .git 后缀、比对小写);
|
||||||
|
调用者无成员项目或项目未填 repo_url 时返回空列表。
|
||||||
手填 URL 的能力不变——没进 App 安装的公开仓库仍可手动填。"""
|
手填 URL 的能力不变——没进 App 安装的公开仓库仍可手动填。"""
|
||||||
conn = self._db()
|
conn = self._db()
|
||||||
try:
|
try:
|
||||||
rows = list_installations(conn)
|
rows = list_installations(conn)
|
||||||
|
member_ids = {
|
||||||
|
row["project_id"]
|
||||||
|
for row in conn.execute(
|
||||||
|
"SELECT project_id FROM project_members"
|
||||||
|
" WHERE user_id = ?",
|
||||||
|
(user_id,)).fetchall()
|
||||||
|
}
|
||||||
|
allowed = set()
|
||||||
|
for row in conn.execute(
|
||||||
|
"SELECT id, repo_url FROM projects").fetchall():
|
||||||
|
if row["id"] not in member_ids or not row["repo_url"]:
|
||||||
|
continue
|
||||||
|
url = str(row["repo_url"]).strip().lower()
|
||||||
|
if url.endswith(".git"):
|
||||||
|
url = url[:-4]
|
||||||
|
allowed.add(url)
|
||||||
finally:
|
finally:
|
||||||
conn.close()
|
conn.close()
|
||||||
repos = [{
|
repos = []
|
||||||
"installation_id": r["installation_id"],
|
for r in rows:
|
||||||
"account_login": r["account_login"],
|
clone_url = "https://github.com/%s.git" % r["repo_full_name"]
|
||||||
"repo_full_name": r["repo_full_name"],
|
normalized = clone_url.lower()
|
||||||
"clone_url": "https://github.com/%s.git" % r["repo_full_name"],
|
if normalized.endswith(".git"):
|
||||||
} for r in rows]
|
normalized = normalized[:-4]
|
||||||
|
if normalized not in allowed:
|
||||||
|
continue
|
||||||
|
repos.append({
|
||||||
|
"installation_id": r["installation_id"],
|
||||||
|
"account_login": r["account_login"],
|
||||||
|
"repo_full_name": r["repo_full_name"],
|
||||||
|
"clone_url": clone_url,
|
||||||
|
})
|
||||||
self._send_json(200, {"repos": repos})
|
self._send_json(200, {"repos": repos})
|
||||||
|
|
||||||
def _list_projects(self, user_id):
|
def _list_projects(self, user_id):
|
||||||
conn = self._db()
|
conn = self._db()
|
||||||
try:
|
try:
|
||||||
member_ids = {
|
member_rows = conn.execute(
|
||||||
row["project_id"]
|
"SELECT project_id, role FROM project_members"
|
||||||
for row in conn.execute(
|
" WHERE user_id = ?",
|
||||||
"SELECT project_id FROM project_members WHERE user_id = ?",
|
(user_id,)).fetchall()
|
||||||
(user_id,),
|
member_ids = {row["project_id"] for row in member_rows}
|
||||||
).fetchall()
|
roles = {row["project_id"]: row["role"]
|
||||||
}
|
for row in member_rows}
|
||||||
rows = conn.execute(
|
rows = conn.execute(
|
||||||
"SELECT id, name, owner_id, created_at, repo_url, auto_pr"
|
"SELECT id, name, owner_id, created_at, repo_url, auto_pr"
|
||||||
" FROM projects ORDER BY created_at, id"
|
" FROM projects ORDER BY created_at, id"
|
||||||
).fetchall()
|
).fetchall()
|
||||||
finally:
|
finally:
|
||||||
conn.close()
|
conn.close()
|
||||||
projects = [_project_to_dict(r) for r in rows
|
projects = []
|
||||||
if r["id"] in member_ids]
|
for r in rows:
|
||||||
|
if r["id"] not in member_ids:
|
||||||
|
continue
|
||||||
|
# 附加字段 my_role(只增字段,不改已有成功语义):
|
||||||
|
# 控制台据此按角色禁用重试/合并/提 PR 等按钮;服务端
|
||||||
|
# 权限仍以每次请求的 check_permission 为准,前端禁用
|
||||||
|
# 只是防误点,不是鉴权。
|
||||||
|
item = _project_to_dict(r)
|
||||||
|
item["my_role"] = roles.get(r["id"])
|
||||||
|
projects.append(item)
|
||||||
self._send_json(200, {"projects": projects})
|
self._send_json(200, {"projects": projects})
|
||||||
|
|
||||||
# -- 项目子资源:记忆 / 知识库 / 成本 -------------------------------
|
# -- 项目子资源:记忆 / 知识库 / 成本 -------------------------------
|
||||||
@@ -2119,6 +2210,11 @@ class CloudControlPlaneHandler(BaseHTTPRequestHandler):
|
|||||||
return
|
return
|
||||||
deleted = _push_delete(self.server.db_path,
|
deleted = _push_delete(self.server.db_path,
|
||||||
subscription_id=parts[3])
|
subscription_id=parts[3])
|
||||||
|
if deleted:
|
||||||
|
with _auth_store(self.server.db_path) as store:
|
||||||
|
store.audit(
|
||||||
|
user_id, project_id, "push_subscription_deleted",
|
||||||
|
"subscription_id=%s" % parts[3])
|
||||||
self._send_json(200, {"deleted": bool(deleted)})
|
self._send_json(200, {"deleted": bool(deleted)})
|
||||||
return
|
return
|
||||||
# /projects/{id}/automations
|
# /projects/{id}/automations
|
||||||
@@ -2389,31 +2485,33 @@ class CloudControlPlaneHandler(BaseHTTPRequestHandler):
|
|||||||
# 缓存(见 cloud/relay.py)。单测可经 server.relay_fetcher
|
# 缓存(见 cloud/relay.py)。单测可经 server.relay_fetcher
|
||||||
# 注入假取数函数;中转挂了或未配 key 时 502,不编造列表。
|
# 注入假取数函数;中转挂了或未配 key 时 502,不编造列表。
|
||||||
# 网关为 sub2api 时:模型目录因用户而异(key 绑定的组
|
# 网关为 sub2api 时:模型目录因用户而异(key 绑定的组
|
||||||
# 决定),取调用者自己的网关 key 现查;无 key 时返回空
|
# 决定)。P1 收紧:首次调模型目录即做真正的懒开通/
|
||||||
# 列表 + provisioning 标记(控制台据此提示开通中),不
|
# 健康检查,不再只回 provisioning: pending——无行则开通,
|
||||||
# 在页面加载时悄悄开通(开通在建号与任务时发生)。
|
# 旧 key 经 /v1/models 实测已死(401/403,含余额为 0 的
|
||||||
|
# 403)则轮换重开;网关本身不可用时回 502、保留旧行。
|
||||||
|
# key 明文绝不进响应、日志、缓存。
|
||||||
if _gateway_name() == "sub2api":
|
if _gateway_name() == "sub2api":
|
||||||
conn = self._db()
|
conn = self._db()
|
||||||
try:
|
try:
|
||||||
row = _get_sub2api_key(conn, user_id)
|
|
||||||
try:
|
try:
|
||||||
conn.commit()
|
status, api_key, _sub, _kid = _validate_sub2api_key(
|
||||||
except Exception:
|
conn, user_id)
|
||||||
pass
|
except _Sub2apiError as exc:
|
||||||
|
self._send_json(502, {"error": str(exc)})
|
||||||
|
return
|
||||||
|
except Exception as exc:
|
||||||
|
self._send_json(502, {"error": str(exc)})
|
||||||
|
return
|
||||||
finally:
|
finally:
|
||||||
conn.close()
|
conn.close()
|
||||||
if row is None or not row.get("api_key"):
|
|
||||||
self._send_json(200, {"models": [], "gateway": "sub2api",
|
|
||||||
"provisioning": "pending"})
|
|
||||||
return
|
|
||||||
try:
|
try:
|
||||||
models = _list_sub2api_models(_gateway_origin(),
|
models = _list_sub2api_models(_gateway_origin(), api_key)
|
||||||
row["api_key"])
|
|
||||||
except Exception as exc:
|
except Exception as exc:
|
||||||
self._send_json(502, {"error": str(exc)})
|
self._send_json(502, {"error": str(exc)})
|
||||||
return
|
return
|
||||||
self._send_json(200, {"models": models, "gateway": "sub2api",
|
self._send_json(200, {"models": models, "gateway": "sub2api",
|
||||||
"cached": False, "updated_at": None})
|
"cached": False, "updated_at": None,
|
||||||
|
"provisioning": status})
|
||||||
return
|
return
|
||||||
fetcher = getattr(self.server, "relay_fetcher", None)
|
fetcher = getattr(self.server, "relay_fetcher", None)
|
||||||
try:
|
try:
|
||||||
@@ -2647,6 +2745,12 @@ class CloudControlPlaneHandler(BaseHTTPRequestHandler):
|
|||||||
self._send_json(400, {
|
self._send_json(400, {
|
||||||
"error": "goal_max_rounds must be >= 1"})
|
"error": "goal_max_rounds must be >= 1"})
|
||||||
return
|
return
|
||||||
|
# P1:旧验收形态轮次设上限 100(与 worker/sandbox 两道门
|
||||||
|
# 一致;新 goal_mode=1 不走本分支,仍由配额兜底)。
|
||||||
|
if goal_max_rounds > 100:
|
||||||
|
self._send_json(400, {
|
||||||
|
"error": "goal_max_rounds 超过上限 100 轮"})
|
||||||
|
return
|
||||||
# 协调员:coordinator_parts 非空时任务本身不跑模型回合,
|
# 协调员:coordinator_parts 非空时任务本身不跑模型回合,
|
||||||
# worker 认领后按分工建并行子任务、等齐、合流;与 goal/
|
# worker 认领后按分工建并行子任务、等齐、合流;与 goal/
|
||||||
# plan/验收命令语义冲突(那些都是“本任务跑模型”的意思),
|
# plan/验收命令语义冲突(那些都是“本任务跑模型”的意思),
|
||||||
@@ -2673,6 +2777,33 @@ class CloudControlPlaneHandler(BaseHTTPRequestHandler):
|
|||||||
return
|
return
|
||||||
if self._reject_if_over_quota(user_id):
|
if self._reject_if_over_quota(user_id):
|
||||||
return
|
return
|
||||||
|
# 协调员预留(P1):父任务准入只查一次时,N 个分工会
|
||||||
|
# 解锁 N 倍消费。父任务创建时要求剩余美元配额至少覆盖
|
||||||
|
# 分工数 × 每分工预留额,否则直接 402、父任务不建;
|
||||||
|
# 子任务归属 = 父任务创建人(worker 建子任务时
|
||||||
|
# created_by 直接取父任务创建人,成本 JOIN 据此归属,
|
||||||
|
# 子任务认领时 worker 侧配额门再查一次)。不限额用户
|
||||||
|
# (quota 为 None)不受此预留约束。
|
||||||
|
if coord_parts and not self._check_coordinator_reserve(
|
||||||
|
user_id, len(coord_parts)):
|
||||||
|
return
|
||||||
|
# sub2api 懒开通/健康检查(P1):建任务前先真校验网关
|
||||||
|
# key——无行则开通,死 key 则轮换重开;网关不可用时
|
||||||
|
# 502 且不建任务(不把死 key/待开通当成已开通发给
|
||||||
|
# worker)。xcmapi 网关下跳过本段。
|
||||||
|
if _gateway_name() == "sub2api":
|
||||||
|
_gw_conn = self._db()
|
||||||
|
try:
|
||||||
|
try:
|
||||||
|
_validate_sub2api_key(_gw_conn, user_id)
|
||||||
|
except _Sub2apiError as exc:
|
||||||
|
self._send_json(502, {"error": str(exc)})
|
||||||
|
return
|
||||||
|
except Exception as exc:
|
||||||
|
self._send_json(502, {"error": str(exc)})
|
||||||
|
return
|
||||||
|
finally:
|
||||||
|
_gw_conn.close()
|
||||||
# 环境绑定:environment_id 非空时必须是本项目、已 ready
|
# 环境绑定:environment_id 非空时必须是本项目、已 ready
|
||||||
# 的环境;creating/failed/archived 一律拒绝并把状态写进
|
# 的环境;creating/failed/archived 一律拒绝并把状态写进
|
||||||
# 报错(creating 时用户就知道该等 worker 准备好再建任务,
|
# 报错(creating 时用户就知道该等 worker 准备好再建任务,
|
||||||
@@ -2836,6 +2967,15 @@ class CloudControlPlaneHandler(BaseHTTPRequestHandler):
|
|||||||
conn.close()
|
conn.close()
|
||||||
with _auth_store(self.server.db_path) as store:
|
with _auth_store(self.server.db_path) as store:
|
||||||
store.audit(user_id, project_id, "create_task", task_id)
|
store.audit(user_id, project_id, "create_task", task_id)
|
||||||
|
if coord_parts:
|
||||||
|
# 创建响应里写明子任务配额归属(与 worker 建子任务时
|
||||||
|
# created_by=父任务创建人一致),不是只写文档宣称。
|
||||||
|
task["quota_attribution"] = (
|
||||||
|
"coordinator subtasks are charged to the parent"
|
||||||
|
" task creator")
|
||||||
|
task["quota_reserved_usd_micros"] = (
|
||||||
|
len(coord_parts)
|
||||||
|
* COORDINATOR_RESERVE_PER_PART_USD_MICROS)
|
||||||
self._send_json(201, task)
|
self._send_json(201, task)
|
||||||
|
|
||||||
def _list_tasks(self, user_id, query):
|
def _list_tasks(self, user_id, query):
|
||||||
@@ -4091,6 +4231,34 @@ class CloudControlPlaneHandler(BaseHTTPRequestHandler):
|
|||||||
return True
|
return True
|
||||||
return False
|
return False
|
||||||
|
|
||||||
|
def _check_coordinator_reserve(self, user_id, n_parts):
|
||||||
|
"""协调员预留准入:通过返回 True;被挡时已回 402 返回 False。
|
||||||
|
|
||||||
|
剩余美元配额(quota - used,quota 为 None = 不限额时放行)
|
||||||
|
必须 >= 分工数 × COORDINATOR_RESERVE_PER_PART_USD_MICROS。
|
||||||
|
这是准入下限估计,不是计费承诺:子任务实际花费仍按
|
||||||
|
created_by=父任务创建人全额记创建人账;父任务被挡时
|
||||||
|
一个子任务也不会建,不存在“第 2 个子任务才被挡”的
|
||||||
|
半截状态。"""
|
||||||
|
if not n_parts or n_parts <= 1:
|
||||||
|
return True
|
||||||
|
quota, used, month = self._quota_status(user_id)
|
||||||
|
if quota is None:
|
||||||
|
return True
|
||||||
|
need = n_parts * COORDINATOR_RESERVE_PER_PART_USD_MICROS
|
||||||
|
if quota - used < need:
|
||||||
|
self._send_json(402, {
|
||||||
|
"error": "coordinator_quota_reserve_exceeded",
|
||||||
|
"month": month,
|
||||||
|
"quota_usd_micros": quota,
|
||||||
|
"used_usd_micros": used,
|
||||||
|
"coordinator_parts": n_parts,
|
||||||
|
"reserved_usd_micros": need,
|
||||||
|
"quota_attribution": "coordinator subtasks are charged"
|
||||||
|
" to the parent task creator"})
|
||||||
|
return False
|
||||||
|
return True
|
||||||
|
|
||||||
def _audit_admin(self, user_id, action, detail):
|
def _audit_admin(self, user_id, action, detail):
|
||||||
"""admin 调用审计(成功与被拒都记,失败绝不挡主流程)。"""
|
"""admin 调用审计(成功与被拒都记,失败绝不挡主流程)。"""
|
||||||
try:
|
try:
|
||||||
@@ -4125,10 +4293,17 @@ class CloudControlPlaneHandler(BaseHTTPRequestHandler):
|
|||||||
|
|
||||||
def _user_quota_self(self, user_id):
|
def _user_quota_self(self, user_id):
|
||||||
quota, used, month = self._quota_status(user_id)
|
quota, used, month = self._quota_status(user_id)
|
||||||
self._send_json(200,
|
# 合并取舍(P1 rebase 到 d254240):main 的 _quota_payload
|
||||||
self._quota_payload(
|
# 已带 unpriced_actual_tokens,P1 的附加字段也全留,供控制台
|
||||||
user_id, quota, used, month,
|
# 同时显示美元与无价 token 两条进度;402 时控制台据
|
||||||
self._quota_unpriced_tokens(user_id)))
|
# 响应字段区分“美元配额已满”与“无价 token 已满”。
|
||||||
|
payload = self._quota_payload(
|
||||||
|
user_id, quota, used, month,
|
||||||
|
self._quota_unpriced_tokens(user_id))
|
||||||
|
payload["unpriced_tokens_used"] = self._quota_unpriced_tokens(
|
||||||
|
user_id, month)
|
||||||
|
payload["unpriced_token_cap"] = unpriced_token_cap()
|
||||||
|
self._send_json(200, payload)
|
||||||
|
|
||||||
def _admin_quota_get(self, user_id, query):
|
def _admin_quota_get(self, user_id, query):
|
||||||
if not self._is_global_admin(user_id):
|
if not self._is_global_admin(user_id):
|
||||||
@@ -4204,10 +4379,20 @@ class CloudControlPlaneHandler(BaseHTTPRequestHandler):
|
|||||||
|
|
||||||
# -- worker 池 -----------------------------------------------------
|
# -- worker 池 -----------------------------------------------------
|
||||||
def _workers_list(self, user_id):
|
def _workers_list(self, user_id):
|
||||||
"""列出注册 worker:任何已认证用户可读(worker 是基础设施
|
"""列出注册 worker:全局管理员看完整清单;其他已认证用户
|
||||||
信息,不属于某个项目的数据,不按项目成员过滤)。"""
|
只看在线/离线聚合(P1 收紧:worker_id、执行器、容量、
|
||||||
|
心跳时间等内部细节不对普通用户暴露,控制台的 Workers
|
||||||
|
区据此只显示聚合数字)。"""
|
||||||
workers = _registry_list(self.server.db_path)
|
workers = _registry_list(self.server.db_path)
|
||||||
self._send_json(200, {"workers": workers})
|
if self._is_global_admin(user_id):
|
||||||
|
self._send_json(200, {"workers": workers})
|
||||||
|
return
|
||||||
|
online = sum(1 for w in workers if w.get("alive"))
|
||||||
|
self._send_json(200, {
|
||||||
|
"workers_online": online,
|
||||||
|
"workers_offline": len(workers) - online,
|
||||||
|
"workers_total": len(workers),
|
||||||
|
})
|
||||||
|
|
||||||
def _workers_heartbeat(self, user_id):
|
def _workers_heartbeat(self, user_id):
|
||||||
"""worker 心跳:body 含 worker_id(必填,缺了 400)与可选
|
"""worker 心跳:body 含 worker_id(必填,缺了 400)与可选
|
||||||
@@ -4429,11 +4614,15 @@ class CloudControlPlaneHandler(BaseHTTPRequestHandler):
|
|||||||
"""POST /admin/backup:备份当前后端数据库,返回路径与规模信息。
|
"""POST /admin/backup:备份当前后端数据库,返回路径与规模信息。
|
||||||
|
|
||||||
权限:只认 CODEX_GLOBAL_ADMINS(见 _is_global_admin),
|
权限:只认 CODEX_GLOBAL_ADMINS(见 _is_global_admin),
|
||||||
未配置时一律 403。返回只含 backend、备份文件路径、
|
未配置时一律 403;非运维调用方根本到不了这里,绝对
|
||||||
文件大小与任务行数,绝不返回数据库密码、备份文件内容
|
路径只回给全局管理员。out_dir 限白名单目录(P1 收紧):
|
||||||
或任何凭据。out_dir 可由 body 指定,否则走环境变量
|
白名单根来自 CLOUD_BACKUP_ROOT(逗号分隔多根),未设
|
||||||
CLOUD_BACKUP_DIR,未设则用 cloud-backups。成功与被拒
|
则 CLOUD_BACKUP_DIR,未设则固定根 cloud-backups;请求
|
||||||
都记审计(审计里只记路径与规模,不记库内容)。
|
的 out_dir 经实路径解析后必须等于某根或在其下,含 ..
|
||||||
|
逃逸、指向 /、/etc、/opt 等系统目录的一律 400(白名单
|
||||||
|
外的绝对路径即使存在也不放行)。备份文件落盘后 chmod
|
||||||
|
0600。成功与被拒都记审计(调用人由审计的 user_id 列
|
||||||
|
记,detail 含目标目录与大小,不记库内容)。
|
||||||
"""
|
"""
|
||||||
if not self._is_global_admin(user_id):
|
if not self._is_global_admin(user_id):
|
||||||
self._audit_admin(user_id, "admin_backup_denied", "backup")
|
self._audit_admin(user_id, "admin_backup_denied", "backup")
|
||||||
@@ -4443,11 +4632,33 @@ class CloudControlPlaneHandler(BaseHTTPRequestHandler):
|
|||||||
if err:
|
if err:
|
||||||
self._send_json(400, {"error": err})
|
self._send_json(400, {"error": err})
|
||||||
return
|
return
|
||||||
out_dir = body.get("out_dir") or os.environ.get(
|
requested = body.get("out_dir")
|
||||||
"CLOUD_BACKUP_DIR") or "cloud-backups"
|
if requested is not None and (
|
||||||
if not isinstance(out_dir, str) or not out_dir:
|
not isinstance(requested, str) or not requested):
|
||||||
self._send_json(400, {"error": "out_dir must be a string"})
|
self._send_json(400, {"error": "out_dir must be a string"})
|
||||||
return
|
return
|
||||||
|
roots_raw = os.environ.get("CLOUD_BACKUP_ROOT", "")
|
||||||
|
if roots_raw.strip():
|
||||||
|
roots = [os.path.realpath(p.strip())
|
||||||
|
for p in roots_raw.split(",") if p.strip()]
|
||||||
|
else:
|
||||||
|
roots = [os.path.realpath(
|
||||||
|
os.environ.get("CLOUD_BACKUP_DIR") or "cloud-backups")]
|
||||||
|
if requested:
|
||||||
|
candidate = os.path.realpath(requested)
|
||||||
|
else:
|
||||||
|
candidate = roots[0]
|
||||||
|
allowed = any(candidate == root
|
||||||
|
or candidate.startswith(root + os.sep)
|
||||||
|
for root in roots)
|
||||||
|
if not allowed:
|
||||||
|
self._audit_admin(
|
||||||
|
user_id, "admin_backup_denied",
|
||||||
|
"out_dir outside backup roots: %s" % requested)
|
||||||
|
self._send_json(400, {
|
||||||
|
"error": "out_dir outside allowed backup roots"})
|
||||||
|
return
|
||||||
|
out_dir = candidate
|
||||||
backend = _db_backend_name()
|
backend = _db_backend_name()
|
||||||
try:
|
try:
|
||||||
if backend == "postgres":
|
if backend == "postgres":
|
||||||
@@ -4456,6 +4667,10 @@ class CloudControlPlaneHandler(BaseHTTPRequestHandler):
|
|||||||
else:
|
else:
|
||||||
from cloud.backup import backup_db as _backup_sqlite
|
from cloud.backup import backup_db as _backup_sqlite
|
||||||
dest = _backup_sqlite(self.server.db_path, out_dir, keep=3)
|
dest = _backup_sqlite(self.server.db_path, out_dir, keep=3)
|
||||||
|
try:
|
||||||
|
os.chmod(dest, 0o600)
|
||||||
|
except OSError:
|
||||||
|
pass
|
||||||
size_bytes = os.path.getsize(dest)
|
size_bytes = os.path.getsize(dest)
|
||||||
conn = self._db()
|
conn = self._db()
|
||||||
try:
|
try:
|
||||||
@@ -4468,8 +4683,8 @@ class CloudControlPlaneHandler(BaseHTTPRequestHandler):
|
|||||||
return
|
return
|
||||||
self._audit_admin(
|
self._audit_admin(
|
||||||
user_id, "admin_backup",
|
user_id, "admin_backup",
|
||||||
"backend=%s path=%s size=%d tasks=%d"
|
"caller=%s backend=%s out_dir=%s path=%s size=%d tasks=%d"
|
||||||
% (backend, dest, size_bytes, task_count))
|
% (user_id, backend, out_dir, dest, size_bytes, task_count))
|
||||||
self._send_json(200, {
|
self._send_json(200, {
|
||||||
"backend": backend, "path": dest,
|
"backend": backend, "path": dest,
|
||||||
"size_bytes": size_bytes, "task_count": task_count,
|
"size_bytes": size_bytes, "task_count": task_count,
|
||||||
@@ -5165,6 +5380,9 @@ class CloudControlPlaneHandler(BaseHTTPRequestHandler):
|
|||||||
label = "api-key"
|
label = "api-key"
|
||||||
with _auth_store(self.server.db_path) as auth:
|
with _auth_store(self.server.db_path) as auth:
|
||||||
key_id, api_key_plain = auth.create_api_key(user_id, label)
|
key_id, api_key_plain = auth.create_api_key(user_id, label)
|
||||||
|
# 审计只记 key_id 与 label,绝不记明文与哈希。
|
||||||
|
auth.audit(user_id, "", "api_key_created",
|
||||||
|
"key_id=%s label=%s" % (key_id, label))
|
||||||
self._send_json(201, {"id": key_id, "label": label,
|
self._send_json(201, {"id": key_id, "label": label,
|
||||||
"hint": api_key_plain[-4:],
|
"hint": api_key_plain[-4:],
|
||||||
"api_key": api_key_plain})
|
"api_key": api_key_plain})
|
||||||
@@ -5182,6 +5400,9 @@ class CloudControlPlaneHandler(BaseHTTPRequestHandler):
|
|||||||
return
|
return
|
||||||
with _auth_store(self.server.db_path) as auth:
|
with _auth_store(self.server.db_path) as auth:
|
||||||
ok = auth.revoke_api_key(user_id, key_id)
|
ok = auth.revoke_api_key(user_id, key_id)
|
||||||
|
if ok:
|
||||||
|
auth.audit(user_id, "", "api_key_revoked",
|
||||||
|
"key_id=%s" % key_id)
|
||||||
if not ok:
|
if not ok:
|
||||||
self._send_json(404, {"error": "key not found"})
|
self._send_json(404, {"error": "key not found"})
|
||||||
return
|
return
|
||||||
@@ -5199,8 +5420,45 @@ class CloudControlPlaneHandler(BaseHTTPRequestHandler):
|
|||||||
keep_hash = hashlib.sha256(api_key.encode()).hexdigest()
|
keep_hash = hashlib.sha256(api_key.encode()).hexdigest()
|
||||||
with _auth_store(self.server.db_path) as auth:
|
with _auth_store(self.server.db_path) as auth:
|
||||||
count = auth.revoke_other_api_keys(user_id, keep_hash)
|
count = auth.revoke_other_api_keys(user_id, keep_hash)
|
||||||
|
# 只记撤销数,不记任何 key 明文、哈希或保留 key 的身份。
|
||||||
|
auth.audit(user_id, "", "api_key_revoke_others",
|
||||||
|
"revoked_count=%d" % count)
|
||||||
self._send_json(200, {"revoked_count": count})
|
self._send_json(200, {"revoked_count": count})
|
||||||
|
|
||||||
|
def _user_gateway_rotate(self):
|
||||||
|
"""POST /users/me/gateway/rotate:仅限本人重新开通/轮换
|
||||||
|
网关 key。非 sub2api 网关时 400;网关不可用时 502 且
|
||||||
|
旧行不动(rotate 内部先删旧行再重开,只在重开失败时
|
||||||
|
旧 key 已丢——调用方须知:轮换是显式动作,失败后须
|
||||||
|
重试本端点重新开通)。审计只记 sub2api_user_id 与
|
||||||
|
key_id,绝不记 key 明文;响应不含 key。"""
|
||||||
|
user_id = self._authenticate()
|
||||||
|
if user_id is None:
|
||||||
|
self._send_json(401, {"error": "unauthorized"})
|
||||||
|
return
|
||||||
|
if _gateway_name() != "sub2api":
|
||||||
|
self._send_json(400, {"error": "gateway is not sub2api"})
|
||||||
|
return
|
||||||
|
conn = self._db()
|
||||||
|
try:
|
||||||
|
try:
|
||||||
|
status, _key, sub_id, key_id = _rotate_sub2api_key(
|
||||||
|
conn, user_id)
|
||||||
|
except _Sub2apiError as exc:
|
||||||
|
self._send_json(502, {"error": str(exc)})
|
||||||
|
return
|
||||||
|
except Exception as exc:
|
||||||
|
self._send_json(502, {"error": str(exc)})
|
||||||
|
return
|
||||||
|
finally:
|
||||||
|
conn.close()
|
||||||
|
with _auth_store(self.server.db_path) as store:
|
||||||
|
store.audit(user_id, "", "gateway_key_rotated",
|
||||||
|
"sub2api_user_id=%s key_id=%s"
|
||||||
|
% (sub_id, key_id))
|
||||||
|
self._send_json(200, {"rotated": True, "provisioning": status,
|
||||||
|
"sub2api_user_id": sub_id})
|
||||||
|
|
||||||
# -- 推送订阅与分发 ---------------------------------------------------
|
# -- 推送订阅与分发 ---------------------------------------------------
|
||||||
def _push_create_endpoint(self, user_id, project_id):
|
def _push_create_endpoint(self, user_id, project_id):
|
||||||
body, err = self._read_json()
|
body, err = self._read_json()
|
||||||
@@ -5215,6 +5473,12 @@ class CloudControlPlaneHandler(BaseHTTPRequestHandler):
|
|||||||
except ValueError as e:
|
except ValueError as e:
|
||||||
self._send_json(400, {"error": str(e)})
|
self._send_json(400, {"error": str(e)})
|
||||||
return
|
return
|
||||||
|
# 审计只记订阅 id 与通道:target 全文不记(Slack webhook
|
||||||
|
# URL 本身含 secret 路径,记进审计等于把 secret 落盘)。
|
||||||
|
with _auth_store(self.server.db_path) as store:
|
||||||
|
store.audit(user_id, project_id, "push_subscription_created",
|
||||||
|
"subscription_id=%s channel=%s"
|
||||||
|
% (sub["id"], sub["channel"]))
|
||||||
self._send_json(201, sub)
|
self._send_json(201, sub)
|
||||||
|
|
||||||
def _push_dispatch_endpoint(self, user_id):
|
def _push_dispatch_endpoint(self, user_id):
|
||||||
@@ -5228,8 +5492,9 @@ class CloudControlPlaneHandler(BaseHTTPRequestHandler):
|
|||||||
self._send_json(400, {"error": "event_type and project_id required"})
|
self._send_json(400, {"error": "event_type and project_id required"})
|
||||||
return
|
return
|
||||||
# P0 安全批后:/admin/push/dispatch 是管理端点,只认
|
# P0 安全批后:/admin/push/dispatch 是管理端点,只认
|
||||||
# CODEX_GLOBAL_ADMINS,不再只凭项目 read 就能让服务端
|
# CODEX_GLOBAL_ADMINS——该口径已强于 P1 要求的 create_task,
|
||||||
# 向订阅目标(可能含内网地址)真 POST。被拒与成功都记审计。
|
# 不再降级。被拒与成功都记审计;目标 SSRF 校验在
|
||||||
|
# push.dispatch_event / 真实发送层做。
|
||||||
if not self._is_global_admin(user_id):
|
if not self._is_global_admin(user_id):
|
||||||
self._audit_admin(user_id, "admin_push_denied",
|
self._audit_admin(user_id, "admin_push_denied",
|
||||||
"dispatch %s %s" % (event_type, project_id))
|
"dispatch %s %s" % (event_type, project_id))
|
||||||
@@ -5258,11 +5523,21 @@ class CloudControlPlaneHandler(BaseHTTPRequestHandler):
|
|||||||
"""创建自动化定义:只登记定义与到期状态,不跑模型、
|
"""创建自动化定义:只登记定义与到期状态,不跑模型、
|
||||||
不另起调度器(执行器后续走 claim 门,见 automations
|
不另起调度器(执行器后续走 claim 门,见 automations
|
||||||
模块文档)。body:name、prompt 必填;interval_seconds
|
模块文档)。body:name、prompt 必填;interval_seconds
|
||||||
可选(默认 86400,最小 60);repo_url 可选。"""
|
可选(默认 86400,最小 60);repo_url 可选。
|
||||||
|
|
||||||
|
P1 配额门:创建时先过与建任务同一道配额门(美元配额
|
||||||
|
与无价 token cap,超限 402),避免“定义免费建、到期
|
||||||
|
无限跑”;单自动化的 spend 上限即创建人配额本身——
|
||||||
|
每次到期产出的任务以 created_by=创建人建任务,走
|
||||||
|
POST /tasks 的配额门(触发路径若绕过服务端建任务,
|
||||||
|
则不在本门覆盖内,见残留)。本服务端没有自动化触发
|
||||||
|
端点,触发侧的门不在本次 server.py 改动内。"""
|
||||||
body, err = self._read_json()
|
body, err = self._read_json()
|
||||||
if err:
|
if err:
|
||||||
self._send_json(400, {"error": err})
|
self._send_json(400, {"error": err})
|
||||||
return
|
return
|
||||||
|
if self._reject_if_over_quota(user_id):
|
||||||
|
return
|
||||||
interval = body.get("interval_seconds", 86400)
|
interval = body.get("interval_seconds", 86400)
|
||||||
if interval is None:
|
if interval is None:
|
||||||
interval = 86400
|
interval = 86400
|
||||||
|
|||||||
+101
-8
@@ -441,8 +441,28 @@ def list_gateway_models(origin, api_key, request=None):
|
|||||||
return out
|
return out
|
||||||
|
|
||||||
|
|
||||||
def ensure_user_gateway_key(conn, user_id, admin=None, request=None):
|
def gateway_key_health(origin, api_key, request=None):
|
||||||
"""幂等开通:已有行直接返回 key;否则建号->登录->发 key->授余额->入库。
|
"""真健康检查:用已有 key 调一次 /v1/models。
|
||||||
|
|
||||||
|
返回 True(200,可用)/ False(401/403:key 已失效,或余额
|
||||||
|
为 0——网关在余额为 0 时鉴权层直接 403,两者都算死 key)/
|
||||||
|
"unknown"(其它状态或网络异常:网关本身不可用时不据此轮换,
|
||||||
|
避免把一次抖动当成 key 死亡、反复删号重建)。
|
||||||
|
"""
|
||||||
|
req = request or _default_request
|
||||||
|
try:
|
||||||
|
st, _data = req("GET", origin + "/v1/models", token=api_key)
|
||||||
|
except Exception:
|
||||||
|
return "unknown"
|
||||||
|
if st == 200:
|
||||||
|
return True
|
||||||
|
if st in (401, 403):
|
||||||
|
return False
|
||||||
|
return "unknown"
|
||||||
|
|
||||||
|
|
||||||
|
def _create_fresh_gateway_key(conn, user_id, admin=None, request=None):
|
||||||
|
"""建号->登录->发 key->授余额->入库,返回 (key, sub_id, key_id)。
|
||||||
|
|
||||||
入库失败时尽力清理刚建的 key 与账号(避免孤儿 key),再抛原错;
|
入库失败时尽力清理刚建的 key 与账号(避免孤儿 key),再抛原错;
|
||||||
授余额失败同样走这条清理(余额为 0 的 key 在网关侧直接 403,
|
授余额失败同样走这条清理(余额为 0 的 key 在网关侧直接 403,
|
||||||
@@ -451,11 +471,9 @@ def ensure_user_gateway_key(conn, user_id, admin=None, request=None):
|
|||||||
(同一进程内第二次起免管理员登录,见 shared_admin_session)。
|
(同一进程内第二次起免管理员登录,见 shared_admin_session)。
|
||||||
调用方负责 conn 的提交语义:本函数在入库后
|
调用方负责 conn 的提交语义:本函数在入库后
|
||||||
commit(开通是独立事务,不跟调用方的业务事务绑在一起)。
|
commit(开通是独立事务,不跟调用方的业务事务绑在一起)。
|
||||||
|
调用方须保证调用前本用户无有效行(ensure/rotate 先处理
|
||||||
|
旧行),否则会主键冲突(并发撞号除外,走轮询复用)。
|
||||||
"""
|
"""
|
||||||
ensure_gateway_keys_table(conn)
|
|
||||||
row = get_user_gateway_key(conn, user_id)
|
|
||||||
if row is not None:
|
|
||||||
return row["api_key"]
|
|
||||||
origin = gateway_origin()
|
origin = gateway_origin()
|
||||||
if not origin:
|
if not origin:
|
||||||
raise Sub2apiError("SUB2API_BASE_URL 未配置,无法开通网关账号")
|
raise Sub2apiError("SUB2API_BASE_URL 未配置,无法开通网关账号")
|
||||||
@@ -476,9 +494,16 @@ def ensure_user_gateway_key(conn, user_id, admin=None, request=None):
|
|||||||
time.sleep(2)
|
time.sleep(2)
|
||||||
row = get_user_gateway_key(conn, user_id)
|
row = get_user_gateway_key(conn, user_id)
|
||||||
if row is not None:
|
if row is not None:
|
||||||
return row["api_key"]
|
# 合并修复(P1 rebase 到 d254240):main 的 #92 竞态
|
||||||
|
# 分支原在单体 ensure 里只回一个 key 字符串;P1
|
||||||
|
# 抽出 _create_fresh 后契约是 3 元组,4 个调用点
|
||||||
|
# 都按 3 元组解包,裸字符串会被当成字符序列解包
|
||||||
|
# 而 ValueError。这里按契约回 3 元组,key_id 未知
|
||||||
|
# 故为 None(与 validate_or_rotate 的 ok 路一致)。
|
||||||
|
return (row["api_key"], row["sub2api_user_id"], None)
|
||||||
raise
|
raise
|
||||||
key_id = None
|
key_id = None
|
||||||
|
user_token = None
|
||||||
try:
|
try:
|
||||||
user_token, refresh = user_login(origin, email, password,
|
user_token, refresh = user_login(origin, email, password,
|
||||||
request=req)
|
request=req)
|
||||||
@@ -497,7 +522,7 @@ def ensure_user_gateway_key(conn, user_id, admin=None, request=None):
|
|||||||
(user_id, sub_id, key, refresh, now, now))
|
(user_id, sub_id, key, refresh, now, now))
|
||||||
conn.commit()
|
conn.commit()
|
||||||
except Exception:
|
except Exception:
|
||||||
if key_id is not None:
|
if key_id is not None and user_token is not None:
|
||||||
try:
|
try:
|
||||||
user_delete_key(origin, user_token, key_id, request=req)
|
user_delete_key(origin, user_token, key_id, request=req)
|
||||||
except Exception:
|
except Exception:
|
||||||
@@ -507,4 +532,72 @@ def ensure_user_gateway_key(conn, user_id, admin=None, request=None):
|
|||||||
except Exception:
|
except Exception:
|
||||||
pass
|
pass
|
||||||
raise
|
raise
|
||||||
|
return key, sub_id, key_id
|
||||||
|
|
||||||
|
|
||||||
|
def ensure_user_gateway_key(conn, user_id, admin=None, request=None):
|
||||||
|
"""幂等开通:已有行直接返回 key;否则走 _create_fresh 开通。
|
||||||
|
|
||||||
|
只保证“行存在”,不保证 key 仍有效:要校验有效性用
|
||||||
|
validate_or_rotate_gateway_key(死 key 会触发轮换重开)。
|
||||||
|
"""
|
||||||
|
ensure_gateway_keys_table(conn)
|
||||||
|
row = get_user_gateway_key(conn, user_id)
|
||||||
|
if row is not None:
|
||||||
|
return row["api_key"]
|
||||||
|
key, _sub_id, _key_id = _create_fresh_gateway_key(
|
||||||
|
conn, user_id, admin=admin, request=request)
|
||||||
return key
|
return key
|
||||||
|
|
||||||
|
|
||||||
|
def validate_or_rotate_gateway_key(conn, user_id, admin=None, request=None):
|
||||||
|
"""懒开通/健康检查合一:返回 (status, key, sub_id, key_id)。
|
||||||
|
|
||||||
|
status:ok(旧 key 经 /v1/models 实测可用)/ provisioned
|
||||||
|
(无行,新开通)/ rotated(旧 key 经实测已死:401/403,
|
||||||
|
含余额为 0 的 403;删旧行后重开,死 key 不再被当成已开通)。
|
||||||
|
网关本身不可用(健康检查 unknown)时抛 Sub2apiError,调用方
|
||||||
|
应回 502、保留旧行不轮换;绝不在此时把“待开通”当成可用。
|
||||||
|
"""
|
||||||
|
ensure_gateway_keys_table(conn)
|
||||||
|
row = get_user_gateway_key(conn, user_id)
|
||||||
|
origin = gateway_origin()
|
||||||
|
if not origin:
|
||||||
|
raise Sub2apiError("SUB2API_BASE_URL 未配置,无法开通网关账号")
|
||||||
|
if row is None or not row.get("api_key"):
|
||||||
|
key, sub_id, key_id = _create_fresh_gateway_key(
|
||||||
|
conn, user_id, admin=admin, request=request)
|
||||||
|
return "provisioned", key, sub_id, key_id
|
||||||
|
health = gateway_key_health(origin, row["api_key"], request=request)
|
||||||
|
if health is True:
|
||||||
|
return "ok", row["api_key"], row.get("sub2api_user_id"), None
|
||||||
|
if health is False:
|
||||||
|
conn.execute(
|
||||||
|
"DELETE FROM user_gateway_keys WHERE user_id = ?",
|
||||||
|
(user_id,))
|
||||||
|
conn.commit()
|
||||||
|
key, sub_id, key_id = _create_fresh_gateway_key(
|
||||||
|
conn, user_id, admin=admin, request=request)
|
||||||
|
return "rotated", key, sub_id, key_id
|
||||||
|
raise Sub2apiError("sub2api 网关健康检查暂不可用,保留现有 key")
|
||||||
|
|
||||||
|
|
||||||
|
def rotate_user_gateway_key(conn, user_id, admin=None, request=None):
|
||||||
|
"""仅限本人调用的重新开通/轮换:删旧行后重开。
|
||||||
|
|
||||||
|
无旧行时直接走开通(与 validate 的 provisioned 等价);返回
|
||||||
|
(status, key, sub_id, key_id),调用方记审计时只可用 sub_id /
|
||||||
|
key_id,绝不记 key 明文。旧 key 即使只是“疑似失效”也会被
|
||||||
|
换掉:轮换是用户显式动作,不以健康检查为前提。
|
||||||
|
"""
|
||||||
|
ensure_gateway_keys_table(conn)
|
||||||
|
row = get_user_gateway_key(conn, user_id)
|
||||||
|
if row is not None:
|
||||||
|
conn.execute(
|
||||||
|
"DELETE FROM user_gateway_keys WHERE user_id = ?",
|
||||||
|
(user_id,))
|
||||||
|
conn.commit()
|
||||||
|
key, sub_id, key_id = _create_fresh_gateway_key(
|
||||||
|
conn, user_id, admin=admin, request=request)
|
||||||
|
return ("rotated" if row is not None else "provisioned",
|
||||||
|
key, sub_id, key_id)
|
||||||
@@ -33,6 +33,9 @@ class CapsServerTestCase(unittest.TestCase):
|
|||||||
self.auth = AuthStore(self.db_path)
|
self.auth = AuthStore(self.db_path)
|
||||||
self.owner_id, self.owner_key = self.auth.create_user("owner")
|
self.owner_id, self.owner_key = self.auth.create_user("owner")
|
||||||
self.viewer_id, self.viewer_key = self.auth.create_user("viewer")
|
self.viewer_id, self.viewer_key = self.auth.create_user("viewer")
|
||||||
|
# P1:完整 workers 清单只限全局管理员;owner 在此测试里即管理员。
|
||||||
|
self._old_admins = os.environ.get("CODEX_GLOBAL_ADMINS")
|
||||||
|
os.environ["CODEX_GLOBAL_ADMINS"] = self.owner_id
|
||||||
self.server = create_server(self.db_path, "127.0.0.1", 0)
|
self.server = create_server(self.db_path, "127.0.0.1", 0)
|
||||||
self.port = self.server.server_address[1]
|
self.port = self.server.server_address[1]
|
||||||
self.thread = threading.Thread(
|
self.thread = threading.Thread(
|
||||||
@@ -48,6 +51,10 @@ class CapsServerTestCase(unittest.TestCase):
|
|||||||
except Exception:
|
except Exception:
|
||||||
pass
|
pass
|
||||||
self.tmp.cleanup()
|
self.tmp.cleanup()
|
||||||
|
if getattr(self, "_old_admins", None) is None:
|
||||||
|
os.environ.pop("CODEX_GLOBAL_ADMINS", None)
|
||||||
|
else:
|
||||||
|
os.environ["CODEX_GLOBAL_ADMINS"] = self._old_admins
|
||||||
|
|
||||||
def _req(self, method, path, body=None, key=None):
|
def _req(self, method, path, body=None, key=None):
|
||||||
conn = http.client.HTTPConnection("127.0.0.1", self.port, timeout=10)
|
conn = http.client.HTTPConnection("127.0.0.1", self.port, timeout=10)
|
||||||
@@ -82,11 +89,21 @@ class CapsServerTestCase(unittest.TestCase):
|
|||||||
"capacity": 4, "running": 2},
|
"capacity": 4, "running": 2},
|
||||||
self.owner_key)
|
self.owner_key)
|
||||||
self.assertEqual(status, 200, body)
|
self.assertEqual(status, 200, body)
|
||||||
# 读权限:任何已认证用户可读(viewer 也能看到基础设施行)。
|
# P1:完整清单只限全局管理员(owner 在此测试里即管理员)。
|
||||||
workers = self._workers(self.viewer_key)
|
workers = self._workers(self.owner_key)
|
||||||
self.assertEqual(workers["w1"]["capacity"], 4)
|
self.assertEqual(workers["w1"]["capacity"], 4)
|
||||||
self.assertEqual(workers["w1"]["running"], 2)
|
self.assertEqual(workers["w1"]["running"], 2)
|
||||||
|
|
||||||
|
def test_workers_list_non_admin_gets_aggregate_only(self):
|
||||||
|
self._req("POST", "/workers/heartbeat",
|
||||||
|
{"worker_id": "w1", "executor": "local",
|
||||||
|
"capacity": 4, "running": 2},
|
||||||
|
self.owner_key)
|
||||||
|
status, body = self._req("GET", "/workers", key=self.viewer_key)
|
||||||
|
self.assertEqual(status, 200, body)
|
||||||
|
self.assertNotIn("workers", body)
|
||||||
|
self.assertIn("workers_total", body)
|
||||||
|
|
||||||
def test_heartbeat_without_running_preserves_old(self):
|
def test_heartbeat_without_running_preserves_old(self):
|
||||||
self._req("POST", "/workers/heartbeat",
|
self._req("POST", "/workers/heartbeat",
|
||||||
{"worker_id": "w1", "capacity": 2, "running": 1},
|
{"worker_id": "w1", "capacity": 2, "running": 1},
|
||||||
|
|||||||
@@ -184,7 +184,16 @@ class WorkerRegistryTestCase(_ServerCase):
|
|||||||
self.assertEqual(status, 200, body)
|
self.assertEqual(status, 200, body)
|
||||||
self.assertTrue(body["alive"])
|
self.assertTrue(body["alive"])
|
||||||
self.assertEqual(body["executor"], "docker")
|
self.assertEqual(body["executor"], "docker")
|
||||||
status, body = self._req("GET", "/workers", None, self.owner_key)
|
# P1:完整 workers 清单只限全局管理员,本测试里 owner 即管理员。
|
||||||
|
old_admins = os.environ.get("CODEX_GLOBAL_ADMINS")
|
||||||
|
os.environ["CODEX_GLOBAL_ADMINS"] = self.owner_id
|
||||||
|
try:
|
||||||
|
status, body = self._req("GET", "/workers", None, self.owner_key)
|
||||||
|
finally:
|
||||||
|
if old_admins is None:
|
||||||
|
os.environ.pop("CODEX_GLOBAL_ADMINS", None)
|
||||||
|
else:
|
||||||
|
os.environ["CODEX_GLOBAL_ADMINS"] = old_admins
|
||||||
self.assertEqual(status, 200, body)
|
self.assertEqual(status, 200, body)
|
||||||
self.assertEqual(len(body["workers"]), 1)
|
self.assertEqual(len(body["workers"]), 1)
|
||||||
self.assertTrue(body["workers"][0]["alive"])
|
self.assertTrue(body["workers"][0]["alive"])
|
||||||
|
|||||||
@@ -95,13 +95,23 @@ class ServerGithubReposTestCase(unittest.TestCase):
|
|||||||
|
|
||||||
def test_list_returns_seeded_installations(self):
|
def test_list_returns_seeded_installations(self):
|
||||||
self._seed_installations()
|
self._seed_installations()
|
||||||
|
# P1:repos 只返回调用者成员项目对应的仓库;先把项目仓库
|
||||||
|
# 绑到其中一个已安装仓库,再断言只返回该仓库,非成员为空。
|
||||||
|
s, d = _req(self.port, "PATCH",
|
||||||
|
"/projects/" + self.project_id,
|
||||||
|
{"repo_url": "https://github.com/openqj/codex-app-server.git"},
|
||||||
|
self.owner_key)
|
||||||
|
self.assertEqual(s, 200, d)
|
||||||
s, d = _req(self.port, "GET", "/github/repos", key=self.owner_key)
|
s, d = _req(self.port, "GET", "/github/repos", key=self.owner_key)
|
||||||
self.assertEqual(s, 200, d)
|
self.assertEqual(s, 200, d)
|
||||||
names = [r["repo_full_name"] for r in d["repos"]]
|
names = [r["repo_full_name"] for r in d["repos"]]
|
||||||
self.assertEqual(names, ["openqj/codex-app-server", "openqj/demo"])
|
self.assertEqual(names, ["openqj/codex-app-server"])
|
||||||
self.assertEqual(
|
self.assertEqual(
|
||||||
d["repos"][0]["clone_url"],
|
d["repos"][0]["clone_url"],
|
||||||
"https://github.com/openqj/codex-app-server.git")
|
"https://github.com/openqj/codex-app-server.git")
|
||||||
|
s, d = _req(self.port, "GET", "/github/repos", key=self.other_key)
|
||||||
|
self.assertEqual(s, 200, d)
|
||||||
|
self.assertEqual(d["repos"], [])
|
||||||
|
|
||||||
def test_patch_repo_url_owner(self):
|
def test_patch_repo_url_owner(self):
|
||||||
url = "https://github.com/openqj/demo.git"
|
url = "https://github.com/openqj/demo.git"
|
||||||
|
|||||||
@@ -0,0 +1,598 @@
|
|||||||
|
"""P1 控制面/控制台单测(server.py / console.html / sub2api.py / push.py)。
|
||||||
|
|
||||||
|
只覆盖本批 P1 控制面改动,不跑真实模型回合、不联网、不碰生产:
|
||||||
|
server 侧走本地临时端口 + 临时 SQLite;网关侧用假 request /
|
||||||
|
假 admin(key、token 全是测试假值);推送 DNS 用注入的
|
||||||
|
resolver。绝不改 worker.py / coordinator.py / cost.py / host.py。
|
||||||
|
"""
|
||||||
|
|
||||||
|
import http.client
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
import sqlite3
|
||||||
|
import stat
|
||||||
|
import sys
|
||||||
|
import tempfile
|
||||||
|
import threading
|
||||||
|
import unittest
|
||||||
|
from pathlib import Path
|
||||||
|
from unittest import mock
|
||||||
|
|
||||||
|
try:
|
||||||
|
from cloud.auth import AuthStore
|
||||||
|
from cloud.push import dispatch_event, validate_push_target
|
||||||
|
from cloud.push import create_subscription
|
||||||
|
from cloud.server import create_server
|
||||||
|
from cloud import server as server_mod
|
||||||
|
from cloud.sub2api import Sub2apiError
|
||||||
|
from cloud.sub2api import ensure_gateway_keys_table
|
||||||
|
from cloud.sub2api import get_user_gateway_key
|
||||||
|
from cloud.sub2api import rotate_user_gateway_key
|
||||||
|
from cloud.sub2api import validate_or_rotate_gateway_key
|
||||||
|
except ImportError:
|
||||||
|
sys.path.insert(0, str(Path(__file__).resolve().parents[2]))
|
||||||
|
from cloud.auth import AuthStore
|
||||||
|
from cloud.push import dispatch_event, validate_push_target
|
||||||
|
from cloud.push import create_subscription
|
||||||
|
from cloud.server import create_server
|
||||||
|
from cloud import server as server_mod
|
||||||
|
from cloud.sub2api import Sub2apiError
|
||||||
|
from cloud.sub2api import ensure_gateway_keys_table
|
||||||
|
from cloud.sub2api import get_user_gateway_key
|
||||||
|
from cloud.sub2api import rotate_user_gateway_key
|
||||||
|
from cloud.sub2api import validate_or_rotate_gateway_key
|
||||||
|
|
||||||
|
|
||||||
|
def _req(port, method, path, body=None, key=None, raw_body=None):
|
||||||
|
conn = http.client.HTTPConnection("127.0.0.1", port, timeout=10)
|
||||||
|
hdrs = {}
|
||||||
|
if raw_body is not None:
|
||||||
|
payload = raw_body
|
||||||
|
elif body is not None:
|
||||||
|
payload = json.dumps(body).encode("utf-8")
|
||||||
|
hdrs["Content-Type"] = "application/json"
|
||||||
|
else:
|
||||||
|
payload = None
|
||||||
|
if key is not None:
|
||||||
|
hdrs["X-API-Key"] = key
|
||||||
|
try:
|
||||||
|
conn.request(method, path, body=payload, headers=hdrs)
|
||||||
|
resp = conn.getresponse()
|
||||||
|
data = resp.read()
|
||||||
|
finally:
|
||||||
|
conn.close()
|
||||||
|
try:
|
||||||
|
parsed = json.loads(data.decode("utf-8")) if data else None
|
||||||
|
except ValueError:
|
||||||
|
parsed = data.decode("utf-8", "replace")
|
||||||
|
return resp.status, parsed
|
||||||
|
|
||||||
|
|
||||||
|
class _FakeGatewayAdmin:
|
||||||
|
"""假网关管理员:建号/授余额只记调用,不联网。"""
|
||||||
|
|
||||||
|
def __init__(self):
|
||||||
|
self.created = []
|
||||||
|
self.deleted = []
|
||||||
|
self.balances = []
|
||||||
|
|
||||||
|
def create_user(self, email, password, username):
|
||||||
|
sub_id = 1000 + len(self.created)
|
||||||
|
self.created.append((email, username))
|
||||||
|
return sub_id
|
||||||
|
|
||||||
|
def delete_user(self, sub_user_id):
|
||||||
|
self.deleted.append(sub_user_id)
|
||||||
|
return True
|
||||||
|
|
||||||
|
def set_balance(self, sub_user_id, amount):
|
||||||
|
self.balances.append((sub_user_id, amount))
|
||||||
|
|
||||||
|
def resolve_group_id(self):
|
||||||
|
return 2
|
||||||
|
|
||||||
|
|
||||||
|
class _FakeGatewayRequest:
|
||||||
|
"""假网关 HTTP:models 状态可调,其余走固定成功路径。"""
|
||||||
|
|
||||||
|
def __init__(self, models_status=200):
|
||||||
|
self.models_status = models_status
|
||||||
|
self.calls = []
|
||||||
|
|
||||||
|
def __call__(self, method, url, body=None, token=None):
|
||||||
|
self.calls.append((method, url))
|
||||||
|
if url.endswith("/v1/models"):
|
||||||
|
return self.models_status, {"data": []}
|
||||||
|
if url.endswith("/api/v1/auth/login"):
|
||||||
|
return 200, {"data": {"access_token": "fake-user-token",
|
||||||
|
"refresh_token": "fake-refresh",
|
||||||
|
"expires_in": 3600}}
|
||||||
|
if url.endswith("/api/v1/keys"):
|
||||||
|
return 201, {"data": {"id": "fake-key-id-1",
|
||||||
|
"key": "fake-gateway-key-1"}}
|
||||||
|
if "/balance" in url:
|
||||||
|
return 200, {"data": {}}
|
||||||
|
return 200, {"data": {}}
|
||||||
|
|
||||||
|
|
||||||
|
class Sub2apiP1TestCase(unittest.TestCase):
|
||||||
|
def setUp(self):
|
||||||
|
self.conn = sqlite3.connect(":memory:")
|
||||||
|
ensure_gateway_keys_table(self.conn)
|
||||||
|
self.admin = _FakeGatewayAdmin()
|
||||||
|
# 假 origin:请求本身全是 fake,不产生真实网络调用。
|
||||||
|
self._old_origin = os.environ.get("SUB2API_BASE_URL")
|
||||||
|
os.environ["SUB2API_BASE_URL"] = "http://127.0.0.1:9"
|
||||||
|
|
||||||
|
def tearDown(self):
|
||||||
|
self.conn.close()
|
||||||
|
if self._old_origin is None:
|
||||||
|
os.environ.pop("SUB2API_BASE_URL", None)
|
||||||
|
else:
|
||||||
|
os.environ["SUB2API_BASE_URL"] = self._old_origin
|
||||||
|
|
||||||
|
def _seed_row(self, key="fake-old-key"):
|
||||||
|
self.conn.execute(
|
||||||
|
"INSERT INTO user_gateway_keys(user_id, sub2api_user_id,"
|
||||||
|
" api_key, refresh_token, created_at, updated_at)"
|
||||||
|
" VALUES(?, ?, ?, ?, ?, ?)",
|
||||||
|
("u1", 42, key, "fake-refresh", 1.0, 1.0))
|
||||||
|
self.conn.commit()
|
||||||
|
|
||||||
|
def test_validate_no_row_provisions(self):
|
||||||
|
req = _FakeGatewayRequest()
|
||||||
|
status, key, sub_id, key_id = validate_or_rotate_gateway_key(
|
||||||
|
self.conn, "u1", admin=self.admin, request=req)
|
||||||
|
self.assertEqual(status, "provisioned")
|
||||||
|
self.assertEqual(key, "fake-gateway-key-1")
|
||||||
|
self.assertEqual(key_id, "fake-key-id-1")
|
||||||
|
self.assertIsNotNone(get_user_gateway_key(self.conn, "u1"))
|
||||||
|
|
||||||
|
def test_validate_healthy_key_not_rotated(self):
|
||||||
|
self._seed_row()
|
||||||
|
req = _FakeGatewayRequest(models_status=200)
|
||||||
|
status, key, _sub, _kid = validate_or_rotate_gateway_key(
|
||||||
|
self.conn, "u1", admin=self.admin, request=req)
|
||||||
|
self.assertEqual(status, "ok")
|
||||||
|
self.assertEqual(key, "fake-old-key")
|
||||||
|
self.assertEqual(self.admin.created, [])
|
||||||
|
|
||||||
|
def test_validate_dead_key_rotates_not_pending(self):
|
||||||
|
self._seed_row()
|
||||||
|
req = _FakeGatewayRequest(models_status=403)
|
||||||
|
status, key, _sub, _kid = validate_or_rotate_gateway_key(
|
||||||
|
self.conn, "u1", admin=self.admin, request=req)
|
||||||
|
self.assertEqual(status, "rotated")
|
||||||
|
self.assertEqual(key, "fake-gateway-key-1")
|
||||||
|
row = get_user_gateway_key(self.conn, "u1")
|
||||||
|
self.assertEqual(row["api_key"], "fake-gateway-key-1")
|
||||||
|
self.assertNotEqual(row["api_key"], "fake-old-key")
|
||||||
|
|
||||||
|
def test_validate_zero_balance_403_rotates(self):
|
||||||
|
# 余额为 0 时网关鉴权层回 403:与 401 同属死 key,必须轮换。
|
||||||
|
self._seed_row()
|
||||||
|
req = _FakeGatewayRequest(models_status=403)
|
||||||
|
status, _key, _sub, _kid = validate_or_rotate_gateway_key(
|
||||||
|
self.conn, "u1", admin=self.admin, request=req)
|
||||||
|
self.assertEqual(status, "rotated")
|
||||||
|
|
||||||
|
def test_validate_gateway_down_keeps_old_row(self):
|
||||||
|
self._seed_row()
|
||||||
|
req = _FakeGatewayRequest(models_status=500)
|
||||||
|
with self.assertRaises(Sub2apiError):
|
||||||
|
validate_or_rotate_gateway_key(
|
||||||
|
self.conn, "u1", admin=self.admin, request=req)
|
||||||
|
row = get_user_gateway_key(self.conn, "u1")
|
||||||
|
self.assertEqual(row["api_key"], "fake-old-key")
|
||||||
|
|
||||||
|
def test_rotate_replaces_row(self):
|
||||||
|
self._seed_row()
|
||||||
|
req = _FakeGatewayRequest()
|
||||||
|
status, key, sub_id, key_id = rotate_user_gateway_key(
|
||||||
|
self.conn, "u1", admin=self.admin, request=req)
|
||||||
|
self.assertEqual(status, "rotated")
|
||||||
|
self.assertEqual(key_id, "fake-key-id-1")
|
||||||
|
self.assertEqual(
|
||||||
|
get_user_gateway_key(self.conn, "u1")["api_key"], key)
|
||||||
|
|
||||||
|
|
||||||
|
class PushP1TestCase(unittest.TestCase):
|
||||||
|
def setUp(self):
|
||||||
|
self.conn = sqlite3.connect(":memory:")
|
||||||
|
|
||||||
|
def tearDown(self):
|
||||||
|
self.conn.close()
|
||||||
|
|
||||||
|
def test_literal_private_and_loopback_blocked(self):
|
||||||
|
for target in (
|
||||||
|
"http://127.0.0.1:9/hook",
|
||||||
|
"http://10.0.0.5/hook",
|
||||||
|
"http://172.16.4.4/hook",
|
||||||
|
"http://192.168.1.10/hook",
|
||||||
|
"http://169.254.169.254/latest/meta-data/",
|
||||||
|
"http://[::1]/hook"):
|
||||||
|
self.assertIsNotNone(
|
||||||
|
validate_push_target(target), target)
|
||||||
|
|
||||||
|
def test_metadata_hostname_blocked(self):
|
||||||
|
self.assertIsNotNone(validate_push_target(
|
||||||
|
"http://metadata.google.internal/hook"))
|
||||||
|
|
||||||
|
def test_hostname_resolved_to_private_blocked(self):
|
||||||
|
err = validate_push_target(
|
||||||
|
"https://hooks.example.com/hook",
|
||||||
|
resolver=lambda host: ["10.1.2.3"])
|
||||||
|
self.assertIsNotNone(err)
|
||||||
|
|
||||||
|
def test_hostname_public_resolves_ok(self):
|
||||||
|
err = validate_push_target(
|
||||||
|
"https://hooks.example.com/hook",
|
||||||
|
resolver=lambda host: ["93.184.216.34"])
|
||||||
|
self.assertIsNone(err)
|
||||||
|
|
||||||
|
def test_dns_failure_rejected(self):
|
||||||
|
def _boom(host):
|
||||||
|
raise OSError("no dns in test")
|
||||||
|
self.assertIsNotNone(validate_push_target(
|
||||||
|
"https://hooks.example.com/hook", resolver=_boom))
|
||||||
|
|
||||||
|
def test_dispatch_loopback_never_posts(self):
|
||||||
|
# 真实发送路径(sender=None,生产服务端走的路径):
|
||||||
|
# loopback 目标在预检即被挡,不产生任何 POST。
|
||||||
|
create_subscription(self.conn, "proj-a", "user-1", "webhook",
|
||||||
|
"http://127.0.0.1:9/hook", "instant")
|
||||||
|
result = dispatch_event(self.conn, "failed", "proj-a",
|
||||||
|
{"task_id": "t1"}, sender=None)
|
||||||
|
self.assertEqual(result["sent"], [])
|
||||||
|
self.assertEqual(len(result["errors"]), 1)
|
||||||
|
self.assertIn("blocked", result["errors"][0]["reason"])
|
||||||
|
|
||||||
|
|
||||||
|
class ServerP1TestCase(unittest.TestCase):
|
||||||
|
def setUp(self):
|
||||||
|
self.tmp = tempfile.TemporaryDirectory()
|
||||||
|
self.db_path = os.path.join(self.tmp.name, "control.db")
|
||||||
|
self.auth = AuthStore(self.db_path)
|
||||||
|
self.owner_id, self.owner_key = self.auth.create_user("owner")
|
||||||
|
self.member_id, self.member_key = self.auth.create_user("member")
|
||||||
|
self.outsider_id, self.outsider_key = self.auth.create_user(
|
||||||
|
"outsider")
|
||||||
|
self._old_admins = os.environ.get("CODEX_GLOBAL_ADMINS")
|
||||||
|
self._old_gateway = os.environ.get("CLOUD_GATEWAY")
|
||||||
|
self._old_backup_root = os.environ.get("CLOUD_BACKUP_ROOT")
|
||||||
|
self._old_backup_dir = os.environ.get("CLOUD_BACKUP_DIR")
|
||||||
|
os.environ["CODEX_GLOBAL_ADMINS"] = self.owner_id
|
||||||
|
os.environ.pop("CLOUD_GATEWAY", None)
|
||||||
|
os.environ.pop("CLOUD_BACKUP_ROOT", None)
|
||||||
|
os.environ.pop("CLOUD_BACKUP_DIR", None)
|
||||||
|
self._old_max_projects = server_mod.MAX_PROJECTS_PER_USER
|
||||||
|
self.server = create_server(self.db_path, "127.0.0.1", 0)
|
||||||
|
self.port = self.server.server_address[1]
|
||||||
|
self.thread = threading.Thread(
|
||||||
|
target=self.server.serve_forever, daemon=True)
|
||||||
|
self.thread.start()
|
||||||
|
s, proj = _req(self.port, "POST", "/projects",
|
||||||
|
{"name": "owner/repo"}, self.owner_key)
|
||||||
|
self.assertEqual(s, 201, proj)
|
||||||
|
self.project_id = proj["id"]
|
||||||
|
self.auth.set_member(
|
||||||
|
self.project_id, self.owner_id, self.member_id, "maintainer")
|
||||||
|
|
||||||
|
def tearDown(self):
|
||||||
|
self.server.shutdown()
|
||||||
|
self.server.server_close()
|
||||||
|
self.thread.join(timeout=5)
|
||||||
|
try:
|
||||||
|
self.auth.conn.close()
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
self.tmp.cleanup()
|
||||||
|
server_mod.MAX_PROJECTS_PER_USER = self._old_max_projects
|
||||||
|
if self._old_admins is None:
|
||||||
|
os.environ.pop("CODEX_GLOBAL_ADMINS", None)
|
||||||
|
else:
|
||||||
|
os.environ["CODEX_GLOBAL_ADMINS"] = self._old_admins
|
||||||
|
if self._old_gateway is None:
|
||||||
|
os.environ.pop("CLOUD_GATEWAY", None)
|
||||||
|
else:
|
||||||
|
os.environ["CLOUD_GATEWAY"] = self._old_gateway
|
||||||
|
if self._old_backup_root is None:
|
||||||
|
os.environ.pop("CLOUD_BACKUP_ROOT", None)
|
||||||
|
else:
|
||||||
|
os.environ["CLOUD_BACKUP_ROOT"] = self._old_backup_root
|
||||||
|
if self._old_backup_dir is None:
|
||||||
|
os.environ.pop("CLOUD_BACKUP_DIR", None)
|
||||||
|
else:
|
||||||
|
os.environ["CLOUD_BACKUP_DIR"] = self._old_backup_dir
|
||||||
|
# 恢复猴子补丁过的 server 模块函数(relay/rotate 相关测试用)。
|
||||||
|
for name in ("_validate_sub2api_key", "_list_sub2api_models",
|
||||||
|
"_rotate_sub2api_key"):
|
||||||
|
try:
|
||||||
|
mock.patch.object(server_mod, name).stop()
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
|
||||||
|
# -- helpers ---------------------------------------------------
|
||||||
|
def _audit_actions(self, user_id=None):
|
||||||
|
conn = sqlite3.connect(self.db_path)
|
||||||
|
try:
|
||||||
|
rows = conn.execute(
|
||||||
|
"SELECT action, detail FROM audit_log"
|
||||||
|
+ (" WHERE user_id = ?" if user_id else ""),
|
||||||
|
(user_id,) if user_id else ()).fetchall()
|
||||||
|
finally:
|
||||||
|
conn.close()
|
||||||
|
return rows
|
||||||
|
|
||||||
|
def _set_quota(self, target, quota):
|
||||||
|
return _req(self.port, "POST", "/admin/quotas",
|
||||||
|
{"user_id": target, "quota_usd_micros": quota},
|
||||||
|
self.owner_key)
|
||||||
|
|
||||||
|
# -- 备份 -------------------------------------------------------
|
||||||
|
def test_backup_whitelist_and_mode(self):
|
||||||
|
root = os.path.join(self.tmp.name, "backup-root")
|
||||||
|
os.environ["CLOUD_BACKUP_ROOT"] = root
|
||||||
|
out_dir = os.path.join(root, "daily")
|
||||||
|
s, body = _req(self.port, "POST", "/admin/backup",
|
||||||
|
{"out_dir": out_dir}, self.owner_key)
|
||||||
|
self.assertEqual(s, 200, body)
|
||||||
|
self.assertTrue(os.path.exists(body["path"]))
|
||||||
|
self.assertEqual(stat.S_IMODE(os.stat(body["path"]).st_mode),
|
||||||
|
0o600)
|
||||||
|
details = [d for _a, d in self._audit_actions(self.owner_id)]
|
||||||
|
self.assertTrue(any("admin_backup" in a
|
||||||
|
for a, _d in self._audit_actions(
|
||||||
|
self.owner_id)))
|
||||||
|
self.assertTrue(any(out_dir in (d or "")
|
||||||
|
for d in details))
|
||||||
|
# 白名单外:系统目录与 .. 逃逸一律 400。
|
||||||
|
for bad in ("/etc", os.path.join(root, "..", "escape")):
|
||||||
|
s, body = _req(self.port, "POST", "/admin/backup",
|
||||||
|
{"out_dir": bad}, self.owner_key)
|
||||||
|
self.assertEqual(s, 400, (bad, body))
|
||||||
|
|
||||||
|
# -- workers 聚合 ------------------------------------------------
|
||||||
|
def test_workers_admin_full_member_aggregate_only(self):
|
||||||
|
s, _b = _req(self.port, "POST", "/workers/heartbeat",
|
||||||
|
{"worker_id": "w-p1", "executor": "docker",
|
||||||
|
"capacity": 2, "running": 0}, self.owner_key)
|
||||||
|
self.assertEqual(s, 200, _b)
|
||||||
|
s, body = _req(self.port, "GET", "/workers", key=self.owner_key)
|
||||||
|
self.assertEqual(s, 200)
|
||||||
|
self.assertIn("workers", body)
|
||||||
|
self.assertTrue(any(w.get("worker_id") == "w-p1"
|
||||||
|
for w in body["workers"]))
|
||||||
|
s, body = _req(self.port, "GET", "/workers", key=self.member_key)
|
||||||
|
self.assertEqual(s, 200)
|
||||||
|
self.assertNotIn("workers", body)
|
||||||
|
self.assertNotIn("w-p1", json.dumps(body))
|
||||||
|
self.assertEqual(body["workers_total"], 1)
|
||||||
|
self.assertEqual(body["workers_online"], 1)
|
||||||
|
self.assertEqual(body["workers_offline"], 0)
|
||||||
|
|
||||||
|
# -- github repos 过滤 -------------------------------------------
|
||||||
|
def test_github_repos_filtered_by_member_projects(self):
|
||||||
|
s, proj = _req(self.port, "POST", "/projects",
|
||||||
|
{"name": "with-repo",
|
||||||
|
"repo_url": "https://github.com/acme/alpha.git"},
|
||||||
|
self.owner_key)
|
||||||
|
self.assertEqual(s, 201, proj)
|
||||||
|
self.auth.set_member(proj["id"], self.owner_id, self.member_id,
|
||||||
|
"maintainer")
|
||||||
|
conn = sqlite3.connect(self.db_path)
|
||||||
|
try:
|
||||||
|
conn.execute(
|
||||||
|
"CREATE TABLE IF NOT EXISTS github_app_installations("
|
||||||
|
"installation_id TEXT, account_login TEXT,"
|
||||||
|
" repo_full_name TEXT, created_at REAL,"
|
||||||
|
" UNIQUE(installation_id, repo_full_name))")
|
||||||
|
conn.execute(
|
||||||
|
"INSERT OR IGNORE INTO github_app_installations"
|
||||||
|
" VALUES(?, ?, ?, ?)", ("7", "acme", "acme/alpha", 1.0))
|
||||||
|
conn.execute(
|
||||||
|
"INSERT OR IGNORE INTO github_app_installations"
|
||||||
|
" VALUES(?, ?, ?, ?)", ("7", "acme", "acme/beta", 1.0))
|
||||||
|
conn.commit()
|
||||||
|
finally:
|
||||||
|
conn.close()
|
||||||
|
s, body = _req(self.port, "GET", "/github/repos",
|
||||||
|
key=self.member_key)
|
||||||
|
self.assertEqual(s, 200, body)
|
||||||
|
names = [r["repo_full_name"] for r in body["repos"]]
|
||||||
|
self.assertEqual(names, ["acme/alpha"])
|
||||||
|
s, body = _req(self.port, "GET", "/github/repos",
|
||||||
|
key=self.outsider_key)
|
||||||
|
self.assertEqual(s, 200, body)
|
||||||
|
self.assertEqual(body["repos"], [])
|
||||||
|
|
||||||
|
# -- key 审计 -----------------------------------------------------
|
||||||
|
def test_api_key_audit_only_id_and_label(self):
|
||||||
|
s, body = _req(self.port, "POST", "/users/me/keys",
|
||||||
|
{"label": "p1-test"}, self.owner_key)
|
||||||
|
self.assertEqual(s, 201, body)
|
||||||
|
key_id = body["id"]
|
||||||
|
plain = body["api_key"]
|
||||||
|
s, _b = _req(self.port, "DELETE",
|
||||||
|
"/users/me/keys/" + key_id, key=self.owner_key)
|
||||||
|
self.assertEqual(s, 200, _b)
|
||||||
|
s, body = _req(self.port, "POST",
|
||||||
|
"/users/me/keys/revoke-others", {},
|
||||||
|
self.owner_key)
|
||||||
|
self.assertEqual(s, 200, body)
|
||||||
|
actions = {a for a, _d in self._audit_actions(self.owner_id)}
|
||||||
|
self.assertIn("api_key_created", actions)
|
||||||
|
self.assertIn("api_key_revoked", actions)
|
||||||
|
self.assertIn("api_key_revoke_others", actions)
|
||||||
|
blob = json.dumps(self._audit_actions(self.owner_id))
|
||||||
|
self.assertNotIn(plain, blob)
|
||||||
|
|
||||||
|
# -- 项目上限 ------------------------------------------------------
|
||||||
|
def test_project_limits(self):
|
||||||
|
s, body = _req(self.port, "POST", "/projects",
|
||||||
|
{"name": "x" * 101}, self.owner_key)
|
||||||
|
self.assertEqual(s, 400, body)
|
||||||
|
self.assertIn("100", body["error"])
|
||||||
|
s, body = _req(self.port, "POST", "/projects",
|
||||||
|
{"name": "ok", "id": "y" * 65}, self.owner_key)
|
||||||
|
self.assertEqual(s, 400, body)
|
||||||
|
self.assertIn("64", body["error"])
|
||||||
|
server_mod.MAX_PROJECTS_PER_USER = 1
|
||||||
|
# setUp 已建 1 个 owner 项目,限额 1 时再建即超限。
|
||||||
|
s, body = _req(self.port, "POST", "/projects",
|
||||||
|
{"name": "second"}, self.owner_key)
|
||||||
|
self.assertEqual(s, 400, body)
|
||||||
|
self.assertIn("max 1", body["error"])
|
||||||
|
|
||||||
|
# -- 协调员预留 ----------------------------------------------------
|
||||||
|
def test_coordinator_reserve_blocks_parent(self):
|
||||||
|
s, _b = self._set_quota(self.owner_id, 1000000)
|
||||||
|
self.assertEqual(s, 200, _b)
|
||||||
|
parts = [{"name": "a", "prompt": "做 a"},
|
||||||
|
{"name": "b", "prompt": "做 b"},
|
||||||
|
{"name": "c", "prompt": "做 c"}]
|
||||||
|
s, body = _req(self.port, "POST", "/tasks",
|
||||||
|
{"project_id": self.project_id, "prompt": "协调",
|
||||||
|
"coordinator_parts": parts}, self.owner_key)
|
||||||
|
self.assertEqual(s, 402, body)
|
||||||
|
self.assertEqual(body["error"],
|
||||||
|
"coordinator_quota_reserve_exceeded")
|
||||||
|
self.assertEqual(body["coordinator_parts"], 3)
|
||||||
|
s, _b = self._set_quota(self.owner_id, 10000000)
|
||||||
|
self.assertEqual(s, 200, _b)
|
||||||
|
s, body = _req(self.port, "POST", "/tasks",
|
||||||
|
{"project_id": self.project_id, "prompt": "协调",
|
||||||
|
"coordinator_parts": parts}, self.owner_key)
|
||||||
|
self.assertEqual(s, 201, body)
|
||||||
|
self.assertIn("parent task creator",
|
||||||
|
body.get("quota_attribution", ""))
|
||||||
|
self.assertEqual(body.get("quota_reserved_usd_micros"), 3000000)
|
||||||
|
|
||||||
|
# -- 自动化配额门 ----------------------------------------------------
|
||||||
|
def test_automation_create_quota_gate(self):
|
||||||
|
s, _b = self._set_quota(self.owner_id, 0)
|
||||||
|
self.assertEqual(s, 200, _b)
|
||||||
|
s, body = _req(self.port, "POST",
|
||||||
|
"/projects/%s/automations" % self.project_id,
|
||||||
|
{"name": "nightly", "prompt": "巡检"},
|
||||||
|
self.owner_key)
|
||||||
|
self.assertEqual(s, 402, body)
|
||||||
|
s, _b = self._set_quota(self.owner_id, 10000000)
|
||||||
|
self.assertEqual(s, 200, _b)
|
||||||
|
s, body = _req(self.port, "POST",
|
||||||
|
"/projects/%s/automations" % self.project_id,
|
||||||
|
{"name": "nightly", "prompt": "巡检"},
|
||||||
|
self.owner_key)
|
||||||
|
self.assertEqual(s, 201, body)
|
||||||
|
|
||||||
|
# -- sub2api 懒开通 / 轮换端点 --------------------------------------
|
||||||
|
def test_relay_models_lazy_provision_not_just_pending(self):
|
||||||
|
os.environ["CLOUD_GATEWAY"] = "sub2api"
|
||||||
|
orig_validate = server_mod._validate_sub2api_key
|
||||||
|
orig_list = server_mod._list_sub2api_models
|
||||||
|
try:
|
||||||
|
server_mod._validate_sub2api_key = (
|
||||||
|
lambda conn, uid: ("provisioned", "fake-key", 7, 9))
|
||||||
|
server_mod._list_sub2api_models = (
|
||||||
|
lambda origin, key: ["model-a"])
|
||||||
|
s, body = _req(self.port, "GET", "/relay/models",
|
||||||
|
key=self.owner_key)
|
||||||
|
self.assertEqual(s, 200, body)
|
||||||
|
self.assertEqual(body["models"], ["model-a"])
|
||||||
|
self.assertEqual(body["provisioning"], "provisioned")
|
||||||
|
server_mod._validate_sub2api_key = (
|
||||||
|
lambda conn, uid: (_ for _ in ()).throw(
|
||||||
|
Sub2apiError("网关暂不可用")))
|
||||||
|
s, body = _req(self.port, "GET", "/relay/models",
|
||||||
|
key=self.owner_key)
|
||||||
|
self.assertEqual(s, 502, body)
|
||||||
|
finally:
|
||||||
|
server_mod._validate_sub2api_key = orig_validate
|
||||||
|
server_mod._list_sub2api_models = orig_list
|
||||||
|
|
||||||
|
def test_gateway_rotate_endpoint(self):
|
||||||
|
s, body = _req(self.port, "POST", "/users/me/gateway/rotate",
|
||||||
|
{}, self.owner_key)
|
||||||
|
self.assertEqual(s, 400, body)
|
||||||
|
os.environ["CLOUD_GATEWAY"] = "sub2api"
|
||||||
|
orig_rotate = server_mod._rotate_sub2api_key
|
||||||
|
try:
|
||||||
|
server_mod._rotate_sub2api_key = (
|
||||||
|
lambda conn, uid: ("rotated", "fake-key", 7, 9))
|
||||||
|
s, body = _req(self.port, "POST", "/users/me/gateway/rotate",
|
||||||
|
{}, self.owner_key)
|
||||||
|
self.assertEqual(s, 200, body)
|
||||||
|
self.assertTrue(body["rotated"])
|
||||||
|
self.assertNotIn("api_key", body)
|
||||||
|
actions = {a for a, _d in self._audit_actions(self.owner_id)}
|
||||||
|
self.assertIn("gateway_key_rotated", actions)
|
||||||
|
finally:
|
||||||
|
server_mod._rotate_sub2api_key = orig_rotate
|
||||||
|
|
||||||
|
# -- 推送审计与分发 SSRF ----------------------------------------------
|
||||||
|
def test_push_audit_and_dispatch_ssrf(self):
|
||||||
|
s, sub = _req(self.port, "POST",
|
||||||
|
"/projects/%s/push/subscriptions" % self.project_id,
|
||||||
|
{"channel": "webhook",
|
||||||
|
"target": "https://example.com/hook",
|
||||||
|
"preference": "instant"}, self.owner_key)
|
||||||
|
self.assertEqual(s, 201, sub)
|
||||||
|
actions = {a for a, _d in self._audit_actions(self.owner_id)}
|
||||||
|
self.assertIn("push_subscription_created", actions)
|
||||||
|
s, _b = _req(self.port, "DELETE",
|
||||||
|
"/projects/%s/push/subscriptions/%s"
|
||||||
|
% (self.project_id, sub["id"]), key=self.owner_key)
|
||||||
|
self.assertEqual(s, 200, _b)
|
||||||
|
actions = {a for a, _d in self._audit_actions(self.owner_id)}
|
||||||
|
self.assertIn("push_subscription_deleted", actions)
|
||||||
|
# 内网目标:分发不 POST、sent 为空、错误含 delivery failed。
|
||||||
|
s, sub = _req(self.port, "POST",
|
||||||
|
"/projects/%s/push/subscriptions" % self.project_id,
|
||||||
|
{"channel": "webhook",
|
||||||
|
"target": "http://127.0.0.1:9/hook",
|
||||||
|
"preference": "instant"}, self.owner_key)
|
||||||
|
self.assertEqual(s, 201, sub)
|
||||||
|
s, body = _req(self.port, "POST", "/admin/push/dispatch",
|
||||||
|
{"event_type": "failed",
|
||||||
|
"project_id": self.project_id,
|
||||||
|
"payload": {"task_id": "t1"}}, self.owner_key)
|
||||||
|
self.assertEqual(s, 200, body)
|
||||||
|
self.assertEqual(body["sent"], [])
|
||||||
|
self.assertEqual(len(body["errors"]), 1)
|
||||||
|
self.assertIn("delivery failed", body["errors"][0]["reason"])
|
||||||
|
actions = {a for a, _d in self._audit_actions(self.owner_id)}
|
||||||
|
self.assertIn("admin_push_dispatch", actions)
|
||||||
|
|
||||||
|
# -- 配额双字段与控制台文案 --------------------------------------------
|
||||||
|
def test_quota_payload_has_unpriced_fields(self):
|
||||||
|
s, body = _req(self.port, "GET", "/users/me/quota",
|
||||||
|
key=self.owner_key)
|
||||||
|
self.assertEqual(s, 200, body)
|
||||||
|
self.assertIn("unpriced_tokens_used", body)
|
||||||
|
self.assertIn("unpriced_token_cap", body)
|
||||||
|
|
||||||
|
def test_console_p1_markers(self):
|
||||||
|
conn = http.client.HTTPConnection(
|
||||||
|
"127.0.0.1", self.port, timeout=10)
|
||||||
|
try:
|
||||||
|
conn.request("GET", "/console")
|
||||||
|
resp = conn.getresponse()
|
||||||
|
text = resp.read().decode("utf-8")
|
||||||
|
finally:
|
||||||
|
conn.close()
|
||||||
|
self.assertEqual(resp.status, 200)
|
||||||
|
for marker in (
|
||||||
|
"登记评审需求",
|
||||||
|
"记录允许",
|
||||||
|
"记录拒绝",
|
||||||
|
"当前版本只留回执,不拦截实时行为",
|
||||||
|
"模型回合结束、改动在 worktree 里尚未合并",
|
||||||
|
"控制面已标记 merged,不代表 git 主分支已合入",
|
||||||
|
"协调员与 goal/计划互斥",
|
||||||
|
"仅 failed/interrupted/merge-failed 可重试",
|
||||||
|
"仅 maintainer 可合并"):
|
||||||
|
self.assertIn(marker, text)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
unittest.main()
|
||||||
@@ -0,0 +1,534 @@
|
|||||||
|
"""P1 执行面针对性单测(只测本路改动,不跑全量三套)。
|
||||||
|
|
||||||
|
运行(在 ~/workspace/codex-app-server-p1security 下):
|
||||||
|
python3 -m unittest discover -s cloud/tests -p "test_p1_security_worker.py" -v
|
||||||
|
|
||||||
|
说明:只用注入的假 runner 与测试假值,不跑真实模型回合、不联网、
|
||||||
|
不登录任何服务;env 里的值只是占位用的假名,绝不是真实密钥。
|
||||||
|
本文件只归本路(执行面)所有,不碰 test_p1_security_server.py。
|
||||||
|
"""
|
||||||
|
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
import sqlite3
|
||||||
|
import sys
|
||||||
|
import tempfile
|
||||||
|
import threading
|
||||||
|
import time
|
||||||
|
import unittest
|
||||||
|
from pathlib import Path
|
||||||
|
from types import SimpleNamespace
|
||||||
|
from unittest import mock
|
||||||
|
|
||||||
|
try:
|
||||||
|
import cloud.sandbox as sandbox_mod
|
||||||
|
from cloud.cost import CostStore
|
||||||
|
from cloud.pr_exec import enqueue_pr_job
|
||||||
|
from cloud.sandbox import DockerExecutor, LocalExecutor
|
||||||
|
from cloud.server import init_db
|
||||||
|
from cloud.worker import Worker, _code_signature
|
||||||
|
except ImportError: # discover -s cloud/tests 时把项目根目录补进 sys.path
|
||||||
|
sys.path.insert(0, str(Path(__file__).resolve().parents[2]))
|
||||||
|
import cloud.sandbox as sandbox_mod
|
||||||
|
from cloud.cost import CostStore
|
||||||
|
from cloud.pr_exec import enqueue_pr_job
|
||||||
|
from cloud.sandbox import DockerExecutor, LocalExecutor
|
||||||
|
from cloud.server import init_db
|
||||||
|
from cloud.worker import Worker, _code_signature
|
||||||
|
|
||||||
|
FAKE_ENV = {"XCM_API_KEY": "test-placeholder"}
|
||||||
|
|
||||||
|
|
||||||
|
def _completed_runner(cmd):
|
||||||
|
if "dispatch" in cmd:
|
||||||
|
name = cmd[cmd.index("--name") + 1]
|
||||||
|
else:
|
||||||
|
name = cmd[cmd.index("--task") + 1]
|
||||||
|
return SimpleNamespace(
|
||||||
|
returncode=0,
|
||||||
|
stdout=json.dumps({"task": name, "status": "completed",
|
||||||
|
"text": "done"}),
|
||||||
|
stderr="")
|
||||||
|
|
||||||
|
|
||||||
|
def _usage_runner(usage, calls):
|
||||||
|
def fake(cmd):
|
||||||
|
calls.append(list(cmd))
|
||||||
|
return SimpleNamespace(
|
||||||
|
returncode=0,
|
||||||
|
stdout=json.dumps({"status": "completed", "text": "done",
|
||||||
|
"usage": usage}),
|
||||||
|
stderr="")
|
||||||
|
return fake
|
||||||
|
|
||||||
|
|
||||||
|
def _insert_task(db_path, task_id, created_by="tester", status="queued",
|
||||||
|
project_id="p1", **cols):
|
||||||
|
now = time.time()
|
||||||
|
conn = sqlite3.connect(db_path)
|
||||||
|
try:
|
||||||
|
conn.execute(
|
||||||
|
"INSERT INTO tasks(id, project_id, prompt, status,"
|
||||||
|
" idempotency_key, depends_on, priority, created_by,"
|
||||||
|
" created_at, updated_at, last_heartbeat)"
|
||||||
|
" VALUES(?, ?, ?, ?, NULL, '[]', 0, ?, ?, ?, NULL)",
|
||||||
|
(task_id, project_id, "do work", status, created_by, now, now),
|
||||||
|
)
|
||||||
|
for key, value in cols.items():
|
||||||
|
conn.execute(
|
||||||
|
"UPDATE tasks SET %s = ? WHERE id = ?" % key,
|
||||||
|
(value, task_id))
|
||||||
|
conn.commit()
|
||||||
|
finally:
|
||||||
|
conn.close()
|
||||||
|
|
||||||
|
|
||||||
|
def _insert_quota(db_path, user_id, micros):
|
||||||
|
conn = sqlite3.connect(db_path)
|
||||||
|
try:
|
||||||
|
conn.execute(
|
||||||
|
"CREATE TABLE IF NOT EXISTS user_quotas("
|
||||||
|
"user_id TEXT PRIMARY KEY, quota_usd_micros BIGINT,"
|
||||||
|
" updated_at DOUBLE PRECISION)")
|
||||||
|
conn.execute(
|
||||||
|
"INSERT INTO user_quotas(user_id, quota_usd_micros, updated_at)"
|
||||||
|
" VALUES(?, ?, ?)",
|
||||||
|
(user_id, micros, time.time()))
|
||||||
|
conn.commit()
|
||||||
|
finally:
|
||||||
|
conn.close()
|
||||||
|
|
||||||
|
|
||||||
|
def _insert_cost(db_path, task_id, model, input_tokens, output_tokens,
|
||||||
|
micros, source="actual"):
|
||||||
|
_insert_task(db_path, task_id, status="completed")
|
||||||
|
store = CostStore(db_path)
|
||||||
|
try:
|
||||||
|
store.record(task_id, "p1", model, input_tokens=input_tokens,
|
||||||
|
output_tokens=output_tokens, usd_micros=micros,
|
||||||
|
source=source)
|
||||||
|
finally:
|
||||||
|
try:
|
||||||
|
store._conn.close()
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
def _status(db_path, task_id):
|
||||||
|
conn = sqlite3.connect(db_path)
|
||||||
|
try:
|
||||||
|
row = conn.execute(
|
||||||
|
"SELECT status FROM tasks WHERE id = ?", (task_id,)).fetchone()
|
||||||
|
finally:
|
||||||
|
conn.close()
|
||||||
|
return row[0] if row else None
|
||||||
|
|
||||||
|
|
||||||
|
def _error_text(db_path, task_id):
|
||||||
|
conn = sqlite3.connect(db_path)
|
||||||
|
try:
|
||||||
|
row = conn.execute(
|
||||||
|
"SELECT error_text FROM tasks WHERE id = ?",
|
||||||
|
(task_id,)).fetchone()
|
||||||
|
finally:
|
||||||
|
conn.close()
|
||||||
|
return row[0] if row else None
|
||||||
|
|
||||||
|
|
||||||
|
class P1WorkerTestCase(unittest.TestCase):
|
||||||
|
def setUp(self):
|
||||||
|
self.tmp = tempfile.TemporaryDirectory()
|
||||||
|
self.db_path = os.path.join(self.tmp.name, "control.db")
|
||||||
|
init_db(self.db_path)
|
||||||
|
self.calls = []
|
||||||
|
|
||||||
|
def tearDown(self):
|
||||||
|
self.tmp.cleanup()
|
||||||
|
|
||||||
|
def _worker(self, runner=None, **kwargs):
|
||||||
|
kwargs.setdefault("env", FAKE_ENV)
|
||||||
|
return Worker(self.db_path, "/fake/host.py",
|
||||||
|
runner=runner or _completed_runner, **kwargs)
|
||||||
|
|
||||||
|
# 1. 无配额行按默认 $10 额度(不再是不限额)
|
||||||
|
def test_missing_quota_row_uses_default_quota(self):
|
||||||
|
_insert_cost(self.db_path, "donor-a", "gpt-6.1-sol",
|
||||||
|
1, 0, 11000000)
|
||||||
|
_insert_cost(self.db_path, "donor-b", "gpt-6.1-sol",
|
||||||
|
1, 0, 9000000)
|
||||||
|
worker = self._worker()
|
||||||
|
over = worker._quota_exceeded(
|
||||||
|
{"created_by": "tester", "project_id": "p1"}, None)
|
||||||
|
# donor-a 归 tester:已花 $11 > 默认 $10,挡住。
|
||||||
|
self.assertIsNotNone(over)
|
||||||
|
self.assertIn("quota_exceeded", over)
|
||||||
|
# donor-b 同样归 tester 会把总额推得更高,单独测另一用户:
|
||||||
|
_insert_task(self.db_path, "donor-c", created_by="rich",
|
||||||
|
status="completed")
|
||||||
|
store = CostStore(self.db_path)
|
||||||
|
try:
|
||||||
|
store.record("donor-c", "p1", "gpt-6.1-sol", input_tokens=1,
|
||||||
|
output_tokens=0, usd_micros=9000000,
|
||||||
|
source="actual")
|
||||||
|
finally:
|
||||||
|
try:
|
||||||
|
store._conn.close()
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
under = worker._quota_exceeded(
|
||||||
|
{"created_by": "rich", "project_id": "p1"}, None)
|
||||||
|
self.assertIsNone(under)
|
||||||
|
|
||||||
|
# 2. 显式 NULL 配额行仍是不限额
|
||||||
|
def test_explicit_null_quota_row_stays_unlimited(self):
|
||||||
|
_insert_quota(self.db_path, "tester", None)
|
||||||
|
_insert_cost(self.db_path, "donor", "gpt-6.1-sol",
|
||||||
|
1, 0, 999000000)
|
||||||
|
worker = self._worker()
|
||||||
|
self.assertIsNone(worker._quota_exceeded(
|
||||||
|
{"created_by": "tester", "project_id": "p1"}, None))
|
||||||
|
|
||||||
|
# 3. 属主为空拒绝新回合(不再返回 None 放行)
|
||||||
|
def test_ownerless_task_refused(self):
|
||||||
|
worker = self._worker()
|
||||||
|
for owner in (None, ""):
|
||||||
|
err = worker._quota_exceeded(
|
||||||
|
{"created_by": owner, "project_id": "p1"}, None)
|
||||||
|
self.assertIsNotNone(err)
|
||||||
|
self.assertIn("quota_exceeded", err)
|
||||||
|
|
||||||
|
# 4. 协调员子任务属主跟父任务走
|
||||||
|
def test_child_inherits_parent_owner(self):
|
||||||
|
_insert_quota(self.db_path, "tester", 100)
|
||||||
|
_insert_cost(self.db_path, "donor", "gpt-6.1-sol",
|
||||||
|
1, 0, 200)
|
||||||
|
_insert_task(self.db_path, "parent", created_by="tester")
|
||||||
|
_insert_task(self.db_path, "parent2", created_by="fresh")
|
||||||
|
worker = self._worker()
|
||||||
|
blocked = worker._quota_exceeded(
|
||||||
|
{"created_by": None, "parent_task_id": "parent",
|
||||||
|
"project_id": "p1"}, None)
|
||||||
|
self.assertIsNotNone(blocked)
|
||||||
|
self.assertIn("quota_exceeded", blocked)
|
||||||
|
# fresh 用户无配额行、未花钱:默认额度内放行。
|
||||||
|
self.assertIsNone(worker._quota_exceeded(
|
||||||
|
{"created_by": None, "parent_task_id": "parent2",
|
||||||
|
"project_id": "p1"}, None))
|
||||||
|
|
||||||
|
# 5. 查库失败 fail-closed(新回合拒绝,不自动放行)
|
||||||
|
def test_quota_db_failure_is_fail_closed(self):
|
||||||
|
worker = self._worker()
|
||||||
|
with mock.patch("cloud.worker._db_connect",
|
||||||
|
side_effect=RuntimeError("test db down")):
|
||||||
|
err = worker._quota_exceeded(
|
||||||
|
{"created_by": "tester", "project_id": "p1"}, None)
|
||||||
|
self.assertIsNotNone(err)
|
||||||
|
self.assertTrue(err.startswith("quota_check_failed:"))
|
||||||
|
|
||||||
|
# 6. Docker 配额看门狗:命中杀容器、未命中不杀
|
||||||
|
def test_docker_quota_watchdog(self):
|
||||||
|
killed = []
|
||||||
|
|
||||||
|
def fake(cmd, timeout=None):
|
||||||
|
killed.append(list(cmd))
|
||||||
|
return SimpleNamespace(returncode=0, stdout="", stderr="")
|
||||||
|
|
||||||
|
executor = DockerExecutor(runner=fake)
|
||||||
|
worker = Worker(self.db_path, "/fake/host.py",
|
||||||
|
env=FAKE_ENV, executor=executor)
|
||||||
|
# 未命中:0.2 秒后停表,kill 不应被调。
|
||||||
|
stop = threading.Event()
|
||||||
|
threading.Timer(0.2, stop.set).start()
|
||||||
|
worker._docker_quota_watchdog(
|
||||||
|
"T1", "codex-test", stop, lambda: False, interval=0.05)
|
||||||
|
self.assertEqual(killed, [])
|
||||||
|
# 命中:立即杀容器并记命中标记。
|
||||||
|
stop2 = threading.Event()
|
||||||
|
worker._docker_quota_watchdog(
|
||||||
|
"T2", "codex-test", stop2, lambda: True, interval=0.05)
|
||||||
|
self.assertTrue(any("kill" in c for c in killed))
|
||||||
|
self.assertTrue(worker._take_quota_hit("T2"))
|
||||||
|
# 命中已消费,第二次取不到(不污染下一轮)。
|
||||||
|
self.assertFalse(worker._take_quota_hit("T2"))
|
||||||
|
|
||||||
|
# 7. usage_cost_usd 按实际成本累加,max_cost_usd 不再是死参数
|
||||||
|
def test_usage_cost_accumulates_and_budget_trips(self):
|
||||||
|
_insert_task(self.db_path, "T1")
|
||||||
|
worker = self._worker(
|
||||||
|
runner=_usage_runner(
|
||||||
|
{"input_tokens": 10000, "cached_input_tokens": 0,
|
||||||
|
"output_tokens": 1000, "total_tokens": 11000},
|
||||||
|
self.calls),
|
||||||
|
model="gpt-6.1-sol", max_cost_usd=0.01)
|
||||||
|
self.assertFalse(worker._budget_exceeded())
|
||||||
|
result = worker.run_one()
|
||||||
|
self.assertEqual(result["status"], "completed")
|
||||||
|
# 10000*$2 + 1000*$10 = $0.03,超过 $0.01 上限。
|
||||||
|
self.assertAlmostEqual(worker.usage_cost_usd, 0.03, places=6)
|
||||||
|
self.assertTrue(worker._budget_exceeded())
|
||||||
|
conn = sqlite3.connect(self.db_path)
|
||||||
|
try:
|
||||||
|
row = conn.execute(
|
||||||
|
"SELECT usd_micros, source FROM cost_records"
|
||||||
|
" WHERE task_id = 'T1'").fetchone()
|
||||||
|
finally:
|
||||||
|
conn.close()
|
||||||
|
self.assertEqual(row, (30000, "actual"))
|
||||||
|
|
||||||
|
# 8. 取消分支先记成本再返回(与配额中断收尾同款记账点)
|
||||||
|
def test_cancel_branch_records_cost(self):
|
||||||
|
worker = self._worker()
|
||||||
|
task = {"id": "T1", "project_id": "p1", "prompt": "do work"}
|
||||||
|
with worker._db_lock:
|
||||||
|
cost_id = worker._record_usage_cost(
|
||||||
|
task, "gpt-6.1-sol",
|
||||||
|
{"input_tokens": 10000, "cached_input_tokens": 0,
|
||||||
|
"output_tokens": 1000, "total_tokens": 11000},
|
||||||
|
"done", 0)
|
||||||
|
self.assertIsNotNone(cost_id)
|
||||||
|
self.assertAlmostEqual(worker.usage_cost_usd, 0.03, places=6)
|
||||||
|
# 无用量时回落估算,不累加 usage_cost_usd。
|
||||||
|
with worker._db_lock:
|
||||||
|
worker._record_usage_cost(
|
||||||
|
{"id": "T2", "project_id": "p1", "prompt": "do work"},
|
||||||
|
"gpt-6.1-sol", None, "done", 0)
|
||||||
|
self.assertAlmostEqual(worker.usage_cost_usd, 0.03, places=6)
|
||||||
|
|
||||||
|
# 9. 估算行不计入无价 token 上限
|
||||||
|
def test_estimated_rows_excluded_from_unpriced_cap(self):
|
||||||
|
_insert_cost(self.db_path, "donor-est", "gpt-5.6-terra",
|
||||||
|
100000, 0, None, source="estimated")
|
||||||
|
worker = self._worker()
|
||||||
|
old = os.environ.get("CLOUD_QUOTA_UNPRICED_TOKEN_CAP")
|
||||||
|
os.environ["CLOUD_QUOTA_UNPRICED_TOKEN_CAP"] = "100"
|
||||||
|
try:
|
||||||
|
# 10 万估算 token 在上限 100 下仍放行(估算不是真花费)。
|
||||||
|
self.assertIsNone(worker._quota_exceeded(
|
||||||
|
{"created_by": "tester", "project_id": "p1"}, None))
|
||||||
|
_insert_cost(self.db_path, "donor-act", "gpt-5.6-terra",
|
||||||
|
75, 75, None, source="actual")
|
||||||
|
err = worker._quota_exceeded(
|
||||||
|
{"created_by": "tester", "project_id": "p1"}, None)
|
||||||
|
self.assertIsNotNone(err)
|
||||||
|
self.assertIn("无价模型 token", err)
|
||||||
|
finally:
|
||||||
|
if old is None:
|
||||||
|
os.environ.pop("CLOUD_QUOTA_UNPRICED_TOKEN_CAP", None)
|
||||||
|
else:
|
||||||
|
os.environ["CLOUD_QUOTA_UNPRICED_TOKEN_CAP"] = old
|
||||||
|
|
||||||
|
# 10. cache_write 不计入无价上限(只 input+output 计)
|
||||||
|
def test_cache_write_excluded_from_unpriced_cap(self):
|
||||||
|
worker = self._worker()
|
||||||
|
old = os.environ.get("CLOUD_QUOTA_UNPRICED_TOKEN_CAP")
|
||||||
|
os.environ["CLOUD_QUOTA_UNPRICED_TOKEN_CAP"] = "100"
|
||||||
|
try:
|
||||||
|
usage = {"input_tokens": 10, "output_tokens": 5,
|
||||||
|
"cache_write_input_tokens": 9999999,
|
||||||
|
"total_tokens": 15}
|
||||||
|
self.assertIsNone(worker._quota_exceeded(
|
||||||
|
{"created_by": "tester", "project_id": "p1",
|
||||||
|
"model": "gpt-5.6-terra"}, usage))
|
||||||
|
usage2 = {"input_tokens": 200, "output_tokens": 0,
|
||||||
|
"cache_write_input_tokens": 0,
|
||||||
|
"total_tokens": 200}
|
||||||
|
err = worker._quota_exceeded(
|
||||||
|
{"created_by": "tester", "project_id": "p1",
|
||||||
|
"model": "gpt-5.6-terra"}, usage2)
|
||||||
|
self.assertIsNotNone(err)
|
||||||
|
finally:
|
||||||
|
if old is None:
|
||||||
|
os.environ.pop("CLOUD_QUOTA_UNPRICED_TOKEN_CAP", None)
|
||||||
|
else:
|
||||||
|
os.environ["CLOUD_QUOTA_UNPRICED_TOKEN_CAP"] = old
|
||||||
|
|
||||||
|
# 11. 协调员子任务继承父任务模型/强度,分工显式值优先
|
||||||
|
def test_coordinator_child_inherits_model_effort(self):
|
||||||
|
worker = self._worker()
|
||||||
|
task = {"id": "coord", "project_id": "p1", "prompt": "coord",
|
||||||
|
"priority": 0, "created_by": "tester", "model": "gpt-6.1-sol",
|
||||||
|
"effort": "high", "executor": None, "environment_id": None}
|
||||||
|
children = worker._coord_sync_children(task, [
|
||||||
|
{"name": "a", "prompt": "do a", "model": None,
|
||||||
|
"effort": None, "executor": None},
|
||||||
|
{"name": "b", "prompt": "do b", "model": "gpt-5.6-terra",
|
||||||
|
"effort": "low", "executor": None},
|
||||||
|
])
|
||||||
|
self.assertEqual(len(children), 2)
|
||||||
|
conn = sqlite3.connect(self.db_path)
|
||||||
|
try:
|
||||||
|
rows = {r[0]: (r[1], r[2]) for r in conn.execute(
|
||||||
|
"SELECT coordinator_part, model, effort FROM tasks"
|
||||||
|
" WHERE parent_task_id = 'coord'").fetchall()}
|
||||||
|
finally:
|
||||||
|
conn.close()
|
||||||
|
self.assertEqual(rows["a"], ("gpt-6.1-sol", "high"))
|
||||||
|
self.assertEqual(rows["b"], ("gpt-5.6-terra", "low"))
|
||||||
|
|
||||||
|
# 12. 断点续跑:Local/Docker/popen 三路都不传 --goal-text
|
||||||
|
def test_goal_resume_breakpoint_omits_goal_text(self):
|
||||||
|
calls = []
|
||||||
|
|
||||||
|
def fake(cmd):
|
||||||
|
calls.append(list(cmd))
|
||||||
|
return SimpleNamespace(
|
||||||
|
returncode=0,
|
||||||
|
stdout="goal met after rounds\n"
|
||||||
|
"usage input=10 cached=0 cache_write=0 output=5 "
|
||||||
|
"reasoning=0 total=15 completions=1\n",
|
||||||
|
stderr="")
|
||||||
|
|
||||||
|
task = {"id": "T1", "project_id": "p1", "prompt": "orig goal",
|
||||||
|
"verify_command": "true", "goal_max_rounds": 3}
|
||||||
|
local = LocalExecutor("/fake/host.py", runner=fake)
|
||||||
|
local.goal_resume_task(task, None)
|
||||||
|
self.assertIn("goal-resume", calls[-1])
|
||||||
|
self.assertNotIn("--goal-text", calls[-1])
|
||||||
|
docker_calls = []
|
||||||
|
|
||||||
|
def dfake(cmd, timeout=None):
|
||||||
|
docker_calls.append(list(cmd))
|
||||||
|
return SimpleNamespace(
|
||||||
|
returncode=0,
|
||||||
|
stdout="goal met after rounds\n"
|
||||||
|
"usage input=10 cached=0 cache_write=0 output=5 "
|
||||||
|
"reasoning=0 total=15 completions=1\n",
|
||||||
|
stderr="")
|
||||||
|
|
||||||
|
docker = DockerExecutor(runner=dfake, env=FAKE_ENV)
|
||||||
|
with mock.patch("cloud.sandbox.shutil.which",
|
||||||
|
return_value="/usr/bin/docker"):
|
||||||
|
docker.goal_resume_task(task, None)
|
||||||
|
self.assertIn("goal-resume", docker_calls[-1])
|
||||||
|
self.assertNotIn("--goal-text", docker_calls[-1])
|
||||||
|
with mock.patch.object(sandbox_mod.subprocess, "Popen") as popen:
|
||||||
|
popen.return_value = SimpleNamespace(pid=9999)
|
||||||
|
local.popen_task(
|
||||||
|
{"id": "T1", "prompt": "resume msg",
|
||||||
|
"_resume_kind": "breakpoint"})
|
||||||
|
# popen_task 默认 kind=dispatch;显式测 goal-resume:
|
||||||
|
local.popen_task(
|
||||||
|
{"id": "T1", "prompt": "resume msg",
|
||||||
|
"_resume_kind": "breakpoint"}, kind="goal-resume")
|
||||||
|
cmd = popen.call_args[0][0]
|
||||||
|
self.assertIn("goal-resume", cmd)
|
||||||
|
self.assertNotIn("--goal-text", cmd)
|
||||||
|
|
||||||
|
# 13. 开新目标:goal-resume 带 --goal-text(旧行为保留)
|
||||||
|
def test_goal_resume_new_goal_keeps_goal_text(self):
|
||||||
|
calls = []
|
||||||
|
|
||||||
|
def fake(cmd):
|
||||||
|
calls.append(list(cmd))
|
||||||
|
return SimpleNamespace(
|
||||||
|
returncode=0,
|
||||||
|
stdout="goal met after rounds\n"
|
||||||
|
"usage input=10 cached=0 cache_write=0 output=5 "
|
||||||
|
"reasoning=0 total=15 completions=1\n",
|
||||||
|
stderr="")
|
||||||
|
|
||||||
|
task = {"id": "T1", "project_id": "p1", "prompt": "new goal"}
|
||||||
|
local = LocalExecutor("/fake/host.py", runner=fake)
|
||||||
|
local.goal_resume_task(task, "new goal")
|
||||||
|
self.assertIn("--goal-text", calls[-1])
|
||||||
|
self.assertIn("new goal", calls[-1])
|
||||||
|
|
||||||
|
# 14. goal 轮次上限:sandbox loud fail + worker 认领后 failed
|
||||||
|
def test_goal_max_rounds_cap(self):
|
||||||
|
task = {"id": "T1", "project_id": "p1", "prompt": "g",
|
||||||
|
"verify_command": "true", "goal_max_rounds": 101}
|
||||||
|
local = LocalExecutor("/fake/host.py", runner=_completed_runner)
|
||||||
|
with self.assertRaises(ValueError):
|
||||||
|
local.goal_task(task)
|
||||||
|
_insert_task(self.db_path, "T1", verify_command="true",
|
||||||
|
goal_max_rounds=101)
|
||||||
|
worker = self._worker()
|
||||||
|
result = worker.run_one()
|
||||||
|
self.assertEqual(result["status"], "failed")
|
||||||
|
self.assertIn("上限", result["error"])
|
||||||
|
self.assertEqual(_status(self.db_path, "T1"), "failed")
|
||||||
|
self.assertEqual(self.calls, [])
|
||||||
|
|
||||||
|
# 15. PR 非 local:显式 needs_local_worker,任务与排队都不动
|
||||||
|
def test_pr_non_local_needs_local_worker(self):
|
||||||
|
conn = sqlite3.connect(self.db_path)
|
||||||
|
try:
|
||||||
|
job_id = enqueue_pr_job(conn, "T1", "p1", "test-host")
|
||||||
|
finally:
|
||||||
|
conn.close()
|
||||||
|
executor = DockerExecutor(runner=lambda cmd, timeout=None:
|
||||||
|
SimpleNamespace(returncode=0, stdout="",
|
||||||
|
stderr=""))
|
||||||
|
worker = Worker(self.db_path, "/fake/host.py", env=FAKE_ENV,
|
||||||
|
executor=executor, host_name="test-host")
|
||||||
|
results = worker._run_pr_jobs()
|
||||||
|
self.assertEqual(len(results), 1)
|
||||||
|
self.assertEqual(results[0]["status"], "needs_local_worker")
|
||||||
|
self.assertEqual(results[0]["pr_job_id"], job_id)
|
||||||
|
conn = sqlite3.connect(self.db_path)
|
||||||
|
try:
|
||||||
|
row = conn.execute(
|
||||||
|
"SELECT status FROM pr_jobs WHERE id = ?",
|
||||||
|
(job_id,)).fetchone()
|
||||||
|
finally:
|
||||||
|
conn.close()
|
||||||
|
self.assertEqual(row[0], "queued")
|
||||||
|
|
||||||
|
# 16. 多文件签名 + 签名缺失/已变时认领 fail-closed
|
||||||
|
def test_code_signature_multi_file_and_claim_refusal(self):
|
||||||
|
sig = _code_signature("/fake/host.py")
|
||||||
|
self.assertIsNotNone(sig)
|
||||||
|
self.assertEqual(len(sig), 6)
|
||||||
|
for entry in sig:
|
||||||
|
self.assertEqual(len(entry), 2)
|
||||||
|
_insert_task(self.db_path, "T1")
|
||||||
|
worker = self._worker()
|
||||||
|
worker._code_signature = ("stale-signature",)
|
||||||
|
result = worker.run_one()
|
||||||
|
self.assertFalse(result["dispatched"])
|
||||||
|
self.assertEqual(result["reason"], "already_claimed")
|
||||||
|
self.assertEqual(_status(self.db_path, "T1"), "queued")
|
||||||
|
worker._code_signature = None
|
||||||
|
result = worker.run_one()
|
||||||
|
self.assertFalse(result["dispatched"])
|
||||||
|
self.assertEqual(_status(self.db_path, "T1"), "queued")
|
||||||
|
|
||||||
|
# 17. goal 首轮撞已存在 → 断点续跑一次,不直接 failed
|
||||||
|
def test_goal_already_exists_falls_back_to_breakpoint(self):
|
||||||
|
calls = []
|
||||||
|
|
||||||
|
def fake(cmd):
|
||||||
|
calls.append(list(cmd))
|
||||||
|
if "goal-resume" in cmd:
|
||||||
|
return SimpleNamespace(
|
||||||
|
returncode=0,
|
||||||
|
stdout="round 1: turn=completed verify_rc=0 "
|
||||||
|
"verify_out=\ngoal met after rounds\n"
|
||||||
|
"usage input=10 cached=0 cache_write=0 output=5 "
|
||||||
|
"reasoning=0 total=15 completions=1\n",
|
||||||
|
stderr="")
|
||||||
|
if "goal" in cmd:
|
||||||
|
return SimpleNamespace(
|
||||||
|
returncode=0,
|
||||||
|
stdout="host 任务已存在:T1 已存在",
|
||||||
|
stderr="")
|
||||||
|
return _completed_runner(cmd)
|
||||||
|
|
||||||
|
_insert_task(self.db_path, "T1", verify_command="true",
|
||||||
|
goal_max_rounds=2)
|
||||||
|
worker = self._worker(runner=fake)
|
||||||
|
result = worker.run_one()
|
||||||
|
self.assertEqual(result["status"], "completed")
|
||||||
|
# 注意:cmd 是元素列表,"goal" in cmd 只匹配恰好等于
|
||||||
|
# "goal" 的元素,goal-resume 命令里并没有 "goal" 元素,
|
||||||
|
# 所以按“任一元素含 goal 子串”过滤。
|
||||||
|
goal_cmds = [c for c in calls
|
||||||
|
if any("goal" in e for e in c)]
|
||||||
|
self.assertTrue(any("goal-resume" in c for c in goal_cmds))
|
||||||
|
resume_cmd = [c for c in goal_cmds if "goal-resume" in c][0]
|
||||||
|
self.assertNotIn("--goal-text", resume_cmd)
|
||||||
|
self.assertEqual(_status(self.db_path, "T1"), "completed")
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
unittest.main()
|
||||||
@@ -307,7 +307,12 @@ class WorkerPrJobsTestCase(unittest.TestCase):
|
|||||||
enqueue_pr_job(conn, "t1", "p1", worker.host_name)
|
enqueue_pr_job(conn, "t1", "p1", worker.host_name)
|
||||||
finally:
|
finally:
|
||||||
conn.close()
|
conn.close()
|
||||||
self.assertEqual(worker._run_pr_jobs(), [])
|
# P1:Docker worker 不再静默返回空,改为显式 needs_local_worker,
|
||||||
|
# 任务保持 queued 等 local worker 认领。
|
||||||
|
jobs = worker._run_pr_jobs()
|
||||||
|
self.assertEqual(len(jobs), 1)
|
||||||
|
self.assertEqual(jobs[0]["status"], "needs_local_worker")
|
||||||
|
self.assertEqual(jobs[0]["task_id"], "t1")
|
||||||
|
|
||||||
def test_autoscaler_forces_host_name(self):
|
def test_autoscaler_forces_host_name(self):
|
||||||
scaler = Autoscaler(self.db_path, [])
|
scaler = Autoscaler(self.db_path, [])
|
||||||
|
|||||||
@@ -41,6 +41,9 @@ class WebhookExecTestCase(unittest.TestCase):
|
|||||||
self.owner_id, self.owner_key = self.auth.create_user("owner")
|
self.owner_id, self.owner_key = self.auth.create_user("owner")
|
||||||
self._old_admins = os.environ.get("CODEX_GLOBAL_ADMINS")
|
self._old_admins = os.environ.get("CODEX_GLOBAL_ADMINS")
|
||||||
os.environ["CODEX_GLOBAL_ADMINS"] = self.owner_id
|
os.environ["CODEX_GLOBAL_ADMINS"] = self.owner_id
|
||||||
|
# P1:备份 out_dir 限白名单根;本测试的 tmp 目录即白名单根。
|
||||||
|
self._old_backup_root = os.environ.get("CLOUD_BACKUP_ROOT")
|
||||||
|
os.environ["CLOUD_BACKUP_ROOT"] = self.tmp.name
|
||||||
self.viewer_id, self.viewer_key = self.auth.create_user("viewer")
|
self.viewer_id, self.viewer_key = self.auth.create_user("viewer")
|
||||||
self.server = create_server(self.db_path, "127.0.0.1", 0)
|
self.server = create_server(self.db_path, "127.0.0.1", 0)
|
||||||
self.port = self.server.server_address[1]
|
self.port = self.server.server_address[1]
|
||||||
@@ -67,6 +70,10 @@ class WebhookExecTestCase(unittest.TestCase):
|
|||||||
os.environ.pop("CODEX_GLOBAL_ADMINS", None)
|
os.environ.pop("CODEX_GLOBAL_ADMINS", None)
|
||||||
else:
|
else:
|
||||||
os.environ["CODEX_GLOBAL_ADMINS"] = self._old_admins
|
os.environ["CODEX_GLOBAL_ADMINS"] = self._old_admins
|
||||||
|
if getattr(self, "_old_backup_root", None) is None:
|
||||||
|
os.environ.pop("CLOUD_BACKUP_ROOT", None)
|
||||||
|
else:
|
||||||
|
os.environ["CLOUD_BACKUP_ROOT"] = self._old_backup_root
|
||||||
|
|
||||||
def _req(self, method, path, body=None, key=None, headers=None,
|
def _req(self, method, path, body=None, key=None, headers=None,
|
||||||
raw_body=None):
|
raw_body=None):
|
||||||
|
|||||||
@@ -336,6 +336,11 @@ class TestRealSenders(PushTestCase):
|
|||||||
|
|
||||||
def setUp(self):
|
def setUp(self):
|
||||||
super().setUp()
|
super().setUp()
|
||||||
|
# P1 SSRF 门默认挡 loopback;本地录制服务器测试显式放行,
|
||||||
|
# 生产默认值仍关闭(不要把本开关当成生产默认)。
|
||||||
|
self._old_push_allow = os.environ.get(
|
||||||
|
"CODEX_PUSH_ALLOW_PRIVATE_TARGETS")
|
||||||
|
os.environ["CODEX_PUSH_ALLOW_PRIVATE_TARGETS"] = "1"
|
||||||
_RecordingHandler.requests = []
|
_RecordingHandler.requests = []
|
||||||
_RecordingHandler.status = 200
|
_RecordingHandler.status = 200
|
||||||
self.server = http.server.ThreadingHTTPServer(
|
self.server = http.server.ThreadingHTTPServer(
|
||||||
@@ -349,6 +354,11 @@ class TestRealSenders(PushTestCase):
|
|||||||
self.server.shutdown()
|
self.server.shutdown()
|
||||||
self.server.server_close()
|
self.server.server_close()
|
||||||
self.thread.join(timeout=5)
|
self.thread.join(timeout=5)
|
||||||
|
if getattr(self, "_old_push_allow", None) is None:
|
||||||
|
os.environ.pop("CODEX_PUSH_ALLOW_PRIVATE_TARGETS", None)
|
||||||
|
else:
|
||||||
|
os.environ["CODEX_PUSH_ALLOW_PRIVATE_TARGETS"] = (
|
||||||
|
self._old_push_allow)
|
||||||
super().tearDown()
|
super().tearDown()
|
||||||
|
|
||||||
def _url(self, path="/hook"):
|
def _url(self, path="/hook"):
|
||||||
|
|||||||
@@ -331,7 +331,8 @@ class TestGoalExecutors(unittest.TestCase):
|
|||||||
{"id": "t1", "prompt": "do it",
|
{"id": "t1", "prompt": "do it",
|
||||||
"verify_command": "true"})
|
"verify_command": "true"})
|
||||||
self.assertEqual(result["status"], "failed")
|
self.assertEqual(result["status"], "failed")
|
||||||
self.assertIn("请新建任务", result["goal_error"])
|
# P1 新语义:goal 失败请用继续而非重试(旧“请新建任务”已废)。
|
||||||
|
self.assertIn("请用继续而非重试", result["goal_error"])
|
||||||
|
|
||||||
def test_local_goal_task_without_verify_is_llm_decided(self):
|
def test_local_goal_task_without_verify_is_llm_decided(self):
|
||||||
# 新语义:没有验收命令不再抛错——走模型自决分支:
|
# 新语义:没有验收命令不再抛错——走模型自决分支:
|
||||||
|
|||||||
@@ -114,6 +114,11 @@ class _Listener:
|
|||||||
|
|
||||||
class GoalPrNotifyTestCase(unittest.TestCase):
|
class GoalPrNotifyTestCase(unittest.TestCase):
|
||||||
def setUp(self):
|
def setUp(self):
|
||||||
|
# P1 SSRF 门默认挡 loopback;本地监听器测试显式放行,
|
||||||
|
# 生产默认值仍关闭。
|
||||||
|
self._old_push_allow = os.environ.get(
|
||||||
|
"CODEX_PUSH_ALLOW_PRIVATE_TARGETS")
|
||||||
|
os.environ["CODEX_PUSH_ALLOW_PRIVATE_TARGETS"] = "1"
|
||||||
self.tmp = tempfile.TemporaryDirectory()
|
self.tmp = tempfile.TemporaryDirectory()
|
||||||
self.db_path = os.path.join(self.tmp.name, "control.db")
|
self.db_path = os.path.join(self.tmp.name, "control.db")
|
||||||
self.auth = AuthStore(self.db_path)
|
self.auth = AuthStore(self.db_path)
|
||||||
@@ -133,6 +138,11 @@ class GoalPrNotifyTestCase(unittest.TestCase):
|
|||||||
self.server.shutdown()
|
self.server.shutdown()
|
||||||
self.server.server_close()
|
self.server.server_close()
|
||||||
self.thread.join(timeout=5)
|
self.thread.join(timeout=5)
|
||||||
|
if getattr(self, "_old_push_allow", None) is None:
|
||||||
|
os.environ.pop("CODEX_PUSH_ALLOW_PRIVATE_TARGETS", None)
|
||||||
|
else:
|
||||||
|
os.environ["CODEX_PUSH_ALLOW_PRIVATE_TARGETS"] = (
|
||||||
|
self._old_push_allow)
|
||||||
try:
|
try:
|
||||||
self.auth.conn.close()
|
self.auth.conn.close()
|
||||||
except Exception:
|
except Exception:
|
||||||
|
|||||||
@@ -74,6 +74,11 @@ def _completed_runner(cmd):
|
|||||||
|
|
||||||
class WorkerTestCase(unittest.TestCase):
|
class WorkerTestCase(unittest.TestCase):
|
||||||
def setUp(self):
|
def setUp(self):
|
||||||
|
# P1 SSRF 门默认挡 loopback;通知测试用本地监听器,显式放行,
|
||||||
|
# 生产默认值仍关闭。
|
||||||
|
self._old_push_allow = os.environ.get(
|
||||||
|
"CODEX_PUSH_ALLOW_PRIVATE_TARGETS")
|
||||||
|
os.environ["CODEX_PUSH_ALLOW_PRIVATE_TARGETS"] = "1"
|
||||||
self.tmp = tempfile.TemporaryDirectory()
|
self.tmp = tempfile.TemporaryDirectory()
|
||||||
self.db_path = os.path.join(self.tmp.name, "control.db")
|
self.db_path = os.path.join(self.tmp.name, "control.db")
|
||||||
init_db(self.db_path)
|
init_db(self.db_path)
|
||||||
@@ -81,6 +86,11 @@ class WorkerTestCase(unittest.TestCase):
|
|||||||
self.calls = []
|
self.calls = []
|
||||||
|
|
||||||
def tearDown(self):
|
def tearDown(self):
|
||||||
|
if getattr(self, "_old_push_allow", None) is None:
|
||||||
|
os.environ.pop("CODEX_PUSH_ALLOW_PRIVATE_TARGETS", None)
|
||||||
|
else:
|
||||||
|
os.environ["CODEX_PUSH_ALLOW_PRIVATE_TARGETS"] = (
|
||||||
|
self._old_push_allow)
|
||||||
self.tmp.cleanup()
|
self.tmp.cleanup()
|
||||||
|
|
||||||
def _worker(self, runner=None, **kwargs):
|
def _worker(self, runner=None, **kwargs):
|
||||||
|
|||||||
+604
-55
@@ -139,6 +139,7 @@ from cloud.coordinator import DEFAULT_POLL_INTERVAL as _COORD_POLL_INTERVAL
|
|||||||
from cloud.coordinator import DEFAULT_TIMEOUT_SECONDS as _COORD_TIMEOUT
|
from cloud.coordinator import DEFAULT_TIMEOUT_SECONDS as _COORD_TIMEOUT
|
||||||
from cloud.coordinator import ensure_columns as _coord_ensure_columns
|
from cloud.coordinator import ensure_columns as _coord_ensure_columns
|
||||||
from cloud.coordinator import parse_parts as _coord_parse_parts
|
from cloud.coordinator import parse_parts as _coord_parse_parts
|
||||||
|
from cloud.coordinator import nonterminal_children as _coord_nonterminal
|
||||||
from cloud.cost import CostStore, estimate_tokens, price_usd_micros, \
|
from cloud.cost import CostStore, estimate_tokens, price_usd_micros, \
|
||||||
sum_usage, unpriced_token_cap
|
sum_usage, unpriced_token_cap
|
||||||
from cloud.db import connect as _db_connect
|
from cloud.db import connect as _db_connect
|
||||||
@@ -170,19 +171,68 @@ DEFAULT_ENV_KEY = "XCM_API_KEY"
|
|||||||
# gpt-6.1-sol 路由当日实测持续 503,terra 同日实测 1 秒完成;
|
# gpt-6.1-sol 路由当日实测持续 503,terra 同日实测 1 秒完成;
|
||||||
# 任务未指定模型时 worker 与 host.py 都落到这里。
|
# 任务未指定模型时 worker 与 host.py 都落到这里。
|
||||||
DEFAULT_MODEL = "gpt-5.6-terra"
|
DEFAULT_MODEL = "gpt-5.6-terra"
|
||||||
|
# P1 执行面:goal 轮次上限(有限轮次形态)。goal_mode=1 的 0 轮
|
||||||
|
# (模型宣告才结束)不在此限——它的上界是配额(配额看门狗与
|
||||||
|
# 认领/边界门),不是轮次;旧验收形态的 goal_max_rounds 必须
|
||||||
|
# 在 1..GOAL_MAX_ROUNDS_CAP 之间,超限的任务 worker 在认领后
|
||||||
|
# 直接 failed(建任务时的 400 校验在 server.py,属另一路文件,
|
||||||
|
# 本路只做执行侧防线,见 run_one 的 goal 上限门)。
|
||||||
|
GOAL_MAX_ROUNDS_CAP = 100
|
||||||
|
# P1 执行面:无配额行时的默认配额(微美元),与 server.py
|
||||||
|
# _default_quota_usd_micros() 同值($10/月,环境变量
|
||||||
|
# CLOUD_DEFAULT_QUOTA_USD_MICROS 可覆盖)。P0 前“无行 =
|
||||||
|
# 不限额”已废:worker 侧查不到行时按此默认额度算,显式
|
||||||
|
# NULL 行才是不限额(与服务端 _quota_status 同口径)。
|
||||||
# host.py dispatch 的原话是 "task 已存在: <name>,先换名或 discard 它"
|
# host.py dispatch 的原话是 "task 已存在: <name>,先换名或 discard 它"
|
||||||
#(英文 task + 已存在),取共有子串做判断,不依赖标点。
|
#(英文 task + 已存在),取共有子串做判断,不依赖标点。
|
||||||
_ALREADY_EXISTS_MARKER = "已存在"
|
_ALREADY_EXISTS_MARKER = "已存在"
|
||||||
|
|
||||||
|
|
||||||
def _code_signature():
|
def _default_quota_usd_micros():
|
||||||
"""本文件签名(mtime_ns, size):部署 staging 换目录后必变;
|
"""无配额行时的默认配额(微美元),与 server 侧同值。
|
||||||
取不到时返回 None(此时自退检查跳过,绝不误杀)。"""
|
|
||||||
|
server.py 的 _default_quota_usd_micros() 在另一路文件里,
|
||||||
|
worker 不跨模块 import 它(避免循环 import),这里复刻
|
||||||
|
同款解析:环境变量 CLOUD_DEFAULT_QUOTA_USD_MICROS,非法
|
||||||
|
值回落 10_000_000。"""
|
||||||
try:
|
try:
|
||||||
st = os.stat(os.path.abspath(__file__))
|
return max(0, int(os.environ.get(
|
||||||
return (st.st_mtime_ns, st.st_size)
|
"CLOUD_DEFAULT_QUOTA_USD_MICROS", "10000000")))
|
||||||
|
except (TypeError, ValueError):
|
||||||
|
return 10000000
|
||||||
|
|
||||||
|
|
||||||
|
def _code_signature(host_py_path=None):
|
||||||
|
"""多文件签名:worker/sandbox/coordinator/cost/server/host
|
||||||
|
各取 (mtime_ns, size) 组成的元组;任一文件取不到即 None。
|
||||||
|
|
||||||
|
P1 执行面扩面:旧版只签 worker.py,部署只换了 sandbox 或
|
||||||
|
host.py 时旧 worker 会继续用旧代码抢任务。host.py 取
|
||||||
|
host_py_path(真实文件时用它,否则用 cloud/ 上一级的
|
||||||
|
host.py);单测常传 /fake/host.py 这种不存在的路径,此时
|
||||||
|
回落到仓库 host.py,不因此判签名缺失。取不到时返回 None,
|
||||||
|
调用方必须 fail-closed(不认领、不执行),绝不按“没变”
|
||||||
|
放行——旧版“取不到当没变”是 fail-open,已废。"""
|
||||||
|
here = os.path.dirname(os.path.abspath(__file__))
|
||||||
|
candidates = [
|
||||||
|
os.path.join(here, "worker.py"),
|
||||||
|
os.path.join(here, "sandbox.py"),
|
||||||
|
os.path.join(here, "coordinator.py"),
|
||||||
|
os.path.join(here, "cost.py"),
|
||||||
|
os.path.join(here, "server.py"),
|
||||||
|
]
|
||||||
|
if host_py_path and os.path.isfile(host_py_path):
|
||||||
|
candidates.append(os.path.abspath(host_py_path))
|
||||||
|
else:
|
||||||
|
candidates.append(os.path.join(os.path.dirname(here), "host.py"))
|
||||||
|
sig = []
|
||||||
|
try:
|
||||||
|
for path in candidates:
|
||||||
|
st = os.stat(path)
|
||||||
|
sig.append((st.st_mtime_ns, st.st_size))
|
||||||
except OSError:
|
except OSError:
|
||||||
return None
|
return None
|
||||||
|
return tuple(sig)
|
||||||
|
|
||||||
|
|
||||||
def _missing_key_error(env_key):
|
def _missing_key_error(env_key):
|
||||||
@@ -447,16 +497,46 @@ class Worker:
|
|||||||
# 注入的普通 dict 同样可读环境变量(无该键时默认 1)。
|
# 注入的普通 dict 同样可读环境变量(无该键时默认 1)。
|
||||||
self.capacity = resolve_capacity(capacity, env=self._env)
|
self.capacity = resolve_capacity(capacity, env=self._env)
|
||||||
# 常驻启动时的代码签名:run_loop 每轮边界比对,变了就自退
|
# 常驻启动时的代码签名:run_loop 每轮边界比对,变了就自退
|
||||||
# (见 run_loop 的代码变化自退检查)。
|
# (见 run_loop 的代码变化自退检查);_claim 每次认领前
|
||||||
self._code_signature = _code_signature()
|
# 也比对(见 _code_signature_valid),不要只在轮次边界查。
|
||||||
|
self._code_signature = _code_signature(host_py_path)
|
||||||
|
# 配额看门狗命中过的任务(Docker 同步路径用):看门狗杀
|
||||||
|
# 容器后,同步返回的 failed 要被 _run_attempt 改写成
|
||||||
|
# quota_exceeded,否则 run_one 会把它当普通失败重试,
|
||||||
|
# 配额中断就续跑下去了。受 _quota_hit_lock 保护。
|
||||||
|
self._quota_hit_tasks = set()
|
||||||
|
self._quota_hit_lock = threading.Lock()
|
||||||
|
|
||||||
def _code_changed(self):
|
def _code_changed(self):
|
||||||
"""启动后 worker.py 是否被换过(部署换目录/手动改代码):
|
"""启动后关键代码是否被换过(部署换目录/手动改代码):
|
||||||
签名取不到时一律当没变,不误杀。"""
|
签名取不到时返回 False(是否“变了”无从判断),但这
|
||||||
now = _code_signature()
|
不等于放行——放行与否看 _code_signature_valid。"""
|
||||||
|
now = _code_signature(self.host_py_path)
|
||||||
return (now is not None and self._code_signature is not None
|
return (now is not None and self._code_signature is not None
|
||||||
and now != self._code_signature)
|
and now != self._code_signature)
|
||||||
|
|
||||||
|
def _code_signature_valid(self):
|
||||||
|
"""签名是否可信且未变:启动签名与现签名都必须取得到
|
||||||
|
且相等,否则 False。False 时 _claim 拒绝认领、run_loop
|
||||||
|
自退(75)、_run_pr_jobs 不碰任务——fail-closed,绝不
|
||||||
|
让签名缺失的 worker 抢任务(旧版取不到当没变是 fail-open)。"""
|
||||||
|
now = _code_signature(self.host_py_path)
|
||||||
|
return (now is not None and self._code_signature is not None
|
||||||
|
and now == self._code_signature)
|
||||||
|
|
||||||
|
def _mark_quota_hit(self, task_id):
|
||||||
|
with self._quota_hit_lock:
|
||||||
|
self._quota_hit_tasks.add(task_id)
|
||||||
|
|
||||||
|
def _take_quota_hit(self, task_id):
|
||||||
|
"""取走一次配额看门狗命中(命中即消费,避免旧命中污染
|
||||||
|
同一任务的下一轮):返回 True 表示本轮确被看门狗杀过。"""
|
||||||
|
with self._quota_hit_lock:
|
||||||
|
if task_id in self._quota_hit_tasks:
|
||||||
|
self._quota_hit_tasks.discard(task_id)
|
||||||
|
return True
|
||||||
|
return False
|
||||||
|
|
||||||
def register(self):
|
def register(self):
|
||||||
"""向控制面注册本 worker(写 workers 表,失败只记日志)。
|
"""向控制面注册本 worker(写 workers 表,失败只记日志)。
|
||||||
|
|
||||||
@@ -573,7 +653,17 @@ class Worker:
|
|||||||
里清(本地 _execute_via_process / Docker 同步路径),
|
里清(本地 _execute_via_process / Docker 同步路径),
|
||||||
认领后提前返回的三道门(缺 key / 仓库 / 配额)在
|
认领后提前返回的三道门(缺 key / 仓库 / 配额)在
|
||||||
run_one 里显式清占位;CAS 语义本身一句未改。
|
run_one 里显式清占位;CAS 语义本身一句未改。
|
||||||
|
P1 签名门:占位之前先比对多文件签名,签名缺失或已变
|
||||||
|
时直接返回 False(fail-closed,不抢任务、不占槽),
|
||||||
|
并记日志;不要只在 run_loop 轮次边界查——并行池线程
|
||||||
|
直接调 run_one/_claim,轮次边界查不到它们。
|
||||||
"""
|
"""
|
||||||
|
if not self._code_signature_valid():
|
||||||
|
log_event(self.logger, "worker_claim_refused_code_signature",
|
||||||
|
task_id=task_id,
|
||||||
|
message="code signature missing or changed,"
|
||||||
|
" refusing claim")
|
||||||
|
return False
|
||||||
with self._running_lock:
|
with self._running_lock:
|
||||||
already_running = task_id in self._running
|
already_running = task_id in self._running
|
||||||
if not already_running and \
|
if not already_running and \
|
||||||
@@ -865,14 +955,51 @@ class Worker:
|
|||||||
_check_unpriced_token_cap 同口径):USD 配额不计量无价
|
_check_unpriced_token_cap 同口径):USD 配额不计量无价
|
||||||
模型,默认模型 terra 无价,若无此上限则配额对默认模型
|
模型,默认模型 terra 无价,若无此上限则配额对默认模型
|
||||||
永远不生效;有配额行的用户当月无价 token(input +
|
永远不生效;有配额行的用户当月无价 token(input +
|
||||||
output,已记账 + 本轮)达到上限即停。无配额行 = 不限额。
|
output,已记账 + 本轮)达到上限即停。无价上限只计
|
||||||
查询本身异常时按不限额处理(fail-open 并记一条警告):
|
source='actual' 的行:估算行(estimated)不是真实花费,
|
||||||
配额是第二道网,绝不能因查询抖动把全部任务挡在门外;
|
计入上限会把“没花钱”算成“花超了”,且与 cost.py
|
||||||
服务端建任务/分支/重试时的准入仍是第一道。
|
_report 的 unpriced_actual_tokens 口径一致。
|
||||||
|
P1 执行面三处收口(旧版三处都是 fail-open,已废):
|
||||||
|
- 属主:任务 created_by 为空时先查父任务(协调员子任务
|
||||||
|
行可能无 created_by,属主跟父任务走);仍查不到时
|
||||||
|
拒绝新回合,不再返回 None 放行。created_by 字面为
|
||||||
|
"system" 的系统任务按普通属主算(无行时同样走默认
|
||||||
|
额度,不另设无限额的系统预算)。
|
||||||
|
- 无配额行:不再是不限额,按默认配额
|
||||||
|
_default_quota_usd_micros()($10/月)算;只有显式
|
||||||
|
NULL 行才是不限额(与 P0 的服务端口径一致,管理员
|
||||||
|
显式设 NULL 的行仍被尊重)。
|
||||||
|
- 查库失败:返回 "quota_check_failed: ..."(fail-closed),
|
||||||
|
绝不返回 None 放行。调用方分流:新回合边界(认领门、
|
||||||
|
重试/fallback/后续消息边界)见此串即拒绝新回合并记
|
||||||
|
worker_quota_stop 审计;执行中轮询(popen 轮询、
|
||||||
|
Docker 配额看门狗)见此串只记警告、不中途杀当前轮
|
||||||
|
——查询抖动不应杀掉已在跑的回合,但下一轮边界仍会
|
||||||
|
拒绝(见 run_one 的 _quota_hit 过滤)。查库失败的
|
||||||
|
审计由边界调用方记,这里只记日志,避免轮询高频写审计。
|
||||||
"""
|
"""
|
||||||
owner = (task or {}).get("created_by")
|
owner = (task or {}).get("created_by")
|
||||||
|
if not owner and (task or {}).get("parent_task_id"):
|
||||||
|
# 协调员子任务属主跟父任务走(与 _gateway_context
|
||||||
|
# 同款回查);查不到时不猜,直接走下面的拒绝分支。
|
||||||
|
try:
|
||||||
|
conn = _db_connect(self.db_path)
|
||||||
|
try:
|
||||||
|
row = conn.execute(
|
||||||
|
"SELECT created_by FROM tasks WHERE id = ?",
|
||||||
|
(task["parent_task_id"],)).fetchone()
|
||||||
|
finally:
|
||||||
|
conn.close()
|
||||||
|
if row is not None:
|
||||||
|
try:
|
||||||
|
owner = row[0]
|
||||||
|
except (TypeError, IndexError):
|
||||||
|
owner = None
|
||||||
|
except Exception:
|
||||||
|
owner = None
|
||||||
if not owner:
|
if not owner:
|
||||||
return None
|
return ("quota_exceeded: 任务缺少属主,无法查配额,"
|
||||||
|
"拒绝新回合(请归属真实用户后再排队)")
|
||||||
try:
|
try:
|
||||||
month = datetime.now(timezone.utc).strftime("%Y-%m")
|
month = datetime.now(timezone.utc).strftime("%Y-%m")
|
||||||
conn = _db_connect(self.db_path)
|
conn = _db_connect(self.db_path)
|
||||||
@@ -887,19 +1014,33 @@ class Worker:
|
|||||||
" WHERE user_id = ?",
|
" WHERE user_id = ?",
|
||||||
(owner,)).fetchone()
|
(owner,)).fetchone()
|
||||||
if row is None:
|
if row is None:
|
||||||
return None
|
# P1:无行按默认额度,不是不限额(P0 服务端
|
||||||
quota = row[0]
|
# 已在建号/登录时补行,worker 侧仍可能遇到
|
||||||
|
# 老库未补行的用户,这里按默认额度兜底,
|
||||||
|
# 不写行——写行是服务端的职责,worker 不抢)。
|
||||||
|
quota = _default_quota_usd_micros()
|
||||||
|
else:
|
||||||
|
quota = row[0]
|
||||||
if quota is None:
|
if quota is None:
|
||||||
return None
|
return None
|
||||||
# cost_records 由 CostStore 懒建:全新库上它还不存
|
# cost_records 由 CostStore 懒建:全新库上它还不存
|
||||||
# 在,直接 JOIN 会报错——先实例化一次建表再查(与
|
# 在,直接 JOIN 会报错——先实例化一次建表再查(与
|
||||||
# 服务端 _quota_status 同款做法),否则 quota=0
|
# 服务端 _quota_status 同款做法),否则 quota=0
|
||||||
#(一分钱也不能花)在全新库上会被误放行。
|
#(一分钱也不能花)在全新库上会被误放行。
|
||||||
|
# P1 补:建表必须落在本连接上,不只 CostStore 的
|
||||||
|
# 连接上——文件库两者指向同一文件所以旧代码碰巧
|
||||||
|
# 能用,:memory: 库两者是不同空库,本连接查 JOIN
|
||||||
|
# 会直接报错并(P1 fail-closed 后)误拒全部任务。
|
||||||
_cost_store = CostStore(self.db_path)
|
_cost_store = CostStore(self.db_path)
|
||||||
try:
|
try:
|
||||||
_cost_store._conn.close()
|
_cost_store._conn.close()
|
||||||
except Exception:
|
except Exception:
|
||||||
pass
|
pass
|
||||||
|
try:
|
||||||
|
conn.execute(CostStore._COST_DDL)
|
||||||
|
conn.commit()
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
used_row = conn.execute(
|
used_row = conn.execute(
|
||||||
"SELECT COALESCE(SUM(c.usd_micros), 0)"
|
"SELECT COALESCE(SUM(c.usd_micros), 0)"
|
||||||
" FROM cost_records c JOIN tasks t"
|
" FROM cost_records c JOIN tasks t"
|
||||||
@@ -912,7 +1053,7 @@ class Worker:
|
|||||||
" FROM cost_records c JOIN tasks t"
|
" FROM cost_records c JOIN tasks t"
|
||||||
" ON c.task_id = t.id"
|
" ON c.task_id = t.id"
|
||||||
" WHERE t.created_by = ? AND c.day LIKE ?"
|
" WHERE t.created_by = ? AND c.day LIKE ?"
|
||||||
" AND c.usd_micros IS NULL",
|
" AND c.usd_micros IS NULL AND c.source = 'actual'",
|
||||||
(owner, month + "%")).fetchone()
|
(owner, month + "%")).fetchone()
|
||||||
unpriced_used = unpriced_row[0] if unpriced_row else 0
|
unpriced_used = unpriced_row[0] if unpriced_row else 0
|
||||||
finally:
|
finally:
|
||||||
@@ -940,11 +1081,15 @@ class Worker:
|
|||||||
% (unpriced_used, cap, month))
|
% (unpriced_used, cap, month))
|
||||||
return None
|
return None
|
||||||
except Exception as exc:
|
except Exception as exc:
|
||||||
|
# P1 fail-closed:查库失败不再返回 None。调用方按
|
||||||
|
# 前缀分流(见本函数 docstring):quota_check_failed
|
||||||
|
# 在新回合边界 = 拒绝,在执行中轮询 = 只警告不杀轮。
|
||||||
try:
|
try:
|
||||||
self.logger.warning("quota check failed: %s", exc)
|
self.logger.warning("quota check failed: %s", exc)
|
||||||
except Exception:
|
except Exception:
|
||||||
pass
|
pass
|
||||||
return None
|
return ("quota_check_failed: 配额查询失败,拒绝新回合"
|
||||||
|
"(查库异常,非配额用完)")
|
||||||
|
|
||||||
def _ready_tasks(self):
|
def _ready_tasks(self):
|
||||||
"""当前就绪任务列表(pick_next 的批量版)。
|
"""当前就绪任务列表(pick_next 的批量版)。
|
||||||
@@ -1123,6 +1268,41 @@ class Worker:
|
|||||||
pass
|
pass
|
||||||
return
|
return
|
||||||
|
|
||||||
|
def _docker_quota_watchdog(self, task_id, container_name,
|
||||||
|
stop_event, quota_check, interval=30.0):
|
||||||
|
"""Docker 同步路径的配额看门狗(P1,复用取消看门狗模式)。
|
||||||
|
|
||||||
|
同步路径阻塞在 docker run 里,run_one 的边界检查要等
|
||||||
|
本轮返回才跑得到——配额在别处被花完时,本轮会继续烧
|
||||||
|
到自然结束。本线程每 interval 秒(默认 30 秒,不超过
|
||||||
|
30 秒)用 quota_check 查一次已记账配额:命中时按容器
|
||||||
|
名真杀容器并记 _quota_hit_tasks,_run_attempt 在同步
|
||||||
|
返回后把结果改写成 quota_exceeded,run_one 走统一的
|
||||||
|
配额中断收尾(interrupted + 配额原因 + 已花成本照记,
|
||||||
|
下一轮不再续跑)。查库失败时 quota_check 返回 False
|
||||||
|
(run_one 的 _quota_hit 会把 quota_check_failed 前缀
|
||||||
|
过滤掉,见 _quota_exceeded docstring),本线程只继续
|
||||||
|
轮询、不杀当前轮;下一轮边界仍会拒绝。诚实边界:看
|
||||||
|
门狗只能看到已记账用量 + 往轮累计用量,本轮容器内尚
|
||||||
|
未回传的 token 它看不到——那部分在结果返回后由边界
|
||||||
|
门拦住,不会悄悄续跑下一轮。
|
||||||
|
"""
|
||||||
|
while not stop_event.wait(interval):
|
||||||
|
try:
|
||||||
|
over = bool(quota_check()) if quota_check else False
|
||||||
|
except Exception:
|
||||||
|
over = False
|
||||||
|
if not over:
|
||||||
|
continue
|
||||||
|
self._mark_quota_hit(task_id)
|
||||||
|
try:
|
||||||
|
kill = getattr(self.executor, "kill_container", None)
|
||||||
|
if kill is not None and container_name:
|
||||||
|
kill(container_name)
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
return
|
||||||
|
|
||||||
def cleanup_stale_containers(self):
|
def cleanup_stale_containers(self):
|
||||||
"""启动清理兜底:清掉上一轮残留的 codex- 容器。
|
"""启动清理兜底:清掉上一轮残留的 codex- 容器。
|
||||||
|
|
||||||
@@ -2069,6 +2249,7 @@ class Worker:
|
|||||||
# 按容器名真杀;非 Docker 路径不启动,避免空线程)。
|
# 按容器名真杀;非 Docker 路径不启动,避免空线程)。
|
||||||
_watchdog_stop = None
|
_watchdog_stop = None
|
||||||
_watchdog_thread = None
|
_watchdog_thread = None
|
||||||
|
_quota_watchdog_thread = None
|
||||||
if is_docker and hasattr(self.executor, "kill_container"):
|
if is_docker and hasattr(self.executor, "kill_container"):
|
||||||
try:
|
try:
|
||||||
_container_name = self.executor.container_name(task)
|
_container_name = self.executor.container_name(task)
|
||||||
@@ -2081,6 +2262,17 @@ class Worker:
|
|||||||
args=(task_id, _container_name, _watchdog_stop),
|
args=(task_id, _container_name, _watchdog_stop),
|
||||||
daemon=True)
|
daemon=True)
|
||||||
_watchdog_thread.start()
|
_watchdog_thread.start()
|
||||||
|
# P1 配额看门狗:只在同步 Docker 路径起(进程
|
||||||
|
# 模式的 _execute_via_process 已每 30 秒轮询
|
||||||
|
# quota_check,不重复起)。quota_check 为 None
|
||||||
|
# 时不起(无配额检查可做,不空转线程)。
|
||||||
|
if quota_check is not None and not self._use_popen():
|
||||||
|
_quota_watchdog_thread = threading.Thread(
|
||||||
|
target=self._docker_quota_watchdog,
|
||||||
|
args=(task_id, _container_name,
|
||||||
|
_watchdog_stop, quota_check),
|
||||||
|
daemon=True)
|
||||||
|
_quota_watchdog_thread.start()
|
||||||
try:
|
try:
|
||||||
gateway_provider, gateway_env = self._ensure_gateway(task)
|
gateway_provider, gateway_env = self._ensure_gateway(task)
|
||||||
if self._use_popen():
|
if self._use_popen():
|
||||||
@@ -2123,8 +2315,14 @@ class Worker:
|
|||||||
prep_err)
|
prep_err)
|
||||||
if goal_mode:
|
if goal_mode:
|
||||||
if task.get("_resume"):
|
if task.get("_resume"):
|
||||||
|
# P1 断点续跑传 None(不传 --goal-text,
|
||||||
|
# host 沿用原 goal+verify);开新目标
|
||||||
|
# 才传本轮 prompt 当新 goal_text。
|
||||||
|
_resume_text = None if task.get(
|
||||||
|
"_resume_kind") == "breakpoint" \
|
||||||
|
else task["prompt"]
|
||||||
result = self.executor.goal_resume_task(
|
result = self.executor.goal_resume_task(
|
||||||
task, task["prompt"],
|
task, _resume_text,
|
||||||
timeout=effective_docker_timeout,
|
timeout=effective_docker_timeout,
|
||||||
extra_mounts=tenant_mounts,
|
extra_mounts=tenant_mounts,
|
||||||
network=tenant_network,
|
network=tenant_network,
|
||||||
@@ -2165,8 +2363,11 @@ class Worker:
|
|||||||
|
|
||||||
if task.get("_resume") and isinstance(
|
if task.get("_resume") and isinstance(
|
||||||
self.executor, LocalExecutor):
|
self.executor, LocalExecutor):
|
||||||
|
_resume_text = None if task.get(
|
||||||
|
"_resume_kind") == "breakpoint" \
|
||||||
|
else task["prompt"]
|
||||||
result = self.executor.goal_resume_task(
|
result = self.executor.goal_resume_task(
|
||||||
task, task["prompt"],
|
task, _resume_text,
|
||||||
extra_env=gateway_env,
|
extra_env=gateway_env,
|
||||||
model_provider=gateway_provider)
|
model_provider=gateway_provider)
|
||||||
elif isinstance(self.executor, LocalExecutor):
|
elif isinstance(self.executor, LocalExecutor):
|
||||||
@@ -2220,6 +2421,8 @@ class Worker:
|
|||||||
_watchdog_stop.set()
|
_watchdog_stop.set()
|
||||||
if _watchdog_thread is not None:
|
if _watchdog_thread is not None:
|
||||||
_watchdog_thread.join(timeout=2)
|
_watchdog_thread.join(timeout=2)
|
||||||
|
if _quota_watchdog_thread is not None:
|
||||||
|
_quota_watchdog_thread.join(timeout=2)
|
||||||
# 同步路径执行结束:清掉取消用的容器标记(取消路径里
|
# 同步路径执行结束:清掉取消用的容器标记(取消路径里
|
||||||
# 若已清过,这里再清一次无害)。进程模式的 proc 由
|
# 若已清过,这里再清一次无害)。进程模式的 proc 由
|
||||||
# _execute_via_process 自己的 finally 清,这里只清
|
# _execute_via_process 自己的 finally 清,这里只清
|
||||||
@@ -2236,6 +2439,18 @@ class Worker:
|
|||||||
if result is not None and result.get("status") == "failed" and \
|
if result is not None and result.get("status") == "failed" and \
|
||||||
"docker run timeout" in (result.get("stdout", "") or ""):
|
"docker run timeout" in (result.get("stdout", "") or ""):
|
||||||
result = dict(result, status="timeout")
|
result = dict(result, status="timeout")
|
||||||
|
# P1 配额看门狗命中改写:同步路径被看门狗杀容器后,
|
||||||
|
# 执行器只返回普通 failed,run_one 无法区分“模型失败”
|
||||||
|
# 与“配额杀容器”——命中时改写成 quota_exceeded,run_one
|
||||||
|
# 走统一配额中断收尾(interrupted + 配额原因 + 成本照记,
|
||||||
|
# 不重试、不续跑下一轮)。改写前先取走命中标记,避免
|
||||||
|
# 旧命中污染同一任务的后续尝试;超时标记优先于配额
|
||||||
|
# 改写的情况不存在(看门狗杀容器时执行器不会同时报超时)。
|
||||||
|
if result is not None and self._take_quota_hit(task_id):
|
||||||
|
result = dict(result, status="quota_exceeded")
|
||||||
|
if error is None:
|
||||||
|
error = ("quota_exceeded: Docker 配额看门狗已杀容器"
|
||||||
|
"(配额中断,下一轮不再续跑)")
|
||||||
return result, error
|
return result, error
|
||||||
|
|
||||||
def _attempt_error(self, result, error, token):
|
def _attempt_error(self, result, error, token):
|
||||||
@@ -2384,7 +2599,12 @@ class Worker:
|
|||||||
task.get("priority", 0) or 0, task["created_by"],
|
task.get("priority", 0) or 0, task["created_by"],
|
||||||
now, now,
|
now, now,
|
||||||
part.get("executor") or task.get("executor"),
|
part.get("executor") or task.get("executor"),
|
||||||
part.get("model"), part.get("effort"),
|
# P1:分工未指定模型/强度时继承父任务(旧版
|
||||||
|
# 直接写 part 的 None,子任务会落到 worker
|
||||||
|
# 默认模型/强度,与父任务的显式选择脱节,
|
||||||
|
# 既改成本也改行为,且排查时看不出继承关系)。
|
||||||
|
part.get("model") or task.get("model"),
|
||||||
|
part.get("effort") or task.get("effort"),
|
||||||
task.get("environment_id"), task["id"], name))
|
task.get("environment_id"), task["id"], name))
|
||||||
self._audit(
|
self._audit(
|
||||||
task["project_id"], "coordinator_child_created",
|
task["project_id"], "coordinator_child_created",
|
||||||
@@ -2422,6 +2642,109 @@ class Worker:
|
|||||||
conn.close()
|
conn.close()
|
||||||
return rows
|
return rows
|
||||||
|
|
||||||
|
def _coord_mark_orphans(self, task, child_ids):
|
||||||
|
"""把仍未终态的子任务标 interrupted 并记审计(P1 孤儿收口)。
|
||||||
|
|
||||||
|
只对 CAS 改到行的子任务记审计(rowcount==1):已被别的
|
||||||
|
worker 终态化的子任务不动,避免覆盖它的结果。返回被标
|
||||||
|
记的子任务 id 列表。正在跑的子任务被标 interrupted 后,
|
||||||
|
跑它的 worker 会在轮询里看到库状态并杀进程/容器(popen
|
||||||
|
每 1 秒查库、Docker 取消看门狗每 1 秒查库),不会无声
|
||||||
|
继续烧钱;若那个 worker 也已死,子任务至少已是终态,
|
||||||
|
不会再被认领烧钱。
|
||||||
|
"""
|
||||||
|
marked = []
|
||||||
|
rows = self._coord_child_rows(child_ids)
|
||||||
|
pending = _coord_nonterminal([
|
||||||
|
{"id": cid, "status": (rows.get(cid) or {}).get("status")}
|
||||||
|
for cid in child_ids])
|
||||||
|
if not pending:
|
||||||
|
return marked
|
||||||
|
conn = _db_connect(self.db_path)
|
||||||
|
try:
|
||||||
|
for ch in pending:
|
||||||
|
cur = conn.execute(
|
||||||
|
"UPDATE tasks SET status = 'interrupted',"
|
||||||
|
" updated_at = ? WHERE id = ? AND status NOT IN"
|
||||||
|
" ('completed', 'failed', 'interrupted', 'merged',"
|
||||||
|
" 'merge-failed')",
|
||||||
|
(time.time(), ch["id"]))
|
||||||
|
if cur.rowcount == 1:
|
||||||
|
marked.append(ch["id"])
|
||||||
|
self._audit(
|
||||||
|
task["project_id"], "coordinator_child_orphaned",
|
||||||
|
"%s -> %s (parent %s)"
|
||||||
|
% (ch["id"], "interrupted", task["id"]))
|
||||||
|
conn.commit()
|
||||||
|
finally:
|
||||||
|
conn.close()
|
||||||
|
return marked
|
||||||
|
|
||||||
|
def sweep_orphaned_coordinator_children(self):
|
||||||
|
"""常驻循环每轮的孤儿扫尾(P1):父已终态或父行失踪、子
|
||||||
|
仍未终态时,把子标 interrupted 并记审计。
|
||||||
|
|
||||||
|
父仍在 running/queued 时绝不动子(协调员等齐期间子任务
|
||||||
|
未终态是正常状态)。父行失踪(被删)同样算孤儿:没有
|
||||||
|
父会再来合流,子任务继续跑只是烧钱。返回被标记的
|
||||||
|
(parent_id, child_id) 对列表;查库失败时返回 [] 且只记
|
||||||
|
日志,不炸循环。
|
||||||
|
"""
|
||||||
|
marked = []
|
||||||
|
try:
|
||||||
|
conn = _db_connect(self.db_path)
|
||||||
|
try:
|
||||||
|
rows = conn.execute(
|
||||||
|
"SELECT id, parent_task_id, project_id FROM tasks"
|
||||||
|
" WHERE parent_task_id IS NOT NULL AND status NOT IN"
|
||||||
|
" ('completed', 'failed', 'interrupted', 'merged',"
|
||||||
|
" 'merge-failed')").fetchall()
|
||||||
|
parents = {}
|
||||||
|
for row in rows:
|
||||||
|
try:
|
||||||
|
pid = row["parent_task_id"]
|
||||||
|
except (KeyError, IndexError, TypeError):
|
||||||
|
continue
|
||||||
|
if pid not in parents:
|
||||||
|
prow = conn.execute(
|
||||||
|
"SELECT status FROM tasks WHERE id = ?",
|
||||||
|
(pid,)).fetchone()
|
||||||
|
try:
|
||||||
|
parents[pid] = prow["status"] \
|
||||||
|
if prow is not None else None
|
||||||
|
except (KeyError, IndexError, TypeError):
|
||||||
|
parents[pid] = None
|
||||||
|
for row in rows:
|
||||||
|
try:
|
||||||
|
cid = row["id"]
|
||||||
|
pid = row["parent_task_id"]
|
||||||
|
proj = row["project_id"]
|
||||||
|
except (KeyError, IndexError, TypeError):
|
||||||
|
continue
|
||||||
|
pstatus = parents.get(pid)
|
||||||
|
if pstatus is not None and \
|
||||||
|
pstatus not in TERMINAL_STATUSES:
|
||||||
|
continue
|
||||||
|
cur = conn.execute(
|
||||||
|
"UPDATE tasks SET status = 'interrupted',"
|
||||||
|
" updated_at = ? WHERE id = ? AND status NOT IN"
|
||||||
|
" ('completed', 'failed', 'interrupted', 'merged',"
|
||||||
|
" 'merge-failed')",
|
||||||
|
(time.time(), cid))
|
||||||
|
if cur.rowcount == 1:
|
||||||
|
marked.append((pid, cid))
|
||||||
|
self._audit(
|
||||||
|
proj, "coordinator_child_orphaned",
|
||||||
|
"%s -> interrupted (parent %s, parent"
|
||||||
|
" status %s)" % (cid, pid, pstatus))
|
||||||
|
conn.commit()
|
||||||
|
finally:
|
||||||
|
conn.close()
|
||||||
|
except Exception as exc:
|
||||||
|
log_event(self.logger, "worker_orphan_sweep_failed",
|
||||||
|
message="orphan sweep failed: %s" % exc)
|
||||||
|
return marked
|
||||||
|
|
||||||
def _run_coordinator(self, task, started_at, poll_interval=None,
|
def _run_coordinator(self, task, started_at, poll_interval=None,
|
||||||
timeout=None):
|
timeout=None):
|
||||||
"""协调员执行:建子任务 → 等齐 → 合流汇总写回。
|
"""协调员执行:建子任务 → 等齐 → 合流汇总写回。
|
||||||
@@ -2460,6 +2783,7 @@ class Worker:
|
|||||||
timed_out = True
|
timed_out = True
|
||||||
break
|
break
|
||||||
time.sleep(poll_interval)
|
time.sleep(poll_interval)
|
||||||
|
orphaned = []
|
||||||
if cancelled or timed_out:
|
if cancelled or timed_out:
|
||||||
rows = self._coord_child_rows(child_ids)
|
rows = self._coord_child_rows(child_ids)
|
||||||
for cid in child_ids:
|
for cid in child_ids:
|
||||||
@@ -2469,6 +2793,15 @@ class Worker:
|
|||||||
self.cancel(cid)
|
self.cancel(cid)
|
||||||
except Exception:
|
except Exception:
|
||||||
pass
|
pass
|
||||||
|
# P1:cancel() 对已终态/不存在的子任务返回 False,
|
||||||
|
# 跨 worker 在跑的子任务也可能因竞态仍未终态——
|
||||||
|
# cancel 之后仍未终态的一律直接标 interrupted 并记
|
||||||
|
# 审计(_coord_mark_orphans),绝不留孤儿无声烧钱;
|
||||||
|
# 标记结果写进汇总报告,调用方在详情里看得到。
|
||||||
|
try:
|
||||||
|
orphaned = self._coord_mark_orphans(task, child_ids)
|
||||||
|
except Exception:
|
||||||
|
orphaned = []
|
||||||
rows = self._coord_child_rows(child_ids)
|
rows = self._coord_child_rows(child_ids)
|
||||||
merged = []
|
merged = []
|
||||||
for ch in children:
|
for ch in children:
|
||||||
@@ -2479,6 +2812,9 @@ class Worker:
|
|||||||
"result_text": row.get("result_text"),
|
"result_text": row.get("result_text"),
|
||||||
"error_text": row.get("error_text")})
|
"error_text": row.get("error_text")})
|
||||||
report, overall = _coord_build_report(merged)
|
report, overall = _coord_build_report(merged)
|
||||||
|
if orphaned:
|
||||||
|
report += ("\n\n孤儿子任务已标 interrupted:%s"
|
||||||
|
% "、".join(orphaned))
|
||||||
if cancelled:
|
if cancelled:
|
||||||
final = "interrupted"
|
final = "interrupted"
|
||||||
error = "cancelled"
|
error = "cancelled"
|
||||||
@@ -2531,6 +2867,47 @@ class Worker:
|
|||||||
"executor": self._executor_name(), "stdout": report,
|
"executor": self._executor_name(), "stdout": report,
|
||||||
"error": error, "events": [], "cost_record_id": None}
|
"error": error, "events": [], "cost_record_id": None}
|
||||||
|
|
||||||
|
def _record_usage_cost(self, task, actual_model, usage_total,
|
||||||
|
stdout, fallback_count):
|
||||||
|
"""记一条任务成本并累加 worker 级用量(调用方须持 _db_lock)。
|
||||||
|
|
||||||
|
P1 成本修齐:旧版 max_cost_usd 参数存在但 usage_cost_usd
|
||||||
|
从未累加——_budget_exceeded 的成本门永远是 0,等于没设。
|
||||||
|
本函数是唯一记账点(终局块与取消分支共用):actual 行
|
||||||
|
按价目表算 usd_micros(无价模型为 None),有价行才累加
|
||||||
|
usage_cost_usd(无价行不折算成 0 美元,仍受无价上限管);
|
||||||
|
无用量时回落字符估算记 estimated,绝不冒充 actual。
|
||||||
|
cache_write_input_tokens 不进这里:它只在 usage dict 里
|
||||||
|
做观测(见 cost.py 的不计费声明),不加进 total、不用
|
||||||
|
于定价。返回 cost record id。
|
||||||
|
"""
|
||||||
|
task_id = task["id"]
|
||||||
|
if usage_total and (usage_total.get("total_tokens") or 0) > 0:
|
||||||
|
micros = price_usd_micros(
|
||||||
|
actual_model, usage_total.get("input_tokens", 0),
|
||||||
|
usage_total.get("cached_input_tokens", 0),
|
||||||
|
usage_total.get("output_tokens", 0))
|
||||||
|
cost_id = self.costs.record(
|
||||||
|
task_id, task["project_id"], actual_model,
|
||||||
|
input_tokens=usage_total.get("input_tokens", 0),
|
||||||
|
output_tokens=usage_total.get("output_tokens", 0),
|
||||||
|
cached_input_tokens=usage_total.get(
|
||||||
|
"cached_input_tokens", 0),
|
||||||
|
usd_micros=micros,
|
||||||
|
source="actual", fallback_count=fallback_count)
|
||||||
|
else:
|
||||||
|
micros = None
|
||||||
|
total = estimate_tokens(task.get("prompt") or "", stdout)
|
||||||
|
cost_id = self.costs.record(
|
||||||
|
task_id, task["project_id"], actual_model,
|
||||||
|
input_tokens=total, output_tokens=0, usd_micros=None,
|
||||||
|
source="estimated", fallback_count=fallback_count)
|
||||||
|
if micros is not None:
|
||||||
|
# max_cost_usd 不再是死参数:只有有价花费才累加,
|
||||||
|
# _budget_exceeded 据此在下一认领前挡住超支 worker。
|
||||||
|
self.usage_cost_usd += micros / 1000000.0
|
||||||
|
return cost_id
|
||||||
|
|
||||||
def run_one(self, task=None):
|
def run_one(self, task=None):
|
||||||
"""认领并执行一个任务,返回结果 dict(见模块 docstring)。
|
"""认领并执行一个任务,返回结果 dict(见模块 docstring)。
|
||||||
|
|
||||||
@@ -2552,11 +2929,32 @@ class Worker:
|
|||||||
# 成它(只换内存里的副本,原 prompt 不动);_resume 标记
|
# 成它(只换内存里的副本,原 prompt 不动);_resume 标记
|
||||||
# 让 _run_attempt 在 goal 模式下走 goal-resume 而不是 goal
|
# 让 _run_attempt 在 goal 模式下走 goal-resume 而不是 goal
|
||||||
# (goal 会撞已存在,只有 goal-resume 能在旧线程上开新循环)。
|
# (goal 会撞已存在,只有 goal-resume 能在旧线程上开新循环)。
|
||||||
|
# P1 goal-resume 分流(断点 vs 开新目标,三处一致的 worker 侧):
|
||||||
|
# - 旧验收形态(verify_command 非空):一律断点续跑——
|
||||||
|
# 绝不能把续跑消息当新 goal_text 传给 host.py,否则
|
||||||
|
# host 的 goal-resume --goal-text 分支会用 verify=None
|
||||||
|
# 开新循环,把原 verify_command 覆盖掉;断点调用不传
|
||||||
|
# --goal-text,host 沿用原 goal + 原 verify + 原轮次,
|
||||||
|
# 从 goal_rounds_done 的下一轮续跑。续跑消息本轮不
|
||||||
|
# 当 prompt 用(断点语义就是续原目标),跑完照常清掉。
|
||||||
|
# - goal_mode=1(无验收):开新目标——续跑消息就是新
|
||||||
|
# goal_text(服务端 /continue 带 goal_mode=1 时已把
|
||||||
|
# task.goal 写成该消息,这里与之对齐)。
|
||||||
|
# - 非 goal 任务:沿用旧行为(prompt 换成续跑消息,走
|
||||||
|
# 已存在续跑 run 路径)。
|
||||||
resume_message = task.get("resume_message")
|
resume_message = task.get("resume_message")
|
||||||
if resume_message:
|
if resume_message:
|
||||||
task = dict(task)
|
task = dict(task)
|
||||||
task["prompt"] = resume_message
|
if task.get("verify_command"):
|
||||||
task["_resume"] = True
|
task["_resume"] = True
|
||||||
|
task["_resume_kind"] = "breakpoint"
|
||||||
|
elif task.get("goal_mode"):
|
||||||
|
task["prompt"] = resume_message
|
||||||
|
task["_resume"] = True
|
||||||
|
task["_resume_kind"] = "new-goal"
|
||||||
|
else:
|
||||||
|
task["prompt"] = resume_message
|
||||||
|
task["_resume"] = True
|
||||||
self._audit(task["project_id"], "worker_claim",
|
self._audit(task["project_id"], "worker_claim",
|
||||||
"%s executor=%s" % (task_id, self._executor_name()))
|
"%s executor=%s" % (task_id, self._executor_name()))
|
||||||
# log_event 只接受 task_id/project_id/status/message 四个
|
# log_event 只接受 task_id/project_id/status/message 四个
|
||||||
@@ -2647,6 +3045,56 @@ class Worker:
|
|||||||
"stdout": "", "error": quota_error, "events": [],
|
"stdout": "", "error": quota_error, "events": [],
|
||||||
"cost_record_id": None}
|
"cost_record_id": None}
|
||||||
|
|
||||||
|
# P1 goal 轮次上限门(执行侧防线):旧验收形态的
|
||||||
|
# goal_max_rounds 超过 GOAL_MAX_ROUNDS_CAP 时直接 failed,
|
||||||
|
# 不进 host.py goal 循环烧轮次;goal_mode=1 的 0(模型宣告
|
||||||
|
# 才结束)豁免轮次上限(上界是配额),但 0 配上旧验收命令
|
||||||
|
# 时同样 failed(服务端建任务时 0 本就 400,老行若有 0
|
||||||
|
# 验收形态属于脏数据,不按不限轮次跑)。建任务时的 400
|
||||||
|
# 校验在 server.py(另一路文件),这里是执行侧第二道门,
|
||||||
|
# 错误文本写明上限与“请新建合规任务”,不静默截断轮次
|
||||||
|
# (截断会改任务语义,调用方看不出被改过)。
|
||||||
|
_goal_rounds = task.get("goal_max_rounds")
|
||||||
|
if (task.get("verify_command") or task.get("goal_mode")) \
|
||||||
|
and _goal_rounds is not None:
|
||||||
|
_goal_cap_error = None
|
||||||
|
try:
|
||||||
|
_goal_rounds_int = int(_goal_rounds)
|
||||||
|
except (TypeError, ValueError):
|
||||||
|
_goal_rounds_int = None
|
||||||
|
if _goal_rounds_int is not None \
|
||||||
|
and _goal_rounds_int > GOAL_MAX_ROUNDS_CAP:
|
||||||
|
_goal_cap_error = (
|
||||||
|
"goal_max_rounds 超过上限 %d 轮(当前 %d),"
|
||||||
|
"请新建轮次合规的任务" % (
|
||||||
|
GOAL_MAX_ROUNDS_CAP, _goal_rounds_int))
|
||||||
|
elif _goal_rounds_int == 0 and task.get("verify_command") \
|
||||||
|
and not task.get("goal_mode"):
|
||||||
|
_goal_cap_error = (
|
||||||
|
"旧验收形态 goal_max_rounds=0 表示不限轮次,"
|
||||||
|
"已禁用(上限 %d 轮),请新建轮次合规的任务"
|
||||||
|
% GOAL_MAX_ROUNDS_CAP)
|
||||||
|
if _goal_cap_error is not None:
|
||||||
|
self._set_status(task_id, "failed")
|
||||||
|
self._write_execution_result(
|
||||||
|
task_id, error_text=_goal_cap_error,
|
||||||
|
actual_model=self.model,
|
||||||
|
started_at=started_at, finished_at=time.time(),
|
||||||
|
only_if_status="failed")
|
||||||
|
self._audit(task["project_id"], "worker_goal_cap_rejected",
|
||||||
|
"%s: %s" % (task_id, _goal_cap_error))
|
||||||
|
log_event(self.logger, "worker_goal_cap_rejected",
|
||||||
|
task_id=task_id, project_id=task["project_id"],
|
||||||
|
status="failed")
|
||||||
|
self._running_clear(task_id)
|
||||||
|
return {"dispatched": True, "task_id": task_id,
|
||||||
|
"host_name": self.executor._host_name(task)
|
||||||
|
if hasattr(self.executor, "_host_name") else task_id,
|
||||||
|
"status": "failed",
|
||||||
|
"executor": self._executor_name(),
|
||||||
|
"stdout": "", "error": _goal_cap_error, "events": [],
|
||||||
|
"cost_record_id": None}
|
||||||
|
|
||||||
# 协调员分支:不跑模型回合,改为按分工建并行子任务、等齐、
|
# 协调员分支:不跑模型回合,改为按分工建并行子任务、等齐、
|
||||||
# 合流汇总。已过 key 门与配额门;仓库门已对协调员任务本身
|
# 合流汇总。已过 key 门与配额门;仓库门已对协调员任务本身
|
||||||
# 跑过(项目仓库解析失败会在前面直接 failed,不会建一堆
|
# 跑过(项目仓库解析失败会在前面直接 failed,不会建一堆
|
||||||
@@ -2721,7 +3169,14 @@ class Worker:
|
|||||||
def _quota_hit():
|
def _quota_hit():
|
||||||
# 闭包读当前值:usage_total 与 task 在重试/fallback
|
# 闭包读当前值:usage_total 与 task 在重试/fallback
|
||||||
# 时会被重绑,轮询掐断要按最新值判断。
|
# 时会被重绑,轮询掐断要按最新值判断。
|
||||||
return self._quota_exceeded(task, usage_total) is not None
|
# P1 分流:执行中轮询只认真正的 quota_exceeded,
|
||||||
|
# quota_check_failed(查库失败)不杀当前轮——查询
|
||||||
|
# 抖动不应杀掉已在跑的回合;但它在重试/fallback/
|
||||||
|
# 后续消息等新回合边界仍是拒绝(那些边界直接调
|
||||||
|
# _quota_exceeded,不走这个过滤闭包)。
|
||||||
|
err = self._quota_exceeded(task, usage_total)
|
||||||
|
return err is not None and \
|
||||||
|
not err.startswith("quota_check_failed:")
|
||||||
|
|
||||||
# 分支任务:先 fork 再跑首轮(见 _run_fork);首轮失败
|
# 分支任务:先 fork 再跑首轮(见 _run_fork);首轮失败
|
||||||
# 不自动重试/fallback,失败后走重试入口重跑首轮(已存在
|
# 不自动重试/fallback,失败后走重试入口重跑首轮(已存在
|
||||||
@@ -2763,6 +3218,39 @@ class Worker:
|
|||||||
# 喂回,或新形态的模型自决多轮),worker 级重试/fallback
|
# 喂回,或新形态的模型自决多轮),worker 级重试/fallback
|
||||||
# 不再叠加——同任务重跑会撞 host 任务名已存在,第二
|
# 不再叠加——同任务重跑会撞 host 任务名已存在,第二
|
||||||
# 次尝试必失败且污染错误文本。
|
# 次尝试必失败且污染错误文本。
|
||||||
|
# P1 唯一例外:goal 首轮撞“已存在”时(重试入口
|
||||||
|
# 重排同一 goal 任务的典型死法),不再直接 failed,
|
||||||
|
# 而是走一次断点续跑(goal-resume 不传 --goal-text,
|
||||||
|
# 沿用原 goal+verify,从断点续跑);断点续跑的
|
||||||
|
# 结果即本轮结果(成功则完成,仍失败则以续跑的
|
||||||
|
# 错误为准,不再拼“已存在”)。已是续跑的任务
|
||||||
|
# (_resume)不再二次回落,避免断点循环。
|
||||||
|
# 分支任务(forked_from)不走这里:fork 首轮的
|
||||||
|
# 已存在语义是 dispatch 续跑,已在 _run_attempt
|
||||||
|
# 内处理,不叠加 goal 断点。
|
||||||
|
if not task.get("_resume") and not task.get("forked_from"):
|
||||||
|
_combined = ((result or {}).get("stdout", "") or "") \
|
||||||
|
+ ((result or {}).get("stderr", "") or "")
|
||||||
|
_goal_err = (result or {}).get("goal_error") or ""
|
||||||
|
if _ALREADY_EXISTS_MARKER in _combined or \
|
||||||
|
_ALREADY_EXISTS_MARKER in _goal_err or \
|
||||||
|
_ALREADY_EXISTS_MARKER in (error or ""):
|
||||||
|
self._audit(task["project_id"],
|
||||||
|
"worker_goal_resume_fallback",
|
||||||
|
"%s: goal 已存在,走断点续跑" % task_id)
|
||||||
|
_resume_task = dict(task)
|
||||||
|
_resume_task["_resume"] = True
|
||||||
|
_resume_task["_resume_kind"] = "breakpoint"
|
||||||
|
result, error = self._run_attempt(
|
||||||
|
_resume_task, quota_check=_quota_hit)
|
||||||
|
usage_total = sum_usage(
|
||||||
|
usage_total, (result or {}).get("usage"))
|
||||||
|
self._deliver_peer_messages(task, result)
|
||||||
|
if (result or {}).get("status") == "quota_exceeded":
|
||||||
|
quota_error = self._quota_exceeded(
|
||||||
|
task, usage_total) or \
|
||||||
|
"quota_exceeded: 本月配额已用完"
|
||||||
|
quota_stop = True
|
||||||
break
|
break
|
||||||
if result.get("status") == "completed":
|
if result.get("status") == "completed":
|
||||||
break
|
break
|
||||||
@@ -2900,6 +3388,18 @@ class Worker:
|
|||||||
turn_id=(result or {}).get("turn_id"),
|
turn_id=(result or {}).get("turn_id"),
|
||||||
root_turn_id=(result or {}).get("root_turn_id"),
|
root_turn_id=(result or {}).get("root_turn_id"),
|
||||||
only_if_status="interrupted")
|
only_if_status="interrupted")
|
||||||
|
# P1:取消分支先记一条成本再返回(与配额中断收尾
|
||||||
|
# 同款记账点)。旧版这里直接返回 cost_record_id=None,
|
||||||
|
# 被取消前已花的模型费用就从账上丢了;取消不等于没
|
||||||
|
# 花钱。记账失败不挡取消返回(costs.record 异常时
|
||||||
|
# cost_id 置 None,任务状态仍是 interrupted)。
|
||||||
|
try:
|
||||||
|
with self._db_lock:
|
||||||
|
_cancel_cost_id = self._record_usage_cost(
|
||||||
|
task, actual_model, usage_total,
|
||||||
|
(result or {}).get("stdout", ""), fallback_count)
|
||||||
|
except Exception:
|
||||||
|
_cancel_cost_id = None
|
||||||
log_event(self.logger, "worker_cancelled", task_id=task_id,
|
log_event(self.logger, "worker_cancelled", task_id=task_id,
|
||||||
project_id=task["project_id"], status="interrupted")
|
project_id=task["project_id"], status="interrupted")
|
||||||
return {"dispatched": True, "task_id": task_id,
|
return {"dispatched": True, "task_id": task_id,
|
||||||
@@ -2908,7 +3408,7 @@ class Worker:
|
|||||||
"status": "interrupted", "executor": self._executor_name(),
|
"status": "interrupted", "executor": self._executor_name(),
|
||||||
"stdout": (result or {}).get("stdout", ""),
|
"stdout": (result or {}).get("stdout", ""),
|
||||||
"error": "cancelled", "events": [],
|
"error": "cancelled", "events": [],
|
||||||
"cost_record_id": None}
|
"cost_record_id": _cancel_cost_id}
|
||||||
|
|
||||||
stdout = result.get("stdout", "") or ""
|
stdout = result.get("stdout", "") or ""
|
||||||
host_status = result.get("host_status")
|
host_status = result.get("host_status")
|
||||||
@@ -2979,32 +3479,12 @@ class Worker:
|
|||||||
self._cleanup_task_seed(task)
|
self._cleanup_task_seed(task)
|
||||||
with self._db_lock:
|
with self._db_lock:
|
||||||
self.usage_turns += 1
|
self.usage_turns += 1
|
||||||
# 成本:执行器带回上游真实用量时记 actual(input/output/
|
# 成本记账走 _record_usage_cost(终局与取消分支共用
|
||||||
# cached 全是实测值,usd 按价目表算、无价模型记 None,
|
# 的唯一记账点):actual/estimated 分流、usage_cost_usd
|
||||||
# 报表里进 unpriced_actual_tokens 而不是 0 美元);
|
# 累加都在里面;cost 记录共用一条连接,并发下必须
|
||||||
# 没带回用量时回落字符估算记 estimated,绝不冒充 actual。
|
# 持锁写,否则两个线程共用同一 sqlite 连接会直接报错。
|
||||||
# fallback_count 复用同一记录;模型记最后真正跑的那个
|
cost_id = self._record_usage_cost(
|
||||||
# (fallback 成功时就是 fallback 模型)。cost 记录共用
|
task, actual_model, usage_total, stdout, fallback_count)
|
||||||
# 一条连接,并发下必须持锁写,否则两个线程共用同一
|
|
||||||
# sqlite 连接会直接报错。
|
|
||||||
if usage_total and (usage_total.get("total_tokens") or 0) > 0:
|
|
||||||
cost_id = self.costs.record(
|
|
||||||
task_id, task["project_id"], actual_model,
|
|
||||||
input_tokens=usage_total.get("input_tokens", 0),
|
|
||||||
output_tokens=usage_total.get("output_tokens", 0),
|
|
||||||
cached_input_tokens=usage_total.get(
|
|
||||||
"cached_input_tokens", 0),
|
|
||||||
usd_micros=price_usd_micros(
|
|
||||||
actual_model, usage_total.get("input_tokens", 0),
|
|
||||||
usage_total.get("cached_input_tokens", 0),
|
|
||||||
usage_total.get("output_tokens", 0)),
|
|
||||||
source="actual", fallback_count=fallback_count)
|
|
||||||
else:
|
|
||||||
total = estimate_tokens(task.get("prompt") or "", stdout)
|
|
||||||
cost_id = self.costs.record(
|
|
||||||
task_id, task["project_id"], actual_model,
|
|
||||||
input_tokens=total, output_tokens=0, usd_micros=None,
|
|
||||||
source="estimated", fallback_count=fallback_count)
|
|
||||||
if final_written:
|
if final_written:
|
||||||
self._audit(task["project_id"],
|
self._audit(task["project_id"],
|
||||||
"worker_quota_stop" if quota_stop else
|
"worker_quota_stop" if quota_stop else
|
||||||
@@ -3064,9 +3544,65 @@ class Worker:
|
|||||||
发 review-requested/merge-failed,这里只写回任务终态。
|
发 review-requested/merge-failed,这里只写回任务终态。
|
||||||
跳过(PrSkipped,如任务已不是 merged)与落盘失败都记
|
跳过(PrSkipped,如任务已不是 merged)与落盘失败都记
|
||||||
failed + error,任务本身保持 merged、可重调 /pr 再排队。
|
failed + error,任务本身保持 merged、可重调 /pr 再排队。
|
||||||
|
P1 非 local 通道:旧版非 local worker 直接返回 [],
|
||||||
|
排队的 pr 任务看起来像“没人处理”且无任何痕迹——
|
||||||
|
静默未完成。现在改为显式 needs_local_worker:只读
|
||||||
|
peek 本主机 queued 的 pr 任务,逐条记审计 + 日志并
|
||||||
|
返回 needs_local_worker 条目,任务本身保持 merged、
|
||||||
|
pr 任务保持 queued(留给 local worker 认领),绝不
|
||||||
|
把它标 failed(failed 会让调用方以为 PR 已失败,且
|
||||||
|
任务仍可重调 /pr 的语义会被混淆)。为什么不直接把
|
||||||
|
pr_jobs 行写成 needs_local_worker 状态:pr_jobs 的认
|
||||||
|
领只认 queued(claim_pr_job 在 pr_exec.py,本路不改
|
||||||
|
该文件),写成新状态后 local worker 也认领不到,任务
|
||||||
|
会被永久搁置;真要落库状态,需 pr_exec.py 的认领同
|
||||||
|
步改成含 needs_local_worker 或 local 重排队——该改动
|
||||||
|
留给协调员(见汇报残留),本路不伪造一个“写了状态
|
||||||
|
却没人能接”的通道。签名无效时同样不碰任务(返回 [],
|
||||||
|
只记日志),fail-closed。
|
||||||
"""
|
"""
|
||||||
if self._executor_name() != "local":
|
if not self._code_signature_valid():
|
||||||
|
log_event(self.logger, "worker_pr_refused_code_signature",
|
||||||
|
message="code signature missing or changed,"
|
||||||
|
" refusing pr jobs")
|
||||||
return []
|
return []
|
||||||
|
if self._executor_name() != "local":
|
||||||
|
results = []
|
||||||
|
try:
|
||||||
|
from cloud.pr_exec import ensure_pr_jobs_table
|
||||||
|
conn = _db_connect(self.db_path)
|
||||||
|
try:
|
||||||
|
ensure_pr_jobs_table(conn)
|
||||||
|
rows = conn.execute(
|
||||||
|
"SELECT id, task_id, project_id FROM pr_jobs"
|
||||||
|
" WHERE host = ? AND status = 'queued'",
|
||||||
|
(self.host_name,)).fetchall()
|
||||||
|
finally:
|
||||||
|
conn.close()
|
||||||
|
for row in rows:
|
||||||
|
try:
|
||||||
|
job_id = row["id"]
|
||||||
|
task_id = row["task_id"]
|
||||||
|
project_id = row["project_id"]
|
||||||
|
except (KeyError, IndexError, TypeError):
|
||||||
|
continue
|
||||||
|
reason = ("PR 执行需要 local worker(host-worktrees"
|
||||||
|
" 只在主机上可见),本 worker 为 %s,"
|
||||||
|
"任务保持 queued 等 local worker 认领"
|
||||||
|
% self._executor_name())
|
||||||
|
self._audit(project_id, "pr_job_needs_local_worker",
|
||||||
|
"%s: %s" % (job_id, reason))
|
||||||
|
log_event(self.logger, "pr_job_needs_local_worker",
|
||||||
|
task_id=task_id, status="needs_local_worker",
|
||||||
|
message=reason)
|
||||||
|
results.append({"pr_job_id": job_id,
|
||||||
|
"task_id": task_id,
|
||||||
|
"status": "needs_local_worker",
|
||||||
|
"error": reason})
|
||||||
|
except Exception as exc:
|
||||||
|
log_event(self.logger, "pr_job_peek_failed",
|
||||||
|
message="pr job peek failed: %s" % exc)
|
||||||
|
return results
|
||||||
results = []
|
results = []
|
||||||
while True:
|
while True:
|
||||||
conn = _db_connect(self.db_path)
|
conn = _db_connect(self.db_path)
|
||||||
@@ -3312,8 +3848,12 @@ class Worker:
|
|||||||
# 任务;退出码 75(EX_TEMPFAIL,失败类):unit 托管
|
# 任务;退出码 75(EX_TEMPFAIL,失败类):unit 托管
|
||||||
# 的由 systemd on-failure 重拉新版,autoscaler 托管
|
# 的由 systemd on-failure 重拉新版,autoscaler 托管
|
||||||
# 的由 reconcile 重拉,孤儿直接消失。单次 run_all
|
# 的由 reconcile 重拉,孤儿直接消失。单次 run_all
|
||||||
# 不走这里(进程本来就退出)。
|
# 不走这里(进程本来就退出)。P1 fail-closed:
|
||||||
if self._code_changed():
|
# 签名缺失同样自退(旧版只在“变了”时退,取不到
|
||||||
|
# 签名时继续跑是 fail-open);每次认领前的签名门
|
||||||
|
# 在 _claim 里(池线程不走轮次边界也拦得住)。
|
||||||
|
if self._code_changed() or \
|
||||||
|
not self._code_signature_valid():
|
||||||
log_event(self.logger, "worker_code_changed_exit",
|
log_event(self.logger, "worker_code_changed_exit",
|
||||||
message="worker code changed, exiting "
|
message="worker code changed, exiting "
|
||||||
"for restart")
|
"for restart")
|
||||||
@@ -3336,6 +3876,15 @@ class Worker:
|
|||||||
except Exception as exc:
|
except Exception as exc:
|
||||||
log_event(self.logger, "worker_sweep_failed",
|
log_event(self.logger, "worker_sweep_failed",
|
||||||
message="worker sweep failed: %s" % exc)
|
message="worker sweep failed: %s" % exc)
|
||||||
|
# P1 孤儿扫尾(接在陈旧回收之后):协调员本身
|
||||||
|
# 被回收标 interrupted 后,它的未终态子任务
|
||||||
|
# 在这里被统一标 interrupted;协调员仍在跑时
|
||||||
|
# 本函数不动子任务,见 sweep 方法 docstring。
|
||||||
|
try:
|
||||||
|
self.sweep_orphaned_coordinator_children()
|
||||||
|
except Exception as exc:
|
||||||
|
log_event(self.logger, "worker_orphan_sweep_failed",
|
||||||
|
message="orphan sweep failed: %s" % exc)
|
||||||
# 每轮结束写心跳:哪怕本轮一个任务都没派,worker 活着
|
# 每轮结束写心跳:哪怕本轮一个任务都没派,worker 活着
|
||||||
# 这件事本身也要被 server 看到(文件心跳 + 注册表心跳)。
|
# 这件事本身也要被 server 看到(文件心跳 + 注册表心跳)。
|
||||||
self._write_heartbeat()
|
self._write_heartbeat()
|
||||||
|
|||||||
@@ -2164,9 +2164,11 @@ def main():
|
|||||||
p = sub.add_parser("goal-resume")
|
p = sub.add_parser("goal-resume")
|
||||||
p.add_argument("--task", required=True)
|
p.add_argument("--task", required=True)
|
||||||
p.add_argument("--goal-text", default=None,
|
p.add_argument("--goal-text", default=None,
|
||||||
help="新目标文本:旧任务接续 goal 模式时用——跳过状态与"
|
help="新目标文本:仅开新目标时用——跳过状态与"
|
||||||
"goal 记录守卫,在旧线程上从第 1 轮开新目标循环"
|
"goal 记录守卫,在旧线程上从第 1 轮开新目标循环"
|
||||||
"(新语义:无验收、不传 --max-rounds 时不限轮次)")
|
"(新语义:无验收、不传 --max-rounds 时不限轮次)。"
|
||||||
|
"断点续跑绝不能传它(不传时沿用原 goal+原验收命令),"
|
||||||
|
"传了会把原验收命令覆盖成 None")
|
||||||
p.add_argument("--max-rounds", type=int, default=None,
|
p.add_argument("--max-rounds", type=int, default=None,
|
||||||
help="不传时沿用原任务的 goal_max_rounds;传了则覆盖总轮次")
|
help="不传时沿用原任务的 goal_max_rounds;传了则覆盖总轮次")
|
||||||
p.add_argument("--verify-timeout", type=int, default=600,
|
p.add_argument("--verify-timeout", type=int, default=600,
|
||||||
@@ -2811,12 +2813,23 @@ def main():
|
|||||||
verify_timeout=args.verify_timeout)
|
verify_timeout=args.verify_timeout)
|
||||||
elif args.cmd == "goal-resume":
|
elif args.cmd == "goal-resume":
|
||||||
t = state["tasks"][args.task]
|
t = state["tasks"][args.task]
|
||||||
|
# P1 goal-resume 契约(与 cloud/sandbox.py、cloud/worker.py
|
||||||
|
# 三处一致,绝不能混用):
|
||||||
|
# - 开新目标:调用方传 --goal-text 时走下面分支,在旧线程
|
||||||
|
# 上从第 1 轮开新循环(verify=None,新语义)。这个分支
|
||||||
|
# 只能由“开新目标”的调用方用,断点续跑绝不能传 --goal-text
|
||||||
|
# ——传了就会把原 verify_command 覆盖成 None。
|
||||||
|
# - 断点续跑:调用方不传 --goal-text 时走下面守卫分支,
|
||||||
|
# 沿用原 goal + 原 verify_command + 原 goal_max_rounds,
|
||||||
|
# 从 goal_rounds_done 的下一轮续跑,verify 原样传回
|
||||||
|
# run_goal_loop(run_goal_loop 会把它再写回同一值,
|
||||||
|
# 不是覆盖成 None)。
|
||||||
if args.goal_text:
|
if args.goal_text:
|
||||||
# 旧任务接续 goal 模式(云端 /continue 带 goal_mode=1
|
# 旧任务接续 goal 模式(云端 /continue 带 goal_mode=1
|
||||||
# 时走这里):旧任务已终局、没有 goal 记录是正常情况,
|
# 时走这里):旧任务已终局、没有 goal 记录是正常情况,
|
||||||
# 不能按断点续跑的守卫拒绝。只要线程还在,就在旧线程
|
# 不能按断点续跑的守卫拒绝。只要线程还在,就在旧线程
|
||||||
# 上从第 1 轮开新目标循环(新语义:无验收、不传轮次
|
# 上从第 1 轮开新目标循环(新语义:无验收、不传轮次
|
||||||
# 时不限轮次,模型宣告完成才结束)。
|
# 时不限轮次,模型宣告完成才结束,上界是云端配额)。
|
||||||
if not t.get("thread_id"):
|
if not t.get("thread_id"):
|
||||||
print(f"goal-resume 拒绝:{args.task} 没有会话线程,"
|
print(f"goal-resume 拒绝:{args.task} 没有会话线程,"
|
||||||
f"不能接续")
|
f"不能接续")
|
||||||
@@ -2851,6 +2864,12 @@ def main():
|
|||||||
max_rounds = t["goal_max_rounds"]
|
max_rounds = t["goal_max_rounds"]
|
||||||
else:
|
else:
|
||||||
max_rounds = 3
|
max_rounds = 3
|
||||||
|
# P1:断点续跑时旧验收形态的 0 轮次(脏数据,0 在旧形态
|
||||||
|
# 里表示不限轮次)不按不限轮次跑,钳制到 100 轮(与云端
|
||||||
|
# GOAL_MAX_ROUNDS_CAP 同值);新形态(verify 为空)的 0
|
||||||
|
# 保持不限轮次,上界是云端配额。
|
||||||
|
if max_rounds == 0 and t.get("verify_command"):
|
||||||
|
max_rounds = 100
|
||||||
srv.resume_thread(t["thread_id"])
|
srv.resume_thread(t["thread_id"])
|
||||||
last_turn = (t.get("turns") or [{}])[-1]
|
last_turn = (t.get("turns") or [{}])[-1]
|
||||||
last_result = last_turn.get("result") or {}
|
last_result = last_turn.get("result") or {}
|
||||||
|
|||||||
Reference in new issue
Block a user