feat: cost/quota set — dual-column priced reports, creator split, hard cutoff, approval-timeout auto-threshold
- CostStore: per-model breakdown, report_by_creator (JOIN tasks.created_by), dual-column reports with priced_usd_micros, unpriced_models listed separately and a billing note (proxy price, not the relay bill); _report itself untouched, old exact-key tests keep passing. - Hard cutoff: cost.quota_cutoff_reason is the single USD/unpriced-cap decision; worker._quota_exceeded now delegates to it (identical messages); claim gate, turn boundary and poll kill stop spend and record interrupted + quota reason + audit; auto-resume re-checks live quota before requeueing. - Approval timeout (G7 ops metric): cloud/approval_timeout.py — default stays 600s, >20% timeout rate recommends 1800s, <5% keeps, mid-range keeps; stats + recommendation + info log; wired into /metrics (member projects, no log spam) and a global-admin /admin/ops/approval-timeout endpoint (sweeps overdue first). - Server: task/project cost endpoints return dual-column reports; new GET /users/me/cost and GET /admin/cost-by-creator. - Tests: cloud/tests/test_cost_quota.py, 10 tests.
This commit is contained in:
1 parent
0072e1ae6a
commit
143ab30094
5 files changed
+871
-12
No files matched your search
+204
@@ -135,6 +135,39 @@ def price_usd_micros(model, input_tokens, cached_input_tokens,
|
||||
return int(round(usd * 1000000))
|
||||
|
||||
|
||||
BILLING_NOTE = (
|
||||
"价格表代理价(经 CLOUD_MODEL_PRICES_JSON / CLOUD_MODEL_PRICES_FILE"
|
||||
" 覆盖层),非中转实际账单;未定价 token 不计入美元、不等于免费。"
|
||||
)
|
||||
|
||||
|
||||
def quota_cutoff_reason(quota_usd_micros, used_usd_micros,
|
||||
unpriced_used_tokens, month, unpriced_cap=None):
|
||||
"""配额硬切断的唯一判定函数(worker 执行侧与单测共用)。
|
||||
|
||||
返回 None 表示未超限、可继续;返回 quota_exceeded 文本表示
|
||||
必须立即停:调用方不再开新回合、不再派生花钱路径,任务以
|
||||
interrupted + 该原因收尾(原因写进 error_text 并记审计),
|
||||
不是只告警。quota 为 None(管理员显式不限额)时不挡;USD
|
||||
配额与无价 token 上限是两道独立的门,任一超限即停。
|
||||
文本格式与 worker 旧分支逐字一致,旧单测按此前缀断言。
|
||||
"""
|
||||
if quota_usd_micros is None:
|
||||
return None
|
||||
if int(used_usd_micros or 0) >= int(quota_usd_micros):
|
||||
return ("quota_exceeded: 本月配额已用完(已用 %d / "
|
||||
"配额 %d 微美元,%s)"
|
||||
% (int(used_usd_micros or 0), int(quota_usd_micros),
|
||||
month))
|
||||
cap = unpriced_token_cap() if unpriced_cap is None \
|
||||
else int(unpriced_cap)
|
||||
if int(unpriced_used_tokens or 0) >= cap:
|
||||
return ("quota_exceeded: 本月无价模型 token 已用完"
|
||||
"(已用 %d / 上限 %d,%s)"
|
||||
% (int(unpriced_used_tokens or 0), cap, month))
|
||||
return None
|
||||
|
||||
|
||||
def unpriced_token_cap():
|
||||
"""Monthly unpriced-token ceiling for users who have a quota row.
|
||||
|
||||
@@ -397,3 +430,174 @@ class CostStore:
|
||||
def report_by_day(self, day) -> dict:
|
||||
"""Aggregate cost records for one UTC day (YYYY-MM-DD)."""
|
||||
return self._report("day = ?", (day,))
|
||||
|
||||
def _report_joined(self, where, params):
|
||||
"""_report 的 JOIN 版:按 tasks.created_by 归属汇总。
|
||||
|
||||
cost_records 本身没有创建者列,归属只能经 tasks JOIN;
|
||||
协调员子任务行无 created_by 时跟父任务走的口径在
|
||||
worker 侧,库层这里只按行上的 created_by 计——无属主
|
||||
任务的成本不归任何人,不会被静默算进某个创建者。
|
||||
"""
|
||||
conn = self._new_conn()
|
||||
try:
|
||||
row = conn.execute(
|
||||
"""SELECT
|
||||
COALESCE(SUM(c.total_tokens), 0),
|
||||
COALESCE(SUM(c.usd_micros), 0),
|
||||
COUNT(*),
|
||||
COALESCE(SUM(c.fallback_count), 0),
|
||||
COALESCE(SUM(CASE WHEN c.source = 'actual'
|
||||
THEN c.total_tokens ELSE 0 END), 0),
|
||||
COALESCE(SUM(CASE WHEN c.source = 'estimated'
|
||||
THEN c.total_tokens ELSE 0 END), 0),
|
||||
COALESCE(SUM(c.cached_input_tokens), 0),
|
||||
COALESCE(SUM(CASE WHEN c.source = 'actual'
|
||||
AND c.usd_micros IS NULL
|
||||
THEN c.total_tokens ELSE 0 END), 0),
|
||||
COALESCE(SUM(c.completions), 0)
|
||||
FROM cost_records c JOIN tasks t
|
||||
ON c.task_id = t.id WHERE %s""" % where,
|
||||
params,
|
||||
).fetchone()
|
||||
finally:
|
||||
if conn is not self._conn:
|
||||
conn.close()
|
||||
(total_tokens, usd_micros, turns, fallback_count, actual,
|
||||
estimated, cached, unpriced, completions) = row
|
||||
return {
|
||||
"total_tokens": int(total_tokens),
|
||||
"usd_micros": int(usd_micros),
|
||||
"turns": int(turns),
|
||||
"fallback_count": int(fallback_count),
|
||||
"actual_tokens": int(actual),
|
||||
"estimated_tokens": int(estimated),
|
||||
"cached_input_tokens": int(cached),
|
||||
"unpriced_actual_tokens": int(unpriced),
|
||||
"llm_requests": int(completions),
|
||||
}
|
||||
|
||||
def report_by_creator(self, creator_id, month=None) -> dict:
|
||||
"""按任务创建者汇总:当月(UTC,YYYY-MM)或全部。
|
||||
|
||||
month 为 None 时不过滤月份;month 格式须为 YYYY-MM,
|
||||
否则 ValueError(调用方转 400)。返回 _report 同款键
|
||||
外加 creator_id 与 month,方便分账报表直接按人对账。
|
||||
"""
|
||||
if month is not None:
|
||||
if not isinstance(month, str) or len(month) != 7 \
|
||||
or month[4] != "-":
|
||||
raise ValueError("month must be YYYY-MM")
|
||||
report = self._report_joined(
|
||||
"t.created_by = ? AND c.day LIKE ?",
|
||||
(creator_id, month + "%"))
|
||||
else:
|
||||
report = self._report_joined(
|
||||
"t.created_by = ?", (creator_id,))
|
||||
report["creator_id"] = creator_id
|
||||
report["month"] = month
|
||||
return report
|
||||
|
||||
def model_breakdown(self, task_id=None, project_id=None,
|
||||
creator_id=None, month=None) -> list:
|
||||
"""按模型分列:恰好指定 task / project / creator 其中一种。
|
||||
|
||||
每行:model、total_tokens、turns、llm_requests、
|
||||
priced_usd_micros(该模型无任何有价行时为 None,不写 0)、
|
||||
unpriced_tokens(source='actual' 且 usd IS NULL 的行,
|
||||
估算行不计)、priced(是否有任一有价行)。未定价模型
|
||||
由 unpriced_tokens > 0 识别,调用方用它单独列出,绝不
|
||||
把 priced_usd_micros=None 的模型按 0 美元合并进合计。
|
||||
creator 口径经 tasks JOIN,month 同 report_by_creator。
|
||||
"""
|
||||
scopes = [s for s in (task_id, project_id, creator_id)
|
||||
if s is not None]
|
||||
if len(scopes) != 1:
|
||||
raise ValueError(
|
||||
"exactly one of task_id, project_id, creator_id required")
|
||||
join = ""
|
||||
if task_id is not None:
|
||||
where, params = "c.task_id = ?", (task_id,)
|
||||
elif project_id is not None:
|
||||
where, params = "c.project_id = ?", (project_id,)
|
||||
else:
|
||||
join = " JOIN tasks t ON c.task_id = t.id"
|
||||
where, params = "t.created_by = ?", (creator_id,)
|
||||
if month is not None:
|
||||
if not isinstance(month, str) or len(month) != 7 \
|
||||
or month[4] != "-":
|
||||
raise ValueError("month must be YYYY-MM")
|
||||
where += " AND c.day LIKE ?"
|
||||
params = params + (month + "%",)
|
||||
conn = self._new_conn()
|
||||
try:
|
||||
rows = conn.execute(
|
||||
"""SELECT c.model,
|
||||
COALESCE(SUM(c.total_tokens), 0),
|
||||
COUNT(*),
|
||||
COALESCE(SUM(c.completions), 0),
|
||||
SUM(c.usd_micros),
|
||||
SUM(CASE WHEN c.usd_micros IS NOT NULL
|
||||
THEN 1 ELSE 0 END),
|
||||
COALESCE(SUM(CASE WHEN c.source = 'actual'
|
||||
AND c.usd_micros IS NULL
|
||||
THEN c.total_tokens ELSE 0 END), 0)
|
||||
FROM cost_records c%s WHERE %s
|
||||
GROUP BY c.model
|
||||
ORDER BY SUM(c.total_tokens) DESC, c.model"""
|
||||
% (join, where),
|
||||
params,
|
||||
).fetchall()
|
||||
finally:
|
||||
if conn is not self._conn:
|
||||
conn.close()
|
||||
out = []
|
||||
for row in rows:
|
||||
(model, total, turns, completions, usd_sum, priced_rows,
|
||||
unpriced) = row
|
||||
out.append({
|
||||
"model": model or "unknown",
|
||||
"total_tokens": int(total),
|
||||
"turns": int(turns),
|
||||
"llm_requests": int(completions),
|
||||
"priced_usd_micros": (int(usd_sum) if priced_rows else None),
|
||||
"unpriced_tokens": int(unpriced),
|
||||
"priced": bool(priced_rows),
|
||||
})
|
||||
return out
|
||||
|
||||
def _dual_column(self, base, breakdown):
|
||||
"""双列报告:在 _report 结果上加第二列与分列明细。
|
||||
|
||||
_report 本身不动(旧单测按其精确键集断言);双列是
|
||||
新方法返回的扩展:priced_usd_micros 与 usd_micros 同值
|
||||
(只计有价行之和),unpriced_models 单独列出未定价
|
||||
模型,billing_note 明确标注代理价、非中转实际账单。
|
||||
"""
|
||||
report = dict(base)
|
||||
report["priced_usd_micros"] = report["usd_micros"]
|
||||
report["unpriced_models"] = sorted(
|
||||
row["model"] for row in breakdown
|
||||
if row["unpriced_tokens"] > 0)
|
||||
report["by_model"] = breakdown
|
||||
report["billing_note"] = BILLING_NOTE
|
||||
return report
|
||||
|
||||
def dual_column_report_by_task(self, task_id) -> dict:
|
||||
"""单任务双列报告(见 _dual_column)。"""
|
||||
return self._dual_column(
|
||||
self.report_by_task(task_id),
|
||||
self.model_breakdown(task_id=task_id))
|
||||
|
||||
def dual_column_report_by_project(self, project_id) -> dict:
|
||||
"""单项目双列报告(见 _dual_column)。"""
|
||||
return self._dual_column(
|
||||
self.report_by_project(project_id),
|
||||
self.model_breakdown(project_id=project_id))
|
||||
|
||||
def dual_column_report_by_creator(self, creator_id,
|
||||
month=None) -> dict:
|
||||
"""按创建者双列报告(见 _dual_column)。"""
|
||||
return self._dual_column(
|
||||
self.report_by_creator(creator_id, month=month),
|
||||
self.model_breakdown(creator_id=creator_id, month=month))
|
||||
Reference in new issue
Block a user