feat: cost/quota set — dual-column priced reports, creator split, hard cutoff, approval-timeout auto-threshold

- CostStore: per-model breakdown, report_by_creator (JOIN tasks.created_by),
  dual-column reports with priced_usd_micros, unpriced_models listed
  separately and a billing note (proxy price, not the relay bill);
  _report itself untouched, old exact-key tests keep passing.
- Hard cutoff: cost.quota_cutoff_reason is the single USD/unpriced-cap
  decision; worker._quota_exceeded now delegates to it (identical
  messages); claim gate, turn boundary and poll kill stop spend and
  record interrupted + quota reason + audit; auto-resume re-checks
  live quota before requeueing.
- Approval timeout (G7 ops metric): cloud/approval_timeout.py —
  default stays 600s, >20% timeout rate recommends 1800s, <5% keeps,
  mid-range keeps; stats + recommendation + info log; wired into
  /metrics (member projects, no log spam) and a global-admin
  /admin/ops/approval-timeout endpoint (sweeps overdue first).
- Server: task/project cost endpoints return dual-column reports;
  new GET /users/me/cost and GET /admin/cost-by-creator.
- Tests: cloud/tests/test_cost_quota.py, 10 tests.
This commit is contained in:
host committed 2026-11-03 02:49:25 +00:00
1 parent 0072e1ae6a
commit 143ab30094
5 files changed
+871 -12

No files matched your search

+204
View File
@@ -135,6 +135,39 @@ def price_usd_micros(model, input_tokens, cached_input_tokens,
return int(round(usd * 1000000))
BILLING_NOTE = (
"价格表代理价(经 CLOUD_MODEL_PRICES_JSON / CLOUD_MODEL_PRICES_FILE"
" 覆盖层),非中转实际账单;未定价 token 不计入美元、不等于免费。"
)
def quota_cutoff_reason(quota_usd_micros, used_usd_micros,
unpriced_used_tokens, month, unpriced_cap=None):
"""配额硬切断的唯一判定函数(worker 执行侧与单测共用)。
返回 None 表示未超限、可继续;返回 quota_exceeded 文本表示
必须立即停:调用方不再开新回合、不再派生花钱路径,任务以
interrupted + 该原因收尾(原因写进 error_text 并记审计),
不是只告警。quota 为 None(管理员显式不限额)时不挡;USD
配额与无价 token 上限是两道独立的门,任一超限即停。
文本格式与 worker 旧分支逐字一致,旧单测按此前缀断言。
"""
if quota_usd_micros is None:
return None
if int(used_usd_micros or 0) >= int(quota_usd_micros):
return ("quota_exceeded: 本月配额已用完(已用 %d / "
"配额 %d 微美元,%s)"
% (int(used_usd_micros or 0), int(quota_usd_micros),
month))
cap = unpriced_token_cap() if unpriced_cap is None \
else int(unpriced_cap)
if int(unpriced_used_tokens or 0) >= cap:
return ("quota_exceeded: 本月无价模型 token 已用完"
"(已用 %d / 上限 %d,%s)"
% (int(unpriced_used_tokens or 0), cap, month))
return None
def unpriced_token_cap():
"""Monthly unpriced-token ceiling for users who have a quota row.
@@ -397,3 +430,174 @@ class CostStore:
def report_by_day(self, day) -> dict:
"""Aggregate cost records for one UTC day (YYYY-MM-DD)."""
return self._report("day = ?", (day,))
def _report_joined(self, where, params):
"""_report 的 JOIN 版:按 tasks.created_by 归属汇总。
cost_records 本身没有创建者列,归属只能经 tasks JOIN;
协调员子任务行无 created_by 时跟父任务走的口径在
worker 侧,库层这里只按行上的 created_by 计——无属主
任务的成本不归任何人,不会被静默算进某个创建者。
"""
conn = self._new_conn()
try:
row = conn.execute(
"""SELECT
COALESCE(SUM(c.total_tokens), 0),
COALESCE(SUM(c.usd_micros), 0),
COUNT(*),
COALESCE(SUM(c.fallback_count), 0),
COALESCE(SUM(CASE WHEN c.source = 'actual'
THEN c.total_tokens ELSE 0 END), 0),
COALESCE(SUM(CASE WHEN c.source = 'estimated'
THEN c.total_tokens ELSE 0 END), 0),
COALESCE(SUM(c.cached_input_tokens), 0),
COALESCE(SUM(CASE WHEN c.source = 'actual'
AND c.usd_micros IS NULL
THEN c.total_tokens ELSE 0 END), 0),
COALESCE(SUM(c.completions), 0)
FROM cost_records c JOIN tasks t
ON c.task_id = t.id WHERE %s""" % where,
params,
).fetchone()
finally:
if conn is not self._conn:
conn.close()
(total_tokens, usd_micros, turns, fallback_count, actual,
estimated, cached, unpriced, completions) = row
return {
"total_tokens": int(total_tokens),
"usd_micros": int(usd_micros),
"turns": int(turns),
"fallback_count": int(fallback_count),
"actual_tokens": int(actual),
"estimated_tokens": int(estimated),
"cached_input_tokens": int(cached),
"unpriced_actual_tokens": int(unpriced),
"llm_requests": int(completions),
}
def report_by_creator(self, creator_id, month=None) -> dict:
"""按任务创建者汇总:当月(UTC,YYYY-MM)或全部。
month 为 None 时不过滤月份;month 格式须为 YYYY-MM,
否则 ValueError(调用方转 400)。返回 _report 同款键
外加 creator_id 与 month,方便分账报表直接按人对账。
"""
if month is not None:
if not isinstance(month, str) or len(month) != 7 \
or month[4] != "-":
raise ValueError("month must be YYYY-MM")
report = self._report_joined(
"t.created_by = ? AND c.day LIKE ?",
(creator_id, month + "%"))
else:
report = self._report_joined(
"t.created_by = ?", (creator_id,))
report["creator_id"] = creator_id
report["month"] = month
return report
def model_breakdown(self, task_id=None, project_id=None,
creator_id=None, month=None) -> list:
"""按模型分列:恰好指定 task / project / creator 其中一种。
每行:model、total_tokens、turns、llm_requests、
priced_usd_micros(该模型无任何有价行时为 None,不写 0)、
unpriced_tokens(source='actual' 且 usd IS NULL 的行,
估算行不计)、priced(是否有任一有价行)。未定价模型
由 unpriced_tokens > 0 识别,调用方用它单独列出,绝不
把 priced_usd_micros=None 的模型按 0 美元合并进合计。
creator 口径经 tasks JOIN,month 同 report_by_creator。
"""
scopes = [s for s in (task_id, project_id, creator_id)
if s is not None]
if len(scopes) != 1:
raise ValueError(
"exactly one of task_id, project_id, creator_id required")
join = ""
if task_id is not None:
where, params = "c.task_id = ?", (task_id,)
elif project_id is not None:
where, params = "c.project_id = ?", (project_id,)
else:
join = " JOIN tasks t ON c.task_id = t.id"
where, params = "t.created_by = ?", (creator_id,)
if month is not None:
if not isinstance(month, str) or len(month) != 7 \
or month[4] != "-":
raise ValueError("month must be YYYY-MM")
where += " AND c.day LIKE ?"
params = params + (month + "%",)
conn = self._new_conn()
try:
rows = conn.execute(
"""SELECT c.model,
COALESCE(SUM(c.total_tokens), 0),
COUNT(*),
COALESCE(SUM(c.completions), 0),
SUM(c.usd_micros),
SUM(CASE WHEN c.usd_micros IS NOT NULL
THEN 1 ELSE 0 END),
COALESCE(SUM(CASE WHEN c.source = 'actual'
AND c.usd_micros IS NULL
THEN c.total_tokens ELSE 0 END), 0)
FROM cost_records c%s WHERE %s
GROUP BY c.model
ORDER BY SUM(c.total_tokens) DESC, c.model"""
% (join, where),
params,
).fetchall()
finally:
if conn is not self._conn:
conn.close()
out = []
for row in rows:
(model, total, turns, completions, usd_sum, priced_rows,
unpriced) = row
out.append({
"model": model or "unknown",
"total_tokens": int(total),
"turns": int(turns),
"llm_requests": int(completions),
"priced_usd_micros": (int(usd_sum) if priced_rows else None),
"unpriced_tokens": int(unpriced),
"priced": bool(priced_rows),
})
return out
def _dual_column(self, base, breakdown):
"""双列报告:在 _report 结果上加第二列与分列明细。
_report 本身不动(旧单测按其精确键集断言);双列是
新方法返回的扩展:priced_usd_micros 与 usd_micros 同值
(只计有价行之和),unpriced_models 单独列出未定价
模型,billing_note 明确标注代理价、非中转实际账单。
"""
report = dict(base)
report["priced_usd_micros"] = report["usd_micros"]
report["unpriced_models"] = sorted(
row["model"] for row in breakdown
if row["unpriced_tokens"] > 0)
report["by_model"] = breakdown
report["billing_note"] = BILLING_NOTE
return report
def dual_column_report_by_task(self, task_id) -> dict:
"""单任务双列报告(见 _dual_column)。"""
return self._dual_column(
self.report_by_task(task_id),
self.model_breakdown(task_id=task_id))
def dual_column_report_by_project(self, project_id) -> dict:
"""单项目双列报告(见 _dual_column)。"""
return self._dual_column(
self.report_by_project(project_id),
self.model_breakdown(project_id=project_id))
def dual_column_report_by_creator(self, creator_id,
month=None) -> dict:
"""按创建者双列报告(见 _dual_column)。"""
return self._dual_column(
self.report_by_creator(creator_id, month=month),
self.model_breakdown(creator_id=creator_id, month=month))