feat: cost/quota set — dual-column priced reports, creator split, hard cutoff, approval-timeout auto-threshold

- CostStore: per-model breakdown, report_by_creator (JOIN tasks.created_by),
  dual-column reports with priced_usd_micros, unpriced_models listed
  separately and a billing note (proxy price, not the relay bill);
  _report itself untouched, old exact-key tests keep passing.
- Hard cutoff: cost.quota_cutoff_reason is the single USD/unpriced-cap
  decision; worker._quota_exceeded now delegates to it (identical
  messages); claim gate, turn boundary and poll kill stop spend and
  record interrupted + quota reason + audit; auto-resume re-checks
  live quota before requeueing.
- Approval timeout (G7 ops metric): cloud/approval_timeout.py —
  default stays 600s, >20% timeout rate recommends 1800s, <5% keeps,
  mid-range keeps; stats + recommendation + info log; wired into
  /metrics (member projects, no log spam) and a global-admin
  /admin/ops/approval-timeout endpoint (sweeps overdue first).
- Server: task/project cost endpoints return dual-column reports;
  new GET /users/me/cost and GET /admin/cost-by-creator.
- Tests: cloud/tests/test_cost_quota.py, 10 tests.
This commit is contained in:
host committed 2026-11-03 02:49:25 +00:00
1 parent 0072e1ae6a
commit 143ab30094
5 files changed
+871 -12

No files matched your search

+116 -2
View File
@@ -132,6 +132,9 @@ from cloud.todos import get_todo as _todo_get
from cloud.todos import list_todos as _todo_list
from cloud.todos import update_todo as _todo_update
from cloud.automations import triage_inbox as _auto_triage
from cloud.approval_timeout import approval_timeout_stats as _appr_timeout_stats
from cloud.approval_timeout import evaluate_approval_timeout as _appr_timeout_eval
from cloud.approval_timeout import recommend_approval_timeout as _appr_timeout_recommend
from cloud.approvals import create_approval as _appr_create
from cloud.approvals import decide_approval as _appr_decide
from cloud.approvals import ensure_approvals_table as _appr_ensure
@@ -3020,6 +3023,12 @@ class CloudControlPlaneHandler(BaseHTTPRequestHandler):
self._send_json(404, {"error": "not found"})
elif path == "/users/me/quota" and method == "GET":
self._user_quota_self(user_id)
elif path == "/users/me/cost" and method == "GET":
self._user_cost_self(user_id, query)
elif path == "/admin/cost-by-creator" and method == "GET":
self._admin_cost_by_creator(user_id, query)
elif path == "/admin/ops/approval-timeout" and method == "GET":
self._admin_approval_timeout_ops(user_id)
elif path == "/users/me/profile" and method == "GET":
self._identity_profile_get(user_id)
elif path == "/users/me/profile" and method in ("PUT", "POST"):
@@ -4637,7 +4646,9 @@ class CloudControlPlaneHandler(BaseHTTPRequestHandler):
def _project_cost(self, project_id, query):
store = CostStore(self.server.db_path)
try:
overall = store.report_by_project(project_id)
# 双列:美元只计有价行,未定价模型单独列在
# unpriced_models,billing_note 标注代理价、非账单。
overall = store.dual_column_report_by_project(project_id)
conn = store._conn
conn.row_factory = sqlite3.Row
days = (query.get("days") or [None])[0]
@@ -4712,7 +4723,8 @@ class CloudControlPlaneHandler(BaseHTTPRequestHandler):
return
store = CostStore(self.server.db_path)
try:
report = store.report_by_task(task_id)
# 双列:与项目成本同款,旧键全保留,只加双列键。
report = store.dual_column_report_by_task(task_id)
finally:
try:
store._conn.close()
@@ -8875,6 +8887,93 @@ class CloudControlPlaneHandler(BaseHTTPRequestHandler):
payload["unpriced_token_cap"] = unpriced_token_cap()
self._send_json(200, payload)
def _cost_by_creator_payload(self, target, month):
"""创建者分账双列载荷(自查与管理端共用)。
month 为 None 时不过滤月份;否则须为 YYYY-MM,否则
ValueError(调用方转 400)。配额进度与成本双列同回:
quota 是当月美元配额(与 _quota_status 同口径),cost
是该创建者名下任务的成本(经 tasks.created_by JOIN)。
"""
quota, used, cur_month = self._quota_status(target)
# 端点默认当月(与配额窗口一致),不传 month 时成本与
# 配额同为当月;CostStore 层 month=None 仍表示全部时间。
eff_month = month if month is not None else cur_month
store = CostStore(self.server.db_path)
try:
cost = store.dual_column_report_by_creator(target,
month=eff_month)
finally:
try:
store._conn.close()
except Exception:
pass
return {
"user_id": target,
"month": eff_month,
"quota_usd_micros": quota,
"used_usd_micros": used,
"remaining_usd_micros": (quota - used)
if quota is not None else None,
"unpriced_tokens_used": self._quota_unpriced_tokens(
target, eff_month),
"unpriced_token_cap": unpriced_token_cap(),
"cost": cost,
}
def _user_cost_self(self, user_id, query):
month = (query.get("month") or [None])[0]
try:
payload = self._cost_by_creator_payload(user_id, month)
except ValueError:
self._send_json(400, {"error": "month must be YYYY-MM"})
return
self._send_json(200, payload)
def _admin_cost_by_creator(self, user_id, query):
if not self._is_global_admin(user_id):
self._audit_admin(user_id, "admin_quota_denied",
"get cost by creator")
self._send_json(403, {"error": "forbidden"})
return
target = (query.get("user_id") or [None])[0]
if not target or not isinstance(target, str):
self._send_json(400, {"error": "user_id required"})
return
if not self._user_exists(target):
self._send_json(404, {"error": "user not found"})
return
month = (query.get("month") or [None])[0]
try:
payload = self._cost_by_creator_payload(target, month)
except ValueError:
self._send_json(400, {"error": "month must be YYYY-MM"})
return
self._send_json(200, payload)
def _admin_approval_timeout_ops(self, user_id):
"""审批超时调阈运维指标(全局管理员只读)。
先扫荡过期 pending(与审批列表同款:过期即 expired,
计入超时率),再统计 + 建议 + 记日志。返回里
recommended_seconds 是建议值,不是已生效值——默认
仍 600 秒,调阈生效须运维按建议改配置,见
cloud/approval_timeout 模块 docstring。
"""
if not self._is_global_admin(user_id):
self._audit_admin(user_id, "admin_quota_denied",
"get approval timeout ops")
self._send_json(403, {"error": "forbidden"})
return
conn = self._db()
try:
_appr_ensure(conn)
_appr_expire(conn)
result = _appr_timeout_eval(conn)
finally:
conn.close()
self._send_json(200, result)
def _admin_quota_get(self, user_id, query):
if not self._is_global_admin(user_id):
self._audit_admin(user_id, "admin_quota_denied",
@@ -9361,6 +9460,20 @@ class CloudControlPlaneHandler(BaseHTTPRequestHandler):
resident_stats = _res_metrics(conn, member_ids, now)
except Exception: # noqa: BLE001 - 观测端点不因 resident 失败
resident_stats = {"available": False}
# G7 运维指标:审批超时调阈(成员项目口径,只加键)。
# 统计失败时置 None(不是零),绝不把“算不出来”写成
# “超时率为零”;未扫荡的过期 pending 不计入,见
# approval_timeout 模块 docstring。这里不用 evaluate
#(它写日志,/metrics 高频轮询会刷屏),只算不记。
try:
_at_stats = _appr_timeout_stats(
conn, project_ids=sorted(member_ids))
approval_timeout = _appr_timeout_recommend(
_at_stats["expired_count"],
_at_stats["decided_count"])
approval_timeout.update(_at_stats)
except Exception:
approval_timeout = None
finally:
conn.close()
try:
@@ -9393,6 +9506,7 @@ class CloudControlPlaneHandler(BaseHTTPRequestHandler):
"queued": by_status.get("queued", 0),
"running": by_status.get("running", 0),
},
"approval_timeout": approval_timeout,
"failures_last_24h": failures_last_24h,
"running_over_300s": running_over_300s,
"workers": {