feat: cost/quota set — dual-column priced reports, creator split, hard cutoff, approval-timeout auto-threshold
- CostStore: per-model breakdown, report_by_creator (JOIN tasks.created_by), dual-column reports with priced_usd_micros, unpriced_models listed separately and a billing note (proxy price, not the relay bill); _report itself untouched, old exact-key tests keep passing. - Hard cutoff: cost.quota_cutoff_reason is the single USD/unpriced-cap decision; worker._quota_exceeded now delegates to it (identical messages); claim gate, turn boundary and poll kill stop spend and record interrupted + quota reason + audit; auto-resume re-checks live quota before requeueing. - Approval timeout (G7 ops metric): cloud/approval_timeout.py — default stays 600s, >20% timeout rate recommends 1800s, <5% keeps, mid-range keeps; stats + recommendation + info log; wired into /metrics (member projects, no log spam) and a global-admin /admin/ops/approval-timeout endpoint (sweeps overdue first). - Server: task/project cost endpoints return dual-column reports; new GET /users/me/cost and GET /admin/cost-by-creator. - Tests: cloud/tests/test_cost_quota.py, 10 tests.
This commit is contained in:
1 parent
cc32f51502
commit
e079b9b44a
5 files changed
+871
-12
No files matched your search
+116
-2
@@ -143,6 +143,9 @@ from cloud.todos import get_todo as _todo_get
|
||||
from cloud.todos import list_todos as _todo_list
|
||||
from cloud.todos import update_todo as _todo_update
|
||||
from cloud.automations import triage_inbox as _auto_triage
|
||||
from cloud.approval_timeout import approval_timeout_stats as _appr_timeout_stats
|
||||
from cloud.approval_timeout import evaluate_approval_timeout as _appr_timeout_eval
|
||||
from cloud.approval_timeout import recommend_approval_timeout as _appr_timeout_recommend
|
||||
from cloud.approvals import create_approval as _appr_create
|
||||
from cloud.approvals import decide_approval as _appr_decide
|
||||
from cloud.approvals import ensure_approvals_table as _appr_ensure
|
||||
@@ -3107,6 +3110,12 @@ class CloudControlPlaneHandler(BaseHTTPRequestHandler):
|
||||
self._send_json(404, {"error": "not found"})
|
||||
elif path == "/users/me/quota" and method == "GET":
|
||||
self._user_quota_self(user_id)
|
||||
elif path == "/users/me/cost" and method == "GET":
|
||||
self._user_cost_self(user_id, query)
|
||||
elif path == "/admin/cost-by-creator" and method == "GET":
|
||||
self._admin_cost_by_creator(user_id, query)
|
||||
elif path == "/admin/ops/approval-timeout" and method == "GET":
|
||||
self._admin_approval_timeout_ops(user_id)
|
||||
elif path == "/users/me/profile" and method == "GET":
|
||||
self._identity_profile_get(user_id)
|
||||
elif path == "/users/me/profile" and method in ("PUT", "POST"):
|
||||
@@ -4726,7 +4735,9 @@ class CloudControlPlaneHandler(BaseHTTPRequestHandler):
|
||||
def _project_cost(self, project_id, query):
|
||||
store = CostStore(self.server.db_path)
|
||||
try:
|
||||
overall = store.report_by_project(project_id)
|
||||
# 双列:美元只计有价行,未定价模型单独列在
|
||||
# unpriced_models,billing_note 标注代理价、非账单。
|
||||
overall = store.dual_column_report_by_project(project_id)
|
||||
conn = store._conn
|
||||
conn.row_factory = sqlite3.Row
|
||||
days = (query.get("days") or [None])[0]
|
||||
@@ -5034,7 +5045,8 @@ class CloudControlPlaneHandler(BaseHTTPRequestHandler):
|
||||
return
|
||||
store = CostStore(self.server.db_path)
|
||||
try:
|
||||
report = store.report_by_task(task_id)
|
||||
# 双列:与项目成本同款,旧键全保留,只加双列键。
|
||||
report = store.dual_column_report_by_task(task_id)
|
||||
finally:
|
||||
try:
|
||||
store._conn.close()
|
||||
@@ -9362,6 +9374,93 @@ class CloudControlPlaneHandler(BaseHTTPRequestHandler):
|
||||
payload["unpriced_token_cap"] = unpriced_token_cap()
|
||||
self._send_json(200, payload)
|
||||
|
||||
def _cost_by_creator_payload(self, target, month):
|
||||
"""创建者分账双列载荷(自查与管理端共用)。
|
||||
|
||||
month 为 None 时不过滤月份;否则须为 YYYY-MM,否则
|
||||
ValueError(调用方转 400)。配额进度与成本双列同回:
|
||||
quota 是当月美元配额(与 _quota_status 同口径),cost
|
||||
是该创建者名下任务的成本(经 tasks.created_by JOIN)。
|
||||
"""
|
||||
quota, used, cur_month = self._quota_status(target)
|
||||
# 端点默认当月(与配额窗口一致),不传 month 时成本与
|
||||
# 配额同为当月;CostStore 层 month=None 仍表示全部时间。
|
||||
eff_month = month if month is not None else cur_month
|
||||
store = CostStore(self.server.db_path)
|
||||
try:
|
||||
cost = store.dual_column_report_by_creator(target,
|
||||
month=eff_month)
|
||||
finally:
|
||||
try:
|
||||
store._conn.close()
|
||||
except Exception:
|
||||
pass
|
||||
return {
|
||||
"user_id": target,
|
||||
"month": eff_month,
|
||||
"quota_usd_micros": quota,
|
||||
"used_usd_micros": used,
|
||||
"remaining_usd_micros": (quota - used)
|
||||
if quota is not None else None,
|
||||
"unpriced_tokens_used": self._quota_unpriced_tokens(
|
||||
target, eff_month),
|
||||
"unpriced_token_cap": unpriced_token_cap(),
|
||||
"cost": cost,
|
||||
}
|
||||
|
||||
def _user_cost_self(self, user_id, query):
|
||||
month = (query.get("month") or [None])[0]
|
||||
try:
|
||||
payload = self._cost_by_creator_payload(user_id, month)
|
||||
except ValueError:
|
||||
self._send_json(400, {"error": "month must be YYYY-MM"})
|
||||
return
|
||||
self._send_json(200, payload)
|
||||
|
||||
def _admin_cost_by_creator(self, user_id, query):
|
||||
if not self._is_global_admin(user_id):
|
||||
self._audit_admin(user_id, "admin_quota_denied",
|
||||
"get cost by creator")
|
||||
self._send_json(403, {"error": "forbidden"})
|
||||
return
|
||||
target = (query.get("user_id") or [None])[0]
|
||||
if not target or not isinstance(target, str):
|
||||
self._send_json(400, {"error": "user_id required"})
|
||||
return
|
||||
if not self._user_exists(target):
|
||||
self._send_json(404, {"error": "user not found"})
|
||||
return
|
||||
month = (query.get("month") or [None])[0]
|
||||
try:
|
||||
payload = self._cost_by_creator_payload(target, month)
|
||||
except ValueError:
|
||||
self._send_json(400, {"error": "month must be YYYY-MM"})
|
||||
return
|
||||
self._send_json(200, payload)
|
||||
|
||||
def _admin_approval_timeout_ops(self, user_id):
|
||||
"""审批超时调阈运维指标(全局管理员只读)。
|
||||
|
||||
先扫荡过期 pending(与审批列表同款:过期即 expired,
|
||||
计入超时率),再统计 + 建议 + 记日志。返回里
|
||||
recommended_seconds 是建议值,不是已生效值——默认
|
||||
仍 600 秒,调阈生效须运维按建议改配置,见
|
||||
cloud/approval_timeout 模块 docstring。
|
||||
"""
|
||||
if not self._is_global_admin(user_id):
|
||||
self._audit_admin(user_id, "admin_quota_denied",
|
||||
"get approval timeout ops")
|
||||
self._send_json(403, {"error": "forbidden"})
|
||||
return
|
||||
conn = self._db()
|
||||
try:
|
||||
_appr_ensure(conn)
|
||||
_appr_expire(conn)
|
||||
result = _appr_timeout_eval(conn)
|
||||
finally:
|
||||
conn.close()
|
||||
self._send_json(200, result)
|
||||
|
||||
def _admin_quota_get(self, user_id, query):
|
||||
if not self._is_global_admin(user_id):
|
||||
self._audit_admin(user_id, "admin_quota_denied",
|
||||
@@ -9848,6 +9947,20 @@ class CloudControlPlaneHandler(BaseHTTPRequestHandler):
|
||||
resident_stats = _res_metrics(conn, member_ids, now)
|
||||
except Exception: # noqa: BLE001 - 观测端点不因 resident 失败
|
||||
resident_stats = {"available": False}
|
||||
# G7 运维指标:审批超时调阈(成员项目口径,只加键)。
|
||||
# 统计失败时置 None(不是零),绝不把“算不出来”写成
|
||||
# “超时率为零”;未扫荡的过期 pending 不计入,见
|
||||
# approval_timeout 模块 docstring。这里不用 evaluate
|
||||
#(它写日志,/metrics 高频轮询会刷屏),只算不记。
|
||||
try:
|
||||
_at_stats = _appr_timeout_stats(
|
||||
conn, project_ids=sorted(member_ids))
|
||||
approval_timeout = _appr_timeout_recommend(
|
||||
_at_stats["expired_count"],
|
||||
_at_stats["decided_count"])
|
||||
approval_timeout.update(_at_stats)
|
||||
except Exception:
|
||||
approval_timeout = None
|
||||
finally:
|
||||
conn.close()
|
||||
try:
|
||||
@@ -9880,6 +9993,7 @@ class CloudControlPlaneHandler(BaseHTTPRequestHandler):
|
||||
"queued": by_status.get("queued", 0),
|
||||
"running": by_status.get("running", 0),
|
||||
},
|
||||
"approval_timeout": approval_timeout,
|
||||
"failures_last_24h": failures_last_24h,
|
||||
"running_over_300s": running_over_300s,
|
||||
"workers": {
|
||||
|
||||
Reference in new issue
Block a user