feat: cost/quota set — dual-column priced reports, creator split, hard cutoff, approval-timeout auto-threshold
- CostStore: per-model breakdown, report_by_creator (JOIN tasks.created_by), dual-column reports with priced_usd_micros, unpriced_models listed separately and a billing note (proxy price, not the relay bill); _report itself untouched, old exact-key tests keep passing. - Hard cutoff: cost.quota_cutoff_reason is the single USD/unpriced-cap decision; worker._quota_exceeded now delegates to it (identical messages); claim gate, turn boundary and poll kill stop spend and record interrupted + quota reason + audit; auto-resume re-checks live quota before requeueing. - Approval timeout (G7 ops metric): cloud/approval_timeout.py — default stays 600s, >20% timeout rate recommends 1800s, <5% keeps, mid-range keeps; stats + recommendation + info log; wired into /metrics (member projects, no log spam) and a global-admin /admin/ops/approval-timeout endpoint (sweeps overdue first). - Server: task/project cost endpoints return dual-column reports; new GET /users/me/cost and GET /admin/cost-by-creator. - Tests: cloud/tests/test_cost_quota.py, 10 tests.
This commit is contained in:
1 parent
cc32f51502
commit
e079b9b44a
5 files changed
+871
-12
No files matched your search
@@ -0,0 +1,145 @@
|
||||
"""审批超时率自动调阈(G7 运维指标,v1.1 契约口径)。
|
||||
|
||||
只依赖标准库与 cloud.approvals 的建表函数,不 import server /
|
||||
worker,避免循环 import。
|
||||
|
||||
口径(与 v1.1 冻结条一致):
|
||||
- 默认审批超时仍是 10 分钟(600 秒)。注意:这是本调阈口径
|
||||
的默认值,不是 approvals.py elicitation 队列的 3600 秒
|
||||
默认值——两条审批线各有默认值,本模块绝不改写后者。
|
||||
- 超时率 = 已终态审批项中 status='expired' 的占比;已终态 =
|
||||
approved + denied + expired。pending 不计入分母(还没结果
|
||||
的项不能算超时也不能算准时);未扫荡的过期 pending 同样
|
||||
不计入——扫荡发生在审批列表与运维端点,统计前应先扫荡。
|
||||
- 超时率 > 20% 时建议调到 30 分钟(1800 秒);< 5% 时不动;
|
||||
5%~20% 之间同样不动(只升不降,降阈须人工决定,本模块
|
||||
绝不自动把超时缩回去)。边界值本身不动:>0.20 才升,
|
||||
==0.20 与 ==0.05 都按不动处理。
|
||||
- 本模块只算建议并记日志,不自动改写任何默认值或已建审批
|
||||
项的 expires_at:调阈生效须运维按建议改配置后重启/重载,
|
||||
建议值进 /metrics 与运维端点,审计靠日志行(见
|
||||
evaluate_approval_timeout 的 info 日志)。
|
||||
"""
|
||||
|
||||
import logging
|
||||
|
||||
from cloud.approvals import ensure_approvals_table
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
DEFAULT_TIMEOUT_SECONDS = 600
|
||||
RAISED_TIMEOUT_SECONDS = 1800
|
||||
RAISE_THRESHOLD = 0.20
|
||||
KEEP_THRESHOLD = 0.05
|
||||
|
||||
|
||||
def timeout_rate(expired_count, decided_count):
|
||||
"""超时率:已终态为 0 时回 None(不是 0%,是算不出来)。"""
|
||||
try:
|
||||
decided = int(decided_count)
|
||||
expired = int(expired_count)
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
if decided <= 0:
|
||||
return None
|
||||
if expired < 0:
|
||||
expired = 0
|
||||
if expired > decided:
|
||||
expired = decided
|
||||
return expired / decided
|
||||
|
||||
|
||||
def recommend_approval_timeout(expired_count, decided_count,
|
||||
current_seconds=DEFAULT_TIMEOUT_SECONDS):
|
||||
"""纯计算函数:输入计数,输出建议(无 I/O,可单测三档)。
|
||||
|
||||
返回键:expired_count、decided_count、timeout_rate、
|
||||
current_seconds、recommended_seconds、action(raise_to_30m
|
||||
或 keep)、reason。current_seconds 非法时按默认值计,不抛。
|
||||
"""
|
||||
try:
|
||||
current = int(current_seconds)
|
||||
except (TypeError, ValueError):
|
||||
current = DEFAULT_TIMEOUT_SECONDS
|
||||
if current <= 0:
|
||||
current = DEFAULT_TIMEOUT_SECONDS
|
||||
rate = timeout_rate(expired_count, decided_count)
|
||||
if rate is not None and rate > RAISE_THRESHOLD:
|
||||
action = "raise_to_30m"
|
||||
recommended = RAISED_TIMEOUT_SECONDS
|
||||
reason = ("approval timeout rate %.4f > %.2f, "
|
||||
"recommend %d seconds" % (rate, RAISE_THRESHOLD,
|
||||
recommended))
|
||||
else:
|
||||
action = "keep"
|
||||
recommended = current
|
||||
if rate is None:
|
||||
reason = "no decided approvals, keep %d seconds" % current
|
||||
else:
|
||||
reason = ("approval timeout rate %.4f <= %.2f, "
|
||||
"keep %d seconds" % (rate, RAISE_THRESHOLD,
|
||||
current))
|
||||
return {
|
||||
"expired_count": int(expired_count or 0),
|
||||
"decided_count": int(decided_count or 0),
|
||||
"timeout_rate": rate,
|
||||
"current_seconds": current,
|
||||
"recommended_seconds": recommended,
|
||||
"action": action,
|
||||
"reason": reason,
|
||||
}
|
||||
|
||||
|
||||
def approval_timeout_stats(conn, project_ids=None):
|
||||
"""从 task_approvals 统计已终态与超时数(纯读,不扫荡)。
|
||||
|
||||
project_ids 为 None 时全库统计;否则只计这些项目(/metrics
|
||||
的成员项目口径)。表不存在时先幂等建表(空表即零统计)。
|
||||
"""
|
||||
ensure_approvals_table(conn)
|
||||
query = ("SELECT COUNT(*), COALESCE(SUM(CASE WHEN status = 'expired'"
|
||||
" THEN 1 ELSE 0 END), 0), COALESCE(SUM(CASE WHEN"
|
||||
" status = 'pending' THEN 1 ELSE 0 END), 0)"
|
||||
" FROM task_approvals WHERE status IN"
|
||||
" ('approved', 'denied', 'expired', 'pending')")
|
||||
args = []
|
||||
if project_ids is not None:
|
||||
ids = sorted(set(project_ids))
|
||||
if not ids:
|
||||
return {"decided_count": 0, "expired_count": 0,
|
||||
"pending_count": 0}
|
||||
query += " AND project_id IN (%s)" % ", ".join(
|
||||
"?" for _ in ids)
|
||||
args.extend(ids)
|
||||
row = conn.execute(query, args).fetchone()
|
||||
decided = int(row[0] or 0) - int(row[2] or 0)
|
||||
return {
|
||||
"decided_count": decided,
|
||||
"expired_count": int(row[1] or 0),
|
||||
"pending_count": int(row[2] or 0),
|
||||
}
|
||||
|
||||
|
||||
def evaluate_approval_timeout(conn, current_seconds=DEFAULT_TIMEOUT_SECONDS,
|
||||
project_ids=None, log=None):
|
||||
"""统计 + 建议 + 日志的一站式运维入口。
|
||||
|
||||
返回 approval_timeout_stats 与 recommend_approval_timeout
|
||||
的合并 dict,并写一条 info 日志(log 未给时用模块 logger)。
|
||||
调用方在需要“过期 pending 也算超时”时须先调 approvals
|
||||
的 expire_overdue 扫荡——本函数自己不写库。
|
||||
"""
|
||||
stats = approval_timeout_stats(conn, project_ids=project_ids)
|
||||
result = recommend_approval_timeout(
|
||||
stats["expired_count"], stats["decided_count"],
|
||||
current_seconds=current_seconds)
|
||||
result.update(stats)
|
||||
(log or logger).info(
|
||||
"approval_timeout metric: decided=%d expired=%d pending=%d "
|
||||
"rate=%s action=%s recommended_seconds=%d",
|
||||
result["decided_count"], result["expired_count"],
|
||||
result["pending_count"],
|
||||
("%.4f" % result["timeout_rate"]
|
||||
if result["timeout_rate"] is not None else "unknown"),
|
||||
result["action"], result["recommended_seconds"])
|
||||
return result
|
||||
Reference in new issue
Block a user