- CostStore: per-model breakdown, report_by_creator (JOIN tasks.created_by), dual-column reports with priced_usd_micros, unpriced_models listed separately and a billing note (proxy price, not the relay bill); _report itself untouched, old exact-key tests keep passing. - Hard cutoff: cost.quota_cutoff_reason is the single USD/unpriced-cap decision; worker._quota_exceeded now delegates to it (identical messages); claim gate, turn boundary and poll kill stop spend and record interrupted + quota reason + audit; auto-resume re-checks live quota before requeueing. - Approval timeout (G7 ops metric): cloud/approval_timeout.py — default stays 600s, >20% timeout rate recommends 1800s, <5% keeps, mid-range keeps; stats + recommendation + info log; wired into /metrics (member projects, no log spam) and a global-admin /admin/ops/approval-timeout endpoint (sweeps overdue first). - Server: task/project cost endpoints return dual-column reports; new GET /users/me/cost and GET /admin/cost-by-creator. - Tests: cloud/tests/test_cost_quota.py, 10 tests.
146 lines
5.8 KiB
Python
146 lines
5.8 KiB
Python
"""审批超时率自动调阈(G7 运维指标,v1.1 契约口径)。
|
||
|
||
只依赖标准库与 cloud.approvals 的建表函数,不 import server /
|
||
worker,避免循环 import。
|
||
|
||
口径(与 v1.1 冻结条一致):
|
||
- 默认审批超时仍是 10 分钟(600 秒)。注意:这是本调阈口径
|
||
的默认值,不是 approvals.py elicitation 队列的 3600 秒
|
||
默认值——两条审批线各有默认值,本模块绝不改写后者。
|
||
- 超时率 = 已终态审批项中 status='expired' 的占比;已终态 =
|
||
approved + denied + expired。pending 不计入分母(还没结果
|
||
的项不能算超时也不能算准时);未扫荡的过期 pending 同样
|
||
不计入——扫荡发生在审批列表与运维端点,统计前应先扫荡。
|
||
- 超时率 > 20% 时建议调到 30 分钟(1800 秒);< 5% 时不动;
|
||
5%~20% 之间同样不动(只升不降,降阈须人工决定,本模块
|
||
绝不自动把超时缩回去)。边界值本身不动:>0.20 才升,
|
||
==0.20 与 ==0.05 都按不动处理。
|
||
- 本模块只算建议并记日志,不自动改写任何默认值或已建审批
|
||
项的 expires_at:调阈生效须运维按建议改配置后重启/重载,
|
||
建议值进 /metrics 与运维端点,审计靠日志行(见
|
||
evaluate_approval_timeout 的 info 日志)。
|
||
"""
|
||
|
||
import logging
|
||
|
||
from cloud.approvals import ensure_approvals_table
|
||
|
||
logger = logging.getLogger(__name__)
|
||
|
||
DEFAULT_TIMEOUT_SECONDS = 600
|
||
RAISED_TIMEOUT_SECONDS = 1800
|
||
RAISE_THRESHOLD = 0.20
|
||
KEEP_THRESHOLD = 0.05
|
||
|
||
|
||
def timeout_rate(expired_count, decided_count):
|
||
"""超时率:已终态为 0 时回 None(不是 0%,是算不出来)。"""
|
||
try:
|
||
decided = int(decided_count)
|
||
expired = int(expired_count)
|
||
except (TypeError, ValueError):
|
||
return None
|
||
if decided <= 0:
|
||
return None
|
||
if expired < 0:
|
||
expired = 0
|
||
if expired > decided:
|
||
expired = decided
|
||
return expired / decided
|
||
|
||
|
||
def recommend_approval_timeout(expired_count, decided_count,
|
||
current_seconds=DEFAULT_TIMEOUT_SECONDS):
|
||
"""纯计算函数:输入计数,输出建议(无 I/O,可单测三档)。
|
||
|
||
返回键:expired_count、decided_count、timeout_rate、
|
||
current_seconds、recommended_seconds、action(raise_to_30m
|
||
或 keep)、reason。current_seconds 非法时按默认值计,不抛。
|
||
"""
|
||
try:
|
||
current = int(current_seconds)
|
||
except (TypeError, ValueError):
|
||
current = DEFAULT_TIMEOUT_SECONDS
|
||
if current <= 0:
|
||
current = DEFAULT_TIMEOUT_SECONDS
|
||
rate = timeout_rate(expired_count, decided_count)
|
||
if rate is not None and rate > RAISE_THRESHOLD:
|
||
action = "raise_to_30m"
|
||
recommended = RAISED_TIMEOUT_SECONDS
|
||
reason = ("approval timeout rate %.4f > %.2f, "
|
||
"recommend %d seconds" % (rate, RAISE_THRESHOLD,
|
||
recommended))
|
||
else:
|
||
action = "keep"
|
||
recommended = current
|
||
if rate is None:
|
||
reason = "no decided approvals, keep %d seconds" % current
|
||
else:
|
||
reason = ("approval timeout rate %.4f <= %.2f, "
|
||
"keep %d seconds" % (rate, RAISE_THRESHOLD,
|
||
current))
|
||
return {
|
||
"expired_count": int(expired_count or 0),
|
||
"decided_count": int(decided_count or 0),
|
||
"timeout_rate": rate,
|
||
"current_seconds": current,
|
||
"recommended_seconds": recommended,
|
||
"action": action,
|
||
"reason": reason,
|
||
}
|
||
|
||
|
||
def approval_timeout_stats(conn, project_ids=None):
|
||
"""从 task_approvals 统计已终态与超时数(纯读,不扫荡)。
|
||
|
||
project_ids 为 None 时全库统计;否则只计这些项目(/metrics
|
||
的成员项目口径)。表不存在时先幂等建表(空表即零统计)。
|
||
"""
|
||
ensure_approvals_table(conn)
|
||
query = ("SELECT COUNT(*), COALESCE(SUM(CASE WHEN status = 'expired'"
|
||
" THEN 1 ELSE 0 END), 0), COALESCE(SUM(CASE WHEN"
|
||
" status = 'pending' THEN 1 ELSE 0 END), 0)"
|
||
" FROM task_approvals WHERE status IN"
|
||
" ('approved', 'denied', 'expired', 'pending')")
|
||
args = []
|
||
if project_ids is not None:
|
||
ids = sorted(set(project_ids))
|
||
if not ids:
|
||
return {"decided_count": 0, "expired_count": 0,
|
||
"pending_count": 0}
|
||
query += " AND project_id IN (%s)" % ", ".join(
|
||
"?" for _ in ids)
|
||
args.extend(ids)
|
||
row = conn.execute(query, args).fetchone()
|
||
decided = int(row[0] or 0) - int(row[2] or 0)
|
||
return {
|
||
"decided_count": decided,
|
||
"expired_count": int(row[1] or 0),
|
||
"pending_count": int(row[2] or 0),
|
||
}
|
||
|
||
|
||
def evaluate_approval_timeout(conn, current_seconds=DEFAULT_TIMEOUT_SECONDS,
|
||
project_ids=None, log=None):
|
||
"""统计 + 建议 + 日志的一站式运维入口。
|
||
|
||
返回 approval_timeout_stats 与 recommend_approval_timeout
|
||
的合并 dict,并写一条 info 日志(log 未给时用模块 logger)。
|
||
调用方在需要“过期 pending 也算超时”时须先调 approvals
|
||
的 expire_overdue 扫荡——本函数自己不写库。
|
||
"""
|
||
stats = approval_timeout_stats(conn, project_ids=project_ids)
|
||
result = recommend_approval_timeout(
|
||
stats["expired_count"], stats["decided_count"],
|
||
current_seconds=current_seconds)
|
||
result.update(stats)
|
||
(log or logger).info(
|
||
"approval_timeout metric: decided=%d expired=%d pending=%d "
|
||
"rate=%s action=%s recommended_seconds=%d",
|
||
result["decided_count"], result["expired_count"],
|
||
result["pending_count"],
|
||
("%.4f" % result["timeout_rate"]
|
||
if result["timeout_rate"] is not None else "unknown"),
|
||
result["action"], result["recommended_seconds"])
|
||
return result
|