Files
codex-app-server/cloud/approval_timeout.py
host 143ab30094 feat: cost/quota set — dual-column priced reports, creator split, hard cutoff, approval-timeout auto-threshold
- CostStore: per-model breakdown, report_by_creator (JOIN tasks.created_by),
  dual-column reports with priced_usd_micros, unpriced_models listed
  separately and a billing note (proxy price, not the relay bill);
  _report itself untouched, old exact-key tests keep passing.
- Hard cutoff: cost.quota_cutoff_reason is the single USD/unpriced-cap
  decision; worker._quota_exceeded now delegates to it (identical
  messages); claim gate, turn boundary and poll kill stop spend and
  record interrupted + quota reason + audit; auto-resume re-checks
  live quota before requeueing.
- Approval timeout (G7 ops metric): cloud/approval_timeout.py —
  default stays 600s, >20% timeout rate recommends 1800s, <5% keeps,
  mid-range keeps; stats + recommendation + info log; wired into
  /metrics (member projects, no log spam) and a global-admin
  /admin/ops/approval-timeout endpoint (sweeps overdue first).
- Server: task/project cost endpoints return dual-column reports;
  new GET /users/me/cost and GET /admin/cost-by-creator.
- Tests: cloud/tests/test_cost_quota.py, 10 tests.
2026-11-03 02:49:25 +00:00

146 lines
5.8 KiB
Python
Raw Permalink Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
"""审批超时率自动调阈(G7 运维指标,v1.1 契约口径)。
只依赖标准库与 cloud.approvals 的建表函数,不 import server /
worker,避免循环 import。
口径(与 v1.1 冻结条一致):
- 默认审批超时仍是 10 分钟(600 秒)。注意:这是本调阈口径
的默认值,不是 approvals.py elicitation 队列的 3600 秒
默认值——两条审批线各有默认值,本模块绝不改写后者。
- 超时率 = 已终态审批项中 status='expired' 的占比;已终态 =
approved + denied + expired。pending 不计入分母(还没结果
的项不能算超时也不能算准时);未扫荡的过期 pending 同样
不计入——扫荡发生在审批列表与运维端点,统计前应先扫荡。
- 超时率 > 20% 时建议调到 30 分钟(1800 秒);< 5% 时不动;
5%~20% 之间同样不动(只升不降,降阈须人工决定,本模块
绝不自动把超时缩回去)。边界值本身不动:>0.20 才升,
==0.20 与 ==0.05 都按不动处理。
- 本模块只算建议并记日志,不自动改写任何默认值或已建审批
项的 expires_at:调阈生效须运维按建议改配置后重启/重载,
建议值进 /metrics 与运维端点,审计靠日志行(见
evaluate_approval_timeout 的 info 日志)。
"""
import logging
from cloud.approvals import ensure_approvals_table
logger = logging.getLogger(__name__)
DEFAULT_TIMEOUT_SECONDS = 600
RAISED_TIMEOUT_SECONDS = 1800
RAISE_THRESHOLD = 0.20
KEEP_THRESHOLD = 0.05
def timeout_rate(expired_count, decided_count):
"""超时率:已终态为 0 时回 None(不是 0%,是算不出来)。"""
try:
decided = int(decided_count)
expired = int(expired_count)
except (TypeError, ValueError):
return None
if decided <= 0:
return None
if expired < 0:
expired = 0
if expired > decided:
expired = decided
return expired / decided
def recommend_approval_timeout(expired_count, decided_count,
current_seconds=DEFAULT_TIMEOUT_SECONDS):
"""纯计算函数:输入计数,输出建议(无 I/O,可单测三档)。
返回键:expired_count、decided_count、timeout_rate、
current_seconds、recommended_seconds、action(raise_to_30m
或 keep)、reason。current_seconds 非法时按默认值计,不抛。
"""
try:
current = int(current_seconds)
except (TypeError, ValueError):
current = DEFAULT_TIMEOUT_SECONDS
if current <= 0:
current = DEFAULT_TIMEOUT_SECONDS
rate = timeout_rate(expired_count, decided_count)
if rate is not None and rate > RAISE_THRESHOLD:
action = "raise_to_30m"
recommended = RAISED_TIMEOUT_SECONDS
reason = ("approval timeout rate %.4f > %.2f, "
"recommend %d seconds" % (rate, RAISE_THRESHOLD,
recommended))
else:
action = "keep"
recommended = current
if rate is None:
reason = "no decided approvals, keep %d seconds" % current
else:
reason = ("approval timeout rate %.4f <= %.2f, "
"keep %d seconds" % (rate, RAISE_THRESHOLD,
current))
return {
"expired_count": int(expired_count or 0),
"decided_count": int(decided_count or 0),
"timeout_rate": rate,
"current_seconds": current,
"recommended_seconds": recommended,
"action": action,
"reason": reason,
}
def approval_timeout_stats(conn, project_ids=None):
"""从 task_approvals 统计已终态与超时数(纯读,不扫荡)。
project_ids 为 None 时全库统计;否则只计这些项目(/metrics
的成员项目口径)。表不存在时先幂等建表(空表即零统计)。
"""
ensure_approvals_table(conn)
query = ("SELECT COUNT(*), COALESCE(SUM(CASE WHEN status = 'expired'"
" THEN 1 ELSE 0 END), 0), COALESCE(SUM(CASE WHEN"
" status = 'pending' THEN 1 ELSE 0 END), 0)"
" FROM task_approvals WHERE status IN"
" ('approved', 'denied', 'expired', 'pending')")
args = []
if project_ids is not None:
ids = sorted(set(project_ids))
if not ids:
return {"decided_count": 0, "expired_count": 0,
"pending_count": 0}
query += " AND project_id IN (%s)" % ", ".join(
"?" for _ in ids)
args.extend(ids)
row = conn.execute(query, args).fetchone()
decided = int(row[0] or 0) - int(row[2] or 0)
return {
"decided_count": decided,
"expired_count": int(row[1] or 0),
"pending_count": int(row[2] or 0),
}
def evaluate_approval_timeout(conn, current_seconds=DEFAULT_TIMEOUT_SECONDS,
project_ids=None, log=None):
"""统计 + 建议 + 日志的一站式运维入口。
返回 approval_timeout_stats 与 recommend_approval_timeout
的合并 dict,并写一条 info 日志(log 未给时用模块 logger)。
调用方在需要“过期 pending 也算超时”时须先调 approvals
的 expire_overdue 扫荡——本函数自己不写库。
"""
stats = approval_timeout_stats(conn, project_ids=project_ids)
result = recommend_approval_timeout(
stats["expired_count"], stats["decided_count"],
current_seconds=current_seconds)
result.update(stats)
(log or logger).info(
"approval_timeout metric: decided=%d expired=%d pending=%d "
"rate=%s action=%s recommended_seconds=%d",
result["decided_count"], result["expired_count"],
result["pending_count"],
("%.4f" % result["timeout_rate"]
if result["timeout_rate"] is not None else "unknown"),
result["action"], result["recommended_seconds"])
return result