fix: 登录限流下开通突发不再硬失败(节流 + 429 重试)
网关登录端点 20 次/分钟/源 IP,100 路突发开通首测仅 19 成功:同步重试波(2/4/8/16/32 秒)把线程推到下一窗口 开头重测也只到 39 成功。本批两层: - 进程内登录节流:60 秒窗口最多 15 次,满了等最早名额 滑出窗口再进(留 5 次给其它进程与人工登录);注册是 best-effort,失败仍有 worker 首任务懒开通兜底,故不定 时提前放弃; - 429 退避重试保留:用户登录与管理员登录遇 429 时 2/4/8/16/32 秒 + 抖动重试,最多 6 次,兜底跨进程竞争; 管理员登录最终失败才清共享缓存。 测试:新增 100 并发单测(预算临时抬高,只验并发正确性) 与 3 个 429 重试单测、1 个节流窗口单测(fake clock); 单测 setUp/tearDown 清空进程级节流名额,免跨测累积拖慢 套件;全量 cloud 800(4 跳过)、host 69、deploy 19 全绿; 真网关 100 路突发重测 100/100 成功、墙钟 362 秒、临时 用户 100/100 已清。残留:硬保证需共享 Redis 节流,控制 面目前没有 Redis 客户端,未做。
This commit is contained in:
1 parent
494c3847d4
commit
8c27407b4a
2 files changed
+205
-17
No files matched your search
+94
-17
@@ -13,8 +13,10 @@
|
||||
便于单测(见 cloud/tests/test_sub2api.py)。
|
||||
"""
|
||||
|
||||
import collections
|
||||
import json
|
||||
import os
|
||||
import random
|
||||
import secrets
|
||||
import threading
|
||||
import time
|
||||
@@ -162,6 +164,67 @@ def shared_admin_session(request=None):
|
||||
return admin
|
||||
|
||||
|
||||
def _is_rate_limited_error(exc):
|
||||
if getattr(exc, "status", None) == 429:
|
||||
return True
|
||||
return "too many requests" in str(exc).lower()
|
||||
|
||||
|
||||
# 登录重试:网关登录端点 20 次/分钟/IP(固定窗口),突发注册
|
||||
# 必然撞限;退避 2/4/8/16/32 秒(+0-2 秒抖动)可跨过 60 秒窗口。
|
||||
_LOGIN_RETRY_ATTEMPTS = 6
|
||||
_LOGIN_RETRY_BASE_SECONDS = 2.0
|
||||
|
||||
|
||||
def _call_with_login_retry(fn):
|
||||
"""登录类调用:先取进程内节流名额,再遇 429 退避重试。
|
||||
|
||||
节流名额保证单进程内登录不超过预算(网关 20 次/分钟/IP,
|
||||
留 5 次余量给其它进程与人工登录);重试兜底跨进程竞争。
|
||||
其它错误与耗尽次数后原错抛出。
|
||||
"""
|
||||
_acquire_login_slot()
|
||||
delay = _LOGIN_RETRY_BASE_SECONDS
|
||||
for attempt in range(_LOGIN_RETRY_ATTEMPTS):
|
||||
try:
|
||||
return fn()
|
||||
except Sub2apiError as exc:
|
||||
if not _is_rate_limited_error(exc) \
|
||||
or attempt == _LOGIN_RETRY_ATTEMPTS - 1:
|
||||
raise
|
||||
time.sleep(delay + random.uniform(0, 2))
|
||||
delay *= 2
|
||||
|
||||
|
||||
# 进程内登录节流:60 秒窗口内最多 _LOGIN_BUDGET_PER_MINUTE 次。
|
||||
# 只协调本进程;多进程突发仍靠 429 重试兜底(硬保证需共享
|
||||
# Redis 节流,控制面目前没有 Redis 客户端,未做)。
|
||||
_LOGIN_BUDGET_PER_MINUTE = 15
|
||||
_LOGIN_WINDOW_SECONDS = 60.0
|
||||
_login_slots = collections.deque()
|
||||
_login_slots_lock = threading.Lock()
|
||||
|
||||
|
||||
def _acquire_login_slot():
|
||||
"""等一个登录名额:窗口满了就等最早一次滑出窗口再进。
|
||||
|
||||
等待时长由排队位置决定(突发 100 路约 7 分钟);注册是
|
||||
best-effort 语义,失败仍有 worker 首任务懒开通兜底,故不设
|
||||
提前放弃——放弃会把有序排队打回同步重试波。
|
||||
"""
|
||||
while True:
|
||||
with _login_slots_lock:
|
||||
now = time.time()
|
||||
while _login_slots \
|
||||
and _login_slots[0] <= now - _LOGIN_WINDOW_SECONDS:
|
||||
_login_slots.popleft()
|
||||
if len(_login_slots) < _LOGIN_BUDGET_PER_MINUTE:
|
||||
_login_slots.append(now)
|
||||
return
|
||||
wait = _login_slots[0] + _LOGIN_WINDOW_SECONDS - now
|
||||
time.sleep(max(wait, 0))
|
||||
|
||||
|
||||
class Sub2apiAdmin:
|
||||
"""管理员会话:登录、建/删用户、查组。token 自动刷新一次。
|
||||
|
||||
@@ -193,15 +256,22 @@ class Sub2apiAdmin:
|
||||
request=request)
|
||||
|
||||
def _login(self):
|
||||
st, data = self.request(
|
||||
"POST", self.origin + "/api/v1/auth/login",
|
||||
{"email": self.email, "password": self.password})
|
||||
payload = (data or {}).get("data") or {}
|
||||
if st != 200 or not payload.get("access_token"):
|
||||
def _once():
|
||||
st, data = self.request(
|
||||
"POST", self.origin + "/api/v1/auth/login",
|
||||
{"email": self.email, "password": self.password})
|
||||
payload = (data or {}).get("data") or {}
|
||||
if st != 200 or not payload.get("access_token"):
|
||||
raise Sub2apiError(
|
||||
"sub2api 管理员登录失败: %s"
|
||||
% ((data or {}).get("message") or st), status=st)
|
||||
return payload
|
||||
|
||||
try:
|
||||
payload = _call_with_login_retry(_once)
|
||||
except Sub2apiError:
|
||||
_evict_shared_session(self)
|
||||
raise Sub2apiError(
|
||||
"sub2api 管理员登录失败: %s"
|
||||
% ((data or {}).get("message") or st), status=st)
|
||||
raise
|
||||
self._token = payload["access_token"]
|
||||
self._refresh = payload.get("refresh_token")
|
||||
try:
|
||||
@@ -298,16 +368,23 @@ class Sub2apiAdmin:
|
||||
|
||||
|
||||
def user_login(origin, email, password, request=None):
|
||||
"""以 sub2api 用户身份登录:返回 (access_token, refresh_token)。"""
|
||||
"""以 sub2api 用户身份登录:返回 (access_token, refresh_token)。
|
||||
|
||||
遇网关登录限流(429)时退避重试,见 _call_with_login_retry。
|
||||
"""
|
||||
req = request or _default_request
|
||||
st, data = req("POST", origin + "/api/v1/auth/login",
|
||||
{"email": email, "password": password})
|
||||
payload = (data or {}).get("data") or {}
|
||||
if st != 200 or not payload.get("access_token"):
|
||||
raise Sub2apiError(
|
||||
"sub2api 用户登录失败: %s"
|
||||
% ((data or {}).get("message") or st), status=st)
|
||||
return payload["access_token"], payload.get("refresh_token")
|
||||
|
||||
def _once():
|
||||
st, data = req("POST", origin + "/api/v1/auth/login",
|
||||
{"email": email, "password": password})
|
||||
payload = (data or {}).get("data") or {}
|
||||
if st != 200 or not payload.get("access_token"):
|
||||
raise Sub2apiError(
|
||||
"sub2api 用户登录失败: %s"
|
||||
% ((data or {}).get("message") or st), status=st)
|
||||
return payload["access_token"], payload.get("refresh_token")
|
||||
|
||||
return _call_with_login_retry(_once)
|
||||
|
||||
|
||||
def user_create_key(origin, user_token, name, group_id, request=None):
|
||||
|
||||
Reference in new issue
Block a user