fix: 登录限流下开通突发不再硬失败(节流 + 429 重试)

网关登录端点 20 次/分钟/源 IP,100 路突发开通首测仅 19
成功:同步重试波(2/4/8/16/32 秒)把线程推到下一窗口
开头重测也只到 39 成功。本批两层:
- 进程内登录节流:60 秒窗口最多 15 次,满了等最早名额
  滑出窗口再进(留 5 次给其它进程与人工登录);注册是
  best-effort,失败仍有 worker 首任务懒开通兜底,故不定
  时提前放弃;
- 429 退避重试保留:用户登录与管理员登录遇 429 时
  2/4/8/16/32 秒 + 抖动重试,最多 6 次,兜底跨进程竞争;
  管理员登录最终失败才清共享缓存。
测试:新增 100 并发单测(预算临时抬高,只验并发正确性)
与 3 个 429 重试单测、1 个节流窗口单测(fake clock);
单测 setUp/tearDown 清空进程级节流名额,免跨测累积拖慢
套件;全量 cloud 800(4 跳过)、host 69、deploy 19 全绿;
真网关 100 路突发重测 100/100 成功、墙钟 362 秒、临时
用户 100/100 已清。残留:硬保证需共享 Redis 节流,控制
面目前没有 Redis 客户端,未做。
This commit is contained in:
host committed 2026-10-27 23:09:22 +08:00
1 parent 494c3847d4
commit 8c27407b4a
2 files changed
+205 -17

No files matched your search

+111
View File
@@ -43,6 +43,8 @@ class FakeApi:
self.fail_balance = False
self.fail_create_user = False
self.create_user_hook = None
self.login_429_times = 0
self.admin_login_429_times = 0
def __call__(self, method, url, body=None, token=None):
with self._lock:
@@ -52,10 +54,18 @@ class FakeApi:
path = url.split("28084", 1)[-1] if "28084" in url else url
self.calls.append((method, path, body, token))
if path == "/api/v1/auth/login" and body.get("email") == "adm@x":
if self.admin_login_429_times > 0:
self.admin_login_429_times -= 1
return 429, {"message":
"Too many requests, please try again later"}
return 200, {"data": {"access_token": self.admin_token,
"refresh_token": "adm-ref",
"expires_in": 3600}}
if path == "/api/v1/auth/login":
if self.login_429_times > 0:
self.login_429_times -= 1
return 429, {"message":
"Too many requests, please try again later"}
return 200, {"data": {"access_token": self.user_token,
"refresh_token": "usr-ref",
"expires_in": 3600}}
@@ -122,12 +132,15 @@ class TestEnsureGatewayKey(unittest.TestCase):
self.api = FakeApi()
self.admin = Sub2apiAdmin("http://x:28084", "adm@x", "pw",
request=self.api)
# 节流名额是进程级共享状态:每个单测从空开始,避免跨测累积。
sub2api._login_slots.clear()
self.env = {"SUB2API_BASE_URL": "http://x:28084",
"SUB2API_GROUP_NAME": "codex-system"}
self.old = {k: os.environ.get(k) for k in self.env}
os.environ.update(self.env)
def tearDown(self):
sub2api._login_slots.clear()
for k, v in self.old.items():
if v is None:
os.environ.pop(k, None)
@@ -305,6 +318,104 @@ class TestEnsureGatewayKey(unittest.TestCase):
finally:
self._restore_admin_env(old)
def test_shared_admin_session_concurrent_100(self):
# 本测只验并发正确性(各 key 唯一、管理员登录 1 次),不验节流:
# 把预算临时抬高,免得 101 次登录被 15 次/分钟的节流拖到 7 分钟。
orig_budget = sub2api._LOGIN_BUDGET_PER_MINUTE
sub2api._LOGIN_BUDGET_PER_MINUTE = 1000
old = self._admin_env()
barrier = threading.Barrier(100)
results = []
errors = []
def run(i):
try:
barrier.wait(timeout=30)
conn = _conn()
key = ensure_user_gateway_key(
conn, "user-h%d" % i, request=self.api)
results.append(key)
except Exception as exc: # noqa: BLE001 - 汇总后断言
errors.append(exc)
try:
threads = [threading.Thread(target=run, args=(i,))
for i in range(100)]
for t in threads:
t.start()
for t in threads:
t.join(timeout=120)
self.assertEqual(errors, [])
self.assertEqual(len(results), 100)
self.assertTrue(all(k.startswith("sk-probe-")
for k in results))
self.assertEqual(len(set(results)), 100)
# 100 路突发冷缓存也只登录一次。
self.assertEqual(self._admin_login_count(), 1)
finally:
sub2api._LOGIN_BUDGET_PER_MINUTE = orig_budget
self._restore_admin_env(old)
def test_user_login_retries_on_429_then_succeeds(self):
self.api.login_429_times = 2
with mock.patch("time.sleep", return_value=None):
token, refresh = sub2api.user_login(
"http://x:28084", "u@x", "pw", request=self.api)
self.assertEqual(token, "usr-tok")
self.assertEqual(refresh, "usr-ref")
logins = [c for c in self.api.calls
if c[1] == "/api/v1/auth/login"
and (c[2] or {}).get("email") != "adm@x"]
self.assertEqual(len(logins), 3)
def test_user_login_429_exhausted_raises(self):
self.api.login_429_times = 100
with mock.patch("time.sleep", return_value=None):
with self.assertRaises(sub2api.Sub2apiError):
sub2api.user_login("http://x:28084", "u@x", "pw",
request=self.api)
logins = [c for c in self.api.calls
if c[1] == "/api/v1/auth/login"
and (c[2] or {}).get("email") != "adm@x"]
self.assertEqual(len(logins), 6)
def test_admin_login_retries_on_429(self):
self.api.admin_login_429_times = 2
admin = Sub2apiAdmin("http://x:28084", "adm@x", "pw",
request=self.api)
with mock.patch("time.sleep", return_value=None):
uid = admin.create_user("u@x", "pw", "u1")
self.assertEqual(uid, 11)
self.assertEqual(self._admin_login_count(), 3)
def test_login_slot_throttle_waits_for_window(self):
orig_budget = sub2api._LOGIN_BUDGET_PER_MINUTE
orig_window = sub2api._LOGIN_WINDOW_SECONDS
sub2api._LOGIN_BUDGET_PER_MINUTE = 2
sub2api._LOGIN_WINDOW_SECONDS = 60.0
sub2api._login_slots.clear()
now = [1000.0]
sleeps = []
def fake_time():
return now[0]
def fake_sleep(seconds):
sleeps.append(seconds)
now[0] += seconds
try:
with mock.patch("time.time", side_effect=fake_time), \
mock.patch("time.sleep", side_effect=fake_sleep):
sub2api._acquire_login_slot()
sub2api._acquire_login_slot()
sub2api._acquire_login_slot()
self.assertEqual(sleeps, [60.0])
finally:
sub2api._LOGIN_BUDGET_PER_MINUTE = orig_budget
sub2api._LOGIN_WINDOW_SECONDS = orig_window
sub2api._login_slots.clear()
class TestProviderCommand(unittest.TestCase):
def test_server_command_sub2api(self):