多用户生产级缺口批:sweep 鉴权、默认配额、默认 executor、 无价 token 上限
- POST /admin/sweep 改走任一项目 maintainer+ 门槛(与备份管理同口径),否则 403,记 admin_sweep 审计;threshold 下限钳 60 秒 - OAuth 建号 created=True 时写默认配额行(默认 $10/月,CLOUD_DEFAULT_QUOTA_USD_MICROS 可调,INSERT OR IGNORE 幂等);老用户无配额行不受影响 - 建任务未指定且无环境继承时默认 executor=docker(分支继承为空时同理);CLOUD_DEFAULT_EXECUTOR 可覆盖,空字符串恢复旧行为;命令台下拉默认改为 docker - 无价模型 token 上限:价目表外模型(默认 terra)成本 usd 为 NULL,USD 配额永不生效;有配额行的用户当月无价 token 达 CLOUD_QUOTA_UNPRICED_TOKEN_CAP(默认 2000 万/月)即停,服务端 402、worker 认领/重试/后续消息同口径;报表仍记 unpriced,不折假美元 - 测试:新增 test_multiuser_gaps 11 个 + test_worker_quota 3 个;全量 cloud 668、host 64、deploy 19 全过
This commit is contained in:
1 parent
c1f9313c10
commit
7e081211db
7 files changed
+580
-13
No files matched your search
+143
-4
@@ -58,7 +58,7 @@ from urllib.parse import parse_qs, quote, urlparse
|
||||
from cloud.auth import AuthStore
|
||||
from cloud.coordinator import parse_parts as _coord_parse_parts
|
||||
from cloud.coordinator import validate_parts as _coord_validate_parts
|
||||
from cloud.cost import CostStore
|
||||
from cloud.cost import CostStore, unpriced_token_cap
|
||||
from cloud.db import connect as _db_connect
|
||||
from cloud.db import create_database as _create_database
|
||||
from cloud.db import init_postgres_schema as _init_postgres_schema
|
||||
@@ -936,6 +936,61 @@ def _ensure_quotas_table(conn):
|
||||
"updated_at DOUBLE PRECISION)")
|
||||
|
||||
|
||||
def _default_executor():
|
||||
"""新任务未指定执行器时的默认值:docker。
|
||||
|
||||
空 executor(任意 worker 可认领)是多用户下的隔离缺口:
|
||||
任务可能落到隔离较弱的 local worker 上。默认 docker 后,
|
||||
只有显式 local/cloud 或系统创建(webhook/修复任务,executor
|
||||
为 NULL)才走别的路。环境变量 CLOUD_DEFAULT_EXECUTOR 可
|
||||
覆盖:设为空字符串即恢复旧行为(None,任意 worker),
|
||||
供单机运维在没有 docker worker 时兜底;非法值按 docker 计。
|
||||
"""
|
||||
raw = os.environ.get("CLOUD_DEFAULT_EXECUTOR", "docker")
|
||||
if raw is None:
|
||||
return "docker"
|
||||
raw = raw.strip()
|
||||
if raw == "":
|
||||
return None
|
||||
if raw in ("local", "docker", "cloud"):
|
||||
return raw
|
||||
return "docker"
|
||||
|
||||
|
||||
def _default_quota_usd_micros():
|
||||
"""OAuth 新用户建号时写入的默认配额(微美元)。
|
||||
|
||||
无配额行 = 不限额,所以 OAuth 自动建号过去等于无限额入驻。
|
||||
新号现在建号即写一行默认配额(老用户无配额行不受影响,
|
||||
行为不变)。环境变量 CLOUD_DEFAULT_QUOTA_USD_MICROS 可
|
||||
覆盖,默认 10_000_000($10/月);非法值按默认计。
|
||||
"""
|
||||
try:
|
||||
return max(0, int(os.environ.get(
|
||||
"CLOUD_DEFAULT_QUOTA_USD_MICROS", "10000000")))
|
||||
except (TypeError, ValueError):
|
||||
return 10000000
|
||||
|
||||
|
||||
def _apply_default_quota(conn, user_id):
|
||||
"""给新用户写默认配额行(已存在则不动,幂等)。
|
||||
|
||||
只在 OAuth 建号(created=True)时调:INSERT OR IGNORE
|
||||
保证重复回调不覆盖管理员后来改过的额度。返回 True 表示
|
||||
本次写了新行(调用方据此记 quota_default 审计)。
|
||||
"""
|
||||
_ensure_quotas_table(conn)
|
||||
cur = conn.execute(
|
||||
"INSERT OR IGNORE INTO user_quotas(user_id, quota_usd_micros,"
|
||||
" updated_at) VALUES(?, ?, ?)",
|
||||
(user_id, _default_quota_usd_micros(), time.time()))
|
||||
try:
|
||||
conn.commit()
|
||||
except Exception:
|
||||
pass
|
||||
return cur.rowcount == 1
|
||||
|
||||
|
||||
_ENV_BRANCH_RE = re.compile(r"^[A-Za-z0-9_.\-/]+$")
|
||||
|
||||
|
||||
@@ -1465,7 +1520,7 @@ class CloudControlPlaneHandler(BaseHTTPRequestHandler):
|
||||
elif path == "/tasks" and method == "GET":
|
||||
self._list_tasks(user_id, query)
|
||||
elif path == "/admin/sweep" and method == "POST":
|
||||
self._admin_sweep()
|
||||
self._admin_sweep(user_id)
|
||||
elif path == "/admin/backup" and method == "POST":
|
||||
self._admin_backup(user_id)
|
||||
elif path == "/admin/quotas" and method == "GET":
|
||||
@@ -2582,6 +2637,11 @@ class CloudControlPlaneHandler(BaseHTTPRequestHandler):
|
||||
return
|
||||
if executor is None and env["executor"]:
|
||||
executor = env["executor"]
|
||||
if executor is None:
|
||||
# 未指定且无环境继承时默认 docker:空 executor(任意
|
||||
# worker 可认领)会让任务落到隔离较弱的 local worker,
|
||||
# 是多用户下的隔离缺口;显式 local/cloud 不受影响。
|
||||
executor = _default_executor()
|
||||
# 附件绑定(upload_id 由 POST /uploads 发出):包必须存在、
|
||||
# 归属当前用户、未使用、24 小时内、暂存目录还在;cloud
|
||||
# 执行器没有 worktree 可 seed,直接拒绝。校验在 INSERT
|
||||
@@ -3288,7 +3348,8 @@ class CloudControlPlaneHandler(BaseHTTPRequestHandler):
|
||||
" VALUES(?, ?, ?, 'queued', NULL, '[]', ?, ?, ?, ?,"
|
||||
" NULL, ?, NULL, NULL, ?, ?, ?, ?, ?, ?)",
|
||||
(new_id, task["project_id"], message, task["priority"],
|
||||
user_id, now, now, task["executor"],
|
||||
user_id, now, now,
|
||||
task["executor"] or _default_executor(),
|
||||
task["goal_max_rounds"] or 3, task["model"],
|
||||
task["effort"], task["approval"], task["plan"] or 0,
|
||||
task_id),
|
||||
@@ -3809,6 +3870,56 @@ class CloudControlPlaneHandler(BaseHTTPRequestHandler):
|
||||
"remaining_usd_micros": (quota - used)
|
||||
if quota is not None else None}
|
||||
|
||||
def _quota_unpriced_tokens(self, user_id, month):
|
||||
"""当月无价模型 token 数(usd_micros IS NULL 的成本行,
|
||||
input + output 之和;cached 已含在 input 里,不重复计)。
|
||||
|
||||
与 _quota_status 同款 CostStore 懒建表做法,全新库上
|
||||
cost_records 不存在时先建表再查。无配额行时调用方不
|
||||
应以此挡人(见 _check_unpriced_token_cap)。
|
||||
"""
|
||||
conn = self._db()
|
||||
try:
|
||||
_cost_store = CostStore(self.server.db_path)
|
||||
try:
|
||||
_cost_store._conn.close()
|
||||
except Exception:
|
||||
pass
|
||||
row = conn.execute(
|
||||
"SELECT COALESCE(SUM(c.input_tokens + c.output_tokens), 0)"
|
||||
" FROM cost_records c JOIN tasks t ON c.task_id = t.id"
|
||||
" WHERE t.created_by = ? AND c.day LIKE ?"
|
||||
" AND c.usd_micros IS NULL",
|
||||
(user_id, month + "%")).fetchone()
|
||||
return int(row[0] if row is not None else 0)
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
def _check_unpriced_token_cap(self, user_id):
|
||||
"""无价 token 上限准入:超限返回 (cap, used, month),否则 None。
|
||||
|
||||
只有有配额行的用户才受限(无配额行 = 不限额,与 USD
|
||||
口径一致);USD 配额本身不计量无价模型,所以这道是
|
||||
补 USD 口径的洞,不是第二套计费。"""
|
||||
month = self._quota_month()
|
||||
conn = self._db()
|
||||
try:
|
||||
_ensure_quotas_table(conn)
|
||||
conn.commit()
|
||||
row = conn.execute(
|
||||
"SELECT quota_usd_micros FROM user_quotas"
|
||||
" WHERE user_id = ?",
|
||||
(user_id,)).fetchone()
|
||||
if row is None:
|
||||
return None
|
||||
finally:
|
||||
conn.close()
|
||||
used = self._quota_unpriced_tokens(user_id, month)
|
||||
cap = unpriced_token_cap()
|
||||
if used >= cap:
|
||||
return cap, used, month
|
||||
return None
|
||||
|
||||
def _check_quota(self, user_id):
|
||||
"""配额准入:放行返回 None;超限返回 (quota, used,
|
||||
month),调用方一律回 402——这是钱用完了,不是请求
|
||||
@@ -3829,6 +3940,14 @@ class CloudControlPlaneHandler(BaseHTTPRequestHandler):
|
||||
"quota_usd_micros": quota,
|
||||
"used_usd_micros": used})
|
||||
return True
|
||||
capped = self._check_unpriced_token_cap(user_id)
|
||||
if capped is not None:
|
||||
cap, used, month = capped
|
||||
self._send_json(402, {"error": "quota_exceeded",
|
||||
"month": month,
|
||||
"unpriced_token_cap": cap,
|
||||
"unpriced_tokens_used": used})
|
||||
return True
|
||||
return False
|
||||
|
||||
def _is_global_maintainer(self, user_id):
|
||||
@@ -4115,7 +4234,19 @@ class CloudControlPlaneHandler(BaseHTTPRequestHandler):
|
||||
self._send_json(200, task)
|
||||
|
||||
# -- 管理 ----------------------------------------------------------
|
||||
def _admin_sweep(self):
|
||||
def _admin_sweep(self, user_id):
|
||||
"""POST /admin/sweep:把超阈值的陈旧 running 任务标 interrupted。
|
||||
|
||||
权限:与 _admin_backup 同口径——调用方须在任一项目里
|
||||
是 maintainer 以上,否则 403。过去这里没有任何权限检查,
|
||||
任意持 key 用户都能跨用户改任务状态。threshold_seconds
|
||||
下限钳到 60 秒:0 会把所有 running 任务一口气扫掉,
|
||||
只能是管理员的显式紧急操作才需要那么小的值,而 60 秒
|
||||
以下的“陈旧”本就没有意义。扫完记 admin_sweep 审计。
|
||||
"""
|
||||
if not self._is_global_maintainer(user_id):
|
||||
self._send_json(403, {"error": "forbidden"})
|
||||
return
|
||||
body, err = self._read_json()
|
||||
if err:
|
||||
self._send_json(400, {"error": err})
|
||||
@@ -4127,7 +4258,11 @@ class CloudControlPlaneHandler(BaseHTTPRequestHandler):
|
||||
except (TypeError, ValueError):
|
||||
self._send_json(400, {"error": "threshold_seconds must be an integer"})
|
||||
return
|
||||
threshold = max(60, threshold)
|
||||
swept = sweep_stale_tasks(self.server.db_path, threshold)
|
||||
with _auth_store(self.server.db_path) as store:
|
||||
store.audit(user_id, "", "admin_sweep",
|
||||
"threshold=%d swept=%d" % (threshold, len(swept)))
|
||||
self._send_json(200, {"swept": swept})
|
||||
|
||||
def _admin_backup(self, user_id):
|
||||
@@ -4366,6 +4501,10 @@ class CloudControlPlaneHandler(BaseHTTPRequestHandler):
|
||||
auth, provider, identity)
|
||||
auth.audit(local_id, "", "oauth_login",
|
||||
"%s:%s" % (provider, identity.get("external_id")))
|
||||
if created and _apply_default_quota(auth.conn, local_id):
|
||||
auth.audit(local_id, "", "quota_default",
|
||||
"quota_usd_micros=%d"
|
||||
% _default_quota_usd_micros())
|
||||
if not created:
|
||||
auth.audit(local_id, "", "api_key_issued", provider)
|
||||
return local_id, created, api_key_plain, rotated, identity, False
|
||||
|
||||
Reference in new issue
Block a user