feat: G7 分层 fail-open/fail-closed + G8 Docker 轮询掐断 + G9 worker 忽略 host 预算 + G13/G14 归属(第三批)

- G7:认领门 closed 时查配额异常回 quota_check_unavailable、标 interrupted(可重试/续跑);轮询与轮次边界用 open,异常时回 quota_check_failed——新回合边界拒绝,执行中轮询经 _quota_hit 闭包过滤此串、只记 warning 不杀在跑任务,确认超限才掐断;查询函数本身不吞异常冒充不限额。
- G8:Docker 路径加 _docker_call_with_quota_poll(后台线程跑阻塞式 docker 调用,主循环每 quota_poll_interval_seconds 缺省 30 秒查一次,确认超限按容器名真杀容器,结果改写 quota_exceeded 走统一收尾:interrupted、成本照记、续跑消息保留,配额命中后不走 ALREADY_EXISTS 回落);同步路径保持边界检查。
- G9:worker 路径默认忽略 host max_daily_turns(host.py 顶层 --ignore-host-budget 只清该项,Local/Docker 执行器在子命令前追加,worker CLI 缺省开、--no-ignore-host-budget 关);standalone 直接跑 host.py 时预算照常。
- G13/G14:/pr 不再走 system 豁免,建任务前按请求用户查配额(402/503 均不排队不改状态),队列路 pr_jobs.created_by 记请求用户(老表 ALTER 补列),worker 跑 pr 任务时审计操作人用 created_by(老行回落 system)。注意:G14 原案的 system 豁免(服务端与 worker 写死)与 P1 冲突(P1:system 按普通属主走默认额度,不另设无限额系统预算),本次 rebase 未合入该豁免,system 豁免需另行决策。
- 配额口径与 P0/P1 对齐:无配额行按默认额度(非不限额),只有显式 NULL 行才不限额;无价上限只计 source='actual' 行;缺属主(含父任务回查仍无)拒绝新回合。
- README 配额节同步以上口径与 G15 接受现状(best-effort 超支、UTC 月切分、在跑任务只在轮询/轮次边界响应配额改动)。G4 与 G15 不在本批。
This commit is contained in:
host committed 2026-10-28 11:14:10 +08:00
1 parent 5883169c82
commit e31fcc6cba
11 files changed
+786 -176

No files matched your search

+94 -2
View File
@@ -17,14 +17,16 @@ import threading
import unittest
from datetime import date, datetime, timedelta, timezone
from pathlib import Path
from types import SimpleNamespace
from unittest import mock
try:
from cloud.auth import AuthStore
from cloud.server import create_server
from cloud.server import CloudControlPlaneHandler, create_server
except ImportError: # discover -s cloud/tests 时把项目根目录补进 sys.path
sys.path.insert(0, str(Path(__file__).resolve().parents[2]))
from cloud.auth import AuthStore
from cloud.server import create_server
from cloud.server import CloudControlPlaneHandler, create_server
def _req(port, method, path, body=None, key=None):
@@ -355,6 +357,96 @@ class ServerQuotasTestCase(unittest.TestCase):
self.assertEqual(s, 200, q)
self.assertEqual(q["unpriced_actual_tokens"], 150)
# -- G7:准入点 fail-closed ----------------------------------------
def test_create_quota_lookup_failure_503(self):
with mock.patch.object(
CloudControlPlaneHandler, "_quota_status",
side_effect=RuntimeError("db down")):
s, body = self._create_task()
self.assertEqual(s, 503, body)
self.assertEqual(body["error"], "quota_check_unavailable")
# 没建出任务:状态、任务表都不改。
conn = sqlite3.connect(self.db_path)
try:
n = conn.execute(
"SELECT COUNT(*) FROM tasks").fetchone()[0]
finally:
conn.close()
self.assertEqual(n, 0)
# -- G13:system 豁免(server 侧) -----------------------------------
def test_system_quota_exempt_server(self):
conn = sqlite3.connect(self.db_path)
try:
conn.execute(
"INSERT INTO user_quotas(user_id, quota_usd_micros,"
" updated_at) VALUES('system', 0, 0)")
conn.commit()
finally:
conn.close()
handler = object.__new__(CloudControlPlaneHandler)
handler.server = SimpleNamespace(db_path=self.db_path)
self.assertIsNone(handler._check_quota("system"))
self.assertFalse(handler._reject_if_over_quota("system"))
# -- G13/G14:/pr 按请求人过配额门、队列记归属人 ----------------------
def _merged_remote_task(self):
s, task = self._create_task()
self.assertEqual(s, 201, task)
conn = sqlite3.connect(self.db_path)
try:
conn.execute(
"UPDATE tasks SET status = 'merged',"
" worktree_host = 'elsewhere-host-1' WHERE id = ?",
(task["id"],))
conn.commit()
finally:
conn.close()
return task
def test_pr_blocked_at_quota_no_job(self):
task = self._merged_remote_task()
s, _ = self._set_quota(self.owner_id, 0)
self.assertEqual(s, 200)
s, body = _req(self.port, "POST",
"/tasks/%s/pr" % task["id"], {},
self.owner_key)
self.assertEqual(s, 402, body)
self.assertEqual(body["error"], "quota_exceeded")
conn = sqlite3.connect(self.db_path)
try:
has_table = conn.execute(
"SELECT 1 FROM sqlite_master WHERE type='table'"
" AND name='pr_jobs'").fetchone() is not None
rows = conn.execute(
"SELECT COUNT(*) FROM pr_jobs"
" WHERE task_id = ?", (task["id"],)).fetchone()[0] \
if has_table else 0
status = conn.execute(
"SELECT status FROM tasks WHERE id = ?",
(task["id"],)).fetchone()[0]
finally:
conn.close()
self.assertEqual(rows, 0)
self.assertEqual(status, "merged")
def test_pr_queue_records_created_by(self):
task = self._merged_remote_task()
s, body = _req(self.port, "POST",
"/tasks/%s/pr" % task["id"], {},
self.owner_key)
self.assertEqual(s, 200, body)
self.assertTrue(body.get("pr_queued"))
conn = sqlite3.connect(self.db_path)
try:
row = conn.execute(
"SELECT created_by FROM pr_jobs WHERE task_id = ?",
(task["id"],)).fetchone()
finally:
conn.close()
self.assertIsNotNone(row)
self.assertEqual(row[0], self.owner_id)
if __name__ == "__main__":
unittest.main()