feat: G7 分层 fail-open/fail-closed + G8 Docker 轮询掐断 + G9 worker 忽略 host 预算 + G13/G14 归属(第三批)
- G7:认领门 closed 时查配额异常回 quota_check_unavailable、标 interrupted(可重试/续跑);轮询与轮次边界用 open,异常时回 quota_check_failed——新回合边界拒绝,执行中轮询经 _quota_hit 闭包过滤此串、只记 warning 不杀在跑任务,确认超限才掐断;查询函数本身不吞异常冒充不限额。 - G8:Docker 路径加 _docker_call_with_quota_poll(后台线程跑阻塞式 docker 调用,主循环每 quota_poll_interval_seconds 缺省 30 秒查一次,确认超限按容器名真杀容器,结果改写 quota_exceeded 走统一收尾:interrupted、成本照记、续跑消息保留,配额命中后不走 ALREADY_EXISTS 回落);同步路径保持边界检查。 - G9:worker 路径默认忽略 host max_daily_turns(host.py 顶层 --ignore-host-budget 只清该项,Local/Docker 执行器在子命令前追加,worker CLI 缺省开、--no-ignore-host-budget 关);standalone 直接跑 host.py 时预算照常。 - G13/G14:/pr 不再走 system 豁免,建任务前按请求用户查配额(402/503 均不排队不改状态),队列路 pr_jobs.created_by 记请求用户(老表 ALTER 补列),worker 跑 pr 任务时审计操作人用 created_by(老行回落 system)。注意:G14 原案的 system 豁免(服务端与 worker 写死)与 P1 冲突(P1:system 按普通属主走默认额度,不另设无限额系统预算),本次 rebase 未合入该豁免,system 豁免需另行决策。 - 配额口径与 P0/P1 对齐:无配额行按默认额度(非不限额),只有显式 NULL 行才不限额;无价上限只计 source='actual' 行;缺属主(含父任务回查仍无)拒绝新回合。 - README 配额节同步以上口径与 G15 接受现状(best-effort 超支、UTC 月切分、在跑任务只在轮询/轮次边界响应配额改动)。G4 与 G15 不在本批。
This commit is contained in:
1 parent
5883169c82
commit
e31fcc6cba
11 files changed
+786
-176
No files matched your search
+293
-144
@@ -372,7 +372,9 @@ class Worker:
|
||||
upload_dir=None,
|
||||
digest_interval_seconds=3600,
|
||||
outbox_backoff_seconds=60,
|
||||
outbox_max_attempts=3):
|
||||
outbox_max_attempts=3,
|
||||
quota_poll_interval_seconds=30,
|
||||
ignore_host_budget=True):
|
||||
self.db_path = str(db_path)
|
||||
self.host_py_path = host_py_path
|
||||
self.config_path = config_path
|
||||
@@ -385,6 +387,17 @@ class Worker:
|
||||
self.digest_interval_seconds = digest_interval_seconds
|
||||
self.outbox_backoff_seconds = outbox_backoff_seconds
|
||||
self.outbox_max_attempts = outbox_max_attempts
|
||||
# G8:配额轮询间隔(秒),Local popen 与 Docker 路径共用,
|
||||
# 缺省 30;单测可传很小的值加速,生产不要改小(会打爆库)。
|
||||
# None 当 30 用。
|
||||
self.quota_poll_interval_seconds = (
|
||||
30 if quota_poll_interval_seconds is None
|
||||
else quota_poll_interval_seconds)
|
||||
# G9:worker 路径默认忽略 host 的 max_daily_turns 预算。
|
||||
# 云端已有 per-user USD 配额 + worker 认领/轮询门,host
|
||||
# 的单机 turn 数预算再拦一次只会误拦云端已批的任务;
|
||||
# standalone 直接跑 host.py 时不受此开关影响,预算照常。
|
||||
self.ignore_host_budget = bool(ignore_host_budget)
|
||||
self._last_digest_sent = 0.0
|
||||
# env 只用于 key 存在性检查:默认接进程环境,单测可注入
|
||||
# 一个普通 dict;worker 永远不把 env 里的值写出。
|
||||
@@ -416,7 +429,15 @@ class Worker:
|
||||
else:
|
||||
self.executor = LocalExecutor(
|
||||
host_py_path, config_path=config_path, runner=runner,
|
||||
repo=repo)
|
||||
repo=repo, ignore_host_budget=self.ignore_host_budget)
|
||||
# 外部传入的执行器(CLI 的 DockerExecutor 等)同样同步
|
||||
# G9 开关:worker 的 ignore_host_budget 是唯一口径,执行
|
||||
# 器自己的缺省值不另起一套。假执行器没有该属性时跳过。
|
||||
if hasattr(self.executor, "ignore_host_budget"):
|
||||
try:
|
||||
self.executor.ignore_host_budget = self.ignore_host_budget
|
||||
except Exception:
|
||||
pass
|
||||
self._cost_store = cost_store
|
||||
self._cancel_requested = set()
|
||||
# 真取消用:process_factory 可注入(单测用它返回一个假长跑
|
||||
@@ -961,7 +982,73 @@ class Worker:
|
||||
return budget_exceeded(self.usage_turns, self.max_turns,
|
||||
self.usage_cost_usd, self.max_cost_usd)
|
||||
|
||||
def _quota_exceeded(self, task, usage_total):
|
||||
def _quota_limit_and_used(self, owner):
|
||||
"""查某人当月配额与已记账用量,返回
|
||||
(quota, used, month, unpriced_used)。
|
||||
|
||||
quota 为 None 表示不限额(只限行里 quota 为 NULL 的
|
||||
显式不限额;无配额行按默认配额
|
||||
_default_quota_usd_micros() 算,与服务端 _quota_status
|
||||
的 P0 口径一致)。查询失败时抛异常,由调用方按
|
||||
fail-open / fail-closed 分层决定——本函数自己绝不吞
|
||||
异常后冒充“不限额”,否则调用方分不清“真不限额”与
|
||||
“查不出来”。
|
||||
unpriced_used 是当月 usd IS NULL 且 source='actual'
|
||||
的成本行 input + output 之和(与服务端
|
||||
_check_unpriced_token_cap 同口径,估算行不计);
|
||||
quota 为 None 时它仍返回实数,但调用方
|
||||
_quota_exceeded 在 quota 为 None 时先返回 None,
|
||||
所以显式不限额用户不受无价上限约束,与服务端一致。
|
||||
"""
|
||||
month = datetime.now(timezone.utc).strftime("%Y-%m")
|
||||
conn = _db_connect(self.db_path)
|
||||
try:
|
||||
conn.execute(
|
||||
"CREATE TABLE IF NOT EXISTS user_quotas("
|
||||
"user_id TEXT PRIMARY KEY, quota_usd_micros BIGINT,"
|
||||
" updated_at DOUBLE PRECISION)")
|
||||
conn.commit()
|
||||
row = conn.execute(
|
||||
"SELECT quota_usd_micros FROM user_quotas"
|
||||
" WHERE user_id = ?",
|
||||
(owner,)).fetchone()
|
||||
if row is None:
|
||||
# P0/P1 口径:无行按默认额度,不是不限额(只有
|
||||
# 显式 NULL 行才是不限额,见下)。
|
||||
quota = _default_quota_usd_micros()
|
||||
else:
|
||||
quota = row[0]
|
||||
if quota is None:
|
||||
return None, 0, month, 0
|
||||
# cost_records 由 CostStore 懒建:全新库上它还不存
|
||||
# 在,直接 JOIN 会报错——先实例化一次建表再查(与
|
||||
# 服务端 _quota_status 同款做法),否则 quota=0
|
||||
#(一分钱也不能花)在全新库上会被误放行。
|
||||
_cost_store = CostStore(self.db_path)
|
||||
try:
|
||||
_cost_store._conn.close()
|
||||
except Exception:
|
||||
pass
|
||||
used_row = conn.execute(
|
||||
"SELECT COALESCE(SUM(c.usd_micros), 0)"
|
||||
" FROM cost_records c JOIN tasks t"
|
||||
" ON c.task_id = t.id"
|
||||
" WHERE t.created_by = ? AND c.day LIKE ?",
|
||||
(owner, month + "%")).fetchone()
|
||||
used = used_row[0] if used_row else 0
|
||||
unpriced_row = conn.execute(
|
||||
"SELECT COALESCE(SUM(c.input_tokens + c.output_tokens), 0)"
|
||||
" FROM cost_records c JOIN tasks t"
|
||||
" ON c.task_id = t.id"
|
||||
" WHERE t.created_by = ? AND c.day LIKE ?"
|
||||
" AND c.usd_micros IS NULL AND c.source = 'actual'",
|
||||
(owner, month + "%")).fetchone()
|
||||
unpriced_used = unpriced_row[0] if unpriced_row else 0
|
||||
return quota, int(used or 0), month, int(unpriced_used or 0)
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
def _quota_exceeded(self, task, usage_total, on_error="open"):
|
||||
"""配额加固的执行时检查:返回超限时的错误文本,否则 None。
|
||||
|
||||
口径与服务端准入(_quota_status)一致:按任务创建者算,
|
||||
@@ -970,29 +1057,27 @@ class Worker:
|
||||
另有一道无价 token 上限(与服务端
|
||||
_check_unpriced_token_cap 同口径):USD 配额不计量无价
|
||||
模型,默认模型 terra 无价,若无此上限则配额对默认模型
|
||||
永远不生效;有配额行的用户当月无价 token(input +
|
||||
永远不生效;有额度约束的用户(无行按默认额度)当月无价
|
||||
token(input +
|
||||
output,已记账 + 本轮)达到上限即停。无价上限只计
|
||||
source='actual' 的行:估算行(estimated)不是真实花费,
|
||||
计入上限会把“没花钱”算成“花超了”,且与 cost.py
|
||||
_report 的 unpriced_actual_tokens 口径一致。
|
||||
P1 执行面三处收口(旧版三处都是 fail-open,已废):
|
||||
属主与额度(P1 口径,与服务端 _quota_status 一致):
|
||||
- 属主:任务 created_by 为空时先查父任务(协调员子任务
|
||||
行可能无 created_by,属主跟父任务走);仍查不到时
|
||||
拒绝新回合,不再返回 None 放行。created_by 字面为
|
||||
"system" 的系统任务按普通属主算(无行时同样走默认
|
||||
额度,不另设无限额的系统预算)。
|
||||
- 无配额行:不再是不限额,按默认配额
|
||||
_default_quota_usd_micros()($10/月)算;只有显式
|
||||
NULL 行才是不限额(与 P0 的服务端口径一致,管理员
|
||||
显式设 NULL 的行仍被尊重)。
|
||||
- 查库失败:返回 "quota_check_failed: ..."(fail-closed),
|
||||
绝不返回 None 放行。调用方分流:新回合边界(认领门、
|
||||
重试/fallback/后续消息边界)见此串即拒绝新回合并记
|
||||
worker_quota_stop 审计;执行中轮询(popen 轮询、
|
||||
Docker 配额看门狗)见此串只记警告、不中途杀当前轮
|
||||
——查询抖动不应杀掉已在跑的回合,但下一轮边界仍会
|
||||
拒绝(见 run_one 的 _quota_hit 过滤)。查库失败的
|
||||
审计由边界调用方记,这里只记日志,避免轮询高频写审计。
|
||||
"system" 的系统任务按普通属主算,不另设无限额的
|
||||
系统预算(G14 的 system 豁免与 P1 冲突,本次未合入)。
|
||||
- 无配额行:按默认配额 _default_quota_usd_micros()
|
||||
($10/月)算;只有显式 NULL 行才是不限额。
|
||||
查库失败分层(G7 + P1):认领门传 on_error="closed",
|
||||
返回 "quota_check_unavailable: ..."(调用方标
|
||||
interrupted,可重试/续跑);默认 "open" 返回
|
||||
"quota_check_failed: ..."——新回合边界(直接调用)
|
||||
见此串即拒绝,执行中轮询经 _quota_hit 闭包过滤此串、
|
||||
只记警告不杀当前轮。查询抖动不应杀掉已在跑的回合,
|
||||
但下一轮边界仍会拒绝。
|
||||
"""
|
||||
owner = (task or {}).get("created_by")
|
||||
if not owner and (task or {}).get("parent_task_id"):
|
||||
@@ -1017,95 +1102,45 @@ class Worker:
|
||||
return ("quota_exceeded: 任务缺少属主,无法查配额,"
|
||||
"拒绝新回合(请归属真实用户后再排队)")
|
||||
try:
|
||||
month = datetime.now(timezone.utc).strftime("%Y-%m")
|
||||
conn = _db_connect(self.db_path)
|
||||
try:
|
||||
conn.execute(
|
||||
"CREATE TABLE IF NOT EXISTS user_quotas("
|
||||
"user_id TEXT PRIMARY KEY, quota_usd_micros BIGINT,"
|
||||
" updated_at DOUBLE PRECISION)")
|
||||
conn.commit()
|
||||
row = conn.execute(
|
||||
"SELECT quota_usd_micros FROM user_quotas"
|
||||
" WHERE user_id = ?",
|
||||
(owner,)).fetchone()
|
||||
if row is None:
|
||||
# P1:无行按默认额度,不是不限额(P0 服务端
|
||||
# 已在建号/登录时补行,worker 侧仍可能遇到
|
||||
# 老库未补行的用户,这里按默认额度兜底,
|
||||
# 不写行——写行是服务端的职责,worker 不抢)。
|
||||
quota = _default_quota_usd_micros()
|
||||
else:
|
||||
quota = row[0]
|
||||
if quota is None:
|
||||
return None
|
||||
# cost_records 由 CostStore 懒建:全新库上它还不存
|
||||
# 在,直接 JOIN 会报错——先实例化一次建表再查(与
|
||||
# 服务端 _quota_status 同款做法),否则 quota=0
|
||||
#(一分钱也不能花)在全新库上会被误放行。
|
||||
# P1 补:建表必须落在本连接上,不只 CostStore 的
|
||||
# 连接上——文件库两者指向同一文件所以旧代码碰巧
|
||||
# 能用,:memory: 库两者是不同空库,本连接查 JOIN
|
||||
# 会直接报错并(P1 fail-closed 后)误拒全部任务。
|
||||
_cost_store = CostStore(self.db_path)
|
||||
try:
|
||||
_cost_store._conn.close()
|
||||
except Exception:
|
||||
pass
|
||||
try:
|
||||
conn.execute(CostStore._COST_DDL)
|
||||
conn.commit()
|
||||
except Exception:
|
||||
pass
|
||||
used_row = conn.execute(
|
||||
"SELECT COALESCE(SUM(c.usd_micros), 0)"
|
||||
" FROM cost_records c JOIN tasks t"
|
||||
" ON c.task_id = t.id"
|
||||
" WHERE t.created_by = ? AND c.day LIKE ?",
|
||||
(owner, month + "%")).fetchone()
|
||||
used = used_row[0] if used_row else 0
|
||||
unpriced_row = conn.execute(
|
||||
"SELECT COALESCE(SUM(c.input_tokens + c.output_tokens), 0)"
|
||||
" FROM cost_records c JOIN tasks t"
|
||||
" ON c.task_id = t.id"
|
||||
" WHERE t.created_by = ? AND c.day LIKE ?"
|
||||
" AND c.usd_micros IS NULL AND c.source = 'actual'",
|
||||
(owner, month + "%")).fetchone()
|
||||
unpriced_used = unpriced_row[0] if unpriced_row else 0
|
||||
finally:
|
||||
conn.close()
|
||||
if usage_total:
|
||||
model = (task or {}).get("model") or self.model
|
||||
micros = price_usd_micros(
|
||||
model, usage_total.get("input_tokens", 0),
|
||||
usage_total.get("cached_input_tokens", 0),
|
||||
usage_total.get("output_tokens", 0))
|
||||
used += micros or 0
|
||||
if micros is None:
|
||||
# 无价模型:本轮 token 计入无价上限(USD 口
|
||||
# 径里它是 0,不会在上面那道里被拦住)。
|
||||
unpriced_used += (
|
||||
max(0, int(usage_total.get("input_tokens", 0)))
|
||||
+ max(0, int(usage_total.get("output_tokens", 0))))
|
||||
if used >= quota:
|
||||
return ("quota_exceeded: 本月配额已用完(已用 %d / "
|
||||
"配额 %d 微美元,%s)" % (used, quota, month))
|
||||
cap = unpriced_token_cap()
|
||||
if unpriced_used >= cap:
|
||||
return ("quota_exceeded: 本月无价模型 token 已用完"
|
||||
"(已用 %d / 上限 %d,%s)"
|
||||
% (unpriced_used, cap, month))
|
||||
return None
|
||||
quota, used, month, unpriced_used = \
|
||||
self._quota_limit_and_used(owner)
|
||||
except Exception as exc:
|
||||
# P1 fail-closed:查库失败不再返回 None。调用方按
|
||||
# 前缀分流(见本函数 docstring):quota_check_failed
|
||||
# 在新回合边界 = 拒绝,在执行中轮询 = 只警告不杀轮。
|
||||
# 查库失败分层(见本函数 docstring):closed(认领门)
|
||||
# 回 quota_check_unavailable,调用方标 interrupted;
|
||||
# open 回 quota_check_failed,新回合边界拒绝,执行中
|
||||
# 轮询经 _quota_hit 闭包过滤、只警告不杀轮。
|
||||
try:
|
||||
self.logger.warning("quota check failed: %s", exc)
|
||||
except Exception:
|
||||
pass
|
||||
if on_error == "closed":
|
||||
return "quota_check_unavailable: 配额查询失败,任务未开跑"
|
||||
return ("quota_check_failed: 配额查询失败,拒绝新回合"
|
||||
"(查库异常,非配额用完)")
|
||||
if quota is None:
|
||||
return None
|
||||
if usage_total:
|
||||
model = (task or {}).get("model") or self.model
|
||||
micros = price_usd_micros(
|
||||
model, usage_total.get("input_tokens", 0),
|
||||
usage_total.get("cached_input_tokens", 0),
|
||||
usage_total.get("output_tokens", 0))
|
||||
used += micros or 0
|
||||
if micros is None:
|
||||
# 无价模型:本轮 token 计入无价上限(USD 口
|
||||
# 径里它是 0,不会在上面那道里被拦住)。
|
||||
unpriced_used += (
|
||||
max(0, int(usage_total.get("input_tokens", 0)))
|
||||
+ max(0, int(usage_total.get("output_tokens", 0))))
|
||||
if used >= quota:
|
||||
return ("quota_exceeded: 本月配额已用完(已用 %d / "
|
||||
"配额 %d 微美元,%s)" % (used, quota, month))
|
||||
cap = unpriced_token_cap()
|
||||
if unpriced_used >= cap:
|
||||
return ("quota_exceeded: 本月无价模型 token 已用完"
|
||||
"(已用 %d / 上限 %d,%s)"
|
||||
% (unpriced_used, cap, month))
|
||||
return None
|
||||
|
||||
def _ready_tasks(self):
|
||||
"""当前就绪任务列表(pick_next 的批量版)。
|
||||
@@ -1511,11 +1546,13 @@ class Worker:
|
||||
的标记,调用方据此保持 interrupted;deadline_seconds
|
||||
到期时杀进程组并返回 status='timeout' 的标记,调用方
|
||||
据此置 failed(超时是失败,不是取消)。quota_check 不为
|
||||
空时每 30 秒查一次配额(轮询本身 50 毫秒一圈,逐圈查会
|
||||
把库打爆),超限时同样杀进程组并返回 status=
|
||||
'quota_exceeded' 的内部标记——调用方据此走配额中断收尾,
|
||||
绝不能把这个标记原样写进 tasks.status(不在合法状态集
|
||||
里)。Docker/同步路径没有这个轮询钩子,只走边界检查。
|
||||
空时每 quota_poll_interval_seconds(缺省 30 秒)查一次
|
||||
配额(轮询本身 50 毫秒一圈,逐圈查会把库打爆),超限时
|
||||
同样杀进程组并返回 status='quota_exceeded' 的内部标记
|
||||
——调用方据此走配额中断收尾,绝不能把这个标记原样写
|
||||
进 tasks.status(不在合法状态集里)。Docker 路径见
|
||||
_docker_call_with_quota_poll(同款轮询);同步路径没有
|
||||
轮询钩子,只走边界检查。
|
||||
"""
|
||||
task_id = task["id"]
|
||||
gateway_provider, gateway_env = self._ensure_gateway(task)
|
||||
@@ -1539,6 +1576,10 @@ class Worker:
|
||||
started = time.time()
|
||||
last_quota_check = 0.0
|
||||
last_db_cancel_check = 0.0
|
||||
quota_interval = getattr(
|
||||
self, "quota_poll_interval_seconds", 30)
|
||||
if quota_interval is None:
|
||||
quota_interval = 30
|
||||
try:
|
||||
while proc.poll() is None:
|
||||
if task_id in self._cancel_requested or self._loop_stop:
|
||||
@@ -1567,7 +1608,7 @@ class Worker:
|
||||
self._kill_proc_group(proc)
|
||||
break
|
||||
if quota_check is not None and \
|
||||
time.time() - last_quota_check >= 30:
|
||||
time.time() - last_quota_check >= quota_interval:
|
||||
last_quota_check = time.time()
|
||||
try:
|
||||
over = quota_check()
|
||||
@@ -2205,6 +2246,80 @@ class Worker:
|
||||
detail += " (" + "; ".join(rejected) + ")"
|
||||
self._audit(task["project_id"], "worker_peer_messages", detail)
|
||||
|
||||
def _docker_call_with_quota_poll(self, task, fn, quota_check=None):
|
||||
"""Docker 同步调用包一层配额轮询(G8),与 popen 路径同款。
|
||||
|
||||
fn 在后台线程里跑(DockerExecutor 的调用本身是阻塞的
|
||||
docker run,无法像 popen 那样在主循环里轮询);主循环
|
||||
每 quota_poll_interval_seconds(缺省 30 秒)调一次
|
||||
quota_check,确认超限时按任务容器名真杀容器,fn 返回
|
||||
后把结果状态改写成内部标记 'quota_exceeded',调用方
|
||||
(run_one)据此走统一的配额中断收尾:状态 interrupted、
|
||||
已花成本照记、续跑消息保留。quota_check 为 None 时
|
||||
直接调 fn,不起线程。quota_check 本身抛异常时按
|
||||
fail-open 处理(只记 warning,不杀容器),只有确认
|
||||
超限才杀;取消时 cancel() 已按容器名杀过容器,线程
|
||||
随之结束,轮询循环不再多杀一次。fn 抛异常时在主线程
|
||||
原样重抛,由 _run_attempt 外层的 except 写成 failed,
|
||||
与不包轮询时的行为一致。注意:轮询查的是已记账用量
|
||||
(与 popen 路径同款),在跑回合的 token 只在回合结束
|
||||
才回传,所以掐断粒度仍是轮询间隔,不是 token 级。
|
||||
"""
|
||||
if quota_check is None:
|
||||
return fn()
|
||||
box = {}
|
||||
|
||||
def _target():
|
||||
try:
|
||||
box["result"] = fn()
|
||||
except Exception as exc: # noqa: BLE001 -- 主线程重抛
|
||||
box["error"] = exc
|
||||
|
||||
thread = threading.Thread(target=_target, daemon=True)
|
||||
thread.start()
|
||||
interval = getattr(self, "quota_poll_interval_seconds", 30)
|
||||
if interval is None:
|
||||
interval = 30
|
||||
last_check = 0.0
|
||||
quota_hit = False
|
||||
while thread.is_alive():
|
||||
if task["id"] in self._cancel_requested or self._loop_stop:
|
||||
break
|
||||
now = time.time()
|
||||
if now - last_check >= interval:
|
||||
last_check = now
|
||||
try:
|
||||
over = bool(quota_check())
|
||||
except Exception as exc:
|
||||
try:
|
||||
self.logger.warning(
|
||||
"docker quota poll failed: %s", exc)
|
||||
except Exception:
|
||||
pass
|
||||
over = False
|
||||
if over:
|
||||
quota_hit = True
|
||||
container = None
|
||||
try:
|
||||
container = self.executor.container_name(task)
|
||||
except Exception:
|
||||
container = None
|
||||
if container is not None and hasattr(
|
||||
self.executor, "kill_container"):
|
||||
try:
|
||||
self.executor.kill_container(container)
|
||||
except Exception:
|
||||
pass
|
||||
break
|
||||
thread.join(timeout=0.05)
|
||||
thread.join()
|
||||
if "error" in box:
|
||||
raise box["error"]
|
||||
result = box.get("result")
|
||||
if quota_hit and isinstance(result, dict):
|
||||
result = dict(result, status="quota_exceeded")
|
||||
return result
|
||||
|
||||
def _run_attempt(self, task, quota_check=None):
|
||||
"""单次执行尝试(不含认领、key 门、重试、成本与最终审计)。
|
||||
|
||||
@@ -2337,12 +2452,16 @@ class Worker:
|
||||
if task.get("_goal_resume_breakpoint"):
|
||||
# G10 断点:Docker 路用无 --goal-text 的
|
||||
# goal-resume-breakpoint 命令。
|
||||
result = self.executor.goal_resume_breakpoint_task(
|
||||
result = self._docker_call_with_quota_poll(
|
||||
task,
|
||||
timeout=effective_docker_timeout,
|
||||
extra_mounts=tenant_mounts,
|
||||
network=tenant_network,
|
||||
extra_env=proxy_env)
|
||||
lambda: self.executor.goal_resume_breakpoint_task(
|
||||
task,
|
||||
timeout=effective_docker_timeout,
|
||||
extra_mounts=tenant_mounts,
|
||||
network=tenant_network,
|
||||
extra_env=_merge_env(proxy_env,
|
||||
gateway_env)),
|
||||
quota_check)
|
||||
elif task.get("_resume"):
|
||||
# P1 断点续跑传 None(不传 --goal-text,
|
||||
# host 沿用原 goal+verify);开新目标
|
||||
@@ -2350,31 +2469,41 @@ class Worker:
|
||||
_resume_text = None if task.get(
|
||||
"_resume_kind") == "breakpoint" \
|
||||
else task["prompt"]
|
||||
result = self.executor.goal_resume_task(
|
||||
task, _resume_text,
|
||||
timeout=effective_docker_timeout,
|
||||
extra_mounts=tenant_mounts,
|
||||
network=tenant_network,
|
||||
extra_env=_merge_env(proxy_env,
|
||||
gateway_env),
|
||||
model_provider=gateway_provider)
|
||||
result = self._docker_call_with_quota_poll(
|
||||
task,
|
||||
lambda: self.executor.goal_resume_task(
|
||||
task, _resume_text,
|
||||
timeout=effective_docker_timeout,
|
||||
extra_mounts=tenant_mounts,
|
||||
network=tenant_network,
|
||||
extra_env=_merge_env(proxy_env,
|
||||
gateway_env),
|
||||
model_provider=gateway_provider),
|
||||
quota_check)
|
||||
else:
|
||||
result = self.executor.goal_task(
|
||||
result = self._docker_call_with_quota_poll(
|
||||
task,
|
||||
lambda: self.executor.goal_task(
|
||||
task, timeout=effective_docker_timeout,
|
||||
extra_mounts=tenant_mounts,
|
||||
network=tenant_network,
|
||||
extra_env=_merge_env(proxy_env,
|
||||
gateway_env),
|
||||
seed_host_dir=seed_host_dir,
|
||||
model_provider=gateway_provider),
|
||||
quota_check)
|
||||
else:
|
||||
result = self._docker_call_with_quota_poll(
|
||||
task,
|
||||
lambda: self.executor.dispatch_task(
|
||||
task, timeout=effective_docker_timeout,
|
||||
extra_mounts=tenant_mounts,
|
||||
network=tenant_network,
|
||||
extra_env=_merge_env(proxy_env,
|
||||
gateway_env),
|
||||
seed_host_dir=seed_host_dir,
|
||||
model_provider=gateway_provider)
|
||||
else:
|
||||
result = self.executor.dispatch_task(
|
||||
task, timeout=effective_docker_timeout,
|
||||
extra_mounts=tenant_mounts,
|
||||
network=tenant_network,
|
||||
extra_env=_merge_env(proxy_env, gateway_env),
|
||||
seed_host_dir=seed_host_dir,
|
||||
model_provider=gateway_provider)
|
||||
model_provider=gateway_provider),
|
||||
quota_check)
|
||||
else:
|
||||
# seed 参数只传给 LocalExecutor:CloudExecutor
|
||||
# 基类与测试假执行器的 goal_task/dispatch_task
|
||||
@@ -2434,19 +2563,23 @@ class Worker:
|
||||
result = self.executor.dispatch_task(task)
|
||||
combined = (result.get("stdout", "") or "") + \
|
||||
(result.get("stderr", "") or "")
|
||||
if not goal_mode and result["status"] != "completed" and \
|
||||
if not goal_mode and result["status"] not in (
|
||||
"completed", "quota_exceeded") and \
|
||||
_ALREADY_EXISTS_MARKER in combined:
|
||||
if is_docker:
|
||||
result = self.executor.run_task(
|
||||
task, timeout=effective_docker_timeout,
|
||||
extra_mounts=tenant_mounts
|
||||
if self.tenant_isolation else None,
|
||||
network=tenant_network
|
||||
if self.tenant_isolation else None,
|
||||
extra_env=_merge_env(
|
||||
proxy_env if self.egress_proxy_port
|
||||
else None, gateway_env),
|
||||
model_provider=gateway_provider)
|
||||
result = self._docker_call_with_quota_poll(
|
||||
task,
|
||||
lambda: self.executor.run_task(
|
||||
task, timeout=effective_docker_timeout,
|
||||
extra_mounts=tenant_mounts
|
||||
if self.tenant_isolation else None,
|
||||
network=tenant_network
|
||||
if self.tenant_isolation else None,
|
||||
extra_env=_merge_env(
|
||||
proxy_env if self.egress_proxy_port
|
||||
else None, gateway_env),
|
||||
model_provider=gateway_provider),
|
||||
quota_check)
|
||||
else:
|
||||
if gateway_provider and not isinstance(
|
||||
self.executor, LocalExecutor):
|
||||
@@ -3092,8 +3225,11 @@ class Worker:
|
||||
# 再开跑——直接标 interrupted + 配额原因,不记成本(还没花
|
||||
# 钱),续跑消息不消费,顶上来后重试/续跑可再跑。与 key
|
||||
# 门、仓库门同款提前返回形状,区别是状态为 interrupted
|
||||
#(可续跑)而非 failed。
|
||||
quota_error = self._quota_exceeded(task, None)
|
||||
#(可续跑)而非 failed。G7 分层:认领门用 closed——
|
||||
# 查询异常时同样标 interrupted(quota_check_unavailable,
|
||||
# 可重试/续跑),绝不在“不知道超没超”时开跑;轮询与
|
||||
# 轮次边界仍用默认 open,查询抖动不杀在跑任务。
|
||||
quota_error = self._quota_exceeded(task, None, on_error="closed")
|
||||
if quota_error is not None:
|
||||
self._set_status(task_id, "interrupted")
|
||||
self._write_execution_result(
|
||||
@@ -3696,8 +3832,11 @@ class Worker:
|
||||
job_id = job["id"]
|
||||
task_id = job["task_id"]
|
||||
try:
|
||||
# G13:审计操作人用 job 的费用归属人(请求 /pr 的
|
||||
# 用户),老行没有 created_by 时回落 "system"。
|
||||
result = execute_task_pr(
|
||||
self.db_path, task_id, "system")
|
||||
self.db_path, task_id,
|
||||
job.get("created_by") or "system")
|
||||
conn = _db_connect(self.db_path)
|
||||
try:
|
||||
finish_pr_job(conn, job_id, "completed",
|
||||
@@ -4121,6 +4260,15 @@ def main(argv=None):
|
||||
"列表里的第一个模型再跑一次,401/400"
|
||||
"绝不 fallback,取消绝不 fallback)")
|
||||
parser.add_argument("--max-turns", type=int, default=None)
|
||||
parser.add_argument("--ignore-host-budget", dest="ignore_host_budget",
|
||||
action="store_true", default=True,
|
||||
help="G9(默认开):worker 路径忽略 host 的 "
|
||||
"max_daily_turns 预算,云端 per-user USD "
|
||||
"配额 + worker 门才是口径")
|
||||
parser.add_argument("--no-ignore-host-budget",
|
||||
dest="ignore_host_budget", action="store_false",
|
||||
help="G9:关闭忽略,worker 调 host.py 时 host "
|
||||
"的 max_daily_turns 预算照常生效(双重门)")
|
||||
parser.add_argument("--max-tasks", type=int, default=None)
|
||||
parser.add_argument("--executor", default="local",
|
||||
choices=["local", "docker", "cloud"],
|
||||
@@ -4219,6 +4367,7 @@ def main(argv=None):
|
||||
args.fallback_models.split(",")]
|
||||
if args.fallback_models else []),
|
||||
max_turns=args.max_turns,
|
||||
ignore_host_budget=args.ignore_host_budget,
|
||||
executor=_build_cli_executor(args, host_py),
|
||||
task_timeout=args.task_timeout,
|
||||
max_retries=args.max_retries,
|
||||
|
||||
Reference in new issue
Block a user