P1 security batch: quota execution checks, cost accounting, goal/coordinator semantics, PR channel, orphan signature, control-plane and console hardening

- Docker quota watchdog + fail-closed quota checks, system tasks need real owner
- max_cost_usd actually accumulated, cancel records cost, estimated rows excluded from unpriced cap, cache_write declared non-billed
- goal-resume keeps original verify on breakpoint resume, goal retry falls back to resume, goal_max_rounds capped at 100 at create and execution
- coordinator children inherit parent model/effort, orphans marked interrupted with audit
- PR jobs on docker workers return explicit needs_local_worker instead of silent empty
- worker code signature covers 6 files and is checked before claim, fail-closed
- sub2api lazy provisioning with rotation, backup whitelist + 0600, push SSRF block, repos/workers desensitized, key/project audits and limits, console buttons/quota/status wording, coordinator/automation quota reserve
This commit is contained in:
host committed 2026-10-27 16:27:50 +00:00
1 parent d254240d84
commit 1402dc67fd
20 files changed
+2616 -179

No files matched your search

+604 -55
View File
@@ -139,6 +139,7 @@ from cloud.coordinator import DEFAULT_POLL_INTERVAL as _COORD_POLL_INTERVAL
from cloud.coordinator import DEFAULT_TIMEOUT_SECONDS as _COORD_TIMEOUT
from cloud.coordinator import ensure_columns as _coord_ensure_columns
from cloud.coordinator import parse_parts as _coord_parse_parts
from cloud.coordinator import nonterminal_children as _coord_nonterminal
from cloud.cost import CostStore, estimate_tokens, price_usd_micros, \
sum_usage, unpriced_token_cap
from cloud.db import connect as _db_connect
@@ -170,19 +171,68 @@ DEFAULT_ENV_KEY = "XCM_API_KEY"
# gpt-6.1-sol 路由当日实测持续 503,terra 同日实测 1 秒完成;
# 任务未指定模型时 worker 与 host.py 都落到这里。
DEFAULT_MODEL = "gpt-5.6-terra"
# P1 执行面:goal 轮次上限(有限轮次形态)。goal_mode=1 的 0 轮
# (模型宣告才结束)不在此限——它的上界是配额(配额看门狗与
# 认领/边界门),不是轮次;旧验收形态的 goal_max_rounds 必须
# 在 1..GOAL_MAX_ROUNDS_CAP 之间,超限的任务 worker 在认领后
# 直接 failed(建任务时的 400 校验在 server.py,属另一路文件,
# 本路只做执行侧防线,见 run_one 的 goal 上限门)。
GOAL_MAX_ROUNDS_CAP = 100
# P1 执行面:无配额行时的默认配额(微美元),与 server.py
# _default_quota_usd_micros() 同值($10/月,环境变量
# CLOUD_DEFAULT_QUOTA_USD_MICROS 可覆盖)。P0 前“无行 =
# 不限额”已废:worker 侧查不到行时按此默认额度算,显式
# NULL 行才是不限额(与服务端 _quota_status 同口径)。
# host.py dispatch 的原话是 "task 已存在: <name>,先换名或 discard 它"
#(英文 task + 已存在),取共有子串做判断,不依赖标点。
_ALREADY_EXISTS_MARKER = "已存在"
def _code_signature():
"""本文件签名(mtime_ns, size):部署 staging 换目录后必变;
取不到时返回 None(此时自退检查跳过,绝不误杀)。"""
def _default_quota_usd_micros():
"""无配额行时的默认配额(微美元),与 server 侧同值。
server.py 的 _default_quota_usd_micros() 在另一路文件里,
worker 不跨模块 import 它(避免循环 import),这里复刻
同款解析:环境变量 CLOUD_DEFAULT_QUOTA_USD_MICROS,非法
值回落 10_000_000。"""
try:
st = os.stat(os.path.abspath(__file__))
return (st.st_mtime_ns, st.st_size)
return max(0, int(os.environ.get(
"CLOUD_DEFAULT_QUOTA_USD_MICROS", "10000000")))
except (TypeError, ValueError):
return 10000000
def _code_signature(host_py_path=None):
"""多文件签名:worker/sandbox/coordinator/cost/server/host
各取 (mtime_ns, size) 组成的元组;任一文件取不到即 None。
P1 执行面扩面:旧版只签 worker.py,部署只换了 sandbox 或
host.py 时旧 worker 会继续用旧代码抢任务。host.py 取
host_py_path(真实文件时用它,否则用 cloud/ 上一级的
host.py);单测常传 /fake/host.py 这种不存在的路径,此时
回落到仓库 host.py,不因此判签名缺失。取不到时返回 None,
调用方必须 fail-closed(不认领、不执行),绝不按“没变”
放行——旧版“取不到当没变”是 fail-open,已废。"""
here = os.path.dirname(os.path.abspath(__file__))
candidates = [
os.path.join(here, "worker.py"),
os.path.join(here, "sandbox.py"),
os.path.join(here, "coordinator.py"),
os.path.join(here, "cost.py"),
os.path.join(here, "server.py"),
]
if host_py_path and os.path.isfile(host_py_path):
candidates.append(os.path.abspath(host_py_path))
else:
candidates.append(os.path.join(os.path.dirname(here), "host.py"))
sig = []
try:
for path in candidates:
st = os.stat(path)
sig.append((st.st_mtime_ns, st.st_size))
except OSError:
return None
return tuple(sig)
def _missing_key_error(env_key):
@@ -447,16 +497,46 @@ class Worker:
# 注入的普通 dict 同样可读环境变量(无该键时默认 1)。
self.capacity = resolve_capacity(capacity, env=self._env)
# 常驻启动时的代码签名:run_loop 每轮边界比对,变了就自退
# (见 run_loop 的代码变化自退检查)。
self._code_signature = _code_signature()
# (见 run_loop 的代码变化自退检查);_claim 每次认领前
# 也比对(见 _code_signature_valid),不要只在轮次边界查。
self._code_signature = _code_signature(host_py_path)
# 配额看门狗命中过的任务(Docker 同步路径用):看门狗杀
# 容器后,同步返回的 failed 要被 _run_attempt 改写成
# quota_exceeded,否则 run_one 会把它当普通失败重试,
# 配额中断就续跑下去了。受 _quota_hit_lock 保护。
self._quota_hit_tasks = set()
self._quota_hit_lock = threading.Lock()
def _code_changed(self):
"""启动后 worker.py 是否被换过(部署换目录/手动改代码):
签名取不到时一律当没变,不误杀。"""
now = _code_signature()
"""启动后关键代码是否被换过(部署换目录/手动改代码):
签名取不到时返回 False(是否“变了”无从判断),但这
不等于放行——放行与否看 _code_signature_valid。"""
now = _code_signature(self.host_py_path)
return (now is not None and self._code_signature is not None
and now != self._code_signature)
def _code_signature_valid(self):
"""签名是否可信且未变:启动签名与现签名都必须取得到
且相等,否则 False。False 时 _claim 拒绝认领、run_loop
自退(75)、_run_pr_jobs 不碰任务——fail-closed,绝不
让签名缺失的 worker 抢任务(旧版取不到当没变是 fail-open)。"""
now = _code_signature(self.host_py_path)
return (now is not None and self._code_signature is not None
and now == self._code_signature)
def _mark_quota_hit(self, task_id):
with self._quota_hit_lock:
self._quota_hit_tasks.add(task_id)
def _take_quota_hit(self, task_id):
"""取走一次配额看门狗命中(命中即消费,避免旧命中污染
同一任务的下一轮):返回 True 表示本轮确被看门狗杀过。"""
with self._quota_hit_lock:
if task_id in self._quota_hit_tasks:
self._quota_hit_tasks.discard(task_id)
return True
return False
def register(self):
"""向控制面注册本 worker(写 workers 表,失败只记日志)。
@@ -573,7 +653,17 @@ class Worker:
里清(本地 _execute_via_process / Docker 同步路径),
认领后提前返回的三道门(缺 key / 仓库 / 配额)在
run_one 里显式清占位;CAS 语义本身一句未改。
P1 签名门:占位之前先比对多文件签名,签名缺失或已变
时直接返回 False(fail-closed,不抢任务、不占槽),
并记日志;不要只在 run_loop 轮次边界查——并行池线程
直接调 run_one/_claim,轮次边界查不到它们。
"""
if not self._code_signature_valid():
log_event(self.logger, "worker_claim_refused_code_signature",
task_id=task_id,
message="code signature missing or changed,"
" refusing claim")
return False
with self._running_lock:
already_running = task_id in self._running
if not already_running and \
@@ -865,14 +955,51 @@ class Worker:
_check_unpriced_token_cap 同口径):USD 配额不计量无价
模型,默认模型 terra 无价,若无此上限则配额对默认模型
永远不生效;有配额行的用户当月无价 token(input +
output,已记账 + 本轮)达到上限即停。无配额行 = 不限额。
查询本身异常时按不限额处理(fail-open 并记一条警告):
配额是第二道网,绝不能因查询抖动把全部任务挡在门外;
服务端建任务/分支/重试时的准入仍是第一道。
output,已记账 + 本轮)达到上限即停。无价上限只计
source='actual' 的行:估算行(estimated)不是真实花费,
计入上限会把“没花钱”算成“花超了”,且与 cost.py
_report 的 unpriced_actual_tokens 口径一致。
P1 执行面三处收口(旧版三处都是 fail-open,已废):
- 属主:任务 created_by 为空时先查父任务(协调员子任务
行可能无 created_by,属主跟父任务走);仍查不到时
拒绝新回合,不再返回 None 放行。created_by 字面为
"system" 的系统任务按普通属主算(无行时同样走默认
额度,不另设无限额的系统预算)。
- 无配额行:不再是不限额,按默认配额
_default_quota_usd_micros()($10/月)算;只有显式
NULL 行才是不限额(与 P0 的服务端口径一致,管理员
显式设 NULL 的行仍被尊重)。
- 查库失败:返回 "quota_check_failed: ..."(fail-closed),
绝不返回 None 放行。调用方分流:新回合边界(认领门、
重试/fallback/后续消息边界)见此串即拒绝新回合并记
worker_quota_stop 审计;执行中轮询(popen 轮询、
Docker 配额看门狗)见此串只记警告、不中途杀当前轮
——查询抖动不应杀掉已在跑的回合,但下一轮边界仍会
拒绝(见 run_one 的 _quota_hit 过滤)。查库失败的
审计由边界调用方记,这里只记日志,避免轮询高频写审计。
"""
owner = (task or {}).get("created_by")
if not owner and (task or {}).get("parent_task_id"):
# 协调员子任务属主跟父任务走(与 _gateway_context
# 同款回查);查不到时不猜,直接走下面的拒绝分支。
try:
conn = _db_connect(self.db_path)
try:
row = conn.execute(
"SELECT created_by FROM tasks WHERE id = ?",
(task["parent_task_id"],)).fetchone()
finally:
conn.close()
if row is not None:
try:
owner = row[0]
except (TypeError, IndexError):
owner = None
except Exception:
owner = None
if not owner:
return None
return ("quota_exceeded: 任务缺少属主,无法查配额,"
"拒绝新回合(请归属真实用户后再排队)")
try:
month = datetime.now(timezone.utc).strftime("%Y-%m")
conn = _db_connect(self.db_path)
@@ -887,19 +1014,33 @@ class Worker:
" WHERE user_id = ?",
(owner,)).fetchone()
if row is None:
return None
quota = row[0]
# P1:无行按默认额度,不是不限额(P0 服务端
# 已在建号/登录时补行,worker 侧仍可能遇到
# 老库未补行的用户,这里按默认额度兜底,
# 不写行——写行是服务端的职责,worker 不抢)。
quota = _default_quota_usd_micros()
else:
quota = row[0]
if quota is None:
return None
# cost_records 由 CostStore 懒建:全新库上它还不存
# 在,直接 JOIN 会报错——先实例化一次建表再查(与
# 服务端 _quota_status 同款做法),否则 quota=0
#(一分钱也不能花)在全新库上会被误放行。
# P1 补:建表必须落在本连接上,不只 CostStore 的
# 连接上——文件库两者指向同一文件所以旧代码碰巧
# 能用,:memory: 库两者是不同空库,本连接查 JOIN
# 会直接报错并(P1 fail-closed 后)误拒全部任务。
_cost_store = CostStore(self.db_path)
try:
_cost_store._conn.close()
except Exception:
pass
try:
conn.execute(CostStore._COST_DDL)
conn.commit()
except Exception:
pass
used_row = conn.execute(
"SELECT COALESCE(SUM(c.usd_micros), 0)"
" FROM cost_records c JOIN tasks t"
@@ -912,7 +1053,7 @@ class Worker:
" FROM cost_records c JOIN tasks t"
" ON c.task_id = t.id"
" WHERE t.created_by = ? AND c.day LIKE ?"
" AND c.usd_micros IS NULL",
" AND c.usd_micros IS NULL AND c.source = 'actual'",
(owner, month + "%")).fetchone()
unpriced_used = unpriced_row[0] if unpriced_row else 0
finally:
@@ -940,11 +1081,15 @@ class Worker:
% (unpriced_used, cap, month))
return None
except Exception as exc:
# P1 fail-closed:查库失败不再返回 None。调用方按
# 前缀分流(见本函数 docstring):quota_check_failed
# 在新回合边界 = 拒绝,在执行中轮询 = 只警告不杀轮。
try:
self.logger.warning("quota check failed: %s", exc)
except Exception:
pass
return None
return ("quota_check_failed: 配额查询失败,拒绝新回合"
"(查库异常,非配额用完)")
def _ready_tasks(self):
"""当前就绪任务列表(pick_next 的批量版)。
@@ -1123,6 +1268,41 @@ class Worker:
pass
return
def _docker_quota_watchdog(self, task_id, container_name,
stop_event, quota_check, interval=30.0):
"""Docker 同步路径的配额看门狗(P1,复用取消看门狗模式)。
同步路径阻塞在 docker run 里,run_one 的边界检查要等
本轮返回才跑得到——配额在别处被花完时,本轮会继续烧
到自然结束。本线程每 interval 秒(默认 30 秒,不超过
30 秒)用 quota_check 查一次已记账配额:命中时按容器
名真杀容器并记 _quota_hit_tasks,_run_attempt 在同步
返回后把结果改写成 quota_exceeded,run_one 走统一的
配额中断收尾(interrupted + 配额原因 + 已花成本照记,
下一轮不再续跑)。查库失败时 quota_check 返回 False
(run_one 的 _quota_hit 会把 quota_check_failed 前缀
过滤掉,见 _quota_exceeded docstring),本线程只继续
轮询、不杀当前轮;下一轮边界仍会拒绝。诚实边界:看
门狗只能看到已记账用量 + 往轮累计用量,本轮容器内尚
未回传的 token 它看不到——那部分在结果返回后由边界
门拦住,不会悄悄续跑下一轮。
"""
while not stop_event.wait(interval):
try:
over = bool(quota_check()) if quota_check else False
except Exception:
over = False
if not over:
continue
self._mark_quota_hit(task_id)
try:
kill = getattr(self.executor, "kill_container", None)
if kill is not None and container_name:
kill(container_name)
except Exception:
pass
return
def cleanup_stale_containers(self):
"""启动清理兜底:清掉上一轮残留的 codex- 容器。
@@ -2069,6 +2249,7 @@ class Worker:
# 按容器名真杀;非 Docker 路径不启动,避免空线程)。
_watchdog_stop = None
_watchdog_thread = None
_quota_watchdog_thread = None
if is_docker and hasattr(self.executor, "kill_container"):
try:
_container_name = self.executor.container_name(task)
@@ -2081,6 +2262,17 @@ class Worker:
args=(task_id, _container_name, _watchdog_stop),
daemon=True)
_watchdog_thread.start()
# P1 配额看门狗:只在同步 Docker 路径起(进程
# 模式的 _execute_via_process 已每 30 秒轮询
# quota_check,不重复起)。quota_check 为 None
# 时不起(无配额检查可做,不空转线程)。
if quota_check is not None and not self._use_popen():
_quota_watchdog_thread = threading.Thread(
target=self._docker_quota_watchdog,
args=(task_id, _container_name,
_watchdog_stop, quota_check),
daemon=True)
_quota_watchdog_thread.start()
try:
gateway_provider, gateway_env = self._ensure_gateway(task)
if self._use_popen():
@@ -2123,8 +2315,14 @@ class Worker:
prep_err)
if goal_mode:
if task.get("_resume"):
# P1 断点续跑传 None(不传 --goal-text,
# host 沿用原 goal+verify);开新目标
# 才传本轮 prompt 当新 goal_text。
_resume_text = None if task.get(
"_resume_kind") == "breakpoint" \
else task["prompt"]
result = self.executor.goal_resume_task(
task, task["prompt"],
task, _resume_text,
timeout=effective_docker_timeout,
extra_mounts=tenant_mounts,
network=tenant_network,
@@ -2165,8 +2363,11 @@ class Worker:
if task.get("_resume") and isinstance(
self.executor, LocalExecutor):
_resume_text = None if task.get(
"_resume_kind") == "breakpoint" \
else task["prompt"]
result = self.executor.goal_resume_task(
task, task["prompt"],
task, _resume_text,
extra_env=gateway_env,
model_provider=gateway_provider)
elif isinstance(self.executor, LocalExecutor):
@@ -2220,6 +2421,8 @@ class Worker:
_watchdog_stop.set()
if _watchdog_thread is not None:
_watchdog_thread.join(timeout=2)
if _quota_watchdog_thread is not None:
_quota_watchdog_thread.join(timeout=2)
# 同步路径执行结束:清掉取消用的容器标记(取消路径里
# 若已清过,这里再清一次无害)。进程模式的 proc 由
# _execute_via_process 自己的 finally 清,这里只清
@@ -2236,6 +2439,18 @@ class Worker:
if result is not None and result.get("status") == "failed" and \
"docker run timeout" in (result.get("stdout", "") or ""):
result = dict(result, status="timeout")
# P1 配额看门狗命中改写:同步路径被看门狗杀容器后,
# 执行器只返回普通 failed,run_one 无法区分“模型失败”
# 与“配额杀容器”——命中时改写成 quota_exceeded,run_one
# 走统一配额中断收尾(interrupted + 配额原因 + 成本照记,
# 不重试、不续跑下一轮)。改写前先取走命中标记,避免
# 旧命中污染同一任务的后续尝试;超时标记优先于配额
# 改写的情况不存在(看门狗杀容器时执行器不会同时报超时)。
if result is not None and self._take_quota_hit(task_id):
result = dict(result, status="quota_exceeded")
if error is None:
error = ("quota_exceeded: Docker 配额看门狗已杀容器"
"(配额中断,下一轮不再续跑)")
return result, error
def _attempt_error(self, result, error, token):
@@ -2384,7 +2599,12 @@ class Worker:
task.get("priority", 0) or 0, task["created_by"],
now, now,
part.get("executor") or task.get("executor"),
part.get("model"), part.get("effort"),
# P1:分工未指定模型/强度时继承父任务(旧版
# 直接写 part 的 None,子任务会落到 worker
# 默认模型/强度,与父任务的显式选择脱节,
# 既改成本也改行为,且排查时看不出继承关系)。
part.get("model") or task.get("model"),
part.get("effort") or task.get("effort"),
task.get("environment_id"), task["id"], name))
self._audit(
task["project_id"], "coordinator_child_created",
@@ -2422,6 +2642,109 @@ class Worker:
conn.close()
return rows
def _coord_mark_orphans(self, task, child_ids):
"""把仍未终态的子任务标 interrupted 并记审计(P1 孤儿收口)。
只对 CAS 改到行的子任务记审计(rowcount==1):已被别的
worker 终态化的子任务不动,避免覆盖它的结果。返回被标
记的子任务 id 列表。正在跑的子任务被标 interrupted 后,
跑它的 worker 会在轮询里看到库状态并杀进程/容器(popen
每 1 秒查库、Docker 取消看门狗每 1 秒查库),不会无声
继续烧钱;若那个 worker 也已死,子任务至少已是终态,
不会再被认领烧钱。
"""
marked = []
rows = self._coord_child_rows(child_ids)
pending = _coord_nonterminal([
{"id": cid, "status": (rows.get(cid) or {}).get("status")}
for cid in child_ids])
if not pending:
return marked
conn = _db_connect(self.db_path)
try:
for ch in pending:
cur = conn.execute(
"UPDATE tasks SET status = 'interrupted',"
" updated_at = ? WHERE id = ? AND status NOT IN"
" ('completed', 'failed', 'interrupted', 'merged',"
" 'merge-failed')",
(time.time(), ch["id"]))
if cur.rowcount == 1:
marked.append(ch["id"])
self._audit(
task["project_id"], "coordinator_child_orphaned",
"%s -> %s (parent %s)"
% (ch["id"], "interrupted", task["id"]))
conn.commit()
finally:
conn.close()
return marked
def sweep_orphaned_coordinator_children(self):
"""常驻循环每轮的孤儿扫尾(P1):父已终态或父行失踪、子
仍未终态时,把子标 interrupted 并记审计。
父仍在 running/queued 时绝不动子(协调员等齐期间子任务
未终态是正常状态)。父行失踪(被删)同样算孤儿:没有
父会再来合流,子任务继续跑只是烧钱。返回被标记的
(parent_id, child_id) 对列表;查库失败时返回 [] 且只记
日志,不炸循环。
"""
marked = []
try:
conn = _db_connect(self.db_path)
try:
rows = conn.execute(
"SELECT id, parent_task_id, project_id FROM tasks"
" WHERE parent_task_id IS NOT NULL AND status NOT IN"
" ('completed', 'failed', 'interrupted', 'merged',"
" 'merge-failed')").fetchall()
parents = {}
for row in rows:
try:
pid = row["parent_task_id"]
except (KeyError, IndexError, TypeError):
continue
if pid not in parents:
prow = conn.execute(
"SELECT status FROM tasks WHERE id = ?",
(pid,)).fetchone()
try:
parents[pid] = prow["status"] \
if prow is not None else None
except (KeyError, IndexError, TypeError):
parents[pid] = None
for row in rows:
try:
cid = row["id"]
pid = row["parent_task_id"]
proj = row["project_id"]
except (KeyError, IndexError, TypeError):
continue
pstatus = parents.get(pid)
if pstatus is not None and \
pstatus not in TERMINAL_STATUSES:
continue
cur = conn.execute(
"UPDATE tasks SET status = 'interrupted',"
" updated_at = ? WHERE id = ? AND status NOT IN"
" ('completed', 'failed', 'interrupted', 'merged',"
" 'merge-failed')",
(time.time(), cid))
if cur.rowcount == 1:
marked.append((pid, cid))
self._audit(
proj, "coordinator_child_orphaned",
"%s -> interrupted (parent %s, parent"
" status %s)" % (cid, pid, pstatus))
conn.commit()
finally:
conn.close()
except Exception as exc:
log_event(self.logger, "worker_orphan_sweep_failed",
message="orphan sweep failed: %s" % exc)
return marked
def _run_coordinator(self, task, started_at, poll_interval=None,
timeout=None):
"""协调员执行:建子任务 → 等齐 → 合流汇总写回。
@@ -2460,6 +2783,7 @@ class Worker:
timed_out = True
break
time.sleep(poll_interval)
orphaned = []
if cancelled or timed_out:
rows = self._coord_child_rows(child_ids)
for cid in child_ids:
@@ -2469,6 +2793,15 @@ class Worker:
self.cancel(cid)
except Exception:
pass
# P1:cancel() 对已终态/不存在的子任务返回 False,
# 跨 worker 在跑的子任务也可能因竞态仍未终态——
# cancel 之后仍未终态的一律直接标 interrupted 并记
# 审计(_coord_mark_orphans),绝不留孤儿无声烧钱;
# 标记结果写进汇总报告,调用方在详情里看得到。
try:
orphaned = self._coord_mark_orphans(task, child_ids)
except Exception:
orphaned = []
rows = self._coord_child_rows(child_ids)
merged = []
for ch in children:
@@ -2479,6 +2812,9 @@ class Worker:
"result_text": row.get("result_text"),
"error_text": row.get("error_text")})
report, overall = _coord_build_report(merged)
if orphaned:
report += ("\n\n孤儿子任务已标 interrupted:%s"
% "、".join(orphaned))
if cancelled:
final = "interrupted"
error = "cancelled"
@@ -2531,6 +2867,47 @@ class Worker:
"executor": self._executor_name(), "stdout": report,
"error": error, "events": [], "cost_record_id": None}
def _record_usage_cost(self, task, actual_model, usage_total,
stdout, fallback_count):
"""记一条任务成本并累加 worker 级用量(调用方须持 _db_lock)。
P1 成本修齐:旧版 max_cost_usd 参数存在但 usage_cost_usd
从未累加——_budget_exceeded 的成本门永远是 0,等于没设。
本函数是唯一记账点(终局块与取消分支共用):actual 行
按价目表算 usd_micros(无价模型为 None),有价行才累加
usage_cost_usd(无价行不折算成 0 美元,仍受无价上限管);
无用量时回落字符估算记 estimated,绝不冒充 actual。
cache_write_input_tokens 不进这里:它只在 usage dict 里
做观测(见 cost.py 的不计费声明),不加进 total、不用
于定价。返回 cost record id。
"""
task_id = task["id"]
if usage_total and (usage_total.get("total_tokens") or 0) > 0:
micros = price_usd_micros(
actual_model, usage_total.get("input_tokens", 0),
usage_total.get("cached_input_tokens", 0),
usage_total.get("output_tokens", 0))
cost_id = self.costs.record(
task_id, task["project_id"], actual_model,
input_tokens=usage_total.get("input_tokens", 0),
output_tokens=usage_total.get("output_tokens", 0),
cached_input_tokens=usage_total.get(
"cached_input_tokens", 0),
usd_micros=micros,
source="actual", fallback_count=fallback_count)
else:
micros = None
total = estimate_tokens(task.get("prompt") or "", stdout)
cost_id = self.costs.record(
task_id, task["project_id"], actual_model,
input_tokens=total, output_tokens=0, usd_micros=None,
source="estimated", fallback_count=fallback_count)
if micros is not None:
# max_cost_usd 不再是死参数:只有有价花费才累加,
# _budget_exceeded 据此在下一认领前挡住超支 worker。
self.usage_cost_usd += micros / 1000000.0
return cost_id
def run_one(self, task=None):
"""认领并执行一个任务,返回结果 dict(见模块 docstring)。
@@ -2552,11 +2929,32 @@ class Worker:
# 成它(只换内存里的副本,原 prompt 不动);_resume 标记
# 让 _run_attempt 在 goal 模式下走 goal-resume 而不是 goal
# (goal 会撞已存在,只有 goal-resume 能在旧线程上开新循环)。
# P1 goal-resume 分流(断点 vs 开新目标,三处一致的 worker 侧):
# - 旧验收形态(verify_command 非空):一律断点续跑——
# 绝不能把续跑消息当新 goal_text 传给 host.py,否则
# host 的 goal-resume --goal-text 分支会用 verify=None
# 开新循环,把原 verify_command 覆盖掉;断点调用不传
# --goal-text,host 沿用原 goal + 原 verify + 原轮次,
# 从 goal_rounds_done 的下一轮续跑。续跑消息本轮不
# 当 prompt 用(断点语义就是续原目标),跑完照常清掉。
# - goal_mode=1(无验收):开新目标——续跑消息就是新
# goal_text(服务端 /continue 带 goal_mode=1 时已把
# task.goal 写成该消息,这里与之对齐)。
# - 非 goal 任务:沿用旧行为(prompt 换成续跑消息,走
# 已存在续跑 run 路径)。
resume_message = task.get("resume_message")
if resume_message:
task = dict(task)
task["prompt"] = resume_message
task["_resume"] = True
if task.get("verify_command"):
task["_resume"] = True
task["_resume_kind"] = "breakpoint"
elif task.get("goal_mode"):
task["prompt"] = resume_message
task["_resume"] = True
task["_resume_kind"] = "new-goal"
else:
task["prompt"] = resume_message
task["_resume"] = True
self._audit(task["project_id"], "worker_claim",
"%s executor=%s" % (task_id, self._executor_name()))
# log_event 只接受 task_id/project_id/status/message 四个
@@ -2647,6 +3045,56 @@ class Worker:
"stdout": "", "error": quota_error, "events": [],
"cost_record_id": None}
# P1 goal 轮次上限门(执行侧防线):旧验收形态的
# goal_max_rounds 超过 GOAL_MAX_ROUNDS_CAP 时直接 failed,
# 不进 host.py goal 循环烧轮次;goal_mode=1 的 0(模型宣告
# 才结束)豁免轮次上限(上界是配额),但 0 配上旧验收命令
# 时同样 failed(服务端建任务时 0 本就 400,老行若有 0
# 验收形态属于脏数据,不按不限轮次跑)。建任务时的 400
# 校验在 server.py(另一路文件),这里是执行侧第二道门,
# 错误文本写明上限与“请新建合规任务”,不静默截断轮次
# (截断会改任务语义,调用方看不出被改过)。
_goal_rounds = task.get("goal_max_rounds")
if (task.get("verify_command") or task.get("goal_mode")) \
and _goal_rounds is not None:
_goal_cap_error = None
try:
_goal_rounds_int = int(_goal_rounds)
except (TypeError, ValueError):
_goal_rounds_int = None
if _goal_rounds_int is not None \
and _goal_rounds_int > GOAL_MAX_ROUNDS_CAP:
_goal_cap_error = (
"goal_max_rounds 超过上限 %d 轮(当前 %d),"
"请新建轮次合规的任务" % (
GOAL_MAX_ROUNDS_CAP, _goal_rounds_int))
elif _goal_rounds_int == 0 and task.get("verify_command") \
and not task.get("goal_mode"):
_goal_cap_error = (
"旧验收形态 goal_max_rounds=0 表示不限轮次,"
"已禁用(上限 %d 轮),请新建轮次合规的任务"
% GOAL_MAX_ROUNDS_CAP)
if _goal_cap_error is not None:
self._set_status(task_id, "failed")
self._write_execution_result(
task_id, error_text=_goal_cap_error,
actual_model=self.model,
started_at=started_at, finished_at=time.time(),
only_if_status="failed")
self._audit(task["project_id"], "worker_goal_cap_rejected",
"%s: %s" % (task_id, _goal_cap_error))
log_event(self.logger, "worker_goal_cap_rejected",
task_id=task_id, project_id=task["project_id"],
status="failed")
self._running_clear(task_id)
return {"dispatched": True, "task_id": task_id,
"host_name": self.executor._host_name(task)
if hasattr(self.executor, "_host_name") else task_id,
"status": "failed",
"executor": self._executor_name(),
"stdout": "", "error": _goal_cap_error, "events": [],
"cost_record_id": None}
# 协调员分支:不跑模型回合,改为按分工建并行子任务、等齐、
# 合流汇总。已过 key 门与配额门;仓库门已对协调员任务本身
# 跑过(项目仓库解析失败会在前面直接 failed,不会建一堆
@@ -2721,7 +3169,14 @@ class Worker:
def _quota_hit():
# 闭包读当前值:usage_total 与 task 在重试/fallback
# 时会被重绑,轮询掐断要按最新值判断。
return self._quota_exceeded(task, usage_total) is not None
# P1 分流:执行中轮询只认真正的 quota_exceeded,
# quota_check_failed(查库失败)不杀当前轮——查询
# 抖动不应杀掉已在跑的回合;但它在重试/fallback/
# 后续消息等新回合边界仍是拒绝(那些边界直接调
# _quota_exceeded,不走这个过滤闭包)。
err = self._quota_exceeded(task, usage_total)
return err is not None and \
not err.startswith("quota_check_failed:")
# 分支任务:先 fork 再跑首轮(见 _run_fork);首轮失败
# 不自动重试/fallback,失败后走重试入口重跑首轮(已存在
@@ -2763,6 +3218,39 @@ class Worker:
# 喂回,或新形态的模型自决多轮),worker 级重试/fallback
# 不再叠加——同任务重跑会撞 host 任务名已存在,第二
# 次尝试必失败且污染错误文本。
# P1 唯一例外:goal 首轮撞“已存在”时(重试入口
# 重排同一 goal 任务的典型死法),不再直接 failed,
# 而是走一次断点续跑(goal-resume 不传 --goal-text,
# 沿用原 goal+verify,从断点续跑);断点续跑的
# 结果即本轮结果(成功则完成,仍失败则以续跑的
# 错误为准,不再拼“已存在”)。已是续跑的任务
# (_resume)不再二次回落,避免断点循环。
# 分支任务(forked_from)不走这里:fork 首轮的
# 已存在语义是 dispatch 续跑,已在 _run_attempt
# 内处理,不叠加 goal 断点。
if not task.get("_resume") and not task.get("forked_from"):
_combined = ((result or {}).get("stdout", "") or "") \
+ ((result or {}).get("stderr", "") or "")
_goal_err = (result or {}).get("goal_error") or ""
if _ALREADY_EXISTS_MARKER in _combined or \
_ALREADY_EXISTS_MARKER in _goal_err or \
_ALREADY_EXISTS_MARKER in (error or ""):
self._audit(task["project_id"],
"worker_goal_resume_fallback",
"%s: goal 已存在,走断点续跑" % task_id)
_resume_task = dict(task)
_resume_task["_resume"] = True
_resume_task["_resume_kind"] = "breakpoint"
result, error = self._run_attempt(
_resume_task, quota_check=_quota_hit)
usage_total = sum_usage(
usage_total, (result or {}).get("usage"))
self._deliver_peer_messages(task, result)
if (result or {}).get("status") == "quota_exceeded":
quota_error = self._quota_exceeded(
task, usage_total) or \
"quota_exceeded: 本月配额已用完"
quota_stop = True
break
if result.get("status") == "completed":
break
@@ -2900,6 +3388,18 @@ class Worker:
turn_id=(result or {}).get("turn_id"),
root_turn_id=(result or {}).get("root_turn_id"),
only_if_status="interrupted")
# P1:取消分支先记一条成本再返回(与配额中断收尾
# 同款记账点)。旧版这里直接返回 cost_record_id=None,
# 被取消前已花的模型费用就从账上丢了;取消不等于没
# 花钱。记账失败不挡取消返回(costs.record 异常时
# cost_id 置 None,任务状态仍是 interrupted)。
try:
with self._db_lock:
_cancel_cost_id = self._record_usage_cost(
task, actual_model, usage_total,
(result or {}).get("stdout", ""), fallback_count)
except Exception:
_cancel_cost_id = None
log_event(self.logger, "worker_cancelled", task_id=task_id,
project_id=task["project_id"], status="interrupted")
return {"dispatched": True, "task_id": task_id,
@@ -2908,7 +3408,7 @@ class Worker:
"status": "interrupted", "executor": self._executor_name(),
"stdout": (result or {}).get("stdout", ""),
"error": "cancelled", "events": [],
"cost_record_id": None}
"cost_record_id": _cancel_cost_id}
stdout = result.get("stdout", "") or ""
host_status = result.get("host_status")
@@ -2979,32 +3479,12 @@ class Worker:
self._cleanup_task_seed(task)
with self._db_lock:
self.usage_turns += 1
# 成本:执行器带回上游真实用量时记 actual(input/output/
# cached 全是实测值,usd 按价目表算、无价模型记 None,
# 报表里进 unpriced_actual_tokens 而不是 0 美元);
# 没带回用量时回落字符估算记 estimated,绝不冒充 actual。
# fallback_count 复用同一记录;模型记最后真正跑的那个
# (fallback 成功时就是 fallback 模型)。cost 记录共用
# 一条连接,并发下必须持锁写,否则两个线程共用同一
# sqlite 连接会直接报错。
if usage_total and (usage_total.get("total_tokens") or 0) > 0:
cost_id = self.costs.record(
task_id, task["project_id"], actual_model,
input_tokens=usage_total.get("input_tokens", 0),
output_tokens=usage_total.get("output_tokens", 0),
cached_input_tokens=usage_total.get(
"cached_input_tokens", 0),
usd_micros=price_usd_micros(
actual_model, usage_total.get("input_tokens", 0),
usage_total.get("cached_input_tokens", 0),
usage_total.get("output_tokens", 0)),
source="actual", fallback_count=fallback_count)
else:
total = estimate_tokens(task.get("prompt") or "", stdout)
cost_id = self.costs.record(
task_id, task["project_id"], actual_model,
input_tokens=total, output_tokens=0, usd_micros=None,
source="estimated", fallback_count=fallback_count)
# 成本记账走 _record_usage_cost(终局与取消分支共用
# 的唯一记账点):actual/estimated 分流、usage_cost_usd
# 累加都在里面;cost 记录共用一条连接,并发下必须
# 持锁写,否则两个线程共用同一 sqlite 连接会直接报错。
cost_id = self._record_usage_cost(
task, actual_model, usage_total, stdout, fallback_count)
if final_written:
self._audit(task["project_id"],
"worker_quota_stop" if quota_stop else
@@ -3064,9 +3544,65 @@ class Worker:
发 review-requested/merge-failed,这里只写回任务终态。
跳过(PrSkipped,如任务已不是 merged)与落盘失败都记
failed + error,任务本身保持 merged、可重调 /pr 再排队。
P1 非 local 通道:旧版非 local worker 直接返回 [],
排队的 pr 任务看起来像“没人处理”且无任何痕迹——
静默未完成。现在改为显式 needs_local_worker:只读
peek 本主机 queued 的 pr 任务,逐条记审计 + 日志并
返回 needs_local_worker 条目,任务本身保持 merged、
pr 任务保持 queued(留给 local worker 认领),绝不
把它标 failed(failed 会让调用方以为 PR 已失败,且
任务仍可重调 /pr 的语义会被混淆)。为什么不直接把
pr_jobs 行写成 needs_local_worker 状态:pr_jobs 的认
领只认 queued(claim_pr_job 在 pr_exec.py,本路不改
该文件),写成新状态后 local worker 也认领不到,任务
会被永久搁置;真要落库状态,需 pr_exec.py 的认领同
步改成含 needs_local_worker 或 local 重排队——该改动
留给协调员(见汇报残留),本路不伪造一个“写了状态
却没人能接”的通道。签名无效时同样不碰任务(返回 [],
只记日志),fail-closed。
"""
if self._executor_name() != "local":
if not self._code_signature_valid():
log_event(self.logger, "worker_pr_refused_code_signature",
message="code signature missing or changed,"
" refusing pr jobs")
return []
if self._executor_name() != "local":
results = []
try:
from cloud.pr_exec import ensure_pr_jobs_table
conn = _db_connect(self.db_path)
try:
ensure_pr_jobs_table(conn)
rows = conn.execute(
"SELECT id, task_id, project_id FROM pr_jobs"
" WHERE host = ? AND status = 'queued'",
(self.host_name,)).fetchall()
finally:
conn.close()
for row in rows:
try:
job_id = row["id"]
task_id = row["task_id"]
project_id = row["project_id"]
except (KeyError, IndexError, TypeError):
continue
reason = ("PR 执行需要 local worker(host-worktrees"
" 只在主机上可见),本 worker 为 %s,"
"任务保持 queued 等 local worker 认领"
% self._executor_name())
self._audit(project_id, "pr_job_needs_local_worker",
"%s: %s" % (job_id, reason))
log_event(self.logger, "pr_job_needs_local_worker",
task_id=task_id, status="needs_local_worker",
message=reason)
results.append({"pr_job_id": job_id,
"task_id": task_id,
"status": "needs_local_worker",
"error": reason})
except Exception as exc:
log_event(self.logger, "pr_job_peek_failed",
message="pr job peek failed: %s" % exc)
return results
results = []
while True:
conn = _db_connect(self.db_path)
@@ -3312,8 +3848,12 @@ class Worker:
# 任务;退出码 75(EX_TEMPFAIL,失败类):unit 托管
# 的由 systemd on-failure 重拉新版,autoscaler 托管
# 的由 reconcile 重拉,孤儿直接消失。单次 run_all
# 不走这里(进程本来就退出)。
if self._code_changed():
# 不走这里(进程本来就退出)。P1 fail-closed:
# 签名缺失同样自退(旧版只在“变了”时退,取不到
# 签名时继续跑是 fail-open);每次认领前的签名门
# 在 _claim 里(池线程不走轮次边界也拦得住)。
if self._code_changed() or \
not self._code_signature_valid():
log_event(self.logger, "worker_code_changed_exit",
message="worker code changed, exiting "
"for restart")
@@ -3336,6 +3876,15 @@ class Worker:
except Exception as exc:
log_event(self.logger, "worker_sweep_failed",
message="worker sweep failed: %s" % exc)
# P1 孤儿扫尾(接在陈旧回收之后):协调员本身
# 被回收标 interrupted 后,它的未终态子任务
# 在这里被统一标 interrupted;协调员仍在跑时
# 本函数不动子任务,见 sweep 方法 docstring。
try:
self.sweep_orphaned_coordinator_children()
except Exception as exc:
log_event(self.logger, "worker_orphan_sweep_failed",
message="orphan sweep failed: %s" % exc)
# 每轮结束写心跳:哪怕本轮一个任务都没派,worker 活着
# 这件事本身也要被 server 看到(文件心跳 + 注册表心跳)。
self._write_heartbeat()