feat: bounded subagent delegation (v5)
Coordinator parts gain three enforced bounds, validated in validate_parts and carried through the coordinator_parts JSON (no schema change): timeout_seconds (1..86400, per-attempt wall clock from coordinator start; the coordinator poll loop cancels only that child with the same cancel mechanism as the global timeout, other parts keep waiting, audit coordinator_child_bound_exceeded), on_failure fail|tolerate (tolerated failures do not fail the coordinator, are named in the report head, and their failed children are not requeued on coordinator re-run while interrupted children still are), and result_chars (0..2000, default 500, enforced in build_report; the 4000-char report writeback cap still applies). evaluate_outcome is the single outcome rule used by both build_report and the worker error text, so tolerated failures never leak into the coordinator failure reason. Unknown part names fail closed. Deliberately NOT added, after reading the execution layer: per-part token/cost budget (cost is recorded only on completion, so a running child's spend is unobservable and creator quota is monthly, not per part), per-part round cap (normal tasks have no per-task round cap in the worker), and write-scope isolation (sandbox is per-project worktree, a scope string in a prompt would be an instruction, not a guarantee). Tests: test_delegation_bounds.py (8 tests).
This commit is contained in:
1 parent
7f60f12639
commit
c15c224570
4 files changed
+443
-29
No files matched your search
+68
-16
@@ -142,7 +142,9 @@ from cloud.coordinator import build_report as _coord_build_report
|
||||
from cloud.coordinator import DEFAULT_POLL_INTERVAL as _COORD_POLL_INTERVAL
|
||||
from cloud.coordinator import DEFAULT_TIMEOUT_SECONDS as _COORD_TIMEOUT
|
||||
from cloud.coordinator import ensure_columns as _coord_ensure_columns
|
||||
from cloud.coordinator import evaluate_outcome as _coord_evaluate
|
||||
from cloud.coordinator import parse_parts as _coord_parse_parts
|
||||
from cloud.coordinator import parts_by_name as _coord_parts_by_name
|
||||
from cloud.coordinator import nonterminal_children as _coord_nonterminal
|
||||
from cloud.cost import CostStore, estimate_tokens, price_usd_micros, \
|
||||
sum_usage, unpriced_token_cap
|
||||
@@ -5007,10 +5009,14 @@ class Worker:
|
||||
"""按分工建/复用子任务,返回 [{name, id}]。
|
||||
|
||||
已有子任务按 coordinator_part(分工名)匹配:已完成的直接
|
||||
复用;失败/中断的回 queued 重跑(旧输出清掉);未终态的
|
||||
不动(正被别的 worker 跑或排队);缺失的才新建。重跑协调
|
||||
员因此不会重复建子任务。子任务是普通任务:执行器过滤、
|
||||
配额门、key 门都在子任务认领时照常生效。
|
||||
复用;失败/中断的回 queued 重跑(旧输出清掉),但有界
|
||||
委派例外:on_failure=tolerate 分工的 failed 子任务不重
|
||||
排——它的失败已被接受为容忍结果,重排等于撤销容忍;
|
||||
其 interrupted 子任务仍重排,因为中断可能是父任务取消
|
||||
所致、未获公平一轮;未终态的不动(正被别的 worker 跑
|
||||
或排队);缺失的才新建。重跑协调员因此不会重复建子
|
||||
任务。子任务是普通任务:执行器过滤、配额门、key 门都
|
||||
在子任务认领时照常生效。
|
||||
"""
|
||||
conn = _db_connect(self.db_path)
|
||||
try:
|
||||
@@ -5042,15 +5048,24 @@ class Worker:
|
||||
if prev is not None:
|
||||
if prev["status"] in TERMINAL_STATUSES and \
|
||||
prev["status"] != "completed":
|
||||
conn.execute(
|
||||
"UPDATE tasks SET status = 'queued',"
|
||||
" result_text = NULL, error_text = NULL,"
|
||||
" updated_at = ? WHERE id = ?",
|
||||
(now, prev["id"]))
|
||||
self._audit(
|
||||
task["project_id"],
|
||||
"coordinator_child_requeued",
|
||||
"%s -> %s" % (name, prev["id"]))
|
||||
# 有界委派:容忍分工的 failed 子任务接受
|
||||
# 为结果,不重排;interrupted 仍重排。
|
||||
if prev["status"] == "failed" and \
|
||||
part.get("on_failure") == "tolerate":
|
||||
self._audit(
|
||||
task["project_id"],
|
||||
"coordinator_child_tolerated",
|
||||
"%s -> %s" % (name, prev["id"]))
|
||||
else:
|
||||
conn.execute(
|
||||
"UPDATE tasks SET status = 'queued',"
|
||||
" result_text = NULL, error_text = NULL,"
|
||||
" updated_at = ? WHERE id = ?",
|
||||
(now, prev["id"]))
|
||||
self._audit(
|
||||
task["project_id"],
|
||||
"coordinator_child_requeued",
|
||||
"%s -> %s" % (name, prev["id"]))
|
||||
children.append({"name": name, "id": prev["id"]})
|
||||
continue
|
||||
# 自动化入闸(创建时门):子任务认领时虽有配额门,
|
||||
@@ -5379,15 +5394,20 @@ class Worker:
|
||||
task_id = task["id"]
|
||||
project_id = task["project_id"]
|
||||
parts = task.get("coordinator_parts") or []
|
||||
parts_by_name = _coord_parts_by_name(parts)
|
||||
child_by_id = {}
|
||||
if poll_interval is None:
|
||||
poll_interval = _COORD_POLL_INTERVAL
|
||||
if timeout is None:
|
||||
timeout = _COORD_TIMEOUT
|
||||
children = self._coord_sync_children(task, parts)
|
||||
child_ids = [c["id"] for c in children]
|
||||
for ch in children:
|
||||
child_by_id[ch["id"]] = ch["name"]
|
||||
deadline = time.time() + timeout
|
||||
cancelled = False
|
||||
timed_out = False
|
||||
bound_exceeded = []
|
||||
while True:
|
||||
try:
|
||||
self.heartbeat(task_id)
|
||||
@@ -5410,6 +5430,33 @@ class Worker:
|
||||
if all((rows.get(cid) or {}).get("status") in TERMINAL_STATUSES
|
||||
for cid in child_ids):
|
||||
break
|
||||
# 有界委派:单分工超时在本轮内 cancel 该子任务,
|
||||
# 其他分工继续等,不因一个慢分工拖到全局超时。
|
||||
# 计时起点是协调员本轮开始(started_at),重跑协
|
||||
# 调员即新一轮委派、时钟重算。
|
||||
now = time.time()
|
||||
for cid in child_ids:
|
||||
part = parts_by_name.get(child_by_id.get(cid))
|
||||
if part is None:
|
||||
continue
|
||||
limit = part.get("timeout_seconds")
|
||||
if limit is None:
|
||||
continue
|
||||
if (rows.get(cid) or {}).get("status") \
|
||||
in TERMINAL_STATUSES:
|
||||
continue
|
||||
if now - started_at >= limit:
|
||||
try:
|
||||
self.cancel(cid)
|
||||
except Exception:
|
||||
pass
|
||||
name = child_by_id.get(cid)
|
||||
if name not in bound_exceeded:
|
||||
bound_exceeded.append(name)
|
||||
self._audit(
|
||||
project_id,
|
||||
"coordinator_child_bound_exceeded",
|
||||
"%s: timeout %ds" % (name, limit))
|
||||
if time.time() >= deadline:
|
||||
timed_out = True
|
||||
break
|
||||
@@ -5442,7 +5489,10 @@ class Worker:
|
||||
"status": row.get("status") or "?",
|
||||
"result_text": row.get("result_text"),
|
||||
"error_text": row.get("error_text")})
|
||||
report, overall = _coord_build_report(merged)
|
||||
report, overall = _coord_build_report(merged, parts)
|
||||
if bound_exceeded:
|
||||
report += ("\n\n有界委派超时:%s(已取消该子任务,"
|
||||
"其他分工继续等齐)" % "、".join(bound_exceeded))
|
||||
if orphaned:
|
||||
report += ("\n\n孤儿子任务已标 interrupted:%s"
|
||||
% "、".join(orphaned))
|
||||
@@ -5459,8 +5509,10 @@ class Worker:
|
||||
final = overall
|
||||
error = None
|
||||
if final != "completed":
|
||||
failed = [m["name"] for m in merged
|
||||
if m["status"] != "completed"]
|
||||
# 错误里只列非容忍失败名:容忍分工的失败已在
|
||||
# 汇总头部点名,不应再进协调员的失败原因。
|
||||
_overall, failed, _tolerated = _coord_evaluate(
|
||||
merged, parts)
|
||||
error = "coordinator: %s 未完成" % "、".join(failed)
|
||||
final_written = self._set_status_if_running(task_id, final)
|
||||
if not final_written:
|
||||
|
||||
Reference in new issue
Block a user