任务内标记工具：task-create/task-poll/task-read 移植、wait-threads 接通、新增 task-cancel/task-retry
- 新模块 cloud/marker_tools.py：纯逻辑解析、校验、快照格式化、 控制门禁、派生代数与直接上限计数 - sandbox 两处解析提取五种标记块；worker 在 5 处投递点与 _deliver_peer_messages 同款位置执行，报告排成后续消息下一轮回父任务 - wait 每块 cap 到 8 个目标（与服务端 /wait 对齐，spawn 原先的 10 个收紧）， 每轮最多 2 块；取消/重试门禁：同项目、非自己、非 goal、 派生链或同创建人，另查 create_task / approve_merge 权限， 查不到时收紧到仅派生链内 - goal 与协调员任务不解析；单测 30 个，cloud 全量 1043 个、4 跳过、0 失败
This commit is contained in:
host committed
1 parent
eae9884b13
commit
3b2fef95fa
5 files changed
+1819
No files matched your search
+393
@@ -148,7 +148,19 @@ from cloud.models import ModelRouter, is_fallbackable, record_fallback
|
||||
from cloud.auto_memory import AutoMemoryStore
|
||||
from cloud.auto_memory import build_injection_prefix as _auto_prefix
|
||||
from cloud.auto_memory import process_due_tasks as _auto_process_due
|
||||
from cloud.marker_tools import ancestry_relation as _marker_ancestry
|
||||
from cloud.marker_tools import check_control_gate as _marker_gate
|
||||
from cloud.marker_tools import count_direct_spawn_children as _marker_count_direct
|
||||
from cloud.marker_tools import fetch_task_snapshot as _marker_snapshot
|
||||
from cloud.marker_tools import format_snapshot as _marker_format
|
||||
from cloud.marker_tools import marker_report_header as _marker_header
|
||||
from cloud.marker_tools import new_child_id as _marker_new_id
|
||||
from cloud.marker_tools import parent_spawn_generation as _marker_generation
|
||||
from cloud.marker_tools import MAX_DIRECT_CHILDREN as _MARKER_MAX_DIRECT
|
||||
from cloud.marker_tools import MAX_SPAWN_DEPTH as _MARKER_MAX_DEPTH
|
||||
from cloud.marker_tools import TERMINAL_STATUSES as _MARKER_TERMINAL
|
||||
from cloud.peer_messages import deliver_peer_message as _deliver_peer_message
|
||||
from cloud.peer_messages import ensure_followups_table as _ensure_followups_table
|
||||
from cloud.peer_messages import peer_message_prefix as _peer_prefix
|
||||
from cloud.peer_messages import row_get as _row_get
|
||||
from cloud.sub2api import ensure_user_gateway_key as _ensure_sub2api_key
|
||||
@@ -2303,6 +2315,382 @@ class Worker:
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
def _queue_self_followup(self, task, body):
|
||||
"""给父任务自己排一条系统后续消息（标记工具报告 / 轮询快照）。
|
||||
|
||||
sender_task_id 置空，排空时不加“来自任务 X 的消息”前缀，
|
||||
正文自带 [任务工具报告] 抬头即可区分。排空循环（run_one
|
||||
的后续消息段）会在轮次边界认领它：父任务运行中直接跑下
|
||||
一轮，父任务已终态时下次认领走续跑——与任务间消息的复活
|
||||
语义同款，只是不经过 peer 的“不能发给自己”拒绝。"""
|
||||
try:
|
||||
conn = _db_connect(self.db_path)
|
||||
try:
|
||||
_ensure_followups_table(conn)
|
||||
conn.execute(
|
||||
"INSERT INTO task_followups(id, task_id, project_id,"
|
||||
" body, mode, status, created_by, sender_task_id,"
|
||||
" created_at, updated_at)"
|
||||
" VALUES(?, ?, ?, ?, 'queued', 'pending', ?, NULL,"
|
||||
" ?, ?)",
|
||||
(uuid.uuid4().hex, task["id"],
|
||||
task["project_id"], body, task.get("created_by"),
|
||||
time.time(), time.time()),
|
||||
)
|
||||
conn.commit()
|
||||
return True
|
||||
finally:
|
||||
conn.close()
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
def _poll_task_snapshot(self, task, poll):
|
||||
"""轮询一个目标任务：每 interval 查一次库状态，直到终态、
|
||||
超时或父任务被取消。等待期间不烧模型费，但占着本 worker
|
||||
的槽位（与协调员等齐同款代价）。返回报告正文。"""
|
||||
target_id = poll["task_id"]
|
||||
interval = poll["interval_seconds"]
|
||||
timeout = poll["timeout_seconds"]
|
||||
chars = poll["chars"]
|
||||
started = time.time()
|
||||
snapshot = None
|
||||
timed_out = False
|
||||
cancelled = False
|
||||
while True:
|
||||
if self._was_cancelled(task["id"]):
|
||||
cancelled = True
|
||||
break
|
||||
try:
|
||||
conn = _db_connect(self.db_path)
|
||||
try:
|
||||
snapshot = _marker_snapshot(conn, target_id)
|
||||
finally:
|
||||
conn.close()
|
||||
except Exception:
|
||||
snapshot = None
|
||||
if snapshot is None:
|
||||
return "轮询任务 %s：不存在（同项目内也查不到）。" % target_id
|
||||
if snapshot["project_id"] != task["project_id"]:
|
||||
return ("轮询任务 %s：拒绝，跨项目任务不可读。"
|
||||
% target_id)
|
||||
if target_id == task["id"]:
|
||||
return "轮询任务 %s：拒绝，不能轮询自己。" % target_id
|
||||
if snapshot["status"] in _MARKER_TERMINAL:
|
||||
break
|
||||
if timeout <= 0:
|
||||
break
|
||||
elapsed = time.time() - started
|
||||
if elapsed >= timeout:
|
||||
timed_out = True
|
||||
break
|
||||
time.sleep(min(interval, max(timeout - elapsed, 0.05)))
|
||||
if snapshot is None:
|
||||
return "轮询任务 %s：读取失败。" % target_id
|
||||
return _marker_format(snapshot, chars, timed_out=timed_out,
|
||||
cancelled=cancelled)
|
||||
|
||||
def _wait_task_targets(self, task, wait):
|
||||
"""等一批目标任务：每 interval 并行查一次全部目标，直到
|
||||
全部终态、超时或父任务被取消。缺失、跨项目、自等待的
|
||||
目标当场记为已处理（写清原因），不让它们把整批拖到超时。
|
||||
轮询/等待目标是只读的，goal 任务也可以等；取消与重试
|
||||
的 goal 拒绝只在控制门禁里，不在这里。返回报告行列表。"""
|
||||
target_ids = wait["task_ids"]
|
||||
interval = wait["interval_seconds"]
|
||||
timeout = wait["timeout_seconds"]
|
||||
chars = wait["chars"]
|
||||
started = time.time()
|
||||
timed_out = False
|
||||
cancelled = False
|
||||
resolved = {} # task_id -> ("snapshot", snap) | ("note", text)
|
||||
latest = {} # task_id -> 最近一次快照（超时报告用）
|
||||
while True:
|
||||
if self._was_cancelled(task["id"]):
|
||||
cancelled = True
|
||||
break
|
||||
pending = [tid for tid in target_ids if tid not in resolved]
|
||||
if not pending:
|
||||
break
|
||||
try:
|
||||
conn = _db_connect(self.db_path)
|
||||
try:
|
||||
for tid in pending:
|
||||
snap = _marker_snapshot(conn, tid)
|
||||
if snap is None:
|
||||
resolved[tid] = (
|
||||
"note",
|
||||
"等待任务 %s：不存在（同项目内也查不到）。"
|
||||
% tid)
|
||||
elif snap["project_id"] != task["project_id"]:
|
||||
resolved[tid] = (
|
||||
"note",
|
||||
"等待任务 %s：拒绝，跨项目任务不可读。"
|
||||
% tid)
|
||||
elif tid == task["id"]:
|
||||
resolved[tid] = (
|
||||
"note",
|
||||
"等待任务 %s：拒绝，不能等待自己。" % tid)
|
||||
else:
|
||||
latest[tid] = snap
|
||||
if snap["status"] in _MARKER_TERMINAL:
|
||||
resolved[tid] = ("snapshot", snap)
|
||||
finally:
|
||||
conn.close()
|
||||
except Exception:
|
||||
pass
|
||||
if all(tid in resolved for tid in target_ids):
|
||||
break
|
||||
if timeout <= 0:
|
||||
break
|
||||
elapsed = time.time() - started
|
||||
if elapsed >= timeout:
|
||||
timed_out = True
|
||||
break
|
||||
time.sleep(min(interval, max(timeout - elapsed, 0.05)))
|
||||
lines = ["[等待 %d 个任务]" % len(target_ids)]
|
||||
for tid in target_ids:
|
||||
kind, value = resolved.get(tid, (None, None))
|
||||
if kind == "note":
|
||||
lines.append(value)
|
||||
elif kind == "snapshot":
|
||||
lines.append(_marker_format(value, chars))
|
||||
else:
|
||||
snap = latest.get(tid)
|
||||
if snap is None:
|
||||
lines.append("等待任务 %s：读取失败。" % tid)
|
||||
else:
|
||||
lines.append(_marker_format(
|
||||
snap, chars, timed_out=timed_out,
|
||||
cancelled=cancelled))
|
||||
if cancelled:
|
||||
lines.append("说明：等待期间父任务被取消，等待已中止。")
|
||||
elif timed_out:
|
||||
lines.append("说明：等待超时，仍有目标未终态，可再等一次。")
|
||||
return lines
|
||||
|
||||
def _marker_auth(self, task, action):
|
||||
"""标记控制操作的权限层，返回 (ok_or_None, reason)。
|
||||
|
||||
ok=True：发送方创建人有该权限；False：明确无权限；
|
||||
None：权限检查本身抛异常（查不到），调用方必须收紧到
|
||||
仅派生链内可用，绝不能静默放宽。"""
|
||||
creator = task.get("created_by")
|
||||
if not creator:
|
||||
return False, "发送方无创建人，无法查权限"
|
||||
try:
|
||||
store = AuthStore(self.db_path)
|
||||
try:
|
||||
ok = store.check_permission(
|
||||
creator, task["project_id"], action)
|
||||
finally:
|
||||
try:
|
||||
store.conn.close()
|
||||
except Exception:
|
||||
pass
|
||||
except Exception as exc:
|
||||
return None, "权限检查失败：%s" % exc
|
||||
return bool(ok), ""
|
||||
|
||||
def _deliver_marker_tools(self, task, result):
|
||||
"""执行本轮 agent 输出里的标记工具块（见 marker_tools）。
|
||||
|
||||
只执行 completed 轮次的块；goal 任务与协调员任务不解析
|
||||
（只收不发，与 peer 同款）；父任务被取消时不执行。
|
||||
执行顺序：创建 → 取消 → 重试 → 轮询 → 等待——创建在前
|
||||
让同一轮刚创建的任务立刻能被轮询，控制操作在快照之前
|
||||
让轮询/等待读到取消或重试后的新状态。全部报告拼成一条
|
||||
系统后续消息排给父任务自己，由排空循环在轮次边界跑下
|
||||
一轮；拒绝只记报告与审计，不炸 run_one。"""
|
||||
if not result or result.get("status") != "completed":
|
||||
return
|
||||
if self._was_cancelled(task["id"]):
|
||||
return
|
||||
if task.get("verify_command") or task.get("goal_mode"):
|
||||
return
|
||||
if task.get("coordinator_parts"):
|
||||
return
|
||||
creates = result.get("task_creates") or []
|
||||
polls = result.get("task_polls") or []
|
||||
waits = result.get("wait_threads") or []
|
||||
cancels = result.get("task_cancels") or []
|
||||
retries = result.get("task_retries") or []
|
||||
if not (creates or polls or waits or cancels or retries):
|
||||
return
|
||||
reports = []
|
||||
created = 0
|
||||
rejected = 0
|
||||
if creates:
|
||||
quota_error = None
|
||||
try:
|
||||
quota_error = self._quota_exceeded(task, None)
|
||||
except Exception as exc:
|
||||
quota_error = "quota_check_failed: %s" % exc
|
||||
if quota_error is not None:
|
||||
reports.append("创建跳过：父任务配额检查未通过（%s），"
|
||||
"本轮 %d 个创建块全部跳过。"
|
||||
% (quota_error, len(creates)))
|
||||
rejected += len(creates)
|
||||
else:
|
||||
try:
|
||||
conn = _db_connect(self.db_path)
|
||||
try:
|
||||
self._ensure_result_columns(conn)
|
||||
generation = _marker_generation(conn, task["id"])
|
||||
direct = _marker_count_direct(conn, task["id"])
|
||||
now = time.time()
|
||||
for create in creates:
|
||||
new_depth = generation + 1
|
||||
if new_depth > _MARKER_MAX_DEPTH:
|
||||
reports.append(
|
||||
"创建跳过：派生深度超限（上限 %d 代）。"
|
||||
% _MARKER_MAX_DEPTH)
|
||||
rejected += 1
|
||||
continue
|
||||
if direct >= _MARKER_MAX_DIRECT:
|
||||
reports.append(
|
||||
"创建跳过：直接派生子任务已达上限 %d 个。"
|
||||
% _MARKER_MAX_DIRECT)
|
||||
rejected += 1
|
||||
continue
|
||||
child_id = _marker_new_id()
|
||||
conn.execute(
|
||||
"INSERT INTO tasks(id, project_id, prompt,"
|
||||
" status, idempotency_key, depends_on,"
|
||||
" priority, created_by, created_at,"
|
||||
" updated_at, last_heartbeat, executor,"
|
||||
" model, effort, environment_id,"
|
||||
" parent_task_id, coordinator_part)"
|
||||
" VALUES(?, ?, ?, 'queued', NULL, '[]',"
|
||||
" ?, ?, ?, ?, NULL, ?, ?, ?, ?, ?, NULL)",
|
||||
(child_id, task["project_id"],
|
||||
create["prompt"],
|
||||
task.get("priority", 0) or 0,
|
||||
task.get("created_by"), now, now,
|
||||
create["executor"]
|
||||
or task.get("executor"),
|
||||
create["model"] or task.get("model"),
|
||||
create["effort"] or task.get("effort"),
|
||||
task.get("environment_id"),
|
||||
task["id"]),
|
||||
)
|
||||
direct += 1
|
||||
created += 1
|
||||
label = create["name"] or "未命名"
|
||||
reports.append(
|
||||
"已创建任务 %s（%s），状态 queued，"
|
||||
"可用 task-poll 按该 id 轮询。"
|
||||
% (child_id, label))
|
||||
conn.commit()
|
||||
finally:
|
||||
conn.close()
|
||||
except Exception as exc:
|
||||
reports.append("创建失败：%s" % exc)
|
||||
rejected += len(creates) - created
|
||||
for kind, blocks, action in (("取消", cancels, "create_task"),
|
||||
("重试", retries, "approve_merge")):
|
||||
for block in blocks:
|
||||
target_id = block["task_id"]
|
||||
try:
|
||||
target = self.get_task(target_id)
|
||||
except Exception as exc:
|
||||
target = None
|
||||
reports.append("%s任务 %s：读取失败（%s），已跳过。"
|
||||
% (kind, target_id, exc))
|
||||
rejected += 1
|
||||
continue
|
||||
relation = "other"
|
||||
if target is not None:
|
||||
try:
|
||||
conn = _db_connect(self.db_path)
|
||||
try:
|
||||
relation = _marker_ancestry(
|
||||
conn, task, target)
|
||||
finally:
|
||||
conn.close()
|
||||
except Exception:
|
||||
relation = "other"
|
||||
gate_kind = "cancel" if kind == "取消" else "retry"
|
||||
ok, reason = _marker_gate(target, task, relation,
|
||||
gate_kind)
|
||||
if not ok:
|
||||
reports.append("%s任务 %s：拒绝（%s）。"
|
||||
% (kind, target_id, reason))
|
||||
rejected += 1
|
||||
continue
|
||||
auth_ok, auth_reason = self._marker_auth(task, action)
|
||||
if auth_ok is False:
|
||||
reports.append("%s任务 %s：拒绝（发送方创建人无"
|
||||
" %s 权限）。"
|
||||
% (kind, target_id, action))
|
||||
rejected += 1
|
||||
continue
|
||||
if auth_ok is None and relation != "descendant":
|
||||
reports.append("%s任务 %s：拒绝（%s，仅派生链内"
|
||||
"可用）。"
|
||||
% (kind, target_id, auth_reason))
|
||||
rejected += 1
|
||||
continue
|
||||
try:
|
||||
if kind == "取消":
|
||||
done = self.cancel(target_id)
|
||||
if done:
|
||||
reports.append("已取消任务 %s，状态为"
|
||||
" interrupted。" % target_id)
|
||||
else:
|
||||
reports.append("取消任务 %s：未执行（目标"
|
||||
"状态已变或不存在）。"
|
||||
% target_id)
|
||||
rejected += 1
|
||||
else:
|
||||
conn = _db_connect(self.db_path)
|
||||
try:
|
||||
cur = conn.execute(
|
||||
"UPDATE tasks SET status = 'queued',"
|
||||
" result_text = NULL, error_text = NULL,"
|
||||
" actual_model = NULL, updated_at = ?"
|
||||
" WHERE id = ? AND status IN"
|
||||
" ('failed', 'interrupted',"
|
||||
" 'merge-failed')",
|
||||
(time.time(), target_id))
|
||||
conn.commit()
|
||||
done = cur.rowcount == 1
|
||||
finally:
|
||||
conn.close()
|
||||
if done:
|
||||
self._audit(task["project_id"],
|
||||
"worker_marker_retry",
|
||||
target_id)
|
||||
reports.append("已重试任务 %s，状态回"
|
||||
" queued，上一轮结果已清空。"
|
||||
% target_id)
|
||||
else:
|
||||
reports.append("重试任务 %s：未执行（目标"
|
||||
"状态已变）。" % target_id)
|
||||
rejected += 1
|
||||
except Exception as exc:
|
||||
reports.append("%s任务 %s：执行失败（%s）。"
|
||||
% (kind, target_id, exc))
|
||||
rejected += 1
|
||||
for poll in polls:
|
||||
try:
|
||||
reports.append(self._poll_task_snapshot(task, poll))
|
||||
except Exception as exc:
|
||||
reports.append("轮询任务 %s 失败：%s"
|
||||
% (poll["task_id"], exc))
|
||||
for wait in waits:
|
||||
try:
|
||||
reports.extend(self._wait_task_targets(task, wait))
|
||||
except Exception as exc:
|
||||
reports.append("等待任务失败：%s" % exc)
|
||||
if reports:
|
||||
body = _marker_header() + "\n" + "\n".join(reports)
|
||||
self._queue_self_followup(task, body)
|
||||
detail = ("%s: spawned=%d rejected=%d polls=%d waits=%d "
|
||||
"cancels=%d retries=%d"
|
||||
% (task["id"], created, rejected, len(polls),
|
||||
len(waits), len(cancels), len(retries)))
|
||||
self._audit(task["project_id"], "worker_marker_tools", detail)
|
||||
|
||||
def _deliver_peer_messages(self, task, result):
|
||||
"""投递本轮 agent 输出里的互发消息标记块（见 peer_messages）。
|
||||
|
||||
@@ -3509,6 +3897,7 @@ class Worker:
|
||||
usage_total = sum_usage(usage_total,
|
||||
(result or {}).get("usage"))
|
||||
self._deliver_peer_messages(task, result)
|
||||
self._deliver_marker_tools(task, result)
|
||||
if (result or {}).get("status") == "quota_exceeded":
|
||||
quota_error = self._quota_exceeded(task, usage_total) or \
|
||||
"quota_exceeded: 本月配额已用完"
|
||||
@@ -3526,6 +3915,7 @@ class Worker:
|
||||
result, error = self._run_attempt(task, quota_check=_quota_hit)
|
||||
usage_total = sum_usage(usage_total, (result or {}).get("usage"))
|
||||
self._deliver_peer_messages(task, result)
|
||||
self._deliver_marker_tools(task, result)
|
||||
if (result or {}).get("status") == "quota_exceeded":
|
||||
# 轮询掐断：进程已被杀，配额原因以现查为准。
|
||||
quota_error = self._quota_exceeded(task, usage_total) or \
|
||||
@@ -3568,6 +3958,7 @@ class Worker:
|
||||
usage_total = sum_usage(
|
||||
usage_total, (result or {}).get("usage"))
|
||||
self._deliver_peer_messages(task, result)
|
||||
self._deliver_marker_tools(task, result)
|
||||
if (result or {}).get("status") == "quota_exceeded":
|
||||
quota_error = self._quota_exceeded(
|
||||
task, usage_total) or \
|
||||
@@ -3611,6 +4002,7 @@ class Worker:
|
||||
usage_total = sum_usage(usage_total,
|
||||
(result or {}).get("usage"))
|
||||
self._deliver_peer_messages(task, result)
|
||||
self._deliver_marker_tools(task, result)
|
||||
if (result or {}).get("status") == "quota_exceeded":
|
||||
quota_error = self._quota_exceeded(task, usage_total) or \
|
||||
"quota_exceeded: 本月配额已用完"
|
||||
@@ -3673,6 +4065,7 @@ class Worker:
|
||||
usage_total = sum_usage(
|
||||
usage_total, (fu_result or {}).get("usage"))
|
||||
self._deliver_peer_messages(task, fu_result)
|
||||
self._deliver_marker_tools(task, fu_result)
|
||||
if (fu_result or {}).get("status") == "quota_exceeded":
|
||||
self._restore_followup(fu["id"])
|
||||
quota_error = self._quota_exceeded(task, usage_total) or \
|
||||
|
||||
Reference in new issue
Block a user