feat: 协调员任务(并行子任务、等齐、合流汇总)

建任务时传 coordinator_parts(分工列表,1-20 项,分工名唯一)即为协调员任务:本身不跑模型回合,worker 认领后一次建齐子任务(普通任务,并行被各 worker 认领),等全部终态后合流成汇总报告写回。子任务带 parent_task_id 与 coordinator_part,控制台可用 parent_task_id 查到;协调员重跑按分工名复用已完成子任务、失败/中断的回 queued 重跑;取消与超时尽力取消未终态子任务。子任务的配额/key/执行器门照常生效,不构成绕过。协调员任务拒绝与 goal/plan/验收命令同用。控制台:任务列表 [协调]/[协调子任务] 标记、详情分工与子任务区、新建表单子任务 JSON 输入。新单测 26 个。
This commit is contained in:
host committed 2026-10-27 15:45:17 +08:00
1 parent 29efc6c61f
commit 7c46ac8e5b
6 files changed
+892 -10

No files matched your search

+28 -1
View File
@@ -160,6 +160,8 @@
<label><input id="planMode" type="checkbox"> 计划模式(只读)</label> <label><input id="planMode" type="checkbox"> 计划模式(只读)</label>
</div> </div>
<textarea id="prompt" placeholder="任务 prompt(必填)"></textarea> <textarea id="prompt" placeholder="任务 prompt(必填)"></textarea>
<textarea id="coordParts" placeholder="协调员子任务 JSON(可选),留空为普通任务"></textarea>
<div class="muted">协调员格式:[{"name":"后端","prompt":"做后端"},{"name":"前端","prompt":"做前端"}];提交后 worker 并行建子任务、等齐、合流汇总。不能与 goal/计划模式同用。</div>
<div class="bar"> <div class="bar">
<select id="modelSelect"> <select id="modelSelect">
<option value="">模型:默认(跟随配置)</option> <option value="">模型:默认(跟随配置)</option>
@@ -248,6 +250,8 @@
<div class="bar"><button id="btnContinueConfirm" class="primary">继续会话</button><span id="continueMsgSpan" class="muted"></span></div> <div class="bar"><button id="btnContinueConfirm" class="primary">继续会话</button><span id="continueMsgSpan" class="muted"></span></div>
</div> </div>
<pre id="detailBody" style="display:none"></pre> <pre id="detailBody" style="display:none"></pre>
<h3>协调子任务</h3>
<div id="coordBox" class="muted">点任务行查看详情后,这里显示协调员的子任务</div>
<h3>子代理链路</h3> <h3>子代理链路</h3>
<div id="agentBox" class="muted">点任务行查看详情后,这里显示子代理链路</div> <div id="agentBox" class="muted">点任务行查看详情后,这里显示子代理链路</div>
<h3>实时链路</h3> <h3>实时链路</h3>
@@ -485,7 +489,7 @@ async function loadTasks() {
"<td>" + esc(shortId(t.id)) + "</td>" + "<td>" + esc(shortId(t.id)) + "</td>" +
"<td title='" + esc(t.turn_id || "") + "'>" + esc(shortId(t.turn_id)) + "</td>" + "<td title='" + esc(t.turn_id || "") + "'>" + esc(shortId(t.turn_id)) + "</td>" +
"<td title='" + esc(t.root_turn_id || "") + "'>" + esc(shortId(t.root_turn_id)) + "</td>" + "<td title='" + esc(t.root_turn_id || "") + "'>" + esc(shortId(t.root_turn_id)) + "</td>" +
"<td>" + (t.plan ? "[计划] " : "") + (t.forked_from ? "[分支] " : "") + ((t.goal_mode || t.verify_command) ? "[目标] " : "") + (t.environment_id ? "[环境 " + esc(shortId(t.environment_id)) + "] " : "") + ((t.attachments && t.attachments.length) ? "[附件×" + t.attachments.length + "] " : "") + esc(String(t.prompt || "").slice(0, 60)) + "</td>" + "<td>" + (t.plan ? "[计划] " : "") + (t.forked_from ? "[分支] " : "") + (t.coordinator_parts ? "[协调] " : "") + (t.parent_task_id ? "[协调子任务] " : "") + ((t.goal_mode || t.verify_command) ? "[目标] " : "") + (t.environment_id ? "[环境 " + esc(shortId(t.environment_id)) + "] " : "") + ((t.attachments && t.attachments.length) ? "[附件×" + t.attachments.length + "] " : "") + esc(String(t.prompt || "").slice(0, 60)) + "</td>" +
"<td class='st-" + esc(t.status) + "'>" + esc(t.status) + "</td>" + "<td class='st-" + esc(t.status) + "'>" + esc(t.status) + "</td>" +
"<td>" + esc(t.executor || "") + "</td>" + "<td>" + esc(t.executor || "") + "</td>" +
"<td>" + esc(t.worktree_host || "") + "</td>" + "<td>" + esc(t.worktree_host || "") + "</td>" +
@@ -568,6 +572,18 @@ function scheduleAgents(taskId) {
if (agentTimer) clearTimeout(agentTimer); if (agentTimer) clearTimeout(agentTimer);
agentTimer = setTimeout(function () { loadAgents(taskId); }, 1000); agentTimer = setTimeout(function () { loadAgents(taskId); }, 1000);
} }
async function loadCoordChildren(taskId) {
var box = $("#coordBox");
if (!currentTask || !currentTask.coordinator_parts) { box.innerHTML = "<span class='muted'>(非协调员任务)</span>"; return; }
try {
var d = await api("/tasks?project_id=" + encodeURIComponent(currentTask.project_id) + "&parent_task_id=" + encodeURIComponent(taskId), "GET");
var kids = d.tasks || [];
if (!kids.length) { box.innerHTML = "<span class='muted'>(子任务尚未创建)</span>"; return; }
box.innerHTML = kids.map(function (k) {
return "<div>分工 <b>" + esc(k.coordinator_part || "") + "</b> | 状态 <span class='st-" + esc(k.status) + "'>" + esc(k.status) + "</span> | " + esc(shortId(k.id)) + " | " + esc(String(k.prompt || "").slice(0, 60)) + "</div>";
}).join("");
} catch (e) { box.innerHTML = "<span class='err'>协调子任务加载失败:" + esc(e.message) + "</span>"; }
}
async function startTrace(taskId, status) { async function startTrace(taskId, status) {
stopTrace(); stopTrace();
var box = $("#traceBox"); var box = $("#traceBox");
@@ -655,6 +671,8 @@ async function showDetail(taskId) {
(t.plan ? " | <b>计划模式(只读)</b>" : "") + (t.plan ? " | <b>计划模式(只读)</b>" : "") +
(t.forked_from ? " | 分支自 " + esc(shortId(t.forked_from)) : "") + (t.forked_from ? " | 分支自 " + esc(shortId(t.forked_from)) : "") +
(t.goal_mode ? " | <b>goal 模式(不限轮次,模型宣告完成)</b>" : (t.verify_command ? " | goal 模式(旧语义,验收命令)" : "")) + (t.goal_mode ? " | <b>goal 模式(不限轮次,模型宣告完成)</b>" : (t.verify_command ? " | goal 模式(旧语义,验收命令)" : "")) +
(t.coordinator_parts ? " | <b>协调员:" + t.coordinator_parts.length + " 个子任务(" + esc(t.coordinator_parts.map(function (p) { return p.name; }).join("、")) + ")</b>" : "") +
(t.parent_task_id ? " | 协调子任务,父任务 " + esc(shortId(t.parent_task_id)) + (t.coordinator_part ? "(" + esc(t.coordinator_part) + ")" : "") : "") +
((t.attachments && t.attachments.length) ? " | 附件:" + t.attachments.map(esc).join("、") : "") + ((t.attachments && t.attachments.length) ? " | 附件:" + t.attachments.map(esc).join("、") : "") +
(t.pr_url ? " | <a href='" + esc(t.pr_url) + "' target='_blank' rel='noopener'>PR #" + esc(t.pr_number) + "</a>" : "") + (t.pr_url ? " | <a href='" + esc(t.pr_url) + "' target='_blank' rel='noopener'>PR #" + esc(t.pr_number) + "</a>" : "") +
"<br>prompt:<br>" + esc(t.prompt || ""); "<br>prompt:<br>" + esc(t.prompt || "");
@@ -678,6 +696,8 @@ async function showDetail(taskId) {
startTrace(taskId, t.status); startTrace(taskId, t.status);
$("#agentBox").innerHTML = "子代理链路加载中…"; $("#agentBox").innerHTML = "子代理链路加载中…";
loadAgents(taskId); loadAgents(taskId);
$("#coordBox").innerHTML = "协调子任务加载中…";
loadCoordChildren(taskId);
loadFollowups(taskId); loadFollowups(taskId);
loadPeerMessages(taskId); loadPeerMessages(taskId);
$("#peerMsg").textContent = ""; $("#peerMsg").textContent = "";
@@ -853,6 +873,12 @@ $("#createTask").onclick = async function () {
// 验收命令与轮次(服务端 goal_mode=1 时一律不限轮次)。 // 验收命令与轮次(服务端 goal_mode=1 时一律不限轮次)。
body.goal_mode = 1; body.goal_mode = 1;
} }
var coordRaw = $("#coordParts").value.trim();
if (coordRaw) {
try { body.coordinator_parts = JSON.parse(coordRaw); }
catch (e) { $("#taskMsg").textContent = "协调员子任务 JSON 解析失败"; return; }
if (!Array.isArray(body.coordinator_parts)) { $("#taskMsg").textContent = "协调员子任务必须是数组"; return; }
}
var atts = gatherAttachments(); var atts = gatherAttachments();
if (atts.length) { if (atts.length) {
try { try {
@@ -867,6 +893,7 @@ $("#createTask").onclick = async function () {
var t = await api("/tasks", "POST", body); var t = await api("/tasks", "POST", body);
$("#taskMsg").innerHTML = "<span class='ok'>任务已建:" + esc(shortId(t.id)) + "</span>"; $("#taskMsg").innerHTML = "<span class='ok'>任务已建:" + esc(shortId(t.id)) + "</span>";
$("#prompt").value = ""; $("#prompt").value = "";
$("#coordParts").value = "";
$("#attFiles").value = ""; $("#attDir").value = ""; $("#attImg").value = ""; $("#attFiles").value = ""; $("#attDir").value = ""; $("#attImg").value = "";
loadTasks(); loadTasks();
} catch (e) { $("#taskMsg").innerHTML = "<span class='err'>提交失败:" + esc(e.message) + "</span>"; } } catch (e) { $("#taskMsg").innerHTML = "<span class='err'>提交失败:" + esc(e.message) + "</span>"; }
+160
View File
@@ -0,0 +1,160 @@
"""协调员:把一个任务拆成多个并行子任务、等齐、合流汇总。
设计(与现有任务模型对齐,不另起一套执行器):
- 协调员本身是一个云端任务:建任务时传 coordinator_parts(分工
列表),它不跑模型回合;worker 认领后按分工一次建齐子任务
(普通任务,并行被各 worker 认领),等全部终态后合流成一份
汇总报告写回协调员任务。
- 子任务是普通任务:配额门、key 门、执行器过滤、取消语义全部
走现有链路,协调员不构成绕过;子任务带 parent_task_id 与
coordinator_part(分工名),控制台可用 parent_task_id 查到它们。
- 幂等与续跑:协调员任务重跑(重试/续跑)时按分工名复用已有
子任务,不重复建;已完成的子任务直接复用,失败/中断的回
queued 重跑——重试协调员即重试未完成的分工。
- 取消与超时:协调员被取消时尽力取消未终态的子任务;等齐
超时时同样取消挂起的子任务并标 failed,报告里写清哪几项
没完成。取消路径的协调员标 interrupted,不发完成/失败推送
(与普通任务的取消口径一致)。
本模块只放纯逻辑(校验、解析、汇总、建表),不碰 DB 之外的
任何东西,单测不需要 worker/server。
"""
import json
MAX_PARTS = 20
DEFAULT_TIMEOUT_SECONDS = 3600
DEFAULT_POLL_INTERVAL = 5.0
REPORT_RESULT_CHARS = 500
EFFORTS = {"low", "medium", "high", "xhigh", "max"}
EXECUTORS = {"local", "docker", "cloud"}
def validate_parts(parts):
"""校验 coordinator_parts,返回 (normalized, error)。
normalized 每项:{name, prompt, model, effort, executor},
未指定的三项为 None(子任务跟随协调员任务或 worker 默认)。
"""
if not isinstance(parts, list):
return None, "coordinator_parts must be a list"
if not parts:
return None, "coordinator_parts must not be empty"
if len(parts) > MAX_PARTS:
return None, "coordinator_parts exceeds %d parts" % MAX_PARTS
seen = set()
out = []
for part in parts:
if not isinstance(part, dict):
return None, "each coordinator part must be an object"
name = part.get("name")
if not isinstance(name, str) or not name.strip():
return None, "each coordinator part needs a non-empty name"
name = name.strip()
if name in seen:
return None, "duplicate coordinator part name: %s" % name
seen.add(name)
prompt = part.get("prompt")
if not isinstance(prompt, str) or not prompt.strip():
return None, "part %s needs a non-empty prompt" % name
model = part.get("model")
if model is not None and (
not isinstance(model, str) or not model.strip()):
return None, "part %s: model must be a non-empty string" % name
effort = part.get("effort")
if effort is not None and effort not in EFFORTS:
return None, (
"part %s: effort must be low, medium, high, xhigh or max"
% name)
executor = part.get("executor")
if executor is not None and executor not in EXECUTORS:
return None, (
"part %s: executor must be local, docker or cloud" % name)
out.append({
"name": name,
"prompt": prompt.strip(),
"model": model.strip() if model else None,
"effort": effort,
"executor": executor,
})
return out, None
def parse_parts(raw):
"""DB 列里的 coordinator_parts → list;坏值给 [],绝不炸。
raw 可能是 JSON 文本(DB 列)、已解析的 list(内存里),或
None(普通任务)。非法分工一律按空处理,调用方据此判定
“不是协调员任务”,而不是把脏数据当分工执行。
"""
if not raw:
return []
if isinstance(raw, list):
parts, err = validate_parts(raw)
return parts if err is None else []
try:
data = json.loads(raw)
except (TypeError, ValueError):
return []
if not isinstance(data, list):
return []
parts, err = validate_parts(data)
return parts if err is None else []
def ensure_columns(conn):
"""给 tasks 表补协调员三列(与 migrate_tasks_table 同款写法)。
coordinator_parts:协调员任务的分工 JSON(普通任务为 NULL)。
parent_task_id:子任务的协调员父任务 id(非子任务为 NULL)。
coordinator_part:子任务的分工名(复用与续跑按它匹配)。
SQLite 与 Postgres 两支都走它;已存在时 no-op。
"""
try:
cols = {row[1] for row in conn.execute(
"PRAGMA table_info(tasks)").fetchall()}
except Exception:
return
for name, coltype in {
"coordinator_parts": "TEXT",
"parent_task_id": "TEXT",
"coordinator_part": "TEXT"}.items():
if name not in cols:
try:
conn.execute(
"ALTER TABLE tasks ADD COLUMN %s %s" % (name, coltype))
except Exception:
pass
def build_report(children):
"""合流汇总:children 为 [{name, id, status, result_text,
error_text}],返回 (report_text, overall_status)。
overall:全部 completed 才 completed,否则 failed(取消与超时
路径由调用方直接定 interrupted/failed,不走这里)。单项输出
截断到 REPORT_RESULT_CHARS,汇总报告本身仍受 worker 写回的
4000 字上限约束(超长时尾部项会被截断,头部计数不受影响)。
"""
lines = []
done = 0
for ch in children:
st = ch.get("status") or "?"
if st == "completed":
done += 1
lines.append("## %s (%s)" % (ch.get("name"), st))
lines.append("子任务:%s" % ch.get("id"))
res = (ch.get("result_text") or "")[:REPORT_RESULT_CHARS]
err = (ch.get("error_text") or "")[:REPORT_RESULT_CHARS]
if res:
lines.append(res)
if err:
lines.append("错误:%s" % err)
if not res and not err:
lines.append("(无输出)")
lines.append("")
overall = "completed" if children and done == len(children) else "failed"
head = "协调员汇总:%d/%d 个子任务完成,总体 %s" % (
done, len(children), overall)
return head + "\n\n" + "\n".join(lines), overall
+3
View File
@@ -841,6 +841,9 @@ POSTGRES_SCHEMA = (
"resume_message TEXT, " "resume_message TEXT, "
"turn_id TEXT, " "turn_id TEXT, "
"root_turn_id TEXT, " "root_turn_id TEXT, "
"coordinator_parts TEXT, "
"parent_task_id TEXT, "
"coordinator_part TEXT, "
"UNIQUE(project_id, idempotency_key))", "UNIQUE(project_id, idempotency_key))",
"CREATE INDEX IF NOT EXISTS idx_tasks_status ON tasks(status)", "CREATE INDEX IF NOT EXISTS idx_tasks_status ON tasks(status)",
# 附件包与任务附件:与 server._ensure_uploads_tables 同款, # 附件包与任务附件:与 server._ensure_uploads_tables 同款,
+53 -8
View File
@@ -56,6 +56,8 @@ from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
from urllib.parse import parse_qs, quote, urlparse from urllib.parse import parse_qs, quote, urlparse
from cloud.auth import AuthStore from cloud.auth import AuthStore
from cloud.coordinator import parse_parts as _coord_parse_parts
from cloud.coordinator import validate_parts as _coord_validate_parts
from cloud.cost import CostStore from cloud.cost import CostStore
from cloud.db import connect as _db_connect from cloud.db import connect as _db_connect
from cloud.db import create_database as _create_database from cloud.db import create_database as _create_database
@@ -761,6 +763,13 @@ def migrate_tasks_table(conn):
# root_turn_id(worker 写回时只在新值非空时覆盖)。 # root_turn_id(worker 写回时只在新值非空时覆盖)。
"turn_id": "TEXT", "turn_id": "TEXT",
"root_turn_id": "TEXT", "root_turn_id": "TEXT",
# 协调员:coordinator_parts 记分工 JSON(普通任务为 NULL,
# worker 认领后按它建并行子任务、等齐、合流);parent_task_id
# 与 coordinator_part 记在子任务行上(父任务 id 与分工名,
# 续跑按分工名复用已有子任务,不重复建)。
"coordinator_parts": "TEXT",
"parent_task_id": "TEXT",
"coordinator_part": "TEXT",
} }
for name, coltype in new_cols.items(): for name, coltype in new_cols.items():
if name not in cols: if name not in cols:
@@ -1086,6 +1095,11 @@ def _task_to_dict(row, attachments=None):
"environment_id": _row_get(row, "environment_id"), "environment_id": _row_get(row, "environment_id"),
"turn_id": _row_get(row, "turn_id"), "turn_id": _row_get(row, "turn_id"),
"root_turn_id": _row_get(row, "root_turn_id"), "root_turn_id": _row_get(row, "root_turn_id"),
"coordinator_parts": (
_coord_parse_parts(_row_get(row, "coordinator_parts"))
if _row_get(row, "coordinator_parts") else None),
"parent_task_id": _row_get(row, "parent_task_id"),
"coordinator_part": _row_get(row, "coordinator_part"),
} }
@@ -2432,6 +2446,22 @@ class CloudControlPlaneHandler(BaseHTTPRequestHandler):
self._send_json(400, { self._send_json(400, {
"error": "goal_max_rounds must be >= 1"}) "error": "goal_max_rounds must be >= 1"})
return return
# 协调员:coordinator_parts 非空时任务本身不跑模型回合,
# worker 认领后按分工建并行子任务、等齐、合流;与 goal/
# plan/验收命令语义冲突(那些都是“本任务跑模型”的意思),
# 同用时 400 且不建任务。
coord_parts = None
if body.get("coordinator_parts") is not None:
coord_parts, coord_err = _coord_validate_parts(
body.get("coordinator_parts"))
if coord_err:
self._send_json(400, {"error": coord_err})
return
if goal_mode or plan or verify_command:
self._send_json(400, {
"error": "协调员任务不跑模型回合,不能与"
" goal/plan/验收命令同用"})
return
idempotency_key = body.get("idempotency_key") idempotency_key = body.get("idempotency_key")
if idempotency_key is not None and not isinstance(idempotency_key, str): if idempotency_key is not None and not isinstance(idempotency_key, str):
self._send_json(400, {"error": "idempotency_key must be a string"}) self._send_json(400, {"error": "idempotency_key must be a string"})
@@ -2553,14 +2583,17 @@ class CloudControlPlaneHandler(BaseHTTPRequestHandler):
" created_at, updated_at, last_heartbeat, executor," " created_at, updated_at, last_heartbeat, executor,"
" goal, verify_command, goal_max_rounds," " goal, verify_command, goal_max_rounds,"
" model, effort, approval, plan, goal_mode," " model, effort, approval, plan, goal_mode,"
" environment_id)" " environment_id, coordinator_parts,"
" parent_task_id, coordinator_part)"
" VALUES(?, ?, ?, 'queued', ?, ?, ?, ?, ?, ?, NULL, ?," " VALUES(?, ?, ?, 'queued', ?, ?, ?, ?, ?, ?, NULL, ?,"
" ?, ?, ?, ?, ?, ?, ?, ?, ?)", " ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
(task_id, project_id, prompt, idempotency_key, (task_id, project_id, prompt, idempotency_key,
json.dumps(depends_on), priority, user_id, now, now, json.dumps(depends_on), priority, user_id, now, now,
executor, goal, verify_command, goal_max_rounds, executor, goal, verify_command, goal_max_rounds,
model, effort, approval, plan, goal_mode, model, effort, approval, plan, goal_mode,
environment_id), environment_id,
json.dumps(coord_parts) if coord_parts else None,
None, None),
) )
for rel_path, size in bundle_files: for rel_path, size in bundle_files:
conn.execute( conn.execute(
@@ -2609,13 +2642,25 @@ class CloudControlPlaneHandler(BaseHTTPRequestHandler):
if not store.check_permission(user_id, project_id, "read"): if not store.check_permission(user_id, project_id, "read"):
self._send_json(403, {"error": "forbidden"}) self._send_json(403, {"error": "forbidden"})
return return
parent_task_id = None
parent_ids = query.get("parent_task_id")
if parent_ids:
parent_task_id = parent_ids[0]
conn = self._db() conn = self._db()
try: try:
rows = conn.execute( if parent_task_id:
"SELECT * FROM tasks WHERE project_id = ?" rows = conn.execute(
" ORDER BY created_at, id", "SELECT * FROM tasks WHERE project_id = ?"
(project_id,), " AND parent_task_id = ?"
).fetchall() " ORDER BY created_at, id",
(project_id, parent_task_id),
).fetchall()
else:
rows = conn.execute(
"SELECT * FROM tasks WHERE project_id = ?"
" ORDER BY created_at, id",
(project_id,),
).fetchall()
amap = _attachments_map(conn, [r["id"] for r in rows]) amap = _attachments_map(conn, [r["id"] for r in rows])
finally: finally:
conn.close() conn.close()
+417
View File
@@ -0,0 +1,417 @@
"""协调员单测:分工校验、汇总报告、worker 分发/等齐/合流、服务端接口。
不跑真实模型回合、不联网:worker 侧用临时 SQLite + 假 runner,
子任务终态由测试直接写库模拟(等齐逻辑只读子任务行,不跑模型);
服务端侧用本地临时端口 + 临时 SQLite,与 test_peer_messages_server
同款骨架。
"""
import http.client
import json
import os
import sqlite3
import sys
import tempfile
import threading
import time
import unittest
from pathlib import Path
from types import SimpleNamespace
try:
from cloud.auth import AuthStore
from cloud.coordinator import build_report, parse_parts, validate_parts
from cloud.server import create_server, init_db
from cloud.worker import Worker
except ImportError: # discover -s cloud/tests 时把项目根目录补进 sys.path
sys.path.insert(0, str(Path(__file__).resolve().parents[2]))
from cloud.auth import AuthStore
from cloud.coordinator import build_report, parse_parts, validate_parts
from cloud.server import create_server, init_db
from cloud.worker import Worker
def _completed_runner(cmd):
return SimpleNamespace(returncode=0, stdout="{}", stderr="")
def _insert_task(db_path, task_id, prompt="do work", status="queued",
project_id="p1", coordinator_parts=None,
parent_task_id=None, coordinator_part=None,
result_text=None, error_text=None):
now = time.time()
conn = sqlite3.connect(db_path)
try:
conn.execute(
"INSERT INTO tasks(id, project_id, prompt, status,"
" idempotency_key, depends_on, priority, created_by,"
" created_at, updated_at, last_heartbeat, coordinator_parts,"
" parent_task_id, coordinator_part, result_text, error_text)"
" VALUES(?, ?, ?, ?, NULL, '[]', 0, 'tester', ?, ?, NULL,"
" ?, ?, ?, ?, ?)",
(task_id, project_id, prompt, status, now, now,
json.dumps(coordinator_parts) if coordinator_parts else None,
parent_task_id, coordinator_part, result_text, error_text),
)
conn.commit()
finally:
conn.close()
def _row(db_path, task_id):
conn = sqlite3.connect(db_path)
try:
row = conn.execute(
"SELECT id, status, result_text, error_text, parent_task_id,"
" coordinator_part, executor, model, effort FROM tasks"
" WHERE id = ?", (task_id,)).fetchone()
finally:
conn.close()
if row is None:
return None
return {
"id": row[0], "status": row[1], "result_text": row[2],
"error_text": row[3], "parent_task_id": row[4],
"coordinator_part": row[5], "executor": row[6],
"model": row[7], "effort": row[8],
}
def _children(db_path, parent_id):
conn = sqlite3.connect(db_path)
try:
rows = conn.execute(
"SELECT id FROM tasks WHERE parent_task_id = ?"
" ORDER BY coordinator_part",
(parent_id,)).fetchall()
finally:
conn.close()
return [r[0] for r in rows]
class ValidatePartsTestCase(unittest.TestCase):
def test_valid_minimal(self):
parts, err = validate_parts([{"name": "a", "prompt": "做 a"}])
self.assertIsNone(err)
self.assertEqual(parts, [{"name": "a", "prompt": "做 a",
"model": None, "effort": None,
"executor": None}])
def test_valid_with_options(self):
parts, err = validate_parts([{
"name": "b", "prompt": "做 b", "model": "m1",
"effort": "high", "executor": "docker"}])
self.assertIsNone(err)
self.assertEqual(parts[0]["model"], "m1")
self.assertEqual(parts[0]["effort"], "high")
self.assertEqual(parts[0]["executor"], "docker")
def test_rejects_non_list(self):
_parts, err = validate_parts({"name": "a"})
self.assertIsNotNone(err)
def test_rejects_empty(self):
_parts, err = validate_parts([])
self.assertIsNotNone(err)
def test_rejects_too_many(self):
parts = [{"name": "p%d" % i, "prompt": "x"} for i in range(21)]
_parts, err = validate_parts(parts)
self.assertIsNotNone(err)
def test_rejects_duplicate_name(self):
_parts, err = validate_parts([
{"name": "a", "prompt": "x"}, {"name": "a", "prompt": "y"}])
self.assertIsNotNone(err)
def test_rejects_empty_name_or_prompt(self):
_parts, err = validate_parts([{"name": " ", "prompt": "x"}])
self.assertIsNotNone(err)
_parts, err = validate_parts([{"name": "a", "prompt": " "}])
self.assertIsNotNone(err)
def test_rejects_bad_effort_executor_model(self):
_parts, err = validate_parts(
[{"name": "a", "prompt": "x", "effort": "ultra"}])
self.assertIsNotNone(err)
_parts, err = validate_parts(
[{"name": "a", "prompt": "x", "executor": "k8s"}])
self.assertIsNotNone(err)
_parts, err = validate_parts(
[{"name": "a", "prompt": "x", "model": " "}])
self.assertIsNotNone(err)
def test_rejects_non_dict_part(self):
_parts, err = validate_parts(["a"])
self.assertIsNotNone(err)
class ParsePartsTestCase(unittest.TestCase):
def test_none_and_empty(self):
self.assertEqual(parse_parts(None), [])
self.assertEqual(parse_parts(""), [])
def test_bad_json(self):
self.assertEqual(parse_parts("{not json"), [])
self.assertEqual(parse_parts('{"name": "a"}'), [])
def test_valid_json(self):
parts = parse_parts('[{"name": "a", "prompt": "x"}]')
self.assertEqual(len(parts), 1)
self.assertEqual(parts[0]["name"], "a")
def test_list_input(self):
self.assertEqual(len(parse_parts([{"name": "a", "prompt": "x"}])), 1)
self.assertEqual(parse_parts([{"name": "a"}]), [])
class BuildReportTestCase(unittest.TestCase):
def test_all_completed(self):
report, overall = build_report([
{"name": "a", "id": "1", "status": "completed",
"result_text": "A 好了", "error_text": None},
{"name": "b", "id": "2", "status": "completed",
"result_text": "B 好了", "error_text": None}])
self.assertEqual(overall, "completed")
self.assertIn("2/2", report)
self.assertIn("A 好了", report)
def test_mixed_failed(self):
report, overall = build_report([
{"name": "a", "id": "1", "status": "completed",
"result_text": "ok", "error_text": None},
{"name": "b", "id": "2", "status": "failed",
"result_text": None, "error_text": "炸了"}])
self.assertEqual(overall, "failed")
self.assertIn("1/2", report)
self.assertIn("炸了", report)
def test_truncates_long_output(self):
report, _overall = build_report([
{"name": "a", "id": "1", "status": "completed",
"result_text": "x" * 600, "error_text": None}])
self.assertNotIn("x" * 600, report)
self.assertIn("x" * 500, report)
def test_empty_output_note(self):
report, _overall = build_report([
{"name": "a", "id": "1", "status": "completed",
"result_text": None, "error_text": None}])
self.assertIn("无输出", report)
class WorkerCoordinatorTestCase(unittest.TestCase):
def setUp(self):
self.tmp = tempfile.TemporaryDirectory()
self.db_path = os.path.join(self.tmp.name, "control.db")
init_db(self.db_path)
self.env = {"XCM_API_KEY": "test-placeholder"}
def tearDown(self):
self.tmp.cleanup()
def _worker(self):
return Worker(self.db_path, "/fake/host.py",
runner=_completed_runner, env=self.env)
def _parts(self):
return [{"name": "后端", "prompt": "做后端", "model": "m1",
"effort": "high"},
{"name": "前端", "prompt": "做前端"}]
def test_sync_creates_children_once(self):
_insert_task(self.db_path, "coord", coordinator_parts=self._parts())
worker = self._worker()
task = worker.get_task("coord")
first = worker._coord_sync_children(task, task["coordinator_parts"])
second = worker._coord_sync_children(task, task["coordinator_parts"])
self.assertEqual([c["id"] for c in first],
[c["id"] for c in second])
self.assertEqual(len(_children(self.db_path, "coord")), 2)
backend = _row(self.db_path, first[0]["id"])
self.assertEqual(backend["parent_task_id"], "coord")
self.assertEqual(backend["coordinator_part"], "后端")
self.assertEqual(backend["model"], "m1")
self.assertEqual(backend["effort"], "high")
self.assertEqual(backend["status"], "queued")
def test_sync_requeues_failed_keeps_completed(self):
_insert_task(self.db_path, "coord", coordinator_parts=self._parts())
_insert_task(self.db_path, "c-old", status="failed",
parent_task_id="coord", coordinator_part="后端",
result_text="旧输出", error_text="旧错")
_insert_task(self.db_path, "c-done", status="completed",
parent_task_id="coord", coordinator_part="前端",
result_text="已做完")
worker = self._worker()
task = worker.get_task("coord")
children = worker._coord_sync_children(task, task["coordinator_parts"])
self.assertEqual([c["id"] for c in children], ["c-old", "c-done"])
old = _row(self.db_path, "c-old")
self.assertEqual(old["status"], "queued")
self.assertIsNone(old["result_text"])
self.assertIsNone(old["error_text"])
done = _row(self.db_path, "c-done")
self.assertEqual(done["status"], "completed")
self.assertEqual(done["result_text"], "已做完")
def test_run_one_all_completed(self):
_insert_task(self.db_path, "coord", coordinator_parts=self._parts())
_insert_task(self.db_path, "c1", status="completed",
parent_task_id="coord", coordinator_part="后端",
result_text="后端好了")
_insert_task(self.db_path, "c2", status="completed",
parent_task_id="coord", coordinator_part="前端",
result_text="前端好了")
worker = self._worker()
result = worker.run_one(worker.get_task("coord"))
self.assertEqual(result["status"], "completed")
self.assertIn("2/2", result["stdout"])
self.assertIn("后端好了", result["stdout"])
self.assertEqual(_row(self.db_path, "coord")["status"], "completed")
def test_run_timeout_cancels_pending(self):
_insert_task(self.db_path, "coord", coordinator_parts=self._parts())
conn = sqlite3.connect(self.db_path)
conn.execute("UPDATE tasks SET status = 'running' WHERE id = 'coord'")
conn.commit()
conn.close()
worker = self._worker()
task = worker.get_task("coord")
result = worker._run_coordinator(
task, time.time(), poll_interval=0.01, timeout=0.05)
self.assertEqual(result["status"], "failed")
self.assertIn("timeout", result["error"])
self.assertEqual(_row(self.db_path, "coord")["status"], "failed")
for cid in _children(self.db_path, "coord"):
self.assertEqual(_row(self.db_path, cid)["status"], "interrupted")
def test_run_cancelled(self):
_insert_task(self.db_path, "coord", coordinator_parts=self._parts())
_insert_task(self.db_path, "c1", status="completed",
parent_task_id="coord", coordinator_part="后端",
result_text="ok")
_insert_task(self.db_path, "c2", status="completed",
parent_task_id="coord", coordinator_part="前端",
result_text="ok")
conn = sqlite3.connect(self.db_path)
conn.execute("UPDATE tasks SET status = 'running' WHERE id = 'coord'")
conn.commit()
conn.close()
worker = self._worker()
worker._cancel_requested.add("coord")
result = worker._run_coordinator(
worker.get_task("coord"), time.time(),
poll_interval=0.01, timeout=60)
self.assertEqual(result["status"], "interrupted")
self.assertEqual(result["error"], "cancelled")
self.assertEqual(_row(self.db_path, "coord")["status"], "interrupted")
class ServerCoordinatorTestCase(unittest.TestCase):
def setUp(self):
self.tmp = tempfile.TemporaryDirectory()
self.db_path = os.path.join(self.tmp.name, "control.db")
self.auth = AuthStore(self.db_path)
self.owner_id, self.owner_key = self.auth.create_user("owner")
self.server = create_server(self.db_path, "127.0.0.1", 0)
self.port = self.server.server_address[1]
self.thread = threading.Thread(
target=self.server.serve_forever, daemon=True)
self.thread.start()
status, self.project = self._req(
"POST", "/projects", {"name": "demo"}, self.owner_key)
self.assertEqual(status, 201)
def tearDown(self):
self.server.shutdown()
self.server.server_close()
self.thread.join(timeout=5)
try:
self.auth.conn.close()
except Exception:
pass
self.tmp.cleanup()
def _req(self, method, path, body=None, key=None):
conn = http.client.HTTPConnection(
"127.0.0.1", self.port, timeout=10)
hdrs = {}
payload = None
if body is not None:
payload = json.dumps(body).encode("utf-8")
hdrs["Content-Type"] = "application/json"
if key is not None:
hdrs["X-API-Key"] = key
try:
conn.request(method, path, body=payload, headers=hdrs)
resp = conn.getresponse()
data = resp.read()
finally:
conn.close()
try:
parsed = json.loads(data.decode("utf-8")) if data else None
except ValueError:
parsed = None
return resp.status, parsed
def _create_coord(self, parts=None, extra=None):
body = {"project_id": self.project["id"], "prompt": "协调干活",
"coordinator_parts": parts if parts is not None else [
{"name": "后端", "prompt": "做后端"},
{"name": "前端", "prompt": "做前端"}]}
if extra:
body.update(extra)
return self._req("POST", "/tasks", body, self.owner_key)
def test_create_coordinator_ok(self):
status, task = self._create_coord()
self.assertEqual(status, 201)
self.assertEqual(len(task["coordinator_parts"]), 2)
self.assertEqual(task["coordinator_parts"][0]["name"], "后端")
self.assertIsNone(task["parent_task_id"])
status, fetched = self._req(
"GET", "/tasks/" + task["id"], key=self.owner_key)
self.assertEqual(status, 200)
self.assertEqual(len(fetched["coordinator_parts"]), 2)
def test_create_coordinator_invalid(self):
status, body = self._create_coord(parts=[])
self.assertEqual(status, 400)
status, _body = self._create_coord(parts=[
{"name": "a", "prompt": "x"}, {"name": "a", "prompt": "y"}])
self.assertEqual(status, 400)
status, _body = self._create_coord(parts=[
{"name": "a", "prompt": "x", "effort": "ultra"}])
self.assertEqual(status, 400)
def test_create_coordinator_conflicts(self):
status, _body = self._create_coord(extra={"goal_mode": 1})
self.assertEqual(status, 400)
status, _body = self._create_coord(extra={"plan": 1})
self.assertEqual(status, 400)
status, _body = self._create_coord(
extra={"verify_command": "true"})
self.assertEqual(status, 400)
def test_list_children_filter(self):
status, coord = self._create_coord()
self.assertEqual(status, 201)
_insert_task(self.db_path, "kid1", parent_task_id=coord["id"],
coordinator_part="后端",
project_id=self.project["id"])
_insert_task(self.db_path, "other", project_id=self.project["id"])
status, data = self._req(
"GET", "/tasks?project_id=" + self.project["id"] +
"&parent_task_id=" + coord["id"], key=self.owner_key)
self.assertEqual(status, 200)
self.assertEqual([t["id"] for t in data["tasks"]], ["kid1"])
status, data = self._req(
"GET", "/tasks?project_id=" + self.project["id"],
key=self.owner_key)
self.assertEqual(status, 200)
self.assertEqual(len(data["tasks"]), 3)
if __name__ == "__main__":
unittest.main()
+231 -1
View File
@@ -130,9 +130,15 @@ import subprocess
import sys import sys
import threading import threading
import time import time
import uuid
from types import SimpleNamespace from types import SimpleNamespace
from cloud.auth import AuthStore from cloud.auth import AuthStore
from cloud.coordinator import build_report as _coord_build_report
from cloud.coordinator import DEFAULT_POLL_INTERVAL as _COORD_POLL_INTERVAL
from cloud.coordinator import DEFAULT_TIMEOUT_SECONDS as _COORD_TIMEOUT
from cloud.coordinator import ensure_columns as _coord_ensure_columns
from cloud.coordinator import parse_parts as _coord_parse_parts
from cloud.cost import CostStore, estimate_tokens, price_usd_micros, sum_usage from cloud.cost import CostStore, estimate_tokens, price_usd_micros, sum_usage
from cloud.db import connect as _db_connect from cloud.db import connect as _db_connect
from cloud.logging_config import get_logger, log_event, setup_logging from cloud.logging_config import get_logger, log_event, setup_logging
@@ -263,6 +269,11 @@ def _task_from_row(row):
"environment_id": _col("environment_id"), "environment_id": _col("environment_id"),
# 续跑消息:老行缺列时 None = 没有待续跑消息。 # 续跑消息:老行缺列时 None = 没有待续跑消息。
"resume_message": _col("resume_message"), "resume_message": _col("resume_message"),
# 协调员:老行缺列时 coordinator_parts 为 [](= 普通任务),
# parent_task_id/coordinator_part 缺列时为 None。
"coordinator_parts": _coord_parse_parts(_col("coordinator_parts")),
"parent_task_id": _col("parent_task_id"),
"coordinator_part": _col("coordinator_part"),
} }
@@ -612,7 +623,10 @@ class Worker:
# turn_id 的回合编号(local/Docker 两路都经 # turn_id 的回合编号(local/Docker 两路都经
# _result_from_runner/_parse_host_result 提取)。 # _result_from_runner/_parse_host_result 提取)。
"turn_id": "TEXT", "turn_id": "TEXT",
"root_turn_id": "TEXT"}.items(): "root_turn_id": "TEXT",
"coordinator_parts": "TEXT",
"parent_task_id": "TEXT",
"coordinator_part": "TEXT"}.items():
if name not in cols: if name not in cols:
try: try:
conn.execute( conn.execute(
@@ -2031,6 +2045,215 @@ class Worker:
return match.group(1) return match.group(1)
return None return None
# -- 协调员 ---------------------------------------------------------
def _coord_sync_children(self, task, parts):
"""按分工建/复用子任务,返回 [{name, id}]。
已有子任务按 coordinator_part(分工名)匹配:已完成的直接
复用;失败/中断的回 queued 重跑(旧输出清掉);未终态的
不动(正被别的 worker 跑或排队);缺失的才新建。重跑协调
员因此不会重复建子任务。子任务是普通任务:执行器过滤、
配额门、key 门都在子任务认领时照常生效。
"""
conn = _db_connect(self.db_path)
try:
_coord_ensure_columns(conn)
try:
conn.commit()
except Exception:
pass
rows = conn.execute(
"SELECT id, coordinator_part, status FROM tasks"
" WHERE parent_task_id = ?",
(task["id"],)).fetchall()
finally:
conn.close()
existing = {}
for row in rows:
try:
existing[row["coordinator_part"]] = {
"id": row["id"], "status": row["status"]}
except (KeyError, IndexError, TypeError):
continue
now = time.time()
children = []
conn = _db_connect(self.db_path)
try:
for part in parts:
name = part["name"]
prev = existing.get(name)
if prev is not None:
if prev["status"] in TERMINAL_STATUSES and \
prev["status"] != "completed":
conn.execute(
"UPDATE tasks SET status = 'queued',"
" result_text = NULL, error_text = NULL,"
" updated_at = ? WHERE id = ?",
(now, prev["id"]))
self._audit(
task["project_id"],
"coordinator_child_requeued",
"%s -> %s" % (name, prev["id"]))
children.append({"name": name, "id": prev["id"]})
continue
child_id = uuid.uuid4().hex
conn.execute(
"INSERT INTO tasks(id, project_id, prompt, status,"
" idempotency_key, depends_on, priority, created_by,"
" created_at, updated_at, last_heartbeat, executor,"
" model, effort, environment_id, parent_task_id,"
" coordinator_part)"
" VALUES(?, ?, ?, 'queued', NULL, '[]', ?, ?, ?, ?,"
" NULL, ?, ?, ?, ?, ?, ?)",
(child_id, task["project_id"], part["prompt"],
task.get("priority", 0) or 0, task["created_by"],
now, now,
part.get("executor") or task.get("executor"),
part.get("model"), part.get("effort"),
task.get("environment_id"), task["id"], name))
self._audit(
task["project_id"], "coordinator_child_created",
"%s -> %s" % (name, child_id))
children.append({"name": name, "id": child_id})
conn.commit()
finally:
conn.close()
return children
def _coord_child_rows(self, child_ids):
"""子任务当前行:{id: {status, result_text, error_text}}。
已删或查不到的子任务不在结果里,调用方按缺失处理(等齐
判断里缺失不算终态,会一直等到超时——子任务行不应被删,
真删了就是超时失败,不静默当完成)。
"""
rows = {}
conn = _db_connect(self.db_path)
try:
for cid in child_ids:
row = conn.execute(
"SELECT id, status, result_text, error_text FROM tasks"
" WHERE id = ?", (cid,)).fetchone()
if row is None:
continue
try:
rows[row["id"]] = {
"status": row["status"],
"result_text": row["result_text"],
"error_text": row["error_text"]}
except (KeyError, IndexError, TypeError):
continue
finally:
conn.close()
return rows
def _run_coordinator(self, task, started_at, poll_interval=None,
timeout=None):
"""协调员执行:建子任务 → 等齐 → 合流汇总写回。
占一个 worker 槽位直到子任务全部终态:等齐期间每轮刷心跳
(sweep 不会误回收);子任务本身由各 worker 并行认领。
取消与超时都会尽力取消未终态的子任务,不留孤儿继续烧
模型费用。返回 run_one 同款结果 dict(无模型花费,
cost_record_id 为 None)。
"""
task_id = task["id"]
project_id = task["project_id"]
parts = task.get("coordinator_parts") or []
if poll_interval is None:
poll_interval = _COORD_POLL_INTERVAL
if timeout is None:
timeout = _COORD_TIMEOUT
children = self._coord_sync_children(task, parts)
child_ids = [c["id"] for c in children]
deadline = time.time() + timeout
cancelled = False
timed_out = False
while True:
try:
self.heartbeat(task_id)
except Exception:
pass
if self._was_cancelled(task_id):
cancelled = True
break
rows = self._coord_child_rows(child_ids)
if all((rows.get(cid) or {}).get("status") in TERMINAL_STATUSES
for cid in child_ids):
break
if time.time() >= deadline:
timed_out = True
break
time.sleep(poll_interval)
if cancelled or timed_out:
rows = self._coord_child_rows(child_ids)
for cid in child_ids:
if (rows.get(cid) or {}).get("status") \
not in TERMINAL_STATUSES:
try:
self.cancel(cid)
except Exception:
pass
rows = self._coord_child_rows(child_ids)
merged = []
for ch in children:
row = rows.get(ch["id"]) or {}
merged.append({
"name": ch["name"], "id": ch["id"],
"status": row.get("status") or "?",
"result_text": row.get("result_text"),
"error_text": row.get("error_text")})
report, overall = _coord_build_report(merged)
if cancelled:
final = "interrupted"
error = "cancelled"
elif timed_out:
final = "failed"
pending = [m["name"] for m in merged
if m["status"] not in TERMINAL_STATUSES]
error = "coordinator timeout: %s 未完成" % "、".join(pending)
report = error + "\n\n" + report
else:
final = overall
error = None
if final != "completed":
failed = [m["name"] for m in merged
if m["status"] != "completed"]
error = "coordinator: %s 未完成" % "、".join(failed)
final_written = self._set_status_if_running(task_id, final)
if not final_written:
final = "interrupted"
finished_at = time.time()
self._write_execution_result(
task_id, result_text=report, error_text=error,
actual_model=None, started_at=started_at,
finished_at=finished_at)
self._cleanup_task_seed(task)
if final_written:
self._audit(project_id,
"worker_completed" if final == "completed"
else ("worker_interrupted"
if final == "interrupted"
else "worker_failed"),
task_id)
log_event(self.logger,
"worker_completed" if final == "completed"
else ("worker_interrupted"
if final == "interrupted"
else "worker_failed"),
task_id=task_id, project_id=project_id, status=final)
if final in ("completed", "failed"):
self._notify_task_finished(task_id, project_id, final,
error=error)
else:
self._audit(project_id, "worker_interrupted", task_id)
host_name = self.executor._host_name(task) \
if hasattr(self.executor, "_host_name") else task_id
return {"dispatched": True, "task_id": task_id,
"host_name": host_name, "status": final,
"executor": self._executor_name(), "stdout": report,
"error": error, "events": [], "cost_record_id": None}
def run_one(self, task=None): def run_one(self, task=None):
"""认领并执行一个任务,返回结果 dict(见模块 docstring)。 """认领并执行一个任务,返回结果 dict(见模块 docstring)。
@@ -2137,6 +2360,13 @@ class Worker:
"stdout": "", "error": quota_error, "events": [], "stdout": "", "error": quota_error, "events": [],
"cost_record_id": None} "cost_record_id": None}
# 协调员分支:不跑模型回合,改为按分工建并行子任务、等齐、
# 合流汇总。已过 key 门与配额门;仓库门已对协调员任务本身
# 跑过(项目仓库解析失败会在前面直接 failed,不会建一堆
# 注定失败的子任务);子任务各自再过 key/配额/仓库门。
if task.get("coordinator_parts"):
return self._run_coordinator(task, started_at)
events = [] events = []
fallback_count = 0 fallback_count = 0
result = None result = None