feat: 协调员任务(并行子任务、等齐、合流汇总)

建任务时传 coordinator_parts(分工列表,1-20 项,分工名唯一)即为协调员任务:本身不跑模型回合,worker 认领后一次建齐子任务(普通任务,并行被各 worker 认领),等全部终态后合流成汇总报告写回。子任务带 parent_task_id 与 coordinator_part,控制台可用 parent_task_id 查到;协调员重跑按分工名复用已完成子任务、失败/中断的回 queued 重跑;取消与超时尽力取消未终态子任务。子任务的配额/key/执行器门照常生效,不构成绕过。协调员任务拒绝与 goal/plan/验收命令同用。控制台:任务列表 [协调]/[协调子任务] 标记、详情分工与子任务区、新建表单子任务 JSON 输入。新单测 26 个。
This commit is contained in:
host committed 2026-10-27 15:45:17 +08:00
1 parent 29efc6c61f
commit 7c46ac8e5b
6 files changed
+892 -10

No files matched your search

+28 -1
View File
@@ -160,6 +160,8 @@
<label><input id="planMode" type="checkbox"> 计划模式(只读)</label>
</div>
<textarea id="prompt" placeholder="任务 prompt(必填)"></textarea>
<textarea id="coordParts" placeholder="协调员子任务 JSON(可选),留空为普通任务"></textarea>
<div class="muted">协调员格式:[{"name":"后端","prompt":"做后端"},{"name":"前端","prompt":"做前端"}];提交后 worker 并行建子任务、等齐、合流汇总。不能与 goal/计划模式同用。</div>
<div class="bar">
<select id="modelSelect">
<option value="">模型:默认(跟随配置)</option>
@@ -248,6 +250,8 @@
<div class="bar"><button id="btnContinueConfirm" class="primary">继续会话</button><span id="continueMsgSpan" class="muted"></span></div>
</div>
<pre id="detailBody" style="display:none"></pre>
<h3>协调子任务</h3>
<div id="coordBox" class="muted">点任务行查看详情后,这里显示协调员的子任务</div>
<h3>子代理链路</h3>
<div id="agentBox" class="muted">点任务行查看详情后,这里显示子代理链路</div>
<h3>实时链路</h3>
@@ -485,7 +489,7 @@ async function loadTasks() {
"<td>" + esc(shortId(t.id)) + "</td>" +
"<td title='" + esc(t.turn_id || "") + "'>" + esc(shortId(t.turn_id)) + "</td>" +
"<td title='" + esc(t.root_turn_id || "") + "'>" + esc(shortId(t.root_turn_id)) + "</td>" +
"<td>" + (t.plan ? "[计划] " : "") + (t.forked_from ? "[分支] " : "") + ((t.goal_mode || t.verify_command) ? "[目标] " : "") + (t.environment_id ? "[环境 " + esc(shortId(t.environment_id)) + "] " : "") + ((t.attachments && t.attachments.length) ? "[附件×" + t.attachments.length + "] " : "") + esc(String(t.prompt || "").slice(0, 60)) + "</td>" +
"<td>" + (t.plan ? "[计划] " : "") + (t.forked_from ? "[分支] " : "") + (t.coordinator_parts ? "[协调] " : "") + (t.parent_task_id ? "[协调子任务] " : "") + ((t.goal_mode || t.verify_command) ? "[目标] " : "") + (t.environment_id ? "[环境 " + esc(shortId(t.environment_id)) + "] " : "") + ((t.attachments && t.attachments.length) ? "[附件×" + t.attachments.length + "] " : "") + esc(String(t.prompt || "").slice(0, 60)) + "</td>" +
"<td class='st-" + esc(t.status) + "'>" + esc(t.status) + "</td>" +
"<td>" + esc(t.executor || "") + "</td>" +
"<td>" + esc(t.worktree_host || "") + "</td>" +
@@ -568,6 +572,18 @@ function scheduleAgents(taskId) {
if (agentTimer) clearTimeout(agentTimer);
agentTimer = setTimeout(function () { loadAgents(taskId); }, 1000);
}
async function loadCoordChildren(taskId) {
var box = $("#coordBox");
if (!currentTask || !currentTask.coordinator_parts) { box.innerHTML = "<span class='muted'>(非协调员任务)</span>"; return; }
try {
var d = await api("/tasks?project_id=" + encodeURIComponent(currentTask.project_id) + "&parent_task_id=" + encodeURIComponent(taskId), "GET");
var kids = d.tasks || [];
if (!kids.length) { box.innerHTML = "<span class='muted'>(子任务尚未创建)</span>"; return; }
box.innerHTML = kids.map(function (k) {
return "<div>分工 <b>" + esc(k.coordinator_part || "") + "</b> | 状态 <span class='st-" + esc(k.status) + "'>" + esc(k.status) + "</span> | " + esc(shortId(k.id)) + " | " + esc(String(k.prompt || "").slice(0, 60)) + "</div>";
}).join("");
} catch (e) { box.innerHTML = "<span class='err'>协调子任务加载失败:" + esc(e.message) + "</span>"; }
}
async function startTrace(taskId, status) {
stopTrace();
var box = $("#traceBox");
@@ -655,6 +671,8 @@ async function showDetail(taskId) {
(t.plan ? " | <b>计划模式(只读)</b>" : "") +
(t.forked_from ? " | 分支自 " + esc(shortId(t.forked_from)) : "") +
(t.goal_mode ? " | <b>goal 模式(不限轮次,模型宣告完成)</b>" : (t.verify_command ? " | goal 模式(旧语义,验收命令)" : "")) +
(t.coordinator_parts ? " | <b>协调员:" + t.coordinator_parts.length + " 个子任务(" + esc(t.coordinator_parts.map(function (p) { return p.name; }).join("、")) + ")</b>" : "") +
(t.parent_task_id ? " | 协调子任务,父任务 " + esc(shortId(t.parent_task_id)) + (t.coordinator_part ? "(" + esc(t.coordinator_part) + ")" : "") : "") +
((t.attachments && t.attachments.length) ? " | 附件:" + t.attachments.map(esc).join("、") : "") +
(t.pr_url ? " | <a href='" + esc(t.pr_url) + "' target='_blank' rel='noopener'>PR #" + esc(t.pr_number) + "</a>" : "") +
"<br>prompt:<br>" + esc(t.prompt || "");
@@ -678,6 +696,8 @@ async function showDetail(taskId) {
startTrace(taskId, t.status);
$("#agentBox").innerHTML = "子代理链路加载中…";
loadAgents(taskId);
$("#coordBox").innerHTML = "协调子任务加载中…";
loadCoordChildren(taskId);
loadFollowups(taskId);
loadPeerMessages(taskId);
$("#peerMsg").textContent = "";
@@ -853,6 +873,12 @@ $("#createTask").onclick = async function () {
// 验收命令与轮次(服务端 goal_mode=1 时一律不限轮次)。
body.goal_mode = 1;
}
var coordRaw = $("#coordParts").value.trim();
if (coordRaw) {
try { body.coordinator_parts = JSON.parse(coordRaw); }
catch (e) { $("#taskMsg").textContent = "协调员子任务 JSON 解析失败"; return; }
if (!Array.isArray(body.coordinator_parts)) { $("#taskMsg").textContent = "协调员子任务必须是数组"; return; }
}
var atts = gatherAttachments();
if (atts.length) {
try {
@@ -867,6 +893,7 @@ $("#createTask").onclick = async function () {
var t = await api("/tasks", "POST", body);
$("#taskMsg").innerHTML = "<span class='ok'>任务已建:" + esc(shortId(t.id)) + "</span>";
$("#prompt").value = "";
$("#coordParts").value = "";
$("#attFiles").value = ""; $("#attDir").value = ""; $("#attImg").value = "";
loadTasks();
} catch (e) { $("#taskMsg").innerHTML = "<span class='err'>提交失败:" + esc(e.message) + "</span>"; }
+160
View File
@@ -0,0 +1,160 @@
"""协调员:把一个任务拆成多个并行子任务、等齐、合流汇总。
设计(与现有任务模型对齐,不另起一套执行器):
- 协调员本身是一个云端任务:建任务时传 coordinator_parts(分工
列表),它不跑模型回合;worker 认领后按分工一次建齐子任务
(普通任务,并行被各 worker 认领),等全部终态后合流成一份
汇总报告写回协调员任务。
- 子任务是普通任务:配额门、key 门、执行器过滤、取消语义全部
走现有链路,协调员不构成绕过;子任务带 parent_task_id 与
coordinator_part(分工名),控制台可用 parent_task_id 查到它们。
- 幂等与续跑:协调员任务重跑(重试/续跑)时按分工名复用已有
子任务,不重复建;已完成的子任务直接复用,失败/中断的回
queued 重跑——重试协调员即重试未完成的分工。
- 取消与超时:协调员被取消时尽力取消未终态的子任务;等齐
超时时同样取消挂起的子任务并标 failed,报告里写清哪几项
没完成。取消路径的协调员标 interrupted,不发完成/失败推送
(与普通任务的取消口径一致)。
本模块只放纯逻辑(校验、解析、汇总、建表),不碰 DB 之外的
任何东西,单测不需要 worker/server。
"""
import json
MAX_PARTS = 20
DEFAULT_TIMEOUT_SECONDS = 3600
DEFAULT_POLL_INTERVAL = 5.0
REPORT_RESULT_CHARS = 500
EFFORTS = {"low", "medium", "high", "xhigh", "max"}
EXECUTORS = {"local", "docker", "cloud"}
def validate_parts(parts):
"""校验 coordinator_parts,返回 (normalized, error)。
normalized 每项:{name, prompt, model, effort, executor},
未指定的三项为 None(子任务跟随协调员任务或 worker 默认)。
"""
if not isinstance(parts, list):
return None, "coordinator_parts must be a list"
if not parts:
return None, "coordinator_parts must not be empty"
if len(parts) > MAX_PARTS:
return None, "coordinator_parts exceeds %d parts" % MAX_PARTS
seen = set()
out = []
for part in parts:
if not isinstance(part, dict):
return None, "each coordinator part must be an object"
name = part.get("name")
if not isinstance(name, str) or not name.strip():
return None, "each coordinator part needs a non-empty name"
name = name.strip()
if name in seen:
return None, "duplicate coordinator part name: %s" % name
seen.add(name)
prompt = part.get("prompt")
if not isinstance(prompt, str) or not prompt.strip():
return None, "part %s needs a non-empty prompt" % name
model = part.get("model")
if model is not None and (
not isinstance(model, str) or not model.strip()):
return None, "part %s: model must be a non-empty string" % name
effort = part.get("effort")
if effort is not None and effort not in EFFORTS:
return None, (
"part %s: effort must be low, medium, high, xhigh or max"
% name)
executor = part.get("executor")
if executor is not None and executor not in EXECUTORS:
return None, (
"part %s: executor must be local, docker or cloud" % name)
out.append({
"name": name,
"prompt": prompt.strip(),
"model": model.strip() if model else None,
"effort": effort,
"executor": executor,
})
return out, None
def parse_parts(raw):
"""DB 列里的 coordinator_parts → list;坏值给 [],绝不炸。
raw 可能是 JSON 文本(DB 列)、已解析的 list(内存里),或
None(普通任务)。非法分工一律按空处理,调用方据此判定
“不是协调员任务”,而不是把脏数据当分工执行。
"""
if not raw:
return []
if isinstance(raw, list):
parts, err = validate_parts(raw)
return parts if err is None else []
try:
data = json.loads(raw)
except (TypeError, ValueError):
return []
if not isinstance(data, list):
return []
parts, err = validate_parts(data)
return parts if err is None else []
def ensure_columns(conn):
"""给 tasks 表补协调员三列(与 migrate_tasks_table 同款写法)。
coordinator_parts:协调员任务的分工 JSON(普通任务为 NULL)。
parent_task_id:子任务的协调员父任务 id(非子任务为 NULL)。
coordinator_part:子任务的分工名(复用与续跑按它匹配)。
SQLite 与 Postgres 两支都走它;已存在时 no-op。
"""
try:
cols = {row[1] for row in conn.execute(
"PRAGMA table_info(tasks)").fetchall()}
except Exception:
return
for name, coltype in {
"coordinator_parts": "TEXT",
"parent_task_id": "TEXT",
"coordinator_part": "TEXT"}.items():
if name not in cols:
try:
conn.execute(
"ALTER TABLE tasks ADD COLUMN %s %s" % (name, coltype))
except Exception:
pass
def build_report(children):
"""合流汇总:children 为 [{name, id, status, result_text,
error_text}],返回 (report_text, overall_status)。
overall:全部 completed 才 completed,否则 failed(取消与超时
路径由调用方直接定 interrupted/failed,不走这里)。单项输出
截断到 REPORT_RESULT_CHARS,汇总报告本身仍受 worker 写回的
4000 字上限约束(超长时尾部项会被截断,头部计数不受影响)。
"""
lines = []
done = 0
for ch in children:
st = ch.get("status") or "?"
if st == "completed":
done += 1
lines.append("## %s (%s)" % (ch.get("name"), st))
lines.append("子任务:%s" % ch.get("id"))
res = (ch.get("result_text") or "")[:REPORT_RESULT_CHARS]
err = (ch.get("error_text") or "")[:REPORT_RESULT_CHARS]
if res:
lines.append(res)
if err:
lines.append("错误:%s" % err)
if not res and not err:
lines.append("(无输出)")
lines.append("")
overall = "completed" if children and done == len(children) else "failed"
head = "协调员汇总:%d/%d 个子任务完成,总体 %s" % (
done, len(children), overall)
return head + "\n\n" + "\n".join(lines), overall
+3
View File
@@ -841,6 +841,9 @@ POSTGRES_SCHEMA = (
"resume_message TEXT, "
"turn_id TEXT, "
"root_turn_id TEXT, "
"coordinator_parts TEXT, "
"parent_task_id TEXT, "
"coordinator_part TEXT, "
"UNIQUE(project_id, idempotency_key))",
"CREATE INDEX IF NOT EXISTS idx_tasks_status ON tasks(status)",
# 附件包与任务附件:与 server._ensure_uploads_tables 同款,
+53 -8
View File
@@ -56,6 +56,8 @@ from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
from urllib.parse import parse_qs, quote, urlparse
from cloud.auth import AuthStore
from cloud.coordinator import parse_parts as _coord_parse_parts
from cloud.coordinator import validate_parts as _coord_validate_parts
from cloud.cost import CostStore
from cloud.db import connect as _db_connect
from cloud.db import create_database as _create_database
@@ -761,6 +763,13 @@ def migrate_tasks_table(conn):
# root_turn_id(worker 写回时只在新值非空时覆盖)。
"turn_id": "TEXT",
"root_turn_id": "TEXT",
# 协调员:coordinator_parts 记分工 JSON(普通任务为 NULL,
# worker 认领后按它建并行子任务、等齐、合流);parent_task_id
# 与 coordinator_part 记在子任务行上(父任务 id 与分工名,
# 续跑按分工名复用已有子任务,不重复建)。
"coordinator_parts": "TEXT",
"parent_task_id": "TEXT",
"coordinator_part": "TEXT",
}
for name, coltype in new_cols.items():
if name not in cols:
@@ -1086,6 +1095,11 @@ def _task_to_dict(row, attachments=None):
"environment_id": _row_get(row, "environment_id"),
"turn_id": _row_get(row, "turn_id"),
"root_turn_id": _row_get(row, "root_turn_id"),
"coordinator_parts": (
_coord_parse_parts(_row_get(row, "coordinator_parts"))
if _row_get(row, "coordinator_parts") else None),
"parent_task_id": _row_get(row, "parent_task_id"),
"coordinator_part": _row_get(row, "coordinator_part"),
}
@@ -2432,6 +2446,22 @@ class CloudControlPlaneHandler(BaseHTTPRequestHandler):
self._send_json(400, {
"error": "goal_max_rounds must be >= 1"})
return
# 协调员:coordinator_parts 非空时任务本身不跑模型回合,
# worker 认领后按分工建并行子任务、等齐、合流;与 goal/
# plan/验收命令语义冲突(那些都是“本任务跑模型”的意思),
# 同用时 400 且不建任务。
coord_parts = None
if body.get("coordinator_parts") is not None:
coord_parts, coord_err = _coord_validate_parts(
body.get("coordinator_parts"))
if coord_err:
self._send_json(400, {"error": coord_err})
return
if goal_mode or plan or verify_command:
self._send_json(400, {
"error": "协调员任务不跑模型回合,不能与"
" goal/plan/验收命令同用"})
return
idempotency_key = body.get("idempotency_key")
if idempotency_key is not None and not isinstance(idempotency_key, str):
self._send_json(400, {"error": "idempotency_key must be a string"})
@@ -2553,14 +2583,17 @@ class CloudControlPlaneHandler(BaseHTTPRequestHandler):
" created_at, updated_at, last_heartbeat, executor,"
" goal, verify_command, goal_max_rounds,"
" model, effort, approval, plan, goal_mode,"
" environment_id)"
" environment_id, coordinator_parts,"
" parent_task_id, coordinator_part)"
" VALUES(?, ?, ?, 'queued', ?, ?, ?, ?, ?, ?, NULL, ?,"
" ?, ?, ?, ?, ?, ?, ?, ?, ?)",
" ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
(task_id, project_id, prompt, idempotency_key,
json.dumps(depends_on), priority, user_id, now, now,
executor, goal, verify_command, goal_max_rounds,
model, effort, approval, plan, goal_mode,
environment_id),
environment_id,
json.dumps(coord_parts) if coord_parts else None,
None, None),
)
for rel_path, size in bundle_files:
conn.execute(
@@ -2609,13 +2642,25 @@ class CloudControlPlaneHandler(BaseHTTPRequestHandler):
if not store.check_permission(user_id, project_id, "read"):
self._send_json(403, {"error": "forbidden"})
return
parent_task_id = None
parent_ids = query.get("parent_task_id")
if parent_ids:
parent_task_id = parent_ids[0]
conn = self._db()
try:
rows = conn.execute(
"SELECT * FROM tasks WHERE project_id = ?"
" ORDER BY created_at, id",
(project_id,),
).fetchall()
if parent_task_id:
rows = conn.execute(
"SELECT * FROM tasks WHERE project_id = ?"
" AND parent_task_id = ?"
" ORDER BY created_at, id",
(project_id, parent_task_id),
).fetchall()
else:
rows = conn.execute(
"SELECT * FROM tasks WHERE project_id = ?"
" ORDER BY created_at, id",
(project_id,),
).fetchall()
amap = _attachments_map(conn, [r["id"] for r in rows])
finally:
conn.close()
+417
View File
@@ -0,0 +1,417 @@
"""协调员单测:分工校验、汇总报告、worker 分发/等齐/合流、服务端接口。
不跑真实模型回合、不联网:worker 侧用临时 SQLite + 假 runner,
子任务终态由测试直接写库模拟(等齐逻辑只读子任务行,不跑模型);
服务端侧用本地临时端口 + 临时 SQLite,与 test_peer_messages_server
同款骨架。
"""
import http.client
import json
import os
import sqlite3
import sys
import tempfile
import threading
import time
import unittest
from pathlib import Path
from types import SimpleNamespace
try:
from cloud.auth import AuthStore
from cloud.coordinator import build_report, parse_parts, validate_parts
from cloud.server import create_server, init_db
from cloud.worker import Worker
except ImportError: # discover -s cloud/tests 时把项目根目录补进 sys.path
sys.path.insert(0, str(Path(__file__).resolve().parents[2]))
from cloud.auth import AuthStore
from cloud.coordinator import build_report, parse_parts, validate_parts
from cloud.server import create_server, init_db
from cloud.worker import Worker
def _completed_runner(cmd):
return SimpleNamespace(returncode=0, stdout="{}", stderr="")
def _insert_task(db_path, task_id, prompt="do work", status="queued",
project_id="p1", coordinator_parts=None,
parent_task_id=None, coordinator_part=None,
result_text=None, error_text=None):
now = time.time()
conn = sqlite3.connect(db_path)
try:
conn.execute(
"INSERT INTO tasks(id, project_id, prompt, status,"
" idempotency_key, depends_on, priority, created_by,"
" created_at, updated_at, last_heartbeat, coordinator_parts,"
" parent_task_id, coordinator_part, result_text, error_text)"
" VALUES(?, ?, ?, ?, NULL, '[]', 0, 'tester', ?, ?, NULL,"
" ?, ?, ?, ?, ?)",
(task_id, project_id, prompt, status, now, now,
json.dumps(coordinator_parts) if coordinator_parts else None,
parent_task_id, coordinator_part, result_text, error_text),
)
conn.commit()
finally:
conn.close()
def _row(db_path, task_id):
conn = sqlite3.connect(db_path)
try:
row = conn.execute(
"SELECT id, status, result_text, error_text, parent_task_id,"
" coordinator_part, executor, model, effort FROM tasks"
" WHERE id = ?", (task_id,)).fetchone()
finally:
conn.close()
if row is None:
return None
return {
"id": row[0], "status": row[1], "result_text": row[2],
"error_text": row[3], "parent_task_id": row[4],
"coordinator_part": row[5], "executor": row[6],
"model": row[7], "effort": row[8],
}
def _children(db_path, parent_id):
conn = sqlite3.connect(db_path)
try:
rows = conn.execute(
"SELECT id FROM tasks WHERE parent_task_id = ?"
" ORDER BY coordinator_part",
(parent_id,)).fetchall()
finally:
conn.close()
return [r[0] for r in rows]
class ValidatePartsTestCase(unittest.TestCase):
def test_valid_minimal(self):
parts, err = validate_parts([{"name": "a", "prompt": "做 a"}])
self.assertIsNone(err)
self.assertEqual(parts, [{"name": "a", "prompt": "做 a",
"model": None, "effort": None,
"executor": None}])
def test_valid_with_options(self):
parts, err = validate_parts([{
"name": "b", "prompt": "做 b", "model": "m1",
"effort": "high", "executor": "docker"}])
self.assertIsNone(err)
self.assertEqual(parts[0]["model"], "m1")
self.assertEqual(parts[0]["effort"], "high")
self.assertEqual(parts[0]["executor"], "docker")
def test_rejects_non_list(self):
_parts, err = validate_parts({"name": "a"})
self.assertIsNotNone(err)
def test_rejects_empty(self):
_parts, err = validate_parts([])
self.assertIsNotNone(err)
def test_rejects_too_many(self):
parts = [{"name": "p%d" % i, "prompt": "x"} for i in range(21)]
_parts, err = validate_parts(parts)
self.assertIsNotNone(err)
def test_rejects_duplicate_name(self):
_parts, err = validate_parts([
{"name": "a", "prompt": "x"}, {"name": "a", "prompt": "y"}])
self.assertIsNotNone(err)
def test_rejects_empty_name_or_prompt(self):
_parts, err = validate_parts([{"name": " ", "prompt": "x"}])
self.assertIsNotNone(err)
_parts, err = validate_parts([{"name": "a", "prompt": " "}])
self.assertIsNotNone(err)
def test_rejects_bad_effort_executor_model(self):
_parts, err = validate_parts(
[{"name": "a", "prompt": "x", "effort": "ultra"}])
self.assertIsNotNone(err)
_parts, err = validate_parts(
[{"name": "a", "prompt": "x", "executor": "k8s"}])
self.assertIsNotNone(err)
_parts, err = validate_parts(
[{"name": "a", "prompt": "x", "model": " "}])
self.assertIsNotNone(err)
def test_rejects_non_dict_part(self):
_parts, err = validate_parts(["a"])
self.assertIsNotNone(err)
class ParsePartsTestCase(unittest.TestCase):
def test_none_and_empty(self):
self.assertEqual(parse_parts(None), [])
self.assertEqual(parse_parts(""), [])
def test_bad_json(self):
self.assertEqual(parse_parts("{not json"), [])
self.assertEqual(parse_parts('{"name": "a"}'), [])
def test_valid_json(self):
parts = parse_parts('[{"name": "a", "prompt": "x"}]')
self.assertEqual(len(parts), 1)
self.assertEqual(parts[0]["name"], "a")
def test_list_input(self):
self.assertEqual(len(parse_parts([{"name": "a", "prompt": "x"}])), 1)
self.assertEqual(parse_parts([{"name": "a"}]), [])
class BuildReportTestCase(unittest.TestCase):
def test_all_completed(self):
report, overall = build_report([
{"name": "a", "id": "1", "status": "completed",
"result_text": "A 好了", "error_text": None},
{"name": "b", "id": "2", "status": "completed",
"result_text": "B 好了", "error_text": None}])
self.assertEqual(overall, "completed")
self.assertIn("2/2", report)
self.assertIn("A 好了", report)
def test_mixed_failed(self):
report, overall = build_report([
{"name": "a", "id": "1", "status": "completed",
"result_text": "ok", "error_text": None},
{"name": "b", "id": "2", "status": "failed",
"result_text": None, "error_text": "炸了"}])
self.assertEqual(overall, "failed")
self.assertIn("1/2", report)
self.assertIn("炸了", report)
def test_truncates_long_output(self):
report, _overall = build_report([
{"name": "a", "id": "1", "status": "completed",
"result_text": "x" * 600, "error_text": None}])
self.assertNotIn("x" * 600, report)
self.assertIn("x" * 500, report)
def test_empty_output_note(self):
report, _overall = build_report([
{"name": "a", "id": "1", "status": "completed",
"result_text": None, "error_text": None}])
self.assertIn("无输出", report)
class WorkerCoordinatorTestCase(unittest.TestCase):
def setUp(self):
self.tmp = tempfile.TemporaryDirectory()
self.db_path = os.path.join(self.tmp.name, "control.db")
init_db(self.db_path)
self.env = {"XCM_API_KEY": "test-placeholder"}
def tearDown(self):
self.tmp.cleanup()
def _worker(self):
return Worker(self.db_path, "/fake/host.py",
runner=_completed_runner, env=self.env)
def _parts(self):
return [{"name": "后端", "prompt": "做后端", "model": "m1",
"effort": "high"},
{"name": "前端", "prompt": "做前端"}]
def test_sync_creates_children_once(self):
_insert_task(self.db_path, "coord", coordinator_parts=self._parts())
worker = self._worker()
task = worker.get_task("coord")
first = worker._coord_sync_children(task, task["coordinator_parts"])
second = worker._coord_sync_children(task, task["coordinator_parts"])
self.assertEqual([c["id"] for c in first],
[c["id"] for c in second])
self.assertEqual(len(_children(self.db_path, "coord")), 2)
backend = _row(self.db_path, first[0]["id"])
self.assertEqual(backend["parent_task_id"], "coord")
self.assertEqual(backend["coordinator_part"], "后端")
self.assertEqual(backend["model"], "m1")
self.assertEqual(backend["effort"], "high")
self.assertEqual(backend["status"], "queued")
def test_sync_requeues_failed_keeps_completed(self):
_insert_task(self.db_path, "coord", coordinator_parts=self._parts())
_insert_task(self.db_path, "c-old", status="failed",
parent_task_id="coord", coordinator_part="后端",
result_text="旧输出", error_text="旧错")
_insert_task(self.db_path, "c-done", status="completed",
parent_task_id="coord", coordinator_part="前端",
result_text="已做完")
worker = self._worker()
task = worker.get_task("coord")
children = worker._coord_sync_children(task, task["coordinator_parts"])
self.assertEqual([c["id"] for c in children], ["c-old", "c-done"])
old = _row(self.db_path, "c-old")
self.assertEqual(old["status"], "queued")
self.assertIsNone(old["result_text"])
self.assertIsNone(old["error_text"])
done = _row(self.db_path, "c-done")
self.assertEqual(done["status"], "completed")
self.assertEqual(done["result_text"], "已做完")
def test_run_one_all_completed(self):
_insert_task(self.db_path, "coord", coordinator_parts=self._parts())
_insert_task(self.db_path, "c1", status="completed",
parent_task_id="coord", coordinator_part="后端",
result_text="后端好了")
_insert_task(self.db_path, "c2", status="completed",
parent_task_id="coord", coordinator_part="前端",
result_text="前端好了")
worker = self._worker()
result = worker.run_one(worker.get_task("coord"))
self.assertEqual(result["status"], "completed")
self.assertIn("2/2", result["stdout"])
self.assertIn("后端好了", result["stdout"])
self.assertEqual(_row(self.db_path, "coord")["status"], "completed")
def test_run_timeout_cancels_pending(self):
_insert_task(self.db_path, "coord", coordinator_parts=self._parts())
conn = sqlite3.connect(self.db_path)
conn.execute("UPDATE tasks SET status = 'running' WHERE id = 'coord'")
conn.commit()
conn.close()
worker = self._worker()
task = worker.get_task("coord")
result = worker._run_coordinator(
task, time.time(), poll_interval=0.01, timeout=0.05)
self.assertEqual(result["status"], "failed")
self.assertIn("timeout", result["error"])
self.assertEqual(_row(self.db_path, "coord")["status"], "failed")
for cid in _children(self.db_path, "coord"):
self.assertEqual(_row(self.db_path, cid)["status"], "interrupted")
def test_run_cancelled(self):
_insert_task(self.db_path, "coord", coordinator_parts=self._parts())
_insert_task(self.db_path, "c1", status="completed",
parent_task_id="coord", coordinator_part="后端",
result_text="ok")
_insert_task(self.db_path, "c2", status="completed",
parent_task_id="coord", coordinator_part="前端",
result_text="ok")
conn = sqlite3.connect(self.db_path)
conn.execute("UPDATE tasks SET status = 'running' WHERE id = 'coord'")
conn.commit()
conn.close()
worker = self._worker()
worker._cancel_requested.add("coord")
result = worker._run_coordinator(
worker.get_task("coord"), time.time(),
poll_interval=0.01, timeout=60)
self.assertEqual(result["status"], "interrupted")
self.assertEqual(result["error"], "cancelled")
self.assertEqual(_row(self.db_path, "coord")["status"], "interrupted")
class ServerCoordinatorTestCase(unittest.TestCase):
def setUp(self):
self.tmp = tempfile.TemporaryDirectory()
self.db_path = os.path.join(self.tmp.name, "control.db")
self.auth = AuthStore(self.db_path)
self.owner_id, self.owner_key = self.auth.create_user("owner")
self.server = create_server(self.db_path, "127.0.0.1", 0)
self.port = self.server.server_address[1]
self.thread = threading.Thread(
target=self.server.serve_forever, daemon=True)
self.thread.start()
status, self.project = self._req(
"POST", "/projects", {"name": "demo"}, self.owner_key)
self.assertEqual(status, 201)
def tearDown(self):
self.server.shutdown()
self.server.server_close()
self.thread.join(timeout=5)
try:
self.auth.conn.close()
except Exception:
pass
self.tmp.cleanup()
def _req(self, method, path, body=None, key=None):
conn = http.client.HTTPConnection(
"127.0.0.1", self.port, timeout=10)
hdrs = {}
payload = None
if body is not None:
payload = json.dumps(body).encode("utf-8")
hdrs["Content-Type"] = "application/json"
if key is not None:
hdrs["X-API-Key"] = key
try:
conn.request(method, path, body=payload, headers=hdrs)
resp = conn.getresponse()
data = resp.read()
finally:
conn.close()
try:
parsed = json.loads(data.decode("utf-8")) if data else None
except ValueError:
parsed = None
return resp.status, parsed
def _create_coord(self, parts=None, extra=None):
body = {"project_id": self.project["id"], "prompt": "协调干活",
"coordinator_parts": parts if parts is not None else [
{"name": "后端", "prompt": "做后端"},
{"name": "前端", "prompt": "做前端"}]}
if extra:
body.update(extra)
return self._req("POST", "/tasks", body, self.owner_key)
def test_create_coordinator_ok(self):
status, task = self._create_coord()
self.assertEqual(status, 201)
self.assertEqual(len(task["coordinator_parts"]), 2)
self.assertEqual(task["coordinator_parts"][0]["name"], "后端")
self.assertIsNone(task["parent_task_id"])
status, fetched = self._req(
"GET", "/tasks/" + task["id"], key=self.owner_key)
self.assertEqual(status, 200)
self.assertEqual(len(fetched["coordinator_parts"]), 2)
def test_create_coordinator_invalid(self):
status, body = self._create_coord(parts=[])
self.assertEqual(status, 400)
status, _body = self._create_coord(parts=[
{"name": "a", "prompt": "x"}, {"name": "a", "prompt": "y"}])
self.assertEqual(status, 400)
status, _body = self._create_coord(parts=[
{"name": "a", "prompt": "x", "effort": "ultra"}])
self.assertEqual(status, 400)
def test_create_coordinator_conflicts(self):
status, _body = self._create_coord(extra={"goal_mode": 1})
self.assertEqual(status, 400)
status, _body = self._create_coord(extra={"plan": 1})
self.assertEqual(status, 400)
status, _body = self._create_coord(
extra={"verify_command": "true"})
self.assertEqual(status, 400)
def test_list_children_filter(self):
status, coord = self._create_coord()
self.assertEqual(status, 201)
_insert_task(self.db_path, "kid1", parent_task_id=coord["id"],
coordinator_part="后端",
project_id=self.project["id"])
_insert_task(self.db_path, "other", project_id=self.project["id"])
status, data = self._req(
"GET", "/tasks?project_id=" + self.project["id"] +
"&parent_task_id=" + coord["id"], key=self.owner_key)
self.assertEqual(status, 200)
self.assertEqual([t["id"] for t in data["tasks"]], ["kid1"])
status, data = self._req(
"GET", "/tasks?project_id=" + self.project["id"],
key=self.owner_key)
self.assertEqual(status, 200)
self.assertEqual(len(data["tasks"]), 3)
if __name__ == "__main__":
unittest.main()
+231 -1
View File
@@ -130,9 +130,15 @@ import subprocess
import sys
import threading
import time
import uuid
from types import SimpleNamespace
from cloud.auth import AuthStore
from cloud.coordinator import build_report as _coord_build_report
from cloud.coordinator import DEFAULT_POLL_INTERVAL as _COORD_POLL_INTERVAL
from cloud.coordinator import DEFAULT_TIMEOUT_SECONDS as _COORD_TIMEOUT
from cloud.coordinator import ensure_columns as _coord_ensure_columns
from cloud.coordinator import parse_parts as _coord_parse_parts
from cloud.cost import CostStore, estimate_tokens, price_usd_micros, sum_usage
from cloud.db import connect as _db_connect
from cloud.logging_config import get_logger, log_event, setup_logging
@@ -263,6 +269,11 @@ def _task_from_row(row):
"environment_id": _col("environment_id"),
# 续跑消息:老行缺列时 None = 没有待续跑消息。
"resume_message": _col("resume_message"),
# 协调员:老行缺列时 coordinator_parts 为 [](= 普通任务),
# parent_task_id/coordinator_part 缺列时为 None。
"coordinator_parts": _coord_parse_parts(_col("coordinator_parts")),
"parent_task_id": _col("parent_task_id"),
"coordinator_part": _col("coordinator_part"),
}
@@ -612,7 +623,10 @@ class Worker:
# turn_id 的回合编号(local/Docker 两路都经
# _result_from_runner/_parse_host_result 提取)。
"turn_id": "TEXT",
"root_turn_id": "TEXT"}.items():
"root_turn_id": "TEXT",
"coordinator_parts": "TEXT",
"parent_task_id": "TEXT",
"coordinator_part": "TEXT"}.items():
if name not in cols:
try:
conn.execute(
@@ -2031,6 +2045,215 @@ class Worker:
return match.group(1)
return None
# -- 协调员 ---------------------------------------------------------
def _coord_sync_children(self, task, parts):
"""按分工建/复用子任务,返回 [{name, id}]。
已有子任务按 coordinator_part(分工名)匹配:已完成的直接
复用;失败/中断的回 queued 重跑(旧输出清掉);未终态的
不动(正被别的 worker 跑或排队);缺失的才新建。重跑协调
员因此不会重复建子任务。子任务是普通任务:执行器过滤、
配额门、key 门都在子任务认领时照常生效。
"""
conn = _db_connect(self.db_path)
try:
_coord_ensure_columns(conn)
try:
conn.commit()
except Exception:
pass
rows = conn.execute(
"SELECT id, coordinator_part, status FROM tasks"
" WHERE parent_task_id = ?",
(task["id"],)).fetchall()
finally:
conn.close()
existing = {}
for row in rows:
try:
existing[row["coordinator_part"]] = {
"id": row["id"], "status": row["status"]}
except (KeyError, IndexError, TypeError):
continue
now = time.time()
children = []
conn = _db_connect(self.db_path)
try:
for part in parts:
name = part["name"]
prev = existing.get(name)
if prev is not None:
if prev["status"] in TERMINAL_STATUSES and \
prev["status"] != "completed":
conn.execute(
"UPDATE tasks SET status = 'queued',"
" result_text = NULL, error_text = NULL,"
" updated_at = ? WHERE id = ?",
(now, prev["id"]))
self._audit(
task["project_id"],
"coordinator_child_requeued",
"%s -> %s" % (name, prev["id"]))
children.append({"name": name, "id": prev["id"]})
continue
child_id = uuid.uuid4().hex
conn.execute(
"INSERT INTO tasks(id, project_id, prompt, status,"
" idempotency_key, depends_on, priority, created_by,"
" created_at, updated_at, last_heartbeat, executor,"
" model, effort, environment_id, parent_task_id,"
" coordinator_part)"
" VALUES(?, ?, ?, 'queued', NULL, '[]', ?, ?, ?, ?,"
" NULL, ?, ?, ?, ?, ?, ?)",
(child_id, task["project_id"], part["prompt"],
task.get("priority", 0) or 0, task["created_by"],
now, now,
part.get("executor") or task.get("executor"),
part.get("model"), part.get("effort"),
task.get("environment_id"), task["id"], name))
self._audit(
task["project_id"], "coordinator_child_created",
"%s -> %s" % (name, child_id))
children.append({"name": name, "id": child_id})
conn.commit()
finally:
conn.close()
return children
def _coord_child_rows(self, child_ids):
"""子任务当前行:{id: {status, result_text, error_text}}。
已删或查不到的子任务不在结果里,调用方按缺失处理(等齐
判断里缺失不算终态,会一直等到超时——子任务行不应被删,
真删了就是超时失败,不静默当完成)。
"""
rows = {}
conn = _db_connect(self.db_path)
try:
for cid in child_ids:
row = conn.execute(
"SELECT id, status, result_text, error_text FROM tasks"
" WHERE id = ?", (cid,)).fetchone()
if row is None:
continue
try:
rows[row["id"]] = {
"status": row["status"],
"result_text": row["result_text"],
"error_text": row["error_text"]}
except (KeyError, IndexError, TypeError):
continue
finally:
conn.close()
return rows
def _run_coordinator(self, task, started_at, poll_interval=None,
timeout=None):
"""协调员执行:建子任务 → 等齐 → 合流汇总写回。
占一个 worker 槽位直到子任务全部终态:等齐期间每轮刷心跳
(sweep 不会误回收);子任务本身由各 worker 并行认领。
取消与超时都会尽力取消未终态的子任务,不留孤儿继续烧
模型费用。返回 run_one 同款结果 dict(无模型花费,
cost_record_id 为 None)。
"""
task_id = task["id"]
project_id = task["project_id"]
parts = task.get("coordinator_parts") or []
if poll_interval is None:
poll_interval = _COORD_POLL_INTERVAL
if timeout is None:
timeout = _COORD_TIMEOUT
children = self._coord_sync_children(task, parts)
child_ids = [c["id"] for c in children]
deadline = time.time() + timeout
cancelled = False
timed_out = False
while True:
try:
self.heartbeat(task_id)
except Exception:
pass
if self._was_cancelled(task_id):
cancelled = True
break
rows = self._coord_child_rows(child_ids)
if all((rows.get(cid) or {}).get("status") in TERMINAL_STATUSES
for cid in child_ids):
break
if time.time() >= deadline:
timed_out = True
break
time.sleep(poll_interval)
if cancelled or timed_out:
rows = self._coord_child_rows(child_ids)
for cid in child_ids:
if (rows.get(cid) or {}).get("status") \
not in TERMINAL_STATUSES:
try:
self.cancel(cid)
except Exception:
pass
rows = self._coord_child_rows(child_ids)
merged = []
for ch in children:
row = rows.get(ch["id"]) or {}
merged.append({
"name": ch["name"], "id": ch["id"],
"status": row.get("status") or "?",
"result_text": row.get("result_text"),
"error_text": row.get("error_text")})
report, overall = _coord_build_report(merged)
if cancelled:
final = "interrupted"
error = "cancelled"
elif timed_out:
final = "failed"
pending = [m["name"] for m in merged
if m["status"] not in TERMINAL_STATUSES]
error = "coordinator timeout: %s 未完成" % "、".join(pending)
report = error + "\n\n" + report
else:
final = overall
error = None
if final != "completed":
failed = [m["name"] for m in merged
if m["status"] != "completed"]
error = "coordinator: %s 未完成" % "、".join(failed)
final_written = self._set_status_if_running(task_id, final)
if not final_written:
final = "interrupted"
finished_at = time.time()
self._write_execution_result(
task_id, result_text=report, error_text=error,
actual_model=None, started_at=started_at,
finished_at=finished_at)
self._cleanup_task_seed(task)
if final_written:
self._audit(project_id,
"worker_completed" if final == "completed"
else ("worker_interrupted"
if final == "interrupted"
else "worker_failed"),
task_id)
log_event(self.logger,
"worker_completed" if final == "completed"
else ("worker_interrupted"
if final == "interrupted"
else "worker_failed"),
task_id=task_id, project_id=project_id, status=final)
if final in ("completed", "failed"):
self._notify_task_finished(task_id, project_id, final,
error=error)
else:
self._audit(project_id, "worker_interrupted", task_id)
host_name = self.executor._host_name(task) \
if hasattr(self.executor, "_host_name") else task_id
return {"dispatched": True, "task_id": task_id,
"host_name": host_name, "status": final,
"executor": self._executor_name(), "stdout": report,
"error": error, "events": [], "cost_record_id": None}
def run_one(self, task=None):
"""认领并执行一个任务,返回结果 dict(见模块 docstring)。
@@ -2137,6 +2360,13 @@ class Worker:
"stdout": "", "error": quota_error, "events": [],
"cost_record_id": None}
# 协调员分支:不跑模型回合,改为按分工建并行子任务、等齐、
# 合流汇总。已过 key 门与配额门;仓库门已对协调员任务本身
# 跑过(项目仓库解析失败会在前面直接 failed,不会建一堆
# 注定失败的子任务);子任务各自再过 key/配额/仓库门。
if task.get("coordinator_parts"):
return self._run_coordinator(task, started_at)
events = []
fallback_count = 0
result = None