host
c15c224570
Coordinator parts gain three enforced bounds, validated in validate_parts and carried through the coordinator_parts JSON (no schema change): timeout_seconds (1..86400, per-attempt wall clock from coordinator start; the coordinator poll loop cancels only that child with the same cancel mechanism as the global timeout, other parts keep waiting, audit coordinator_child_bound_exceeded), on_failure fail|tolerate (tolerated failures do not fail the coordinator, are named in the report head, and their failed children are not requeued on coordinator re-run while interrupted children still are), and result_chars (0..2000, default 500, enforced in build_report; the 4000-char report writeback cap still applies). evaluate_outcome is the single outcome rule used by both build_report and the worker error text, so tolerated failures never leak into the coordinator failure reason. Unknown part names fail closed. Deliberately NOT added, after reading the execution layer: per-part token/cost budget (cost is recorded only on completion, so a running child's spend is unobservable and creator quota is monthly, not per part), per-part round cap (normal tasks have no per-task round cap in the worker), and write-scope isolation (sandbox is per-project worktree, a scope string in a prompt would be an instruction, not a guarantee). Tests: test_delegation_bounds.py (8 tests).
242 lines
9.5 KiB
Python
242 lines
9.5 KiB
Python
"""有界委派 v5 单测：分工边界校验、容忍评价、回传截断、
|
|
单分工超时取消、容忍分工重跑不重排。
|
|
|
|
运行（在仓库根目录下）：
|
|
python3 -m unittest discover -s cloud/tests -p "test_delegation_bounds.py" -v
|
|
|
|
说明：纯函数部分不碰库；worker 部分用临时 SQLite + 假
|
|
runner，子任务终态由测试直接写库模拟（与
|
|
test_coordinator 同款骨架），不跑真实模型、不联网。
|
|
关键断言：慢分工超时只取消该分工、不拖到全局超时；
|
|
容忍分工失败不拖垮协调员、且重跑时不重排；未知分工
|
|
按 fail 处理；token/轮数/写范围字段不在本轨——查代码
|
|
后确认执行层无法执行，故不以字段冒充（见
|
|
cloud/coordinator.py 模块注释）。
|
|
"""
|
|
|
|
import os
|
|
import sqlite3
|
|
import sys
|
|
import tempfile
|
|
import time
|
|
import unittest
|
|
from pathlib import Path
|
|
|
|
try:
|
|
from cloud.coordinator import build_report, evaluate_outcome
|
|
from cloud.coordinator import validate_parts
|
|
from cloud.server import init_db
|
|
from cloud.worker import Worker
|
|
except ImportError:
|
|
sys.path.insert(0, str(Path(__file__).resolve().parents[2]))
|
|
from cloud.coordinator import build_report, evaluate_outcome
|
|
from cloud.coordinator import validate_parts
|
|
from cloud.server import init_db
|
|
from cloud.worker import Worker
|
|
|
|
|
|
def _insert_task(db_path, task_id, prompt="do work", status="queued",
|
|
project_id="p1", coordinator_parts=None,
|
|
parent_task_id=None, coordinator_part=None,
|
|
result_text=None, error_text=None):
|
|
now = time.time()
|
|
conn = sqlite3.connect(db_path)
|
|
try:
|
|
import json
|
|
conn.execute(
|
|
"INSERT INTO tasks(id, project_id, prompt, status,"
|
|
" idempotency_key, depends_on, priority, created_by,"
|
|
" created_at, updated_at, last_heartbeat, coordinator_parts,"
|
|
" parent_task_id, coordinator_part, result_text, error_text)"
|
|
" VALUES(?, ?, ?, ?, NULL, '[]', 0, 'tester', ?, ?, NULL,"
|
|
" ?, ?, ?, ?, ?)",
|
|
(task_id, project_id, prompt, status, now, now,
|
|
json.dumps(coordinator_parts) if coordinator_parts else None,
|
|
parent_task_id, coordinator_part, result_text, error_text),
|
|
)
|
|
conn.commit()
|
|
finally:
|
|
conn.close()
|
|
|
|
|
|
def _row(db_path, task_id):
|
|
conn = sqlite3.connect(db_path)
|
|
try:
|
|
row = conn.execute(
|
|
"SELECT status, result_text, error_text FROM tasks"
|
|
" WHERE id = ?", (task_id,)).fetchone()
|
|
finally:
|
|
conn.close()
|
|
return {"status": row[0], "result_text": row[1],
|
|
"error_text": row[2]} if row else None
|
|
|
|
|
|
def _children(db_path, parent_id):
|
|
conn = sqlite3.connect(db_path)
|
|
try:
|
|
rows = conn.execute(
|
|
"SELECT id FROM tasks WHERE parent_task_id = ?"
|
|
" ORDER BY coordinator_part", (parent_id,)).fetchall()
|
|
finally:
|
|
conn.close()
|
|
return [r[0] for r in rows]
|
|
|
|
|
|
class ValidateBoundsTestCase(unittest.TestCase):
|
|
def test_valid_bounds(self):
|
|
parts, err = validate_parts([{
|
|
"name": "a", "prompt": "x", "timeout_seconds": 60,
|
|
"on_failure": "tolerate", "result_chars": 100}])
|
|
self.assertIsNone(err)
|
|
self.assertEqual(parts[0]["timeout_seconds"], 60)
|
|
self.assertEqual(parts[0]["on_failure"], "tolerate")
|
|
self.assertEqual(parts[0]["result_chars"], 100)
|
|
|
|
def test_rejects_bad_bounds(self):
|
|
for bad in (
|
|
{"timeout_seconds": 0},
|
|
{"timeout_seconds": 86401},
|
|
{"timeout_seconds": True},
|
|
{"timeout_seconds": 1.5},
|
|
{"on_failure": "retry"},
|
|
{"result_chars": -1},
|
|
{"result_chars": 2001},
|
|
):
|
|
_parts, err = validate_parts(
|
|
[{"name": "a", "prompt": "x", **bad}])
|
|
self.assertIsNotNone(err, bad)
|
|
|
|
def test_old_parts_get_defaults(self):
|
|
parts, err = validate_parts([{"name": "a", "prompt": "x"}])
|
|
self.assertIsNone(err)
|
|
self.assertIsNone(parts[0]["timeout_seconds"])
|
|
self.assertEqual(parts[0]["on_failure"], "fail")
|
|
self.assertEqual(parts[0]["result_chars"], 500)
|
|
|
|
|
|
class OutcomeTestCase(unittest.TestCase):
|
|
def test_tolerated_failure_completes(self):
|
|
parts, _err = validate_parts([
|
|
{"name": "a", "prompt": "x"},
|
|
{"name": "b", "prompt": "x", "on_failure": "tolerate"}])
|
|
overall, failed, tolerated = evaluate_outcome([
|
|
{"name": "a", "status": "completed"},
|
|
{"name": "b", "status": "failed"}], parts)
|
|
self.assertEqual(overall, "completed")
|
|
self.assertEqual(failed, [])
|
|
self.assertEqual(tolerated, ["b"])
|
|
|
|
def test_unknown_part_fails_closed(self):
|
|
parts, _err = validate_parts([{"name": "a", "prompt": "x"}])
|
|
overall, failed, _tol = evaluate_outcome([
|
|
{"name": "ghost", "status": "failed"}], parts)
|
|
self.assertEqual(overall, "failed")
|
|
self.assertEqual(failed, ["ghost"])
|
|
|
|
def test_per_part_result_chars(self):
|
|
parts, _err = validate_parts([
|
|
{"name": "a", "prompt": "x", "result_chars": 10}])
|
|
report, _overall = build_report([
|
|
{"name": "a", "id": "1", "status": "completed",
|
|
"result_text": "x" * 100, "error_text": None}], parts)
|
|
self.assertIn("x" * 10, report)
|
|
self.assertNotIn("x" * 11, report)
|
|
|
|
|
|
class WorkerBoundsTestCase(unittest.TestCase):
|
|
def setUp(self):
|
|
self.tmp = tempfile.TemporaryDirectory()
|
|
self.db_path = os.path.join(self.tmp.name, "control.db")
|
|
init_db(self.db_path)
|
|
self.env = {"XCM_API_KEY": "test-placeholder"}
|
|
|
|
def tearDown(self):
|
|
self.tmp.cleanup()
|
|
|
|
def _worker(self):
|
|
from types import SimpleNamespace
|
|
|
|
def _runner(cmd):
|
|
return SimpleNamespace(returncode=0, stdout="{}",
|
|
stderr="")
|
|
return Worker(self.db_path, "/fake/host.py",
|
|
runner=_runner, env=self.env)
|
|
|
|
def _parts(self):
|
|
return [
|
|
{"name": "快", "prompt": "做快活", "timeout_seconds": 60},
|
|
{"name": "慢", "prompt": "做慢活", "timeout_seconds": 1},
|
|
]
|
|
|
|
def test_per_part_timeout_cancels_only_slow_part(self):
|
|
# 快分工已完成，慢分工排队未终态：协调员本轮开始已
|
|
# 超过慢分工的 1 秒上限（started_at 往前推 2 秒），
|
|
# 循环应只取消慢分工、不等到 60 秒全局超时。
|
|
_insert_task(self.db_path, "coord",
|
|
coordinator_parts=self._parts())
|
|
conn = sqlite3.connect(self.db_path)
|
|
conn.execute("UPDATE tasks SET status = 'running'"
|
|
" WHERE id = 'coord'")
|
|
conn.commit()
|
|
conn.close()
|
|
worker = self._worker()
|
|
task = worker.get_task("coord")
|
|
children = worker._coord_sync_children(task, task["coordinator_parts"])
|
|
self.assertEqual(len(children), 2)
|
|
# 快分工直接标完成，慢分工保持排队：下一轮循环里
|
|
# 慢分工被单分工超时取消、快分工不动，等齐立即结束。
|
|
by_name = {}
|
|
conn = sqlite3.connect(self.db_path)
|
|
try:
|
|
for cid, part in conn.execute(
|
|
"SELECT id, coordinator_part FROM tasks"
|
|
" WHERE parent_task_id = 'coord'").fetchall():
|
|
by_name[part] = cid
|
|
conn.execute(
|
|
"UPDATE tasks SET status = 'completed',"
|
|
" result_text = 'ok' WHERE id = ?",
|
|
(by_name["快"],))
|
|
conn.commit()
|
|
finally:
|
|
conn.close()
|
|
started = time.time()
|
|
result = worker._run_coordinator(
|
|
worker.get_task("coord"), time.time() - 2,
|
|
poll_interval=0.01, timeout=60)
|
|
self.assertLess(time.time() - started, 10)
|
|
self.assertEqual(result["status"], "failed")
|
|
self.assertIn("有界委派超时", result["stdout"] or "")
|
|
self.assertEqual(_row(self.db_path, by_name["快"])["status"],
|
|
"completed")
|
|
self.assertEqual(_row(self.db_path, by_name["慢"])["status"],
|
|
"interrupted")
|
|
|
|
def test_tolerated_failed_not_requeued(self):
|
|
parts = [
|
|
{"name": "主", "prompt": "做主活"},
|
|
{"name": "副", "prompt": "做副活", "on_failure": "tolerate"},
|
|
]
|
|
_insert_task(self.db_path, "coord", coordinator_parts=parts)
|
|
_insert_task(self.db_path, "main", status="completed",
|
|
parent_task_id="coord", coordinator_part="主",
|
|
result_text="ok")
|
|
_insert_task(self.db_path, "side", status="failed",
|
|
parent_task_id="coord", coordinator_part="副",
|
|
error_text="炸了")
|
|
conn = sqlite3.connect(self.db_path)
|
|
conn.execute("UPDATE tasks SET status = 'running'"
|
|
" WHERE id = 'coord'")
|
|
conn.commit()
|
|
conn.close()
|
|
worker = self._worker()
|
|
task = worker.get_task("coord")
|
|
children = worker._coord_sync_children(
|
|
task, task["coordinator_parts"])
|
|
self.assertEqual(len(children), 2)
|
|
# 副分工的 failed 子任务未被回 queued，仍是 failed。
|
|
self.assertEqual(_row(self.db_path, "side")["status"], "failed")
|
|
result = worker._run_coordinator(
|
|
task, time.time(), poll_interval=0.01, timeout=60)
|
|
self.assertEqual(result["status"], "completed")
|
|
self.assertIn("已容忍", result["stdout"] or "")
|