"""协调员审计循环接线测试：合流点 + 包拆分 + 开关落库调整。 只用 init_db 的本地库与注入的假 runner，不跑真实模型回合、 不联网。覆盖（按任务书）：合流推回（失败子任务重排、包只含 自己的 finding、父任务包可读、其余子任务不动）；合流通过 （包清理）；前置门拒绝直接升级、不耗轮次、不重排子任务； 容忍/用户取消/配额缺失子任务不进门、不成 finding；3 轮仍 失败升级；主观与确定性拆分（helper 层）；sync 创建与重排时 的包拆分投递；POST /tasks 协调员带 audit_loop 改为接受。 注意：合流逻辑测试直接调 _coord_audit_loop_merge（合流点本身）， 不跑完整 _run_coordinator 轮询循环——轮询要等子任务终态， 这里子任务行已是终态，合流逻辑与 _run_coordinator 调用的 是同一函数；另有两例线程测试跑完整 _run_coordinator（子任务 在本轮轮询期间终态，模拟 worker 完成），覆盖父任务 CAS 回 queued 与升级终局两条分支。 """ import json import os import sqlite3 import sys import tempfile import time import unittest from pathlib import Path from types import SimpleNamespace try: from cloud.audit_loop import Finding from cloud.server import init_db from cloud.worker import Worker except ImportError: sys.path.insert(0, str(Path(__file__).resolve().parents[2])) from cloud.audit_loop import Finding from cloud.server import init_db from cloud.worker import Worker def _insert(db_path, task_id, status="queued", parent=None, part=None, result=None, error=None, audit_loop=0, paused=0, reason=None, parts=None): now = time.time() conn = sqlite3.connect(db_path) try: conn.execute( "INSERT INTO tasks(id, project_id, prompt, status," " idempotency_key, depends_on, priority, created_by," " created_at, updated_at, last_heartbeat, audit_loop," " parent_task_id, coordinator_part, result_text, error_text," " paused_by_user, interrupted_reason, coordinator_parts)" " VALUES(?, 'p1', 'do', ?, NULL, '[]', 0, 'tester'," " ?, ?, NULL, ?, ?, ?, ?, ?, ?, ?, ?)", (task_id, status, now, now, audit_loop, parent, part, result, error, paused, reason, json.dumps(parts) if parts is not None else None)) conn.commit() finally: conn.close() def _row(db_path, task_id): conn = sqlite3.connect(db_path) try: return conn.execute( "SELECT status, result_text, error_text, audit_loop_packet" " FROM tasks WHERE id = ?", (task_id,)).fetchone() finally: conn.close() def _packet(db_path, task_id): row = _row(db_path, task_id) if row is None or row[3] is None: return None return json.loads(row[3]) def _state(db_path, task_id): conn = sqlite3.connect(db_path) try: row = conn.execute( "SELECT round, last_decision FROM task_audit_loop_state" " WHERE task_id = ?", (task_id,)).fetchone() except sqlite3.OperationalError: return None finally: conn.close() return row def _completed_runner(cmd): return SimpleNamespace(returncode=0, stdout="{}", stderr="") class CoordAuditTestCase(unittest.TestCase): def setUp(self): self._old = os.environ.get("CODEX_PUSH_ALLOW_PRIVATE_TARGETS") os.environ["CODEX_PUSH_ALLOW_PRIVATE_TARGETS"] = "1" self.tmp = tempfile.TemporaryDirectory() self.db_path = os.path.join(self.tmp.name, "control.db") init_db(self.db_path) self.env = {"XCM_API_KEY": "test-placeholder"} self.audits = [] self.worker = self._worker() conn = sqlite3.connect(self.db_path) try: self.worker._ensure_result_columns(conn) conn.commit() finally: conn.close() def tearDown(self): if self._old is None: os.environ.pop("CODEX_PUSH_ALLOW_PRIVATE_TARGETS", None) else: os.environ["CODEX_PUSH_ALLOW_PRIVATE_TARGETS"] = self._old self.tmp.cleanup() def _worker(self): worker = Worker(self.db_path, "/fake/host.py", runner=_completed_runner, env=self.env, max_retries=0, retry_backoff_seconds=0) worker._audit = lambda p, a, d="": self.audits.append((p, a, d)) return worker def _parent(self, task_id="P1"): return {"id": task_id, "project_id": "p1", "audit_loop": 1} def _parts(self, *names, tolerated=()): return [{"name": n, "on_failure": "tolerate" if n in tolerated else "fail"} for n in names] def test_split_findings(self): det = Finding("child:a", "high", "e", "r", "a", kind="deterministic", gate="child:a") sub = Finding("s1", "low", "e", "r", "a", kind="subjective") d, s = Worker._coord_split_findings([det, sub]) self.assertEqual([f.id for f in d], ["child:a"]) self.assertEqual([f.id for f in s], ["s1"]) def test_split_packet_only_own_finding(self): packet = {"task_id": "P1", "round": 2, "findings": [ {"id": "child:a", "severity": "high", "evidence": "ea", "reproduction": "ra", "acceptance": "aa", "kind": "deterministic", "gate": "child:a"}, {"id": "child:b", "severity": "high", "evidence": "eb", "reproduction": "rb", "acceptance": "ab", "kind": "deterministic", "gate": "child:b"}], "criteria": {"child:a": "child:a", "child:b": "child:b"}} split = Worker._coord_split_packet(packet, "a") self.assertEqual([f["id"] for f in split["findings"]], ["child:a"]) self.assertEqual(split["criteria"], {"child:a": "child:a"}) self.assertEqual(split["round"], 2) self.assertIsNone(Worker._coord_split_packet(packet, "zzz")) def test_merge_pushback(self): _insert(self.db_path, "P1", status="running", audit_loop=1) _insert(self.db_path, "C1", status="failed", parent="P1", part="a", error="boom") _insert(self.db_path, "C2", status="completed", parent="P1", part="b", result="done") out = self.worker._coord_audit_loop_merge( self._parent(), self._parts("a", "b"), [{"name": "a", "id": "C1"}, {"name": "b", "id": "C2"}]) self.assertEqual(out["decision"], "pushback") row_a = _row(self.db_path, "C1") self.assertEqual(row_a[0], "queued") self.assertIsNone(row_a[1]) packet_a = _packet(self.db_path, "C1") self.assertEqual([f["id"] for f in packet_a["findings"]], ["child:a"]) self.assertEqual(_row(self.db_path, "C2")[0], "completed") parent_packet = _packet(self.db_path, "P1") self.assertEqual([f["id"] for f in parent_packet["findings"]], ["child:a"]) self.assertEqual(_state(self.db_path, "P1")[0], 1) self.assertTrue(any(a == "worker_coord_audit_loop_pushback" for _, a, _ in self.audits)) def test_merge_pass_clears_packets(self): _insert(self.db_path, "P1", status="running", audit_loop=1) _insert(self.db_path, "C1", status="completed", parent="P1", part="a", result="done") conn = sqlite3.connect(self.db_path) try: conn.execute( "UPDATE tasks SET audit_loop_packet = ? WHERE id = ?", ('{"round": 1, "findings": []}', "C1")) conn.execute( "UPDATE tasks SET audit_loop_packet = ? WHERE id = ?", ('{"round": 1, "findings": []}', "P1")) conn.commit() finally: conn.close() out = self.worker._coord_audit_loop_merge( self._parent(), self._parts("a"), [{"name": "a", "id": "C1"}]) self.assertEqual(out["decision"], "pass") self.assertIsNone(_packet(self.db_path, "C1")) self.assertIsNone(_packet(self.db_path, "P1")) def test_merge_gate_rejected_escalates_without_round(self): _insert(self.db_path, "P1", status="running", audit_loop=1) _insert(self.db_path, "C1", status="failed", parent="P1", part="a", error="gateway exploded") conn = sqlite3.connect(self.db_path) try: conn.execute( "CREATE TABLE IF NOT EXISTS audit_log(user_id TEXT," " project_id TEXT, action TEXT, detail TEXT," " created_at DOUBLE PRECISION)") conn.execute( "INSERT INTO audit_log(user_id, project_id, action," " detail, created_at) VALUES('system', 'p1'," " 'worker_capability_gate_rejected', ?, ?)", ("C1: tier gateway model X; refusing to dispatch", time.time())) conn.commit() finally: conn.close() out = self.worker._coord_audit_loop_merge( self._parent(), self._parts("a"), [{"name": "a", "id": "C1"}]) self.assertEqual(out["decision"], "escalate") self.assertIn("a", out["error"]) # 不耗轮次：状态表里没有该父任务的行；子任务不重排。 self.assertIsNone(_state(self.db_path, "P1")) self.assertEqual(_row(self.db_path, "C1")[0], "failed") self.assertTrue(any(a == "worker_coord_audit_loop_escalated" for _, a, _ in self.audits)) def test_merge_gate_rejected_by_error_marker(self): _insert(self.db_path, "P1", status="running", audit_loop=1) _insert(self.db_path, "C1", status="failed", parent="P1", part="a", error="Missing environment variable: XCM_API_KEY") out = self.worker._coord_audit_loop_merge( self._parent(), self._parts("a"), [{"name": "a", "id": "C1"}]) self.assertEqual(out["decision"], "escalate") def test_merge_excluded_children_pass(self): _insert(self.db_path, "P1", status="running", audit_loop=1) _insert(self.db_path, "C1", status="failed", parent="P1", part="a", error="tolerated boom") _insert(self.db_path, "C2", status="failed", parent="P1", part="b", error="user stopped", paused=1) _insert(self.db_path, "C3", status="completed", parent="P1", part="c", result="done") out = self.worker._coord_audit_loop_merge( self._parent(), self._parts("a", "b", "c", tolerated=("a",)), [{"name": "a", "id": "C1"}, {"name": "b", "id": "C2"}, {"name": "c", "id": "C3"}]) self.assertEqual(out["decision"], "pass") # 配额缺失的分工不在 children 里：不列门、不成 finding。 out2 = self.worker._coord_audit_loop_merge( self._parent("P2"), self._parts("c"), [{"name": "c", "id": "C3"}]) self.assertEqual(out2["decision"], "pass") def test_merge_max_rounds_escalates(self): _insert(self.db_path, "P1", status="running", audit_loop=1) _insert(self.db_path, "C1", status="failed", parent="P1", part="a", error="boom again") conn = sqlite3.connect(self.db_path) try: conn.execute( "INSERT INTO task_audit_loop_state(task_id, round," " passed_gates, criteria, last_decision, updated_at)" " VALUES('P1', 3, '[]', ?, 'pushback', ?)", ('{"child:a": "child:a"}', time.time())) conn.commit() finally: conn.close() out = self.worker._coord_audit_loop_merge( self._parent(), self._parts("a"), [{"name": "a", "id": "C1"}]) self.assertEqual(out["decision"], "escalate") self.assertIsNone(_packet(self.db_path, "P1")) self.assertIsNone(_packet(self.db_path, "C1")) def test_merge_switch_off_skips(self): _insert(self.db_path, "P1", status="running", audit_loop=0) _insert(self.db_path, "C1", status="failed", parent="P1", part="a", error="boom") out = self.worker._coord_audit_loop_merge( {"id": "P1", "project_id": "p1", "audit_loop": 0}, self._parts("a"), [{"name": "a", "id": "C1"}]) self.assertEqual(out["decision"], "skip") self.assertEqual(_row(self.db_path, "C1")[0], "failed") def test_sync_creation_and_requeue_attach_packet(self): parent_packet = { "task_id": "P1", "round": 1, "findings": [{"id": "child:a", "severity": "high", "evidence": "ea", "reproduction": "ra", "acceptance": "aa", "kind": "deterministic", "gate": "child:a"}], "criteria": {"child:a": "child:a"}} parent = {"id": "P1", "project_id": "p1", "audit_loop": 1, "audit_loop_packet": json.dumps(parent_packet), "created_by": "tester", "priority": 0} self.worker._quota_exceeded = lambda *a: None children = self.worker._coord_sync_children( parent, [{"name": "a", "prompt": "do a"}]) self.assertEqual(len(children), 1) packet = _packet(self.db_path, children[0]["id"]) self.assertIsNotNone(packet) self.assertEqual([f["id"] for f in packet["findings"]], ["child:a"]) # 重排分支：已失败的子任务 + 父包 → 重排并贴拆分包。 conn = sqlite3.connect(self.db_path) try: conn.execute( "UPDATE tasks SET status = 'failed', error_text = 'boom'" " WHERE id = ?", (children[0]["id"],)) conn.execute( "UPDATE tasks SET audit_loop_packet = NULL WHERE id = ?", (children[0]["id"],)) conn.commit() finally: conn.close() children2 = self.worker._coord_sync_children( parent, [{"name": "a", "prompt": "do a"}]) self.assertEqual(children2[0]["id"], children[0]["id"]) row = _row(self.db_path, children[0]["id"]) self.assertEqual(row[0], "queued") packet2 = _packet(self.db_path, children[0]["id"]) self.assertEqual([f["id"] for f in packet2["findings"]], ["child:a"]) def test_run_coordinator_pushback_full_path(self): # 完整 _run_coordinator 路径：子任务在本轮轮询期间终态 # （线程里翻状态模拟 worker 完成），合流推回后父任务回 # queued、失败子任务回 queued 且带拆分包。直接插 failed # 行测不到合流——sync 会在轮询前就把它重排走。 import threading _insert(self.db_path, "P1", status="running", audit_loop=1, parts=[{"name": "a", "prompt": "do a"}, {"name": "b", "prompt": "do b"}]) _insert(self.db_path, "C1", status="queued", parent="P1", part="a") _insert(self.db_path, "C2", status="queued", parent="P1", part="b") self.worker._quota_exceeded = lambda *a: None box = {} def _run(): box["result"] = self.worker._run_coordinator( self.worker.get_task("P1"), time.time(), poll_interval=0.01, timeout=10) thread = threading.Thread(target=_run) thread.start() deadline = time.time() + 5 while time.time() < deadline: conn = sqlite3.connect(self.db_path) try: conn.execute( "UPDATE tasks SET status = 'failed'," " error_text = 'boom' WHERE id = 'C1'") conn.execute( "UPDATE tasks SET status = 'completed'," " result_text = 'done' WHERE id = 'C2'") conn.commit() finally: conn.close() time.sleep(0.05) if not thread.is_alive(): break thread.join(timeout=5) self.assertFalse(thread.is_alive()) result = box["result"] self.assertEqual(result["status"], "queued") self.assertEqual(_row(self.db_path, "P1")[0], "queued") self.assertEqual(_row(self.db_path, "C1")[0], "queued") packet = _packet(self.db_path, "C1") self.assertEqual([f["id"] for f in packet["findings"]], ["child:a"]) def test_run_coordinator_escalate_full_path(self): # 门拒绝子任务在本轮终态 → 合流直接升级：父任务 failed、 # 错误点名分工、子任务不重排。 import threading _insert(self.db_path, "P1", status="running", audit_loop=1, parts=[{"name": "a", "prompt": "do a"}]) _insert(self.db_path, "C1", status="queued", parent="P1", part="a") conn = sqlite3.connect(self.db_path) try: conn.execute( "CREATE TABLE IF NOT EXISTS audit_log(user_id TEXT," " project_id TEXT, action TEXT, detail TEXT," " created_at DOUBLE PRECISION)") conn.execute( "INSERT INTO audit_log VALUES('system', 'p1'," " 'worker_tool_gate_rejected'," " 'C1: agent 声明必需工具缺失：websearch', ?)", (time.time(),)) conn.commit() finally: conn.close() self.worker._quota_exceeded = lambda *a: None box = {} def _run(): box["result"] = self.worker._run_coordinator( self.worker.get_task("P1"), time.time(), poll_interval=0.01, timeout=10) thread = threading.Thread(target=_run) thread.start() deadline = time.time() + 5 while time.time() < deadline: conn = sqlite3.connect(self.db_path) try: conn.execute( "UPDATE tasks SET status = 'failed'," " error_text = 'agent 声明必需工具缺失：websearch'" " WHERE id = 'C1'") conn.commit() finally: conn.close() time.sleep(0.05) if not thread.is_alive(): break thread.join(timeout=5) self.assertFalse(thread.is_alive()) result = box["result"] self.assertEqual(result["status"], "failed") self.assertIn("审计循环升级", result["error"]) self.assertEqual(_row(self.db_path, "P1")[0], "failed") self.assertEqual(_row(self.db_path, "C1")[0], "failed") if __name__ == "__main__": unittest.main()