host
a7f77c8a98
- I-1: evaluate_shadow counts dropped rows by cause (unparseable / unknown_tier / explicit / duplicate) with the identity audit_rows_total == routed_task_n + dropped_total; the shadow endpoint passes created_at/id through and reports audit totals next to sample counts. - I-2 (option b): parse logs a warning naming each tier dropped for an invalid mode/requires; load attaches partial_failures to the policy; the worker records a task-level worker_routing_policy_load_failed audit for partial failures (same action as full failure, detail says failure=partial); the shadow report carries the policy fingerprint, dropped tiers and partial failures (no task context on that path, so no task-level audit there). - I-5: coordinator gate audit lookup failure now raises _CoordGateLookupFailed and the merge converts it to escalate (no requeue, no round consumed); persisted-model lookup failure and required-tools check failure each write a named audit row before falling through to dispatch/allow; audit-loop evaluate failure stays escalate (regression test). - Test fixtures: coordinator suites build audit_log via AuthStore as production's first real audit does, since their stubbed _audit never creates the table.
460 lines
20 KiB
Python
460 lines
20 KiB
Python
"""协调员审计循环接线测试：合流点 + 包拆分 + 开关落库调整。
|
||
|
||
只用 init_db 的本地库与注入的假 runner，不跑真实模型回合、
|
||
不联网。覆盖（按任务书）：合流推回（失败子任务重排、包只含
|
||
自己的 finding、父任务包可读、其余子任务不动）；合流通过
|
||
（包清理）；前置门拒绝直接升级、不耗轮次、不重排子任务；
|
||
容忍/用户取消/配额缺失子任务不进门、不成 finding；3 轮仍
|
||
失败升级；主观与确定性拆分（helper 层）；sync 创建与重排时
|
||
的包拆分投递；POST /tasks 协调员带 audit_loop 改为接受。
|
||
|
||
注意：合流逻辑测试直接调 _coord_audit_loop_merge（合流点本身），
|
||
不跑完整 _run_coordinator 轮询循环——轮询要等子任务终态，
|
||
这里子任务行已是终态，合流逻辑与 _run_coordinator 调用的
|
||
是同一函数；另有两例线程测试跑完整 _run_coordinator（子任务
|
||
在本轮轮询期间终态，模拟 worker 完成），覆盖父任务 CAS 回
|
||
queued 与升级终局两条分支。
|
||
"""
|
||
|
||
import json
|
||
import os
|
||
import sqlite3
|
||
import sys
|
||
import tempfile
|
||
import time
|
||
import unittest
|
||
from pathlib import Path
|
||
from types import SimpleNamespace
|
||
|
||
try:
|
||
from cloud.audit_loop import Finding
|
||
from cloud.auth import AuthStore
|
||
from cloud.server import init_db
|
||
from cloud.worker import Worker
|
||
except ImportError:
|
||
sys.path.insert(0, str(Path(__file__).resolve().parents[2]))
|
||
from cloud.audit_loop import Finding
|
||
from cloud.auth import AuthStore
|
||
from cloud.server import init_db
|
||
from cloud.worker import Worker
|
||
|
||
|
||
def _insert(db_path, task_id, status="queued", parent=None, part=None,
|
||
result=None, error=None, audit_loop=0, paused=0, reason=None,
|
||
parts=None):
|
||
now = time.time()
|
||
conn = sqlite3.connect(db_path)
|
||
try:
|
||
conn.execute(
|
||
"INSERT INTO tasks(id, project_id, prompt, status,"
|
||
" idempotency_key, depends_on, priority, created_by,"
|
||
" created_at, updated_at, last_heartbeat, audit_loop,"
|
||
" parent_task_id, coordinator_part, result_text, error_text,"
|
||
" paused_by_user, interrupted_reason, coordinator_parts)"
|
||
" VALUES(?, 'p1', 'do', ?, NULL, '[]', 0, 'tester',"
|
||
" ?, ?, NULL, ?, ?, ?, ?, ?, ?, ?, ?)",
|
||
(task_id, status, now, now, audit_loop, parent, part,
|
||
result, error, paused, reason,
|
||
json.dumps(parts) if parts is not None else None))
|
||
conn.commit()
|
||
finally:
|
||
conn.close()
|
||
|
||
|
||
def _row(db_path, task_id):
|
||
conn = sqlite3.connect(db_path)
|
||
try:
|
||
return conn.execute(
|
||
"SELECT status, result_text, error_text, audit_loop_packet"
|
||
" FROM tasks WHERE id = ?", (task_id,)).fetchone()
|
||
finally:
|
||
conn.close()
|
||
|
||
|
||
def _packet(db_path, task_id):
|
||
row = _row(db_path, task_id)
|
||
if row is None or row[3] is None:
|
||
return None
|
||
return json.loads(row[3])
|
||
|
||
|
||
def _state(db_path, task_id):
|
||
conn = sqlite3.connect(db_path)
|
||
try:
|
||
row = conn.execute(
|
||
"SELECT round, last_decision FROM task_audit_loop_state"
|
||
" WHERE task_id = ?", (task_id,)).fetchone()
|
||
except sqlite3.OperationalError:
|
||
return None
|
||
finally:
|
||
conn.close()
|
||
return row
|
||
|
||
|
||
def _completed_runner(cmd):
|
||
return SimpleNamespace(returncode=0, stdout="{}", stderr="")
|
||
|
||
|
||
class CoordAuditTestCase(unittest.TestCase):
|
||
def setUp(self):
|
||
self._old = os.environ.get("CODEX_PUSH_ALLOW_PRIVATE_TARGETS")
|
||
os.environ["CODEX_PUSH_ALLOW_PRIVATE_TARGETS"] = "1"
|
||
self.tmp = tempfile.TemporaryDirectory()
|
||
self.db_path = os.path.join(self.tmp.name, "control.db")
|
||
init_db(self.db_path)
|
||
# Audit batch 2 (I-5): 合流的门拒绝审计查询失败时走
|
||
# escalate，不再回落空集。生产里 audit_log 由首次
|
||
# 真实 _audit 经 AuthStore 建好，合流时表一定存在；
|
||
# 本文件把 _audit 桩成列表，表永远不会出现，所以
|
||
# 在夹具里按生产实际补建表——否则所有合流测试都会
|
||
# 因缺表改走 escalate，与被测行为无关。
|
||
_store = AuthStore(self.db_path)
|
||
_store.conn.close()
|
||
self.env = {"XCM_API_KEY": "test-placeholder"}
|
||
self.audits = []
|
||
self.worker = self._worker()
|
||
conn = sqlite3.connect(self.db_path)
|
||
try:
|
||
self.worker._ensure_result_columns(conn)
|
||
conn.commit()
|
||
finally:
|
||
conn.close()
|
||
|
||
def tearDown(self):
|
||
if self._old is None:
|
||
os.environ.pop("CODEX_PUSH_ALLOW_PRIVATE_TARGETS", None)
|
||
else:
|
||
os.environ["CODEX_PUSH_ALLOW_PRIVATE_TARGETS"] = self._old
|
||
self.tmp.cleanup()
|
||
|
||
def _worker(self):
|
||
worker = Worker(self.db_path, "/fake/host.py",
|
||
runner=_completed_runner, env=self.env,
|
||
max_retries=0, retry_backoff_seconds=0)
|
||
worker._audit = lambda p, a, d="": self.audits.append((p, a, d))
|
||
return worker
|
||
|
||
def _parent(self, task_id="P1"):
|
||
return {"id": task_id, "project_id": "p1", "audit_loop": 1}
|
||
|
||
def _parts(self, *names, tolerated=()):
|
||
return [{"name": n,
|
||
"on_failure": "tolerate" if n in tolerated else "fail"}
|
||
for n in names]
|
||
|
||
def test_split_findings(self):
|
||
det = Finding("child:a", "high", "e", "r", "a",
|
||
kind="deterministic", gate="child:a")
|
||
sub = Finding("s1", "low", "e", "r", "a", kind="subjective")
|
||
d, s = Worker._coord_split_findings([det, sub])
|
||
self.assertEqual([f.id for f in d], ["child:a"])
|
||
self.assertEqual([f.id for f in s], ["s1"])
|
||
|
||
def test_split_packet_only_own_finding(self):
|
||
packet = {"task_id": "P1", "round": 2,
|
||
"findings": [
|
||
{"id": "child:a", "severity": "high",
|
||
"evidence": "ea", "reproduction": "ra",
|
||
"acceptance": "aa", "kind": "deterministic",
|
||
"gate": "child:a"},
|
||
{"id": "child:b", "severity": "high",
|
||
"evidence": "eb", "reproduction": "rb",
|
||
"acceptance": "ab", "kind": "deterministic",
|
||
"gate": "child:b"}],
|
||
"criteria": {"child:a": "child:a",
|
||
"child:b": "child:b"}}
|
||
split = Worker._coord_split_packet(packet, "a")
|
||
self.assertEqual([f["id"] for f in split["findings"]],
|
||
["child:a"])
|
||
self.assertEqual(split["criteria"], {"child:a": "child:a"})
|
||
self.assertEqual(split["round"], 2)
|
||
self.assertIsNone(Worker._coord_split_packet(packet, "zzz"))
|
||
|
||
def test_merge_pushback(self):
|
||
_insert(self.db_path, "P1", status="running", audit_loop=1)
|
||
_insert(self.db_path, "C1", status="failed", parent="P1",
|
||
part="a", error="boom")
|
||
_insert(self.db_path, "C2", status="completed", parent="P1",
|
||
part="b", result="done")
|
||
out = self.worker._coord_audit_loop_merge(
|
||
self._parent(), self._parts("a", "b"),
|
||
[{"name": "a", "id": "C1"}, {"name": "b", "id": "C2"}])
|
||
self.assertEqual(out["decision"], "pushback")
|
||
row_a = _row(self.db_path, "C1")
|
||
self.assertEqual(row_a[0], "queued")
|
||
self.assertIsNone(row_a[1])
|
||
packet_a = _packet(self.db_path, "C1")
|
||
self.assertEqual([f["id"] for f in packet_a["findings"]],
|
||
["child:a"])
|
||
self.assertEqual(_row(self.db_path, "C2")[0], "completed")
|
||
parent_packet = _packet(self.db_path, "P1")
|
||
self.assertEqual([f["id"] for f in parent_packet["findings"]],
|
||
["child:a"])
|
||
self.assertEqual(_state(self.db_path, "P1")[0], 1)
|
||
self.assertTrue(any(a == "worker_coord_audit_loop_pushback"
|
||
for _, a, _ in self.audits))
|
||
|
||
def test_merge_pass_clears_packets(self):
|
||
_insert(self.db_path, "P1", status="running", audit_loop=1)
|
||
_insert(self.db_path, "C1", status="completed", parent="P1",
|
||
part="a", result="done")
|
||
conn = sqlite3.connect(self.db_path)
|
||
try:
|
||
conn.execute(
|
||
"UPDATE tasks SET audit_loop_packet = ? WHERE id = ?",
|
||
('{"round": 1, "findings": []}', "C1"))
|
||
conn.execute(
|
||
"UPDATE tasks SET audit_loop_packet = ? WHERE id = ?",
|
||
('{"round": 1, "findings": []}', "P1"))
|
||
conn.commit()
|
||
finally:
|
||
conn.close()
|
||
out = self.worker._coord_audit_loop_merge(
|
||
self._parent(), self._parts("a"),
|
||
[{"name": "a", "id": "C1"}])
|
||
self.assertEqual(out["decision"], "pass")
|
||
self.assertIsNone(_packet(self.db_path, "C1"))
|
||
self.assertIsNone(_packet(self.db_path, "P1"))
|
||
|
||
def test_merge_gate_rejected_escalates_without_round(self):
|
||
_insert(self.db_path, "P1", status="running", audit_loop=1)
|
||
_insert(self.db_path, "C1", status="failed", parent="P1",
|
||
part="a", error="gateway exploded")
|
||
conn = sqlite3.connect(self.db_path)
|
||
try:
|
||
conn.execute(
|
||
"CREATE TABLE IF NOT EXISTS audit_log(user_id TEXT,"
|
||
" project_id TEXT, action TEXT, detail TEXT,"
|
||
" created_at DOUBLE PRECISION)")
|
||
conn.execute(
|
||
"INSERT INTO audit_log(user_id, project_id, action,"
|
||
" detail, created_at) VALUES('system', 'p1',"
|
||
" 'worker_capability_gate_rejected', ?, ?)",
|
||
("C1: tier gateway model X; refusing to dispatch", time.time()))
|
||
conn.commit()
|
||
finally:
|
||
conn.close()
|
||
out = self.worker._coord_audit_loop_merge(
|
||
self._parent(), self._parts("a"),
|
||
[{"name": "a", "id": "C1"}])
|
||
self.assertEqual(out["decision"], "escalate")
|
||
self.assertIn("a", out["error"])
|
||
# 不耗轮次：状态表里没有该父任务的行；子任务不重排。
|
||
self.assertIsNone(_state(self.db_path, "P1"))
|
||
self.assertEqual(_row(self.db_path, "C1")[0], "failed")
|
||
self.assertTrue(any(a == "worker_coord_audit_loop_escalated"
|
||
for _, a, _ in self.audits))
|
||
|
||
def test_merge_gate_rejected_by_error_marker(self):
|
||
_insert(self.db_path, "P1", status="running", audit_loop=1)
|
||
_insert(self.db_path, "C1", status="failed", parent="P1",
|
||
part="a", error="Missing environment variable: XCM_API_KEY")
|
||
out = self.worker._coord_audit_loop_merge(
|
||
self._parent(), self._parts("a"),
|
||
[{"name": "a", "id": "C1"}])
|
||
self.assertEqual(out["decision"], "escalate")
|
||
|
||
def test_merge_excluded_children_pass(self):
|
||
_insert(self.db_path, "P1", status="running", audit_loop=1)
|
||
_insert(self.db_path, "C1", status="failed", parent="P1",
|
||
part="a", error="tolerated boom")
|
||
_insert(self.db_path, "C2", status="failed", parent="P1",
|
||
part="b", error="user stopped", paused=1)
|
||
_insert(self.db_path, "C3", status="completed", parent="P1",
|
||
part="c", result="done")
|
||
out = self.worker._coord_audit_loop_merge(
|
||
self._parent(), self._parts("a", "b", "c", tolerated=("a",)),
|
||
[{"name": "a", "id": "C1"}, {"name": "b", "id": "C2"},
|
||
{"name": "c", "id": "C3"}])
|
||
self.assertEqual(out["decision"], "pass")
|
||
# 配额缺失的分工不在 children 里：不列门、不成 finding。
|
||
out2 = self.worker._coord_audit_loop_merge(
|
||
self._parent("P2"),
|
||
self._parts("c"),
|
||
[{"name": "c", "id": "C3"}])
|
||
self.assertEqual(out2["decision"], "pass")
|
||
|
||
def test_merge_max_rounds_escalates(self):
|
||
_insert(self.db_path, "P1", status="running", audit_loop=1)
|
||
_insert(self.db_path, "C1", status="failed", parent="P1",
|
||
part="a", error="boom again")
|
||
conn = sqlite3.connect(self.db_path)
|
||
try:
|
||
conn.execute(
|
||
"INSERT INTO task_audit_loop_state(task_id, round,"
|
||
" passed_gates, criteria, last_decision, updated_at)"
|
||
" VALUES('P1', 3, '[]', ?, 'pushback', ?)",
|
||
('{"child:a": "child:a"}', time.time()))
|
||
conn.commit()
|
||
finally:
|
||
conn.close()
|
||
out = self.worker._coord_audit_loop_merge(
|
||
self._parent(), self._parts("a"),
|
||
[{"name": "a", "id": "C1"}])
|
||
self.assertEqual(out["decision"], "escalate")
|
||
self.assertIsNone(_packet(self.db_path, "P1"))
|
||
self.assertIsNone(_packet(self.db_path, "C1"))
|
||
|
||
def test_merge_switch_off_skips(self):
|
||
_insert(self.db_path, "P1", status="running", audit_loop=0)
|
||
_insert(self.db_path, "C1", status="failed", parent="P1",
|
||
part="a", error="boom")
|
||
out = self.worker._coord_audit_loop_merge(
|
||
{"id": "P1", "project_id": "p1", "audit_loop": 0},
|
||
self._parts("a"), [{"name": "a", "id": "C1"}])
|
||
self.assertEqual(out["decision"], "skip")
|
||
self.assertEqual(_row(self.db_path, "C1")[0], "failed")
|
||
|
||
def test_sync_creation_and_requeue_attach_packet(self):
|
||
parent_packet = {
|
||
"task_id": "P1", "round": 1,
|
||
"findings": [{"id": "child:a", "severity": "high",
|
||
"evidence": "ea", "reproduction": "ra",
|
||
"acceptance": "aa", "kind": "deterministic",
|
||
"gate": "child:a"}],
|
||
"criteria": {"child:a": "child:a"}}
|
||
parent = {"id": "P1", "project_id": "p1", "audit_loop": 1,
|
||
"audit_loop_packet": json.dumps(parent_packet),
|
||
"created_by": "tester", "priority": 0}
|
||
self.worker._quota_exceeded = lambda *a: None
|
||
children = self.worker._coord_sync_children(
|
||
parent, [{"name": "a", "prompt": "do a"}])
|
||
self.assertEqual(len(children), 1)
|
||
packet = _packet(self.db_path, children[0]["id"])
|
||
self.assertIsNotNone(packet)
|
||
self.assertEqual([f["id"] for f in packet["findings"]],
|
||
["child:a"])
|
||
# 重排分支：已失败的子任务 + 父包 → 重排并贴拆分包。
|
||
conn = sqlite3.connect(self.db_path)
|
||
try:
|
||
conn.execute(
|
||
"UPDATE tasks SET status = 'failed', error_text = 'boom'"
|
||
" WHERE id = ?", (children[0]["id"],))
|
||
conn.execute(
|
||
"UPDATE tasks SET audit_loop_packet = NULL WHERE id = ?",
|
||
(children[0]["id"],))
|
||
conn.commit()
|
||
finally:
|
||
conn.close()
|
||
children2 = self.worker._coord_sync_children(
|
||
parent, [{"name": "a", "prompt": "do a"}])
|
||
self.assertEqual(children2[0]["id"], children[0]["id"])
|
||
row = _row(self.db_path, children[0]["id"])
|
||
self.assertEqual(row[0], "queued")
|
||
packet2 = _packet(self.db_path, children[0]["id"])
|
||
self.assertEqual([f["id"] for f in packet2["findings"]],
|
||
["child:a"])
|
||
|
||
|
||
def test_run_coordinator_pushback_full_path(self):
|
||
# 完整 _run_coordinator 路径：子任务在本轮轮询期间终态
|
||
# （线程里翻状态模拟 worker 完成），合流推回后父任务回
|
||
# queued、失败子任务回 queued 且带拆分包。直接插 failed
|
||
# 行测不到合流——sync 会在轮询前就把它重排走。
|
||
import threading
|
||
_insert(self.db_path, "P1", status="running", audit_loop=1,
|
||
parts=[{"name": "a", "prompt": "do a"},
|
||
{"name": "b", "prompt": "do b"}])
|
||
_insert(self.db_path, "C1", status="queued", parent="P1",
|
||
part="a")
|
||
_insert(self.db_path, "C2", status="queued", parent="P1",
|
||
part="b")
|
||
self.worker._quota_exceeded = lambda *a: None
|
||
box = {}
|
||
|
||
def _run():
|
||
box["result"] = self.worker._run_coordinator(
|
||
self.worker.get_task("P1"), time.time(),
|
||
poll_interval=0.01, timeout=10)
|
||
|
||
thread = threading.Thread(target=_run)
|
||
thread.start()
|
||
deadline = time.time() + 5
|
||
while time.time() < deadline:
|
||
conn = sqlite3.connect(self.db_path)
|
||
try:
|
||
conn.execute(
|
||
"UPDATE tasks SET status = 'failed',"
|
||
" error_text = 'boom' WHERE id = 'C1'")
|
||
conn.execute(
|
||
"UPDATE tasks SET status = 'completed',"
|
||
" result_text = 'done' WHERE id = 'C2'")
|
||
conn.commit()
|
||
finally:
|
||
conn.close()
|
||
time.sleep(0.05)
|
||
if not thread.is_alive():
|
||
break
|
||
thread.join(timeout=5)
|
||
self.assertFalse(thread.is_alive())
|
||
result = box["result"]
|
||
self.assertEqual(result["status"], "queued")
|
||
self.assertEqual(_row(self.db_path, "P1")[0], "queued")
|
||
self.assertEqual(_row(self.db_path, "C1")[0], "queued")
|
||
packet = _packet(self.db_path, "C1")
|
||
self.assertEqual([f["id"] for f in packet["findings"]],
|
||
["child:a"])
|
||
|
||
def test_run_coordinator_escalate_full_path(self):
|
||
# 门拒绝子任务在本轮终态 → 合流直接升级：父任务 failed、
|
||
# 错误点名分工、子任务不重排。
|
||
import threading
|
||
_insert(self.db_path, "P1", status="running", audit_loop=1,
|
||
parts=[{"name": "a", "prompt": "do a"}])
|
||
_insert(self.db_path, "C1", status="queued", parent="P1",
|
||
part="a")
|
||
conn = sqlite3.connect(self.db_path)
|
||
try:
|
||
conn.execute(
|
||
"CREATE TABLE IF NOT EXISTS audit_log(user_id TEXT,"
|
||
" project_id TEXT, action TEXT, detail TEXT,"
|
||
" created_at DOUBLE PRECISION)")
|
||
# Audit batch 2 (I-5)：setUp 已按生产实际经 AuthStore
|
||
# 建好含 id 列的 audit_log 表，INSERT 必须显式列名，
|
||
# 否则在 6 列的生产表上会因 5 个值报错；显式列名在
|
||
# 旧 5 列手工表上同样成立。
|
||
conn.execute(
|
||
"INSERT INTO audit_log(user_id, project_id, action,"
|
||
" detail, created_at) VALUES('system', 'p1',"
|
||
" 'worker_tool_gate_rejected',"
|
||
" 'C1: agent 声明必需工具缺失：websearch', ?)",
|
||
(time.time(),))
|
||
conn.commit()
|
||
finally:
|
||
conn.close()
|
||
self.worker._quota_exceeded = lambda *a: None
|
||
box = {}
|
||
|
||
def _run():
|
||
box["result"] = self.worker._run_coordinator(
|
||
self.worker.get_task("P1"), time.time(),
|
||
poll_interval=0.01, timeout=10)
|
||
|
||
thread = threading.Thread(target=_run)
|
||
thread.start()
|
||
deadline = time.time() + 5
|
||
while time.time() < deadline:
|
||
conn = sqlite3.connect(self.db_path)
|
||
try:
|
||
conn.execute(
|
||
"UPDATE tasks SET status = 'failed',"
|
||
" error_text = 'agent 声明必需工具缺失：websearch'"
|
||
" WHERE id = 'C1'")
|
||
conn.commit()
|
||
finally:
|
||
conn.close()
|
||
time.sleep(0.05)
|
||
if not thread.is_alive():
|
||
break
|
||
thread.join(timeout=5)
|
||
self.assertFalse(thread.is_alive())
|
||
result = box["result"]
|
||
self.assertEqual(result["status"], "failed")
|
||
self.assertIn("审计循环升级", result["error"])
|
||
self.assertEqual(_row(self.db_path, "P1")[0], "failed")
|
||
self.assertEqual(_row(self.db_path, "C1")[0], "failed")
|
||
|
||
|
||
if __name__ == "__main__":
|
||
unittest.main()
|