host
a7f77c8a98
- I-1: evaluate_shadow counts dropped rows by cause (unparseable / unknown_tier / explicit / duplicate) with the identity audit_rows_total == routed_task_n + dropped_total; the shadow endpoint passes created_at/id through and reports audit totals next to sample counts. - I-2 (option b): parse logs a warning naming each tier dropped for an invalid mode/requires; load attaches partial_failures to the policy; the worker records a task-level worker_routing_policy_load_failed audit for partial failures (same action as full failure, detail says failure=partial); the shadow report carries the policy fingerprint, dropped tiers and partial failures (no task context on that path, so no task-level audit there). - I-5: coordinator gate audit lookup failure now raises _CoordGateLookupFailed and the merge converts it to escalate (no requeue, no round consumed); persisted-model lookup failure and required-tools check failure each write a named audit row before falling through to dispatch/allow; audit-loop evaluate failure stays escalate (regression test). - Test fixtures: coordinator suites build audit_log via AuthStore as production's first real audit does, since their stubbed _audit never creates the table.
346 lines
15 KiB
Python
346 lines
15 KiB
Python
"""协调员审计循环复审修复测试：B1 + I1 + I2 + I3。
|
||
|
||
只用 init_db 的本地库与注入的假 runner，不跑真实模型回合、
|
||
不联网。覆盖（按任务书）：
|
||
- B1：旧门拒绝审计行（早于本轮 started_at）不再误判新失败，
|
||
子任务执行失败应推回而非升级；本轮门拒绝审计行仍升级；
|
||
started_at 未知时保守保留门拒绝（fail-closed 向升级）。
|
||
- I1：merged 计完成——evaluate_outcome / build_report /
|
||
合流 pass 三处对齐；sync 不把 merged 子任务重排。
|
||
- I2：合流推回事务里父 CAS 失败时，子任务不被孤儿重排、
|
||
包列不写、且有明确审计；推回成功时父任务已由合流
|
||
CAS 回 queued（hook 不再另做父 CAS）。
|
||
- I3：推回存库分支异常时走升级而非回正常终局，error
|
||
写明存库失败、记 escalated 审计；已消耗的轮次不恢复。
|
||
"""
|
||
|
||
import json
|
||
import os
|
||
import sqlite3
|
||
import sys
|
||
import tempfile
|
||
import time
|
||
import unittest
|
||
from pathlib import Path
|
||
from types import SimpleNamespace
|
||
|
||
try:
|
||
from cloud.auth import AuthStore
|
||
from cloud.coordinator import build_report, evaluate_outcome
|
||
from cloud.server import init_db
|
||
from cloud.worker import Worker
|
||
except ImportError:
|
||
sys.path.insert(0, str(Path(__file__).resolve().parents[2]))
|
||
from cloud.auth import AuthStore
|
||
from cloud.coordinator import build_report, evaluate_outcome
|
||
from cloud.server import init_db
|
||
from cloud.worker import Worker
|
||
|
||
|
||
def _insert(db_path, task_id, status="queued", parent=None, part=None,
|
||
result=None, error=None, audit_loop=0, parts=None):
|
||
now = time.time()
|
||
conn = sqlite3.connect(db_path)
|
||
try:
|
||
conn.execute(
|
||
"INSERT INTO tasks(id, project_id, prompt, status,"
|
||
" idempotency_key, depends_on, priority, created_by,"
|
||
" created_at, updated_at, last_heartbeat, audit_loop,"
|
||
" parent_task_id, coordinator_part, result_text, error_text,"
|
||
" coordinator_parts)"
|
||
" VALUES(?, 'p1', 'do', ?, NULL, '[]', 0, 'tester',"
|
||
" ?, ?, NULL, ?, ?, ?, ?, ?, ?)",
|
||
(task_id, status, now, now, audit_loop, parent, part,
|
||
result, error,
|
||
json.dumps(parts) if parts is not None else None))
|
||
conn.commit()
|
||
finally:
|
||
conn.close()
|
||
|
||
|
||
def _row(db_path, task_id):
|
||
conn = sqlite3.connect(db_path)
|
||
try:
|
||
return conn.execute(
|
||
"SELECT status, result_text, error_text, audit_loop_packet"
|
||
" FROM tasks WHERE id = ?", (task_id,)).fetchone()
|
||
finally:
|
||
conn.close()
|
||
|
||
|
||
def _packet(db_path, task_id):
|
||
row = _row(db_path, task_id)
|
||
if row is None or row[3] is None:
|
||
return None
|
||
return json.loads(row[3])
|
||
|
||
|
||
def _set_started(db_path, task_id, started_at):
|
||
conn = sqlite3.connect(db_path)
|
||
try:
|
||
conn.execute("UPDATE tasks SET started_at = ? WHERE id = ?",
|
||
(started_at, task_id))
|
||
conn.commit()
|
||
finally:
|
||
conn.close()
|
||
|
||
|
||
def _gate_audit(db_path, detail, created_at):
|
||
conn = sqlite3.connect(db_path)
|
||
try:
|
||
conn.execute(
|
||
"CREATE TABLE IF NOT EXISTS audit_log(user_id TEXT,"
|
||
" project_id TEXT, action TEXT, detail TEXT,"
|
||
" created_at DOUBLE PRECISION)")
|
||
conn.execute(
|
||
"INSERT INTO audit_log(user_id, project_id, action,"
|
||
" detail, created_at) VALUES('system', 'p1',"
|
||
" 'worker_capability_gate_rejected', ?, ?)",
|
||
(detail, created_at))
|
||
conn.commit()
|
||
finally:
|
||
conn.close()
|
||
|
||
|
||
def _completed_runner(cmd):
|
||
return SimpleNamespace(returncode=0, stdout="{}", stderr="")
|
||
|
||
|
||
class CoordFixTestCase(unittest.TestCase):
|
||
def setUp(self):
|
||
self._old = os.environ.get("CODEX_PUSH_ALLOW_PRIVATE_TARGETS")
|
||
os.environ["CODEX_PUSH_ALLOW_PRIVATE_TARGETS"] = "1"
|
||
self.tmp = tempfile.TemporaryDirectory()
|
||
self.db_path = os.path.join(self.tmp.name, "control.db")
|
||
init_db(self.db_path)
|
||
# Audit batch 2 (I-5)，与 test_audit_loop_coord.py 同款
|
||
# 理由：生产里 audit_log 由首次真实 _audit 经 AuthStore
|
||
# 建好，合流时表一定存在；本文件把 _audit 桩成列表，
|
||
# 表永远不会出现，夹具里按生产实际补建表，否则合流
|
||
# 测试会因缺表改走 escalate，与被测行为无关。
|
||
_store = AuthStore(self.db_path)
|
||
_store.conn.close()
|
||
self.env = {"XCM_API_KEY": "test-placeholder"}
|
||
self.audits = []
|
||
self.worker = Worker(self.db_path, "/fake/host.py",
|
||
runner=_completed_runner, env=self.env,
|
||
max_retries=0, retry_backoff_seconds=0)
|
||
self.worker._audit = lambda p, a, d="": self.audits.append((p, a, d))
|
||
conn = sqlite3.connect(self.db_path)
|
||
try:
|
||
self.worker._ensure_result_columns(conn)
|
||
conn.commit()
|
||
finally:
|
||
conn.close()
|
||
|
||
def tearDown(self):
|
||
if self._old is None:
|
||
os.environ.pop("CODEX_PUSH_ALLOW_PRIVATE_TARGETS", None)
|
||
else:
|
||
os.environ["CODEX_PUSH_ALLOW_PRIVATE_TARGETS"] = self._old
|
||
self.tmp.cleanup()
|
||
|
||
def _parent(self, task_id="P1"):
|
||
return {"id": task_id, "project_id": "p1", "audit_loop": 1}
|
||
|
||
def _parts(self, *names):
|
||
return [{"name": n, "on_failure": "fail"} for n in names]
|
||
|
||
# ---- B1 ----
|
||
def test_b1_old_gate_audit_does_not_block_pushback(self):
|
||
# 门拒绝 → 修复 → 执行失败：旧审计行早于本轮 started_at，
|
||
# 必须被作用域过滤掉，子任务按执行失败推回而非升级。
|
||
_insert(self.db_path, "P1", status="running", audit_loop=1)
|
||
_insert(self.db_path, "C1", status="failed", parent="P1",
|
||
part="a", error="boom")
|
||
now = time.time()
|
||
_set_started(self.db_path, "C1", now)
|
||
_gate_audit(self.db_path, "C1: old gate rejection", now - 100)
|
||
out = self.worker._coord_audit_loop_merge(
|
||
self._parent(), self._parts("a"),
|
||
[{"name": "a", "id": "C1"}])
|
||
self.assertEqual(out["decision"], "pushback")
|
||
self.assertEqual(_row(self.db_path, "C1")[0], "queued")
|
||
|
||
def test_b1_current_gate_audit_still_escalates(self):
|
||
# 本轮门拒绝审计行晚于 started_at，仍应升级、不耗轮次。
|
||
_insert(self.db_path, "P1", status="running", audit_loop=1)
|
||
_insert(self.db_path, "C1", status="failed", parent="P1",
|
||
part="a", error="gateway exploded")
|
||
now = time.time()
|
||
_set_started(self.db_path, "C1", now)
|
||
_gate_audit(self.db_path, "C1: new gate rejection", now + 10)
|
||
out = self.worker._coord_audit_loop_merge(
|
||
self._parent(), self._parts("a"),
|
||
[{"name": "a", "id": "C1"}])
|
||
self.assertEqual(out["decision"], "escalate")
|
||
self.assertEqual(_row(self.db_path, "C1")[0], "failed")
|
||
|
||
def test_b1_unknown_started_at_is_conservative(self):
|
||
# started_at 未知（老行/缺列）时保守保留门拒绝，向升级
|
||
# fail-closed，绝不因起始未知就把门拒绝任务推回。
|
||
_insert(self.db_path, "P1", status="running", audit_loop=1)
|
||
_insert(self.db_path, "C1", status="failed", parent="P1",
|
||
part="a", error="gateway exploded")
|
||
_gate_audit(self.db_path, "C1: gate rejection",
|
||
time.time())
|
||
out = self.worker._coord_audit_loop_merge(
|
||
self._parent(), self._parts("a"),
|
||
[{"name": "a", "id": "C1"}])
|
||
self.assertEqual(out["decision"], "escalate")
|
||
|
||
def test_b1_m1_id_tiebreak_with_id_column(self):
|
||
# M1：生产 audit_log 有 id 列时，_coord_gate_rejected_
|
||
# child_ids 走 ORDER BY created_at DESC, id DESC 的
|
||
# 第一条查询；B1 旧测试用的无 id 列建表只覆盖了回落
|
||
# 路径，tie-break 语义（同 created_at 时 id 倒序）
|
||
# 没有任何断言。
|
||
#
|
||
# 诚实的设计说明：该函数返回的是子任务 id 集合而非
|
||
# 单行，所以“同一子任务多行、同 created_at”时无论
|
||
# 取哪行结果都一样——那样的测试是假测试，抓不到取
|
||
# 错行。tie-break 唯一可观测的后果在 5000 行封顶
|
||
# 处：同 created_at 时 id 倒序保留最新 5000 行、丢
|
||
# 最旧一行；若改成 id 正序或落到无 id 排序的回落
|
||
# 路径，则正好相反（丢最新一行、留最旧一行，见
|
||
# /tmp 实测：三条查询在 5001 行同 created_at 下
|
||
# 的结果互为反面）。因此用 5001 行同 created_at
|
||
# 压封顶边界断言：最新 id 的子任务必须在集合里、
|
||
# 最旧 id 的子任务必须不在，且集合恰好 5000 条。
|
||
conn = sqlite3.connect(self.db_path)
|
||
try:
|
||
# Audit batch 2 (I-5)：setUp 已按生产实际经 AuthStore
|
||
# 建好含 id 自增列的 audit_log 表，建表改 IF NOT
|
||
# EXISTS——生产表与本测试要的表在 id 语义上一致，
|
||
# tie-break 断言不受影响。
|
||
conn.execute(
|
||
"CREATE TABLE IF NOT EXISTS audit_log(id INTEGER"
|
||
" PRIMARY KEY AUTOINCREMENT, user_id TEXT,"
|
||
" project_id TEXT, action TEXT, detail TEXT,"
|
||
" created_at DOUBLE PRECISION)")
|
||
now = time.time()
|
||
conn.executemany(
|
||
"INSERT INTO audit_log(user_id, project_id,"
|
||
" action, detail, created_at)"
|
||
" VALUES('system', 'p1',"
|
||
" 'worker_capability_gate_rejected', ?, ?)",
|
||
[("C%04d: gate rejection" % i, now)
|
||
for i in range(5001)])
|
||
conn.commit()
|
||
finally:
|
||
conn.close()
|
||
child_ids = ["C%04d" % i for i in range(5001)]
|
||
found = self.worker._coord_gate_rejected_child_ids(
|
||
"p1", child_ids,
|
||
attempt_started_at={cid: now - 100
|
||
for cid in child_ids})
|
||
self.assertEqual(len(found), 5000)
|
||
# 最新 id 的行（C5000）必须被 id 倒序保留下来。
|
||
self.assertIn("C5000", found)
|
||
# 最旧 id 的行（C0000）必须被 5000 行封顶挤掉；
|
||
# 若排序改成 id 正序或落到回落路径，这里会反过来
|
||
# （C5000 被挤掉、C0000 留下），断言即失败。
|
||
self.assertNotIn("C0000", found)
|
||
|
||
# ---- I1 ----
|
||
def test_i1_merged_counts_as_completed_everywhere(self):
|
||
overall, failed, _tol = evaluate_outcome(
|
||
[{"name": "a", "status": "merged"},
|
||
{"name": "b", "status": "completed"}],
|
||
[{"name": "a"}, {"name": "b"}])
|
||
self.assertEqual(overall, "completed")
|
||
self.assertEqual(failed, [])
|
||
_report, overall2 = build_report(
|
||
[{"name": "a", "id": "C1", "status": "merged",
|
||
"result_text": "done", "error_text": ""},
|
||
{"name": "b", "id": "C2", "status": "completed",
|
||
"result_text": "done", "error_text": ""}],
|
||
[{"name": "a"}, {"name": "b"}])
|
||
self.assertEqual(overall2, "completed")
|
||
# 合流：merged 子任务按完成计，审计 pass。
|
||
_insert(self.db_path, "P1", status="running", audit_loop=1)
|
||
_insert(self.db_path, "C1", status="merged", parent="P1",
|
||
part="a", result="done")
|
||
out = self.worker._coord_audit_loop_merge(
|
||
self._parent(), self._parts("a"),
|
||
[{"name": "a", "id": "C1"}])
|
||
self.assertEqual(out["decision"], "pass")
|
||
|
||
def test_i1_sync_does_not_requeue_merged_child(self):
|
||
# sync 把 merged 当完成直接复用，绝不回 queued 清结果。
|
||
_insert(self.db_path, "P1", status="running", audit_loop=1)
|
||
_insert(self.db_path, "C1", status="merged", parent="P1",
|
||
part="a", result="done")
|
||
self.worker._quota_exceeded = lambda *a: None
|
||
children = self.worker._coord_sync_children(
|
||
{"id": "P1", "project_id": "p1", "audit_loop": 1,
|
||
"created_by": "tester", "priority": 0},
|
||
[{"name": "a", "prompt": "do a"}])
|
||
self.assertEqual(children[0]["id"], "C1")
|
||
row = _row(self.db_path, "C1")
|
||
self.assertEqual(row[0], "merged")
|
||
self.assertEqual(row[1], "done")
|
||
|
||
# ---- I2 ----
|
||
def test_i2_parent_cas_failure_rolls_back_children(self):
|
||
# 父任务状态已被改动（模拟并发取消）：合流事务回滚，
|
||
# 子任务不被孤儿重排、包列不写、且有明确审计。
|
||
_insert(self.db_path, "P1", status="interrupted",
|
||
audit_loop=1)
|
||
_insert(self.db_path, "C1", status="failed", parent="P1",
|
||
part="a", error="boom")
|
||
out = self.worker._coord_audit_loop_merge(
|
||
self._parent(), self._parts("a"),
|
||
[{"name": "a", "id": "C1"}])
|
||
self.assertEqual(out["decision"], "pushback_parent_cas_failed")
|
||
self.assertEqual(_row(self.db_path, "C1")[0], "failed")
|
||
self.assertIsNone(_packet(self.db_path, "C1"))
|
||
self.assertIsNone(_packet(self.db_path, "P1"))
|
||
self.assertTrue(any(
|
||
a == "worker_coord_audit_loop_pushback_parent_cas_failed"
|
||
for _, a, _ in self.audits))
|
||
|
||
def test_i2_pushback_parent_already_queued_by_merge(self):
|
||
# 推回成功时父任务已由合流事务 CAS 回 queued（hook
|
||
# 不再另做父 CAS）；子任务重排、包可读。
|
||
_insert(self.db_path, "P1", status="running", audit_loop=1)
|
||
_insert(self.db_path, "C1", status="failed", parent="P1",
|
||
part="a", error="boom")
|
||
out = self.worker._coord_audit_loop_merge(
|
||
self._parent(), self._parts("a"),
|
||
[{"name": "a", "id": "C1"}])
|
||
self.assertEqual(out["decision"], "pushback")
|
||
self.assertEqual(_row(self.db_path, "P1")[0], "queued")
|
||
self.assertEqual(_row(self.db_path, "C1")[0], "queued")
|
||
self.assertIsNotNone(_packet(self.db_path, "C1"))
|
||
|
||
# ---- I3 ----
|
||
def test_i3_pushback_store_failure_escalates(self):
|
||
# 推回存库分支内任何异常都走升级而非回正常终局：
|
||
# 用拆分包抛异常模拟存库分支异常（该异常位于同一
|
||
# try 块内，与真实 DB 写失败走同一条 except 路径）。
|
||
_insert(self.db_path, "P1", status="running", audit_loop=1)
|
||
_insert(self.db_path, "C1", status="failed", parent="P1",
|
||
part="a", error="boom")
|
||
orig = Worker._coord_split_packet
|
||
try:
|
||
Worker._coord_split_packet = staticmethod(
|
||
lambda *a: (_ for _ in ()).throw(
|
||
RuntimeError("db write failed")))
|
||
out = self.worker._coord_audit_loop_merge(
|
||
self._parent(), self._parts("a"),
|
||
[{"name": "a", "id": "C1"}])
|
||
finally:
|
||
Worker._coord_split_packet = orig
|
||
self.assertEqual(out["decision"], "escalate")
|
||
self.assertIn("推回存库失败", out["error"])
|
||
self.assertTrue(any(a == "worker_coord_audit_loop_escalated"
|
||
for _, a, _ in self.audits))
|
||
# 升级清包：父包与子包都不应残留。
|
||
self.assertIsNone(_packet(self.db_path, "P1"))
|
||
self.assertIsNone(_packet(self.db_path, "C1"))
|
||
|
||
|
||
if __name__ == "__main__":
|
||
unittest.main()
|