fix: audit batch 2 visibility (I-1 shadow dropped rows, I-2 partial policy failure, I-5 failure policy table)

- I-1: evaluate_shadow counts dropped rows by cause (unparseable /
  unknown_tier / explicit / duplicate) with the identity
  audit_rows_total == routed_task_n + dropped_total; the shadow
  endpoint passes created_at/id through and reports audit totals
  next to sample counts.
- I-2 (option b): parse logs a warning naming each tier dropped for
  an invalid mode/requires; load attaches partial_failures to the
  policy; the worker records a task-level
  worker_routing_policy_load_failed audit for partial failures
  (same action as full failure, detail says failure=partial);
  the shadow report carries the policy fingerprint, dropped tiers
  and partial failures (no task context on that path, so no
  task-level audit there).
- I-5: coordinator gate audit lookup failure now raises
  _CoordGateLookupFailed and the merge converts it to escalate
  (no requeue, no round consumed); persisted-model lookup failure
  and required-tools check failure each write a named audit row
  before falling through to dispatch/allow; audit-loop evaluate
  failure stays escalate (regression test).
- Test fixtures: coordinator suites build audit_log via AuthStore
  as production's first real audit does, since their stubbed
  _audit never creates the table.

cloud/tests/test_audit_batch2.py

@@ -0,0 +1,422 @@
+"""审计第 2 批（I-1、I-2、I-5）单测：可见性三件。
+
+运行（在仓库根目录下）：
+    python3 -m unittest discover -s cloud/tests -p "test_audit_batch2.py" -v
+
+说明：只断言本批新增的可见性行为，不碰 I-3、I-4、I-6～I-14、
+N-1～N-12。I-1 断在报告层（不只断解析返回 None）；I-2 断在
+任务级审计与影子报告字段层；I-5 四处各一例失败注入。
+不跑真实模型回合、不联网。
+"""
+
+import http.client
+import json
+import os
+import sqlite3
+import sys
+import tempfile
+import threading
+import time
+import unittest
+from pathlib import Path
+from types import SimpleNamespace
+
+try:
+    from cloud.auth import AuthStore
+    from cloud.models import load_routing_policy
+    from cloud.models import parse_routing_policy
+    from cloud.routing_shadow import evaluate_shadow
+    from cloud.sandbox import DockerExecutor
+    from cloud.server import create_server
+    from cloud.server import init_db
+    from cloud.worker import Worker
+    from cloud.worker import _CoordGateLookupFailed
+except ImportError:
+    sys.path.insert(0, str(Path(__file__).resolve().parents[2]))
+    from cloud.auth import AuthStore
+    from cloud.models import load_routing_policy
+    from cloud.models import parse_routing_policy
+    from cloud.routing_shadow import evaluate_shadow
+    from cloud.sandbox import DockerExecutor
+    from cloud.server import create_server
+    from cloud.server import init_db
+    from cloud.worker import Worker
+    from cloud.worker import _CoordGateLookupFailed
+
+
+def _completed_runner(cmd):
+    return SimpleNamespace(returncode=0, stdout="{}", stderr="")
+
+
+def _insert_task(db_path, task_id, status="queued", parent=None,
+                 part=None, error=None, audit_loop=0, parts=None,
+                 project_id="p1"):
+    now = time.time()
+    conn = sqlite3.connect(db_path)
+    try:
+        conn.execute(
+            "INSERT INTO tasks(id, project_id, prompt, status,"
+            " idempotency_key, depends_on, priority, created_by,"
+            " created_at, updated_at, last_heartbeat, audit_loop,"
+            " parent_task_id, coordinator_part, result_text, error_text,"
+            " paused_by_user, interrupted_reason, coordinator_parts,"
+            " started_at)"
+            " VALUES(?, ?, 'do', ?, NULL, '[]', 0, 'tester',"
+            " ?, ?, NULL, ?, ?, ?, NULL, ?, 0, NULL, ?, ?)",
+            (task_id, project_id, status, now, now, audit_loop,
+             parent, part, error,
+             json.dumps(parts) if parts is not None else None, now))
+        conn.commit()
+    finally:
+        conn.close()
+
+
+def _audit_rows(db_path, project_id="p1"):
+    store = AuthStore(db_path)
+    try:
+        return [(e["action"], e["detail"]) for e in store.list_audit(
+            project_id)]
+    finally:
+        store.conn.close()
+
+
+def _task_row(db_path, task_id):
+    conn = sqlite3.connect(db_path)
+    try:
+        row = conn.execute(
+            "SELECT status, audit_loop_packet FROM tasks WHERE id = ?",
+            (task_id,)).fetchone()
+        return row
+    finally:
+        conn.close()
+
+
+class DroppedRowsTestCase(unittest.TestCase):
+    def test_buckets_and_identity_at_report_level(self):
+        # 6 行输入：2 个有效样本 + 4 种丢行原因各 1 行。
+        rows = [
+            {"detail": "t1 tier=plan source=shadow applied=False",
+             "created_at": 1.0, "id": 1},
+            {"detail": "t2 tier=goal source=policy applied=True",
+             "created_at": 2.0, "id": 2},
+            {"detail": "this line does not parse at all",
+             "created_at": 3.0, "id": 3},
+            {"detail": "t3 tier=weird source=shadow applied=True",
+             "created_at": 4.0, "id": 4},
+            {"detail": "t4 tier=plan source=explicit applied=False",
+             "created_at": 5.0, "id": 5},
+            {"detail": "t1 tier=plan source=shadow applied=False",
+             "created_at": 6.0, "id": 6},
+        ]
+        tasks = {"t1": {"status": "completed"},
+                 "t2": {"status": "completed"}}
+        report = evaluate_shadow(rows, tasks, {})
+        self.assertEqual(report["dropped_rows"], {
+            "unparseable": 1, "unknown_tier": 1,
+            "explicit": 1, "duplicate": 1})
+        self.assertEqual(report["dropped_total"], 4)
+        self.assertEqual(report["audit_rows_total"], 6)
+        self.assertEqual(report["routed_task_n"], 2)
+        # 行恒等式：总行数 = 样本数 + 丢行总数。
+        self.assertEqual(
+            report["audit_rows_total"],
+            report["routed_task_n"] + report["dropped_total"])
+        # 显式行同时计 per-tier explicit_n（不是丢失）。
+        self.assertEqual(report["tiers"]["plan"]["explicit_n"], 1)
+        self.assertEqual(report["tiers"]["plan"]["routed_n"], 1)
+
+
+class ShadowEndpointPolicyTestCase(unittest.TestCase):
+    def setUp(self):
+        self._old_json = os.environ.get("CLOUD_ROUTING_POLICY_JSON")
+        self._old_file = os.environ.get("CLOUD_ROUTING_POLICY_FILE")
+        self._old_mode = os.environ.get("CLOUD_ROUTING_MODE")
+        # 策略含一个无效档模式的档（plan）与一个有效档（goal）：
+        # 端点报告里必须出现丢弃档 plan，而不是只剩 goal。
+        os.environ["CLOUD_ROUTING_POLICY_JSON"] = json.dumps({
+            "mode": "shadow",
+            "tiers": {"plan": {"model": "m-plan", "mode": "bogus"},
+                      "goal": "m-goal"}})
+        os.environ.pop("CLOUD_ROUTING_POLICY_FILE", None)
+        os.environ.pop("CLOUD_ROUTING_MODE", None)
+        self.tmp = tempfile.TemporaryDirectory()
+        self.db_path = os.path.join(self.tmp.name, "control.db")
+        init_db(self.db_path)
+        self.auth = AuthStore(self.db_path)
+        _owner_id, self.owner_key = self.auth.create_user("owner")
+        self.server = create_server(self.db_path, "127.0.0.1", 0)
+        self.port = self.server.server_address[1]
+        self.thread = threading.Thread(
+            target=self.server.serve_forever, daemon=True)
+        self.thread.start()
+        status, self.project = self._req(
+            "POST", "/projects", {"name": "demo"}, self.owner_key)
+        self.assertEqual(status, 201, self.project)
+
+    def tearDown(self):
+        self.server.shutdown()
+        self.server.server_close()
+        self.thread.join(timeout=5)
+        try:
+            self.auth.conn.close()
+        except Exception:
+            pass
+        self.tmp.cleanup()
+        for key, old in (("CLOUD_ROUTING_POLICY_JSON", self._old_json),
+                         ("CLOUD_ROUTING_POLICY_FILE", self._old_file),
+                         ("CLOUD_ROUTING_MODE", self._old_mode)):
+            if old is None:
+                os.environ.pop(key, None)
+            else:
+                os.environ[key] = old
+
+    def _req(self, method, path, body=None, key=None):
+        conn = http.client.HTTPConnection("127.0.0.1", self.port,
+                                          timeout=10)
+        headers = {"Content-Type": "application/json"}
+        if key:
+            headers["X-API-Key"] = key
+        conn.request(method, path,
+                     json.dumps(body) if body is not None else None,
+                     headers)
+        resp = conn.getresponse()
+        raw = resp.read()
+        conn.close()
+        try:
+            data = json.loads(raw)
+        except ValueError:
+            data = None
+        return resp.status, data
+
+    def test_endpoint_reports_dropped_tiers_and_fingerprint(self):
+        status, report = self._req(
+            "GET", "/routing-shadow?project_id=" + self.project["id"],
+            key=self.owner_key)
+        self.assertEqual(status, 200, report)
+        policy = report["routing_policy"]
+        self.assertEqual(policy["status"], "partial")
+        self.assertEqual(policy["dropped_tiers"], ["plan"])
+        self.assertEqual(len(policy["fingerprint"]), 64)
+        # 无样本时丢行计数仍在报告里（不是只有样本才有）。
+        self.assertEqual(report["audit_rows_total"], 0)
+        self.assertEqual(report["dropped_total"], 0)
+
+
+class PolicyDropWarningTestCase(unittest.TestCase):
+    def test_invalid_mode_warns_with_tier_name_and_keeps_others(self):
+        with self.assertLogs("cloud.models", level="WARNING") as logs:
+            policy = parse_routing_policy({
+                "mode": "shadow",
+                "tiers": {"plan": {"model": "m-plan",
+                                   "mode": "bogus"},
+                          "goal": "m-goal"}})
+        self.assertIsNotNone(policy)
+        self.assertNotIn("plan", policy.tiers)
+        self.assertIn("goal", policy.tiers)
+        self.assertEqual(policy.dropped_tiers, ["plan"])
+        self.assertTrue(any("plan" in m for m in logs.output))
+
+    def test_invalid_requires_drops_tier_with_warning(self):
+        with self.assertLogs("cloud.models", level="WARNING") as logs:
+            policy = parse_routing_policy({
+                "mode": "shadow",
+                "tiers": {"plan": {"model": "m-plan",
+                                   "requires": {"bogus_key": True}},
+                          "goal": "m-goal"}})
+        self.assertIsNotNone(policy)
+        self.assertNotIn("plan", policy.tiers)
+        self.assertIn("goal", policy.tiers)
+        self.assertEqual(policy.dropped_tiers, ["plan"])
+        self.assertTrue(any("plan" in m for m in logs.output))
+
+
+class PartialFailureTaskAuditTestCase(unittest.TestCase):
+    def setUp(self):
+        self.tmp = tempfile.TemporaryDirectory()
+        self.db_path = os.path.join(self.tmp.name, "control.db")
+        init_db(self.db_path)
+        _insert_task(self.db_path, "P1")
+
+    def tearDown(self):
+        self.tmp.cleanup()
+
+    def test_partial_failure_writes_task_audit(self):
+        # 文件来源失败 + JSON 来源可用 = 部分失败：策略可用，
+        # 但必须在任务级审计里可见（与全失败同款 action）。
+        env = {"XCM_API_KEY": "test-placeholder",
+               "CLOUD_ROUTING_POLICY_FILE": "/nonexistent/policy.json",
+               "CLOUD_ROUTING_POLICY_JSON": json.dumps({
+                   "mode": "shadow", "tiers": {"default": "m1"}})}
+        worker = Worker(self.db_path, "/fake/host.py",
+                        runner=_completed_runner, env=env,
+                        max_retries=0, retry_backoff_seconds=0)
+        self.assertFalse(worker.routing_policy_load_failed)
+        self.assertTrue(worker.routing_policy_partial_failure)
+        self.assertTrue(worker.routing_policy.partial_failures)
+        result = worker.run_one()
+        self.assertEqual(result["status"], "completed")
+        actions = _audit_rows(self.db_path)
+        partial = [d for a, d in actions
+                   if a == "worker_routing_policy_load_failed"]
+        self.assertTrue(partial)
+        self.assertIn("failure=partial", partial[0])
+        self.assertIn("policy.json", partial[0])
+
+
+class CoordGateLookupFailureTestCase(unittest.TestCase):
+    def setUp(self):
+        self._old = os.environ.get("CODEX_PUSH_ALLOW_PRIVATE_TARGETS")
+        os.environ["CODEX_PUSH_ALLOW_PRIVATE_TARGETS"] = "1"
+        self.tmp = tempfile.TemporaryDirectory()
+        self.db_path = os.path.join(self.tmp.name, "control.db")
+        init_db(self.db_path)
+        self.env = {"XCM_API_KEY": "test-placeholder"}
+        self.audits = []
+        self.worker = Worker(self.db_path, "/fake/host.py",
+                             runner=_completed_runner, env=self.env,
+                             max_retries=0, retry_backoff_seconds=0)
+        self.worker._audit = lambda p, a, d="": self.audits.append(
+            (p, a, d))
+        conn = sqlite3.connect(self.db_path)
+        try:
+            self.worker._ensure_result_columns(conn)
+            conn.commit()
+        finally:
+            conn.close()
+
+    def tearDown(self):
+        if self._old is None:
+            os.environ.pop("CODEX_PUSH_ALLOW_PRIVATE_TARGETS", None)
+        else:
+            os.environ[CODEX_PUSH_ALLOW_PRIVATE_TARGETS] = self._old
+        self.tmp.cleanup()
+
+    def test_direct_lookup_raises_instead_of_empty_set(self):
+        # init_db 不建 audit_log 表（生产里由 AuthStore 建）：
+        # 查询失败时旧代码返回空集，新代码必须抛错——空集
+        # 会被合流读成“没有门拒绝”而把门拒绝推回。
+        with self.assertRaises(_CoordGateLookupFailed):
+            self.worker._coord_gate_rejected_child_ids("p1", ["C1"])
+
+    def test_merge_escalates_without_requeue_or_round(self):
+        # 父任务审计开启、子任务执行失败；门拒绝审计表缺失
+        # 导致查询失败时，合流必须升级、不重排、不进推回。
+        # 注意：这里故意不建 audit_log 表——缺表即查询失败。
+        _insert_task(self.db_path, "P1", status="running",
+                     audit_loop=1,
+                     parts=[{"name": "a", "prompt": "do a"}])
+        _insert_task(self.db_path, "C1", status="failed",
+                     parent="P1", part="a", error="boom")
+        out = self.worker._coord_audit_loop_merge(
+            {"id": "P1", "project_id": "p1", "audit_loop": 1},
+            [{"name": "a", "prompt": "do a"}],
+            [{"id": "C1", "name": "a"}])
+        self.assertEqual(out["decision"], "escalate")
+        self.assertIn("查询失败", out["error"])
+        self.assertTrue(out.get("gate_lookup_failed"))
+        # 子任务未被重排回 queued，仍是 failed。
+        self.assertEqual(_task_row(self.db_path, "C1")[0], "failed")
+        # 升级审计里点名了查询失败。
+        self.assertTrue(any(
+            a == "worker_coord_audit_loop_escalated"
+            and "gate_lookup_failed" in d
+            for _p, a, d in self.audits))
+
+
+class PersistedModelLookupFailureTestCase(unittest.TestCase):
+    def setUp(self):
+        self.tmp = tempfile.TemporaryDirectory()
+        self.db_path = os.path.join(self.tmp.name, "control.db")
+        init_db(self.db_path)
+
+    def tearDown(self):
+        self.tmp.cleanup()
+
+    def test_lookup_failure_audits_and_still_dispatches(self):
+        # _db_connect 桩成抛错：复核查不到依据时返回 False
+        # （放行），但必须在审计里点名依据缺失，不能只有日志。
+        # _audit 走 AuthStore 独立连接，不受该桩影响。
+        import cloud.worker as worker_mod
+        real_connect = worker_mod._db_connect
+
+        def _boom(_path):
+            raise RuntimeError("db down for lookup")
+
+        worker_mod._db_connect = _boom
+        try:
+            worker = Worker(self.db_path, "/fake/host.py",
+                            runner=_completed_runner,
+                            env={"XCM_API_KEY": "test-placeholder"},
+                            max_retries=0, retry_backoff_seconds=0)
+            self.assertFalse(worker._is_persisted_model(
+                "T1", "p1", "m1"))
+        finally:
+            worker_mod._db_connect = real_connect
+        actions = _audit_rows(self.db_path)
+        found = [d for a, d in actions
+                 if a == "worker_persisted_model_lookup_failed"]
+        self.assertTrue(found)
+        self.assertIn("T1", found[0])
+
+
+class RequiredToolsCheckFailureTestCase(unittest.TestCase):
+    def setUp(self):
+        self.tmp = tempfile.TemporaryDirectory()
+        self.db_path = os.path.join(self.tmp.name, "control.db")
+        init_db(self.db_path)
+
+    def tearDown(self):
+        self.tmp.cleanup()
+
+    def test_check_failure_audits_and_still_allows(self):
+        # 租户目录读取失败时返回 [] 放行不变，但必须在审计
+        # 里点名依据缺失，不能只有 warning 日志。
+        worker = Worker(self.db_path, "/fake/host.py",
+                        runner=_completed_runner,
+                        env={"XCM_API_KEY": "test-placeholder"},
+                        max_retries=0, retry_backoff_seconds=0)
+        worker.tenant_isolation = True
+        worker.executor = DockerExecutor.__new__(DockerExecutor)
+
+        def _boom(_project_id):
+            raise RuntimeError("tenant home unreadable")
+
+        worker._tenant_dirs = _boom
+        result = worker._missing_required_tools(
+            {"id": "T1", "project_id": "p1",
+             "agent_tools": ["websearch"]})
+        self.assertEqual(result, [])
+        actions = _audit_rows(self.db_path)
+        found = [d for a, d in actions
+                 if a == "worker_required_tools_check_failed"]
+        self.assertTrue(found)
+        self.assertIn("T1", found[0])
+
+
+class AuditLoopEvaluateFailureTestCase(unittest.TestCase):
+    def setUp(self):
+        self.tmp = tempfile.TemporaryDirectory()
+        self.db_path = os.path.join(self.tmp.name, "control.db")
+        init_db(self.db_path)
+
+    def tearDown(self):
+        self.tmp.cleanup()
+
+    def test_evaluate_failure_still_escalates(self):
+        # 回归断言：审计循环评估失败仍 fail-closed 到 escalate
+        # （I-5 策略表里评估失败→升级这一路本来就有，不动）。
+        # 非法门结果值让纯函数 evaluate 抛 ValueError，经
+        # audit_loop_evaluate 捕获后应为 escalate。
+        worker = Worker(self.db_path, "/fake/host.py",
+                        runner=_completed_runner,
+                        env={"XCM_API_KEY": "test-placeholder"},
+                        max_retries=0, retry_backoff_seconds=0)
+        out = worker.audit_loop_evaluate(
+            {"id": "T1", "project_id": "p1", "audit_loop": 1},
+            {"child:a": "not-a-bool"})
+        self.assertEqual(out["decision"], "escalate")
+        self.assertEqual(out["reason"], "evaluate_failed")
+
+
+if __name__ == "__main__":
+    unittest.main()

cloud/tests/test_audit_loop_coord.py

@@ -28,11 +28,13 @@ from types import SimpleNamespace
 
 try:
     from cloud.audit_loop import Finding
+    from cloud.auth import AuthStore
     from cloud.server import init_db
     from cloud.worker import Worker
 except ImportError:
     sys.path.insert(0, str(Path(__file__).resolve().parents[2]))
     from cloud.audit_loop import Finding
+    from cloud.auth import AuthStore
     from cloud.server import init_db
     from cloud.worker import Worker
 
@@ -100,6 +102,14 @@ class CoordAuditTestCase(unittest.TestCase):
         self.tmp = tempfile.TemporaryDirectory()
         self.db_path = os.path.join(self.tmp.name, "control.db")
         init_db(self.db_path)
+        # Audit batch 2 (I-5): 合流的门拒绝审计查询失败时走
+        # escalate，不再回落空集。生产里 audit_log 由首次
+        # 真实 _audit 经 AuthStore 建好，合流时表一定存在；
+        # 本文件把 _audit 桩成列表，表永远不会出现，所以
+        # 在夹具里按生产实际补建表——否则所有合流测试都会
+        # 因缺表改走 escalate，与被测行为无关。
+        _store = AuthStore(self.db_path)
+        _store.conn.close()
         self.env = {"XCM_API_KEY": "test-placeholder"}
         self.audits = []
         self.worker = self._worker()
@@ -399,8 +409,13 @@ class CoordAuditTestCase(unittest.TestCase):
                 "CREATE TABLE IF NOT EXISTS audit_log(user_id TEXT,"
                 " project_id TEXT, action TEXT, detail TEXT,"
                 " created_at DOUBLE PRECISION)")
+            # Audit batch 2 (I-5)：setUp 已按生产实际经 AuthStore
+            # 建好含 id 列的 audit_log 表，INSERT 必须显式列名，
+            # 否则在 6 列的生产表上会因 5 个值报错；显式列名在
+            # 旧 5 列手工表上同样成立。
             conn.execute(
-                "INSERT INTO audit_log VALUES('system', 'p1',"
+                "INSERT INTO audit_log(user_id, project_id, action,"
+                " detail, created_at) VALUES('system', 'p1',"
                 " 'worker_tool_gate_rejected',"
                 " 'C1: agent 声明必需工具缺失：websearch', ?)",
                 (time.time(),))

cloud/tests/test_audit_loop_coord_fix.py

@@ -25,11 +25,13 @@ from pathlib import Path
 from types import SimpleNamespace
 
 try:
+    from cloud.auth import AuthStore
     from cloud.coordinator import build_report, evaluate_outcome
     from cloud.server import init_db
     from cloud.worker import Worker
 except ImportError:
     sys.path.insert(0, str(Path(__file__).resolve().parents[2]))
+    from cloud.auth import AuthStore
     from cloud.coordinator import build_report, evaluate_outcome
     from cloud.server import init_db
     from cloud.worker import Worker
@@ -111,6 +113,13 @@ class CoordFixTestCase(unittest.TestCase):
         self.tmp = tempfile.TemporaryDirectory()
         self.db_path = os.path.join(self.tmp.name, "control.db")
         init_db(self.db_path)
+        # Audit batch 2 (I-5)，与 test_audit_loop_coord.py 同款
+        # 理由：生产里 audit_log 由首次真实 _audit 经 AuthStore
+        # 建好，合流时表一定存在；本文件把 _audit 桩成列表，
+        # 表永远不会出现，夹具里按生产实际补建表，否则合流
+        # 测试会因缺表改走 escalate，与被测行为无关。
+        _store = AuthStore(self.db_path)
+        _store.conn.close()
         self.env = {"XCM_API_KEY": "test-placeholder"}
         self.audits = []
         self.worker = Worker(self.db_path, "/fake/host.py",
@@ -200,10 +209,14 @@ class CoordFixTestCase(unittest.TestCase):
         # 最旧 id 的子任务必须不在，且集合恰好 5000 条。
         conn = sqlite3.connect(self.db_path)
         try:
+            # Audit batch 2 (I-5)：setUp 已按生产实际经 AuthStore
+            # 建好含 id 自增列的 audit_log 表，建表改 IF NOT
+            # EXISTS——生产表与本测试要的表在 id 语义上一致，
+            # tie-break 断言不受影响。
             conn.execute(
-                "CREATE TABLE audit_log(id INTEGER PRIMARY KEY"
-                " AUTOINCREMENT, user_id TEXT, project_id TEXT,"
-                " action TEXT, detail TEXT,"
+                "CREATE TABLE IF NOT EXISTS audit_log(id INTEGER"
+                " PRIMARY KEY AUTOINCREMENT, user_id TEXT,"
+                " project_id TEXT, action TEXT, detail TEXT,"
                 " created_at DOUBLE PRECISION)")
             now = time.time()
             conn.executemany(