"""路由影子评估 v0b 单测：解析、按档统计、门禁、服务端点。 运行（在仓库根目录下）： python3 -m unittest discover -s cloud/tests -p "test_routing_shadow.py" -v 说明：纯函数部分用 plain dict，不碰库；服务端点部分用 create_server 本地 HTTP（test_task_options 同款），审计行 直接写进测试库（审计表由 AuthStore 自建），不跑真实模型、 不联网。关键断言：同一任务两次审计只算一个样本；显式 任务不算路由样本；影子从不声称反事实；未满 20 不 ready。 """ import http.client import json import os import sqlite3 import sys import tempfile import threading import unittest from pathlib import Path try: from cloud.routing_shadow import MIN_SAMPLES from cloud.routing_shadow import evaluate_shadow from cloud.routing_shadow import parse_worker_routing_detail from cloud.auth import AuthStore from cloud.server import create_server except ImportError: sys.path.insert(0, str(Path(__file__).resolve().parents[2])) from cloud.routing_shadow import MIN_SAMPLES from cloud.routing_shadow import evaluate_shadow from cloud.routing_shadow import parse_worker_routing_detail from cloud.auth import AuthStore from cloud.server import create_server def _detail(task_id, tier, source, applied, suggested=None, chosen=None): detail = "%s tier=%s source=%s applied=%s" % ( task_id, tier, source, applied) if suggested is not None: detail += " suggested=%s" % suggested if chosen is not None: detail += " chosen=%s" % chosen return detail class ParseTestCase(unittest.TestCase): def test_new_and_old_format(self): parsed = parse_worker_routing_detail( _detail("t1", "plan", "shadow", "False", suggested="gpt-6.1-sol", chosen="gpt-5.6-terra")) self.assertEqual(parsed["suggested_model"], "gpt-6.1-sol") self.assertEqual(parsed["chosen_model"], "gpt-5.6-terra") self.assertFalse(parsed["applied"]) old = parse_worker_routing_detail( _detail("t2", "goal", "shadow", "False")) self.assertIsNone(old["suggested_model"]) self.assertIsNone(old["chosen_model"]) def test_unparseable_returns_none(self): self.assertIsNone(parse_worker_routing_detail("garbage")) self.assertIsNone(parse_worker_routing_detail(None)) class EvaluateTestCase(unittest.TestCase): def test_dedup_explicit_and_gate(self): audit_rows = [ {"detail": _detail("t1", "plan", "shadow", "False", suggested="gpt-6.1-sol")}, # 同一任务重试再记一次：仍只算一个样本。 {"detail": _detail("t1", "plan", "shadow", "False", suggested="gpt-6.1-sol")}, {"detail": _detail("t2", "plan", "shadow", "False")}, {"detail": _detail("t3", "default", "policy", "True", suggested="-", chosen="gpt-6.1-sol")}, ] tasks = {"t1": {"status": "completed"}, "t2": {"status": "failed"}, "t3": {"status": "completed"}} costs = {"t1": {"total_tokens": 100, "usd_micros": 200, "unpriced_tokens": 0}, "t2": {"total_tokens": 50, "usd_micros": None, "unpriced_tokens": 50}} report = evaluate_shadow(audit_rows, tasks, costs, explicit_n=5) plan = report["tiers"]["plan"] self.assertEqual(plan["routed_n"], 2) self.assertEqual(plan["shadow_n"], 2) self.assertEqual(plan["completed"], 1) self.assertEqual(plan["failed"], 1) self.assertEqual(plan["total_tokens"], 150) self.assertEqual(plan["usd_micros"], 200) self.assertEqual(plan["unpriced_tokens"], 50) self.assertEqual(plan["suggested_models"], {"gpt-6.1-sol": 1}) self.assertEqual(plan["suggested_unknown_n"], 1) self.assertEqual(plan["gate"], "not_ready") # Audit fix 3（I2）：就绪门按已完成数计——plan 档 # 2 个路由样本里只有 1 个已完成，缺口是 19 不是 18。 self.assertEqual(plan["sample_shortfall"], MIN_SAMPLES - 1) self.assertEqual(report["explicit_n"], 5) self.assertEqual(report["routed_task_n"], 3) self.assertEqual(report["counterfactual"], "not_measured") self.assertEqual(report["ready_for_review"], []) self.assertIn("Not ready", report["recommendation"]) def test_gate_ready_at_threshold(self): audit_rows = [ {"detail": _detail("t%d" % i, "plan", "shadow", "False", suggested="gpt-6.1-sol")} for i in range(MIN_SAMPLES)] tasks = {"t%d" % i: {"status": "completed"} for i in range(MIN_SAMPLES)} report = evaluate_shadow(audit_rows, tasks, {}) self.assertEqual(report["tiers"]["plan"]["gate"], "ready_for_review") self.assertEqual(report["ready_for_review"], ["plan"]) self.assertIn("manual review", report["recommendation"]) self.assertNotIn("activate plan", report["recommendation"]) class ServerShadowTestCase(unittest.TestCase): def setUp(self): self.tmp = tempfile.TemporaryDirectory() self.db_path = os.path.join(self.tmp.name, "control.db") self.auth = AuthStore(self.db_path) self.owner_id, self.owner_key = self.auth.create_user("owner") self.server = create_server(self.db_path, "127.0.0.1", 0) self.port = self.server.server_address[1] self.thread = threading.Thread( target=self.server.serve_forever, daemon=True) self.thread.start() status, self.project = self._req( "POST", "/projects", {"name": "demo"}, self.owner_key) self.assertEqual(status, 201) # 审计行与任务行直接写库（审计表由 AuthStore 自建）。 conn = sqlite3.connect(self.db_path) try: conn.execute( "INSERT INTO audit_log(user_id, project_id, action," " detail, created_at) VALUES(?, ?, ?, ?, ?)", ("system", self.project["id"], "worker_routing", _detail("t1", "plan", "shadow", "False", suggested="gpt-6.1-sol"), 1.0)) conn.commit() finally: conn.close() def tearDown(self): self.server.shutdown() self.server.server_close() self.thread.join(timeout=5) try: self.auth.conn.close() except Exception: pass self.tmp.cleanup() def _req(self, method, path, body=None, key=None): conn = http.client.HTTPConnection("127.0.0.1", self.port, timeout=10) headers = {"Content-Type": "application/json"} if key: headers["X-API-Key"] = key conn.request(method, path, json.dumps(body) if body is not None else None, headers) resp = conn.getresponse() raw = resp.read() conn.close() try: data = json.loads(raw) except ValueError: data = None return resp.status, data def test_endpoint_and_permission(self): status, report = self._req( "GET", "/routing-shadow?project_id=" + self.project["id"], key=self.owner_key) self.assertEqual(status, 200, report) self.assertEqual(report["tiers"]["plan"]["routed_n"], 1) self.assertEqual(report["counterfactual"], "not_measured") self.assertFalse(report["truncated"]) # 无 project_id 400；无 key 时 _dispatch 的鉴权在前， # 这里只断言缺 project_id 的 400 已由端点处理。 status, _data = self._req( "GET", "/routing-shadow", key=self.owner_key) self.assertEqual(status, 400)