feat: routing shadow evaluation v0b (read-only, never auto-activates)
New pure module cloud/routing_shadow.py: parse worker_routing audit details (old format without suggested model parses with suggested unknown, never guessed), evaluate per-tier routed samples with one-task-one-sample dedup, explicit tasks counted separately, status under the chosen model, token/USD with unpriced tokens kept apart, and a min-20 gate per tier. The report explicitly marks the counterfactual not_measured: shadow suggested models never ran, so no suggested-vs-chosen cost/quality comparison is possible. Worker audit detail now includes suggested/chosen for future rows. GET /routing-shadow serves the report (project read permission, truncated flag over 10000 audit rows, cost table ensured so a fresh DB does not 500). Tests: test_routing_shadow.py (5 tests).
This commit is contained in:
host committed
1 parent
fd37505c48
commit
bac2a39377
4 files changed
+481
-2
No files matched your search
@@ -0,0 +1,184 @@
|
||||
"""路由影子评估 v0b 单测：解析、按档统计、门禁、服务端点。
|
||||
|
||||
运行（在仓库根目录下）：
|
||||
python3 -m unittest discover -s cloud/tests -p "test_routing_shadow.py" -v
|
||||
|
||||
说明：纯函数部分用 plain dict，不碰库；服务端点部分用
|
||||
create_server 本地 HTTP（test_task_options 同款），审计行
|
||||
直接写进测试库（审计表由 AuthStore 自建），不跑真实模型、
|
||||
不联网。关键断言：同一任务两次审计只算一个样本；显式
|
||||
任务不算路由样本；影子从不声称反事实；未满 20 不 ready。
|
||||
"""
|
||||
|
||||
import http.client
|
||||
import json
|
||||
import os
|
||||
import sqlite3
|
||||
import sys
|
||||
import tempfile
|
||||
import threading
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
try:
|
||||
from cloud.routing_shadow import MIN_SAMPLES
|
||||
from cloud.routing_shadow import evaluate_shadow
|
||||
from cloud.routing_shadow import parse_worker_routing_detail
|
||||
from cloud.auth import AuthStore
|
||||
from cloud.server import create_server
|
||||
except ImportError:
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[2]))
|
||||
from cloud.routing_shadow import MIN_SAMPLES
|
||||
from cloud.routing_shadow import evaluate_shadow
|
||||
from cloud.routing_shadow import parse_worker_routing_detail
|
||||
from cloud.auth import AuthStore
|
||||
from cloud.server import create_server
|
||||
|
||||
|
||||
def _detail(task_id, tier, source, applied, suggested=None, chosen=None):
|
||||
detail = "%s tier=%s source=%s applied=%s" % (
|
||||
task_id, tier, source, applied)
|
||||
if suggested is not None:
|
||||
detail += " suggested=%s" % suggested
|
||||
if chosen is not None:
|
||||
detail += " chosen=%s" % chosen
|
||||
return detail
|
||||
|
||||
|
||||
class ParseTestCase(unittest.TestCase):
|
||||
def test_new_and_old_format(self):
|
||||
parsed = parse_worker_routing_detail(
|
||||
_detail("t1", "plan", "shadow", "False",
|
||||
suggested="gpt-6.1-sol", chosen="gpt-5.6-terra"))
|
||||
self.assertEqual(parsed["suggested_model"], "gpt-6.1-sol")
|
||||
self.assertEqual(parsed["chosen_model"], "gpt-5.6-terra")
|
||||
self.assertFalse(parsed["applied"])
|
||||
old = parse_worker_routing_detail(
|
||||
_detail("t2", "goal", "shadow", "False"))
|
||||
self.assertIsNone(old["suggested_model"])
|
||||
self.assertIsNone(old["chosen_model"])
|
||||
|
||||
def test_unparseable_returns_none(self):
|
||||
self.assertIsNone(parse_worker_routing_detail("garbage"))
|
||||
self.assertIsNone(parse_worker_routing_detail(None))
|
||||
|
||||
|
||||
class EvaluateTestCase(unittest.TestCase):
|
||||
def test_dedup_explicit_and_gate(self):
|
||||
audit_rows = [
|
||||
{"detail": _detail("t1", "plan", "shadow", "False",
|
||||
suggested="gpt-6.1-sol")},
|
||||
# 同一任务重试再记一次：仍只算一个样本。
|
||||
{"detail": _detail("t1", "plan", "shadow", "False",
|
||||
suggested="gpt-6.1-sol")},
|
||||
{"detail": _detail("t2", "plan", "shadow", "False")},
|
||||
{"detail": _detail("t3", "default", "policy", "True",
|
||||
suggested="-", chosen="gpt-6.1-sol")},
|
||||
]
|
||||
tasks = {"t1": {"status": "completed"},
|
||||
"t2": {"status": "failed"},
|
||||
"t3": {"status": "completed"}}
|
||||
costs = {"t1": {"total_tokens": 100, "usd_micros": 200,
|
||||
"unpriced_tokens": 0},
|
||||
"t2": {"total_tokens": 50, "usd_micros": None,
|
||||
"unpriced_tokens": 50}}
|
||||
report = evaluate_shadow(audit_rows, tasks, costs, explicit_n=5)
|
||||
plan = report["tiers"]["plan"]
|
||||
self.assertEqual(plan["routed_n"], 2)
|
||||
self.assertEqual(plan["shadow_n"], 2)
|
||||
self.assertEqual(plan["completed"], 1)
|
||||
self.assertEqual(plan["failed"], 1)
|
||||
self.assertEqual(plan["total_tokens"], 150)
|
||||
self.assertEqual(plan["usd_micros"], 200)
|
||||
self.assertEqual(plan["unpriced_tokens"], 50)
|
||||
self.assertEqual(plan["suggested_models"], {"gpt-6.1-sol": 1})
|
||||
self.assertEqual(plan["suggested_unknown_n"], 1)
|
||||
self.assertEqual(plan["gate"], "not_ready")
|
||||
self.assertEqual(plan["sample_shortfall"], MIN_SAMPLES - 2)
|
||||
self.assertEqual(report["explicit_n"], 5)
|
||||
self.assertEqual(report["routed_task_n"], 3)
|
||||
self.assertEqual(report["counterfactual"], "not_measured")
|
||||
self.assertEqual(report["ready_for_review"], [])
|
||||
self.assertIn("Not ready", report["recommendation"])
|
||||
|
||||
def test_gate_ready_at_threshold(self):
|
||||
audit_rows = [
|
||||
{"detail": _detail("t%d" % i, "plan", "shadow", "False",
|
||||
suggested="gpt-6.1-sol")}
|
||||
for i in range(MIN_SAMPLES)]
|
||||
tasks = {"t%d" % i: {"status": "completed"}
|
||||
for i in range(MIN_SAMPLES)}
|
||||
report = evaluate_shadow(audit_rows, tasks, {})
|
||||
self.assertEqual(report["tiers"]["plan"]["gate"],
|
||||
"ready_for_review")
|
||||
self.assertEqual(report["ready_for_review"], ["plan"])
|
||||
self.assertIn("manual review", report["recommendation"])
|
||||
self.assertNotIn("activate plan", report["recommendation"])
|
||||
|
||||
|
||||
class ServerShadowTestCase(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.tmp = tempfile.TemporaryDirectory()
|
||||
self.db_path = os.path.join(self.tmp.name, "control.db")
|
||||
self.auth = AuthStore(self.db_path)
|
||||
self.owner_id, self.owner_key = self.auth.create_user("owner")
|
||||
self.server = create_server(self.db_path, "127.0.0.1", 0)
|
||||
self.port = self.server.server_address[1]
|
||||
self.thread = threading.Thread(
|
||||
target=self.server.serve_forever, daemon=True)
|
||||
self.thread.start()
|
||||
status, self.project = self._req(
|
||||
"POST", "/projects", {"name": "demo"}, self.owner_key)
|
||||
self.assertEqual(status, 201)
|
||||
# 审计行与任务行直接写库（审计表由 AuthStore 自建）。
|
||||
conn = sqlite3.connect(self.db_path)
|
||||
try:
|
||||
conn.execute(
|
||||
"INSERT INTO audit_log(user_id, project_id, action,"
|
||||
" detail, created_at) VALUES(?, ?, ?, ?, ?)",
|
||||
("system", self.project["id"], "worker_routing",
|
||||
_detail("t1", "plan", "shadow", "False",
|
||||
suggested="gpt-6.1-sol"), 1.0))
|
||||
conn.commit()
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
def tearDown(self):
|
||||
self.server.shutdown()
|
||||
self.server.server_close()
|
||||
self.thread.join(timeout=5)
|
||||
try:
|
||||
self.auth.conn.close()
|
||||
except Exception:
|
||||
pass
|
||||
self.tmp.cleanup()
|
||||
|
||||
def _req(self, method, path, body=None, key=None):
|
||||
conn = http.client.HTTPConnection("127.0.0.1", self.port, timeout=10)
|
||||
headers = {"Content-Type": "application/json"}
|
||||
if key:
|
||||
headers["X-API-Key"] = key
|
||||
conn.request(method, path,
|
||||
json.dumps(body) if body is not None else None, headers)
|
||||
resp = conn.getresponse()
|
||||
raw = resp.read()
|
||||
conn.close()
|
||||
try:
|
||||
data = json.loads(raw)
|
||||
except ValueError:
|
||||
data = None
|
||||
return resp.status, data
|
||||
|
||||
def test_endpoint_and_permission(self):
|
||||
status, report = self._req(
|
||||
"GET", "/routing-shadow?project_id=" + self.project["id"],
|
||||
key=self.owner_key)
|
||||
self.assertEqual(status, 200, report)
|
||||
self.assertEqual(report["tiers"]["plan"]["routed_n"], 1)
|
||||
self.assertEqual(report["counterfactual"], "not_measured")
|
||||
self.assertFalse(report["truncated"])
|
||||
# 无 project_id 400；无 key 时 _dispatch 的鉴权在前，
|
||||
# 这里只断言缺 project_id 的 400 已由端点处理。
|
||||
status, _data = self._req(
|
||||
"GET", "/routing-shadow", key=self.owner_key)
|
||||
self.assertEqual(status, 400)
|
||||
Reference in new issue
Block a user