feat: routing shadow evaluation v0b (read-only, never auto-activates)

New pure module cloud/routing_shadow.py: parse worker_routing
audit details (old format without suggested model parses with
suggested unknown, never guessed), evaluate per-tier routed
samples with one-task-one-sample dedup, explicit tasks counted
separately, status under the chosen model, token/USD with
unpriced tokens kept apart, and a min-20 gate per tier. The
report explicitly marks the counterfactual not_measured:
shadow suggested models never ran, so no suggested-vs-chosen
cost/quality comparison is possible. Worker audit detail now
includes suggested/chosen for future rows. GET /routing-shadow
serves the report (project read permission, truncated flag
over 10000 audit rows, cost table ensured so a fresh DB does
not 500).

Tests: test_routing_shadow.py (5 tests).
This commit is contained in:
host committed 2026-11-04 18:01:44 +00:00
1 parent fd37505c48
commit bac2a39377
4 files changed
+481 -2

No files matched your search

+77
View File
@@ -66,6 +66,7 @@ from cloud.auth import AuthStore
from cloud.coordinator import parse_parts as _coord_parse_parts
from cloud.coordinator import validate_parts as _coord_validate_parts
from cloud.cost import CostStore, unpriced_token_cap
from cloud.routing_shadow import evaluate_shadow as _evaluate_shadow
from cloud.db import connect as _db_connect
from cloud.db import create_database as _create_database
from cloud.db import init_postgres_schema as _init_postgres_schema
@@ -2341,6 +2342,8 @@ class CloudControlPlaneHandler(BaseHTTPRequestHandler):
self._create_upload(user_id)
elif path == "/tasks" and method == "GET":
self._list_tasks(user_id, query)
elif path == "/routing-shadow" and method == "GET":
self._routing_shadow(user_id, query)
elif path == "/admin/sweep" and method == "POST":
self._admin_sweep(user_id)
elif path == "/admin/backup" and method == "POST":
@@ -4129,6 +4132,80 @@ class CloudControlPlaneHandler(BaseHTTPRequestHandler):
"daily": daily})
# -- 任务成本与模型总览 ----------------------------------------------
def _routing_shadow(self, user_id, query):
# GET /routing-shadow?project_id=...:路由 v0b 影子评估,
# 只读,不改任何模式、不激活任何档。权限同 _list_tasks:
# 项目 read 权限,没有则 403。审计行按 created_at 升序
# 取(评估里按任务去重,第一次出现即样本),上限 10000
# 行——超过时报告里仍按取到的行算,调用方须知样本被
# 截断(见 response 的 truncated 字段),不假装全量。
project_ids = query.get("project_id")
project_id = project_ids[0] if project_ids else None
if not project_id:
self._send_json(400, {"error": "project_id required"})
return
with _auth_store(self.server.db_path) as store:
if not store.check_permission(user_id, project_id, "read"):
self._send_json(403, {"error": "forbidden"})
return
# cost_records 表只在 CostStore 首次初始化时建:新鲜
# 库上直接查会报 no such table,报告端点不应因此 500。
# 先建表(幂等),再读;建表失败时成本按空算、报告
# 里 token/usd 为 0,不假装成本已计。
try:
_cost_store = CostStore(self.server.db_path)
try:
_cost_store._conn.close()
except Exception:
pass
except Exception:
pass
conn = self._db()
try:
audit_rows = conn.execute(
"SELECT detail FROM audit_log"
" WHERE project_id = ? AND action = 'worker_routing'"
" ORDER BY created_at, id LIMIT 10001",
(project_id,),
).fetchall()
truncated = len(audit_rows) > 10000
audit_rows = audit_rows[:10000]
task_rows = conn.execute(
"SELECT id, status, model FROM tasks"
" WHERE project_id = ?",
(project_id,),
).fetchall()
cost_rows = conn.execute(
"SELECT task_id, total_tokens, usd_micros"
" FROM cost_records WHERE project_id = ?",
(project_id,),
).fetchall()
finally:
conn.close()
tasks_by_id = {}
explicit_n = 0
for row in task_rows:
tasks_by_id[row["id"]] = {"status": row["status"]}
if row["model"]:
explicit_n += 1
costs_by_task = {}
for row in cost_rows:
entry = costs_by_task.setdefault(
row["task_id"],
{"total_tokens": 0, "usd_micros": 0,
"unpriced_tokens": 0})
entry["total_tokens"] += int(row["total_tokens"] or 0)
if row["usd_micros"] is None:
entry["unpriced_tokens"] += int(row["total_tokens"] or 0)
else:
entry["usd_micros"] += int(row["usd_micros"])
report = _evaluate_shadow(
[{"detail": row["detail"]} for row in audit_rows],
tasks_by_id, costs_by_task, explicit_n=explicit_n)
report["project_id"] = project_id
report["truncated"] = truncated
self._send_json(200, report)
def _task_cost(self, user_id, task_id):
row = self._fetch_task(task_id)
if row is None: