feat: routing shadow evaluation v0b (read-only, never auto-activates)
New pure module cloud/routing_shadow.py: parse worker_routing audit details (old format without suggested model parses with suggested unknown, never guessed), evaluate per-tier routed samples with one-task-one-sample dedup, explicit tasks counted separately, status under the chosen model, token/USD with unpriced tokens kept apart, and a min-20 gate per tier. The report explicitly marks the counterfactual not_measured: shadow suggested models never ran, so no suggested-vs-chosen cost/quality comparison is possible. Worker audit detail now includes suggested/chosen for future rows. GET /routing-shadow serves the report (project read permission, truncated flag over 10000 audit rows, cost table ensured so a fresh DB does not 500). Tests: test_routing_shadow.py (5 tests).
This commit is contained in:
1 parent
fd37505c48
commit
bac2a39377
4 files changed
+481
-2
No files matched your search
@@ -66,6 +66,7 @@ from cloud.auth import AuthStore
|
||||
from cloud.coordinator import parse_parts as _coord_parse_parts
|
||||
from cloud.coordinator import validate_parts as _coord_validate_parts
|
||||
from cloud.cost import CostStore, unpriced_token_cap
|
||||
from cloud.routing_shadow import evaluate_shadow as _evaluate_shadow
|
||||
from cloud.db import connect as _db_connect
|
||||
from cloud.db import create_database as _create_database
|
||||
from cloud.db import init_postgres_schema as _init_postgres_schema
|
||||
@@ -2341,6 +2342,8 @@ class CloudControlPlaneHandler(BaseHTTPRequestHandler):
|
||||
self._create_upload(user_id)
|
||||
elif path == "/tasks" and method == "GET":
|
||||
self._list_tasks(user_id, query)
|
||||
elif path == "/routing-shadow" and method == "GET":
|
||||
self._routing_shadow(user_id, query)
|
||||
elif path == "/admin/sweep" and method == "POST":
|
||||
self._admin_sweep(user_id)
|
||||
elif path == "/admin/backup" and method == "POST":
|
||||
@@ -4129,6 +4132,80 @@ class CloudControlPlaneHandler(BaseHTTPRequestHandler):
|
||||
"daily": daily})
|
||||
|
||||
# -- 任务成本与模型总览 ----------------------------------------------
|
||||
def _routing_shadow(self, user_id, query):
|
||||
# GET /routing-shadow?project_id=...:路由 v0b 影子评估,
|
||||
# 只读,不改任何模式、不激活任何档。权限同 _list_tasks:
|
||||
# 项目 read 权限,没有则 403。审计行按 created_at 升序
|
||||
# 取(评估里按任务去重,第一次出现即样本),上限 10000
|
||||
# 行——超过时报告里仍按取到的行算,调用方须知样本被
|
||||
# 截断(见 response 的 truncated 字段),不假装全量。
|
||||
project_ids = query.get("project_id")
|
||||
project_id = project_ids[0] if project_ids else None
|
||||
if not project_id:
|
||||
self._send_json(400, {"error": "project_id required"})
|
||||
return
|
||||
with _auth_store(self.server.db_path) as store:
|
||||
if not store.check_permission(user_id, project_id, "read"):
|
||||
self._send_json(403, {"error": "forbidden"})
|
||||
return
|
||||
# cost_records 表只在 CostStore 首次初始化时建:新鲜
|
||||
# 库上直接查会报 no such table,报告端点不应因此 500。
|
||||
# 先建表(幂等),再读;建表失败时成本按空算、报告
|
||||
# 里 token/usd 为 0,不假装成本已计。
|
||||
try:
|
||||
_cost_store = CostStore(self.server.db_path)
|
||||
try:
|
||||
_cost_store._conn.close()
|
||||
except Exception:
|
||||
pass
|
||||
except Exception:
|
||||
pass
|
||||
conn = self._db()
|
||||
try:
|
||||
audit_rows = conn.execute(
|
||||
"SELECT detail FROM audit_log"
|
||||
" WHERE project_id = ? AND action = 'worker_routing'"
|
||||
" ORDER BY created_at, id LIMIT 10001",
|
||||
(project_id,),
|
||||
).fetchall()
|
||||
truncated = len(audit_rows) > 10000
|
||||
audit_rows = audit_rows[:10000]
|
||||
task_rows = conn.execute(
|
||||
"SELECT id, status, model FROM tasks"
|
||||
" WHERE project_id = ?",
|
||||
(project_id,),
|
||||
).fetchall()
|
||||
cost_rows = conn.execute(
|
||||
"SELECT task_id, total_tokens, usd_micros"
|
||||
" FROM cost_records WHERE project_id = ?",
|
||||
(project_id,),
|
||||
).fetchall()
|
||||
finally:
|
||||
conn.close()
|
||||
tasks_by_id = {}
|
||||
explicit_n = 0
|
||||
for row in task_rows:
|
||||
tasks_by_id[row["id"]] = {"status": row["status"]}
|
||||
if row["model"]:
|
||||
explicit_n += 1
|
||||
costs_by_task = {}
|
||||
for row in cost_rows:
|
||||
entry = costs_by_task.setdefault(
|
||||
row["task_id"],
|
||||
{"total_tokens": 0, "usd_micros": 0,
|
||||
"unpriced_tokens": 0})
|
||||
entry["total_tokens"] += int(row["total_tokens"] or 0)
|
||||
if row["usd_micros"] is None:
|
||||
entry["unpriced_tokens"] += int(row["total_tokens"] or 0)
|
||||
else:
|
||||
entry["usd_micros"] += int(row["usd_micros"])
|
||||
report = _evaluate_shadow(
|
||||
[{"detail": row["detail"]} for row in audit_rows],
|
||||
tasks_by_id, costs_by_task, explicit_n=explicit_n)
|
||||
report["project_id"] = project_id
|
||||
report["truncated"] = truncated
|
||||
self._send_json(200, report)
|
||||
|
||||
def _task_cost(self, user_id, task_id):
|
||||
row = self._fetch_task(task_id)
|
||||
if row is None:
|
||||
|
||||
Reference in new issue
Block a user