host
bac2a39377
New pure module cloud/routing_shadow.py: parse worker_routing audit details (old format without suggested model parses with suggested unknown, never guessed), evaluate per-tier routed samples with one-task-one-sample dedup, explicit tasks counted separately, status under the chosen model, token/USD with unpriced tokens kept apart, and a min-20 gate per tier. The report explicitly marks the counterfactual not_measured: shadow suggested models never ran, so no suggested-vs-chosen cost/quality comparison is possible. Worker audit detail now includes suggested/chosen for future rows. GET /routing-shadow serves the report (project read permission, truncated flag over 10000 audit rows, cost table ensured so a fresh DB does not 500). Tests: test_routing_shadow.py (5 tests).
212 lines
8.4 KiB
Python
212 lines
8.4 KiB
Python
"""Routing v0b: shadow evaluation (read-only, never auto-activates).
|
|
|
|
This module answers the one question routing v0 left unanswered:
|
|
"is there enough shadow evidence to review a tier for activation?"
|
|
It does NOT decide activation, and it does NOT claim a counterfactual.
|
|
|
|
Why the counterfactual is explicitly not measured
|
|
--------------------------------------------------
|
|
In shadow mode the suggested model never runs. The task's actual
|
|
outcome (status, tokens, cost) is the outcome under the chosen
|
|
worker-default model, not under the suggested model. So this report
|
|
can measure, per tier:
|
|
|
|
* how many distinct tasks were routed (one task = one sample,
|
|
retries do not double-count),
|
|
* their status breakdown under the chosen model,
|
|
* the distribution of suggested models,
|
|
* total tokens and priced USD, with unpriced tokens kept separate,
|
|
|
|
but it cannot say the suggested model "would have been cheaper or
|
|
better". Any report that compared shadow suggested cost to actual
|
|
cost as if both had run would be fabrication. The field
|
|
``counterfactual`` in every report says ``not_measured`` for exactly
|
|
this reason, and the top-level recommendation never activates a tier:
|
|
activation stays a manual decision, plan tier first, per the v0 design.
|
|
|
|
Data sources, and their limits
|
|
-------------------------------
|
|
* ``audit_log`` rows with action ``worker_routing``. The v0 format
|
|
was ``<task_id> tier=<tier> source=<source> applied=<bool>``;
|
|
v0b extends it with ``suggested=<model|-> chosen=<model|->``
|
|
(see worker.py). Rows in the old format parse fine, with
|
|
``suggested_model`` reported as unknown and counted separately
|
|
(``suggested_unknown_n``), never guessed.
|
|
* ``tasks`` rows for status and explicit-model separation. A task
|
|
with a creation-time ``model`` is explicit: routing never audits
|
|
it (decide_routing returns source ``explicit`` and the worker
|
|
does not audit), so explicit tasks are counted separately and
|
|
never inflate a tier's sample count.
|
|
* ``cost_records`` aggregated per task. USD here is whatever the
|
|
cost store recorded: with the current proxy-price overlay that is
|
|
the gpt-6.1-sol proxy price, NOT the relay's actual bill, and
|
|
unpriced rows contribute tokens but no USD. The report keeps
|
|
``usd_micros`` (priced only) and ``unpriced_tokens`` apart and
|
|
labels USD as proxy-priced when any unpriced tokens exist.
|
|
|
|
Pure functions only: this module never touches a DB, a provider,
|
|
or an environment variable, so it is unit-testable with plain dicts.
|
|
The server endpoint (GET /routing-shadow) is the only DB reader,
|
|
and it passes already-aggregated rows in.
|
|
"""
|
|
|
|
import re
|
|
|
|
MIN_SAMPLES = 20
|
|
TIERS = ("plan", "goal", "default")
|
|
ACTIVATION_ORDER = ("plan", "goal", "default")
|
|
|
|
_DETAIL_RE = re.compile(
|
|
r"^(?P<task_id>\S+)\s+tier=(?P<tier>\S+)\s+source=(?P<source>\S+)"
|
|
r"\s+applied=(?P<applied>\S+)"
|
|
r"(?:\s+suggested=(?P<suggested>\S+))?"
|
|
r"(?:\s+chosen=(?P<chosen>\S+))?\s*$")
|
|
|
|
|
|
def parse_worker_routing_detail(detail):
|
|
"""Parse one worker_routing audit detail, or None if not parseable.
|
|
|
|
Old format (no suggested/chosen) parses with both models None.
|
|
A ``-`` placeholder also means None, never a model named "-".
|
|
"""
|
|
if not isinstance(detail, str):
|
|
return None
|
|
match = _DETAIL_RE.match(detail.strip())
|
|
if not match:
|
|
return None
|
|
applied_raw = match.group("applied").lower()
|
|
applied = applied_raw in ("true", "1", "yes")
|
|
suggested = match.group("suggested")
|
|
chosen = match.group("chosen")
|
|
return {
|
|
"task_id": match.group("task_id"),
|
|
"tier": match.group("tier"),
|
|
"source": match.group("source"),
|
|
"applied": applied,
|
|
"suggested_model": None if suggested in (None, "-") else suggested,
|
|
"chosen_model": None if chosen in (None, "-") else chosen,
|
|
}
|
|
|
|
|
|
def _empty_tier():
|
|
return {
|
|
"routed_n": 0,
|
|
"shadow_n": 0,
|
|
"policy_n": 0,
|
|
"completed": 0,
|
|
"failed": 0,
|
|
"other_status": 0,
|
|
"unknown_status": 0,
|
|
"total_tokens": 0,
|
|
"usd_micros": 0,
|
|
"unpriced_tokens": 0,
|
|
"suggested_models": {},
|
|
"suggested_unknown_n": 0,
|
|
"gate": "not_ready",
|
|
"sample_shortfall": MIN_SAMPLES,
|
|
}
|
|
|
|
|
|
def evaluate_shadow(audit_rows, tasks_by_id, costs_by_task,
|
|
explicit_n=0, min_samples=MIN_SAMPLES):
|
|
"""Evaluate shadow evidence from already-fetched rows.
|
|
|
|
``audit_rows``: dicts with at least ``detail`` (and optionally
|
|
``created_at`` for earliest-first dedup; without it, first-seen
|
|
wins). ``tasks_by_id``: task_id -> {"status": str}.
|
|
``costs_by_task``: task_id -> {"total_tokens": int,
|
|
"usd_micros": int|None, "unpriced_tokens": int}.
|
|
``explicit_n``: number of tasks with a creation-time model in
|
|
the same project (counted separately, never as routed samples).
|
|
"""
|
|
tiers = {tier: _empty_tier() for tier in TIERS}
|
|
seen = set()
|
|
routed_task_ids = set()
|
|
for row in audit_rows or []:
|
|
parsed = parse_worker_routing_detail(
|
|
(row or {}).get("detail"))
|
|
if parsed is None:
|
|
continue
|
|
task_id = parsed["task_id"]
|
|
if task_id in seen:
|
|
continue
|
|
seen.add(task_id)
|
|
tier = parsed["tier"]
|
|
if tier not in tiers:
|
|
# Unknown tier: do not invent a bucket, do not count it
|
|
# as a sample for any known tier.
|
|
continue
|
|
bucket = tiers[tier]
|
|
routed_task_ids.add(task_id)
|
|
bucket["routed_n"] += 1
|
|
if parsed["source"] == "shadow":
|
|
bucket["shadow_n"] += 1
|
|
elif parsed["source"] == "policy":
|
|
bucket["policy_n"] += 1
|
|
else:
|
|
# Other sources (e.g. a future source) are routed
|
|
# samples, but neither shadow nor applied-policy samples.
|
|
pass
|
|
task = tasks_by_id.get(task_id) if tasks_by_id else None
|
|
status = (task or {}).get("status") if task else None
|
|
if status == "completed":
|
|
bucket["completed"] += 1
|
|
elif status == "failed":
|
|
bucket["failed"] += 1
|
|
elif status is None:
|
|
bucket["unknown_status"] += 1
|
|
else:
|
|
bucket["other_status"] += 1
|
|
cost = costs_by_task.get(task_id) if costs_by_task else None
|
|
if cost:
|
|
bucket["total_tokens"] += int(cost.get("total_tokens") or 0)
|
|
if cost.get("usd_micros") is not None:
|
|
bucket["usd_micros"] += int(cost["usd_micros"])
|
|
bucket["unpriced_tokens"] += int(
|
|
cost.get("unpriced_tokens") or 0)
|
|
if parsed["suggested_model"]:
|
|
models = bucket["suggested_models"]
|
|
models[parsed["suggested_model"]] = \
|
|
models.get(parsed["suggested_model"], 0) + 1
|
|
else:
|
|
bucket["suggested_unknown_n"] += 1
|
|
for tier, bucket in tiers.items():
|
|
bucket["gate"] = (
|
|
"ready_for_review" if bucket["routed_n"] >= min_samples
|
|
else "not_ready")
|
|
bucket["sample_shortfall"] = max(
|
|
0, min_samples - bucket["routed_n"])
|
|
ready = [t for t in ACTIVATION_ORDER
|
|
if tiers[t]["gate"] == "ready_for_review"]
|
|
if ready:
|
|
recommendation = (
|
|
"Ready for manual review (not activation): %s. "
|
|
"Review plan tier first, then goal, then default; "
|
|
"activation stays a manual decision and is never "
|
|
"automatic from this report." % ", ".join(ready))
|
|
else:
|
|
recommendation = (
|
|
"Not ready for review: no tier has %d routed samples. "
|
|
"Keep shadow mode running; do not activate any tier."
|
|
% min_samples)
|
|
return {
|
|
"min_samples": min_samples,
|
|
"tiers": tiers,
|
|
"explicit_n": int(explicit_n or 0),
|
|
"routed_task_n": len(routed_task_ids),
|
|
"counterfactual": "not_measured",
|
|
"counterfactual_note": (
|
|
"Shadow suggested models never ran, so this report cannot "
|
|
"compare suggested vs chosen cost or quality; it measures "
|
|
"sample size, status under the chosen model, and the "
|
|
"suggested-model distribution only."),
|
|
"usd_note": (
|
|
"usd_micros sums priced cost records only; unpriced "
|
|
"tokens are kept separate. With the current proxy-price "
|
|
"overlay, USD is proxy-priced (gpt-6.1-sol), not the "
|
|
"relay's actual bill."),
|
|
"activation_order": list(ACTIVATION_ORDER),
|
|
"ready_for_review": ready,
|
|
"recommendation": recommendation,
|
|
}
|