feat: routing shadow evaluation v0b (read-only, never auto-activates)
New pure module cloud/routing_shadow.py: parse worker_routing audit details (old format without suggested model parses with suggested unknown, never guessed), evaluate per-tier routed samples with one-task-one-sample dedup, explicit tasks counted separately, status under the chosen model, token/USD with unpriced tokens kept apart, and a min-20 gate per tier. The report explicitly marks the counterfactual not_measured: shadow suggested models never ran, so no suggested-vs-chosen cost/quality comparison is possible. Worker audit detail now includes suggested/chosen for future rows. GET /routing-shadow serves the report (project read permission, truncated flag over 10000 audit rows, cost table ensured so a fresh DB does not 500). Tests: test_routing_shadow.py (5 tests).
This commit is contained in:
host committed
1 parent
fd37505c48
commit
bac2a39377
4 files changed
+481
-2
No files matched your search
@@ -0,0 +1,211 @@
|
||||
"""Routing v0b: shadow evaluation (read-only, never auto-activates).
|
||||
|
||||
This module answers the one question routing v0 left unanswered:
|
||||
"is there enough shadow evidence to review a tier for activation?"
|
||||
It does NOT decide activation, and it does NOT claim a counterfactual.
|
||||
|
||||
Why the counterfactual is explicitly not measured
|
||||
--------------------------------------------------
|
||||
In shadow mode the suggested model never runs. The task's actual
|
||||
outcome (status, tokens, cost) is the outcome under the chosen
|
||||
worker-default model, not under the suggested model. So this report
|
||||
can measure, per tier:
|
||||
|
||||
* how many distinct tasks were routed (one task = one sample,
|
||||
retries do not double-count),
|
||||
* their status breakdown under the chosen model,
|
||||
* the distribution of suggested models,
|
||||
* total tokens and priced USD, with unpriced tokens kept separate,
|
||||
|
||||
but it cannot say the suggested model "would have been cheaper or
|
||||
better". Any report that compared shadow suggested cost to actual
|
||||
cost as if both had run would be fabrication. The field
|
||||
``counterfactual`` in every report says ``not_measured`` for exactly
|
||||
this reason, and the top-level recommendation never activates a tier:
|
||||
activation stays a manual decision, plan tier first, per the v0 design.
|
||||
|
||||
Data sources, and their limits
|
||||
-------------------------------
|
||||
* ``audit_log`` rows with action ``worker_routing``. The v0 format
|
||||
was ``<task_id> tier=<tier> source=<source> applied=<bool>``;
|
||||
v0b extends it with ``suggested=<model|-> chosen=<model|->``
|
||||
(see worker.py). Rows in the old format parse fine, with
|
||||
``suggested_model`` reported as unknown and counted separately
|
||||
(``suggested_unknown_n``), never guessed.
|
||||
* ``tasks`` rows for status and explicit-model separation. A task
|
||||
with a creation-time ``model`` is explicit: routing never audits
|
||||
it (decide_routing returns source ``explicit`` and the worker
|
||||
does not audit), so explicit tasks are counted separately and
|
||||
never inflate a tier's sample count.
|
||||
* ``cost_records`` aggregated per task. USD here is whatever the
|
||||
cost store recorded: with the current proxy-price overlay that is
|
||||
the gpt-6.1-sol proxy price, NOT the relay's actual bill, and
|
||||
unpriced rows contribute tokens but no USD. The report keeps
|
||||
``usd_micros`` (priced only) and ``unpriced_tokens`` apart and
|
||||
labels USD as proxy-priced when any unpriced tokens exist.
|
||||
|
||||
Pure functions only: this module never touches a DB, a provider,
|
||||
or an environment variable, so it is unit-testable with plain dicts.
|
||||
The server endpoint (GET /routing-shadow) is the only DB reader,
|
||||
and it passes already-aggregated rows in.
|
||||
"""
|
||||
|
||||
import re
|
||||
|
||||
MIN_SAMPLES = 20
|
||||
TIERS = ("plan", "goal", "default")
|
||||
ACTIVATION_ORDER = ("plan", "goal", "default")
|
||||
|
||||
_DETAIL_RE = re.compile(
|
||||
r"^(?P<task_id>\S+)\s+tier=(?P<tier>\S+)\s+source=(?P<source>\S+)"
|
||||
r"\s+applied=(?P<applied>\S+)"
|
||||
r"(?:\s+suggested=(?P<suggested>\S+))?"
|
||||
r"(?:\s+chosen=(?P<chosen>\S+))?\s*$")
|
||||
|
||||
|
||||
def parse_worker_routing_detail(detail):
|
||||
"""Parse one worker_routing audit detail, or None if not parseable.
|
||||
|
||||
Old format (no suggested/chosen) parses with both models None.
|
||||
A ``-`` placeholder also means None, never a model named "-".
|
||||
"""
|
||||
if not isinstance(detail, str):
|
||||
return None
|
||||
match = _DETAIL_RE.match(detail.strip())
|
||||
if not match:
|
||||
return None
|
||||
applied_raw = match.group("applied").lower()
|
||||
applied = applied_raw in ("true", "1", "yes")
|
||||
suggested = match.group("suggested")
|
||||
chosen = match.group("chosen")
|
||||
return {
|
||||
"task_id": match.group("task_id"),
|
||||
"tier": match.group("tier"),
|
||||
"source": match.group("source"),
|
||||
"applied": applied,
|
||||
"suggested_model": None if suggested in (None, "-") else suggested,
|
||||
"chosen_model": None if chosen in (None, "-") else chosen,
|
||||
}
|
||||
|
||||
|
||||
def _empty_tier():
|
||||
return {
|
||||
"routed_n": 0,
|
||||
"shadow_n": 0,
|
||||
"policy_n": 0,
|
||||
"completed": 0,
|
||||
"failed": 0,
|
||||
"other_status": 0,
|
||||
"unknown_status": 0,
|
||||
"total_tokens": 0,
|
||||
"usd_micros": 0,
|
||||
"unpriced_tokens": 0,
|
||||
"suggested_models": {},
|
||||
"suggested_unknown_n": 0,
|
||||
"gate": "not_ready",
|
||||
"sample_shortfall": MIN_SAMPLES,
|
||||
}
|
||||
|
||||
|
||||
def evaluate_shadow(audit_rows, tasks_by_id, costs_by_task,
|
||||
explicit_n=0, min_samples=MIN_SAMPLES):
|
||||
"""Evaluate shadow evidence from already-fetched rows.
|
||||
|
||||
``audit_rows``: dicts with at least ``detail`` (and optionally
|
||||
``created_at`` for earliest-first dedup; without it, first-seen
|
||||
wins). ``tasks_by_id``: task_id -> {"status": str}.
|
||||
``costs_by_task``: task_id -> {"total_tokens": int,
|
||||
"usd_micros": int|None, "unpriced_tokens": int}.
|
||||
``explicit_n``: number of tasks with a creation-time model in
|
||||
the same project (counted separately, never as routed samples).
|
||||
"""
|
||||
tiers = {tier: _empty_tier() for tier in TIERS}
|
||||
seen = set()
|
||||
routed_task_ids = set()
|
||||
for row in audit_rows or []:
|
||||
parsed = parse_worker_routing_detail(
|
||||
(row or {}).get("detail"))
|
||||
if parsed is None:
|
||||
continue
|
||||
task_id = parsed["task_id"]
|
||||
if task_id in seen:
|
||||
continue
|
||||
seen.add(task_id)
|
||||
tier = parsed["tier"]
|
||||
if tier not in tiers:
|
||||
# Unknown tier: do not invent a bucket, do not count it
|
||||
# as a sample for any known tier.
|
||||
continue
|
||||
bucket = tiers[tier]
|
||||
routed_task_ids.add(task_id)
|
||||
bucket["routed_n"] += 1
|
||||
if parsed["source"] == "shadow":
|
||||
bucket["shadow_n"] += 1
|
||||
elif parsed["source"] == "policy":
|
||||
bucket["policy_n"] += 1
|
||||
else:
|
||||
# Other sources (e.g. a future source) are routed
|
||||
# samples, but neither shadow nor applied-policy samples.
|
||||
pass
|
||||
task = tasks_by_id.get(task_id) if tasks_by_id else None
|
||||
status = (task or {}).get("status") if task else None
|
||||
if status == "completed":
|
||||
bucket["completed"] += 1
|
||||
elif status == "failed":
|
||||
bucket["failed"] += 1
|
||||
elif status is None:
|
||||
bucket["unknown_status"] += 1
|
||||
else:
|
||||
bucket["other_status"] += 1
|
||||
cost = costs_by_task.get(task_id) if costs_by_task else None
|
||||
if cost:
|
||||
bucket["total_tokens"] += int(cost.get("total_tokens") or 0)
|
||||
if cost.get("usd_micros") is not None:
|
||||
bucket["usd_micros"] += int(cost["usd_micros"])
|
||||
bucket["unpriced_tokens"] += int(
|
||||
cost.get("unpriced_tokens") or 0)
|
||||
if parsed["suggested_model"]:
|
||||
models = bucket["suggested_models"]
|
||||
models[parsed["suggested_model"]] = \
|
||||
models.get(parsed["suggested_model"], 0) + 1
|
||||
else:
|
||||
bucket["suggested_unknown_n"] += 1
|
||||
for tier, bucket in tiers.items():
|
||||
bucket["gate"] = (
|
||||
"ready_for_review" if bucket["routed_n"] >= min_samples
|
||||
else "not_ready")
|
||||
bucket["sample_shortfall"] = max(
|
||||
0, min_samples - bucket["routed_n"])
|
||||
ready = [t for t in ACTIVATION_ORDER
|
||||
if tiers[t]["gate"] == "ready_for_review"]
|
||||
if ready:
|
||||
recommendation = (
|
||||
"Ready for manual review (not activation): %s. "
|
||||
"Review plan tier first, then goal, then default; "
|
||||
"activation stays a manual decision and is never "
|
||||
"automatic from this report." % ", ".join(ready))
|
||||
else:
|
||||
recommendation = (
|
||||
"Not ready for review: no tier has %d routed samples. "
|
||||
"Keep shadow mode running; do not activate any tier."
|
||||
% min_samples)
|
||||
return {
|
||||
"min_samples": min_samples,
|
||||
"tiers": tiers,
|
||||
"explicit_n": int(explicit_n or 0),
|
||||
"routed_task_n": len(routed_task_ids),
|
||||
"counterfactual": "not_measured",
|
||||
"counterfactual_note": (
|
||||
"Shadow suggested models never ran, so this report cannot "
|
||||
"compare suggested vs chosen cost or quality; it measures "
|
||||
"sample size, status under the chosen model, and the "
|
||||
"suggested-model distribution only."),
|
||||
"usd_note": (
|
||||
"usd_micros sums priced cost records only; unpriced "
|
||||
"tokens are kept separate. With the current proxy-price "
|
||||
"overlay, USD is proxy-priced (gpt-6.1-sol), not the "
|
||||
"relay's actual bill."),
|
||||
"activation_order": list(ACTIVATION_ORDER),
|
||||
"ready_for_review": ready,
|
||||
"recommendation": recommendation,
|
||||
}
|
||||
Reference in new issue
Block a user