"""Routing v0b: shadow evaluation (read-only, never auto-activates). This module answers the one question routing v0 left unanswered: "is there enough shadow evidence to review a tier for activation?" It does NOT decide activation, and it does NOT claim a counterfactual. Why the counterfactual is explicitly not measured -------------------------------------------------- In shadow mode the suggested model never runs. The task's actual outcome (status, tokens, cost) is the outcome under the chosen worker-default model, not under the suggested model. So this report can measure, per tier: * how many distinct tasks were routed (one task = one sample, retries do not double-count), * their status breakdown under the chosen model, * the distribution of suggested models, * total tokens and priced USD, with unpriced tokens kept separate, but it cannot say the suggested model "would have been cheaper or better". Any report that compared shadow suggested cost to actual cost as if both had run would be fabrication. The field ``counterfactual`` in every report says ``not_measured`` for exactly this reason, and the top-level recommendation never activates a tier: activation stays a manual decision, plan tier first, per the v0 design. Data sources, and their limits ------------------------------- * ``audit_log`` rows with action ``worker_routing``. The v0 format was `` tier= source= applied=``; v0b extends it with ``suggested= chosen=`` (see worker.py). Rows in the old format parse fine, with ``suggested_model`` reported as unknown and counted separately (``suggested_unknown_n``), never guessed. * ``tasks`` rows for status and explicit-model separation. A task with a creation-time ``model`` is explicit: routing never audits it (decide_routing returns source ``explicit`` and the worker does not audit), so explicit tasks are counted separately and never inflate a tier's sample count. * ``cost_records`` aggregated per task. USD here is whatever the cost store recorded: with the current proxy-price overlay that is the gpt-6.1-sol proxy price, NOT the relay's actual bill, and unpriced rows contribute tokens but no USD. The report keeps ``usd_micros`` (priced only) and ``unpriced_tokens`` apart and labels USD as proxy-priced when any unpriced tokens exist. Pure functions only: this module never touches a DB, a provider, or an environment variable, so it is unit-testable with plain dicts. The server endpoint (GET /routing-shadow) is the only DB reader, and it passes already-aggregated rows in. """ import re MIN_SAMPLES = 20 TIERS = ("plan", "goal", "default") ACTIVATION_ORDER = ("plan", "goal", "default") _DETAIL_RE = re.compile( r"^(?P\S+)\s+tier=(?P\S+)\s+source=(?P\S+)" r"\s+applied=(?P\S+)" r"(?:\s+suggested=(?P\S+))?" r"(?:\s+chosen=(?P\S+))?\s*$") def parse_worker_routing_detail(detail): """Parse one worker_routing audit detail, or None if not parseable. Old format (no suggested/chosen) parses with both models None. A ``-`` placeholder also means None, never a model named "-". """ if not isinstance(detail, str): return None match = _DETAIL_RE.match(detail.strip()) if not match: return None applied_raw = match.group("applied").lower() applied = applied_raw in ("true", "1", "yes") suggested = match.group("suggested") chosen = match.group("chosen") return { "task_id": match.group("task_id"), "tier": match.group("tier"), "source": match.group("source"), "applied": applied, "suggested_model": None if suggested in (None, "-") else suggested, "chosen_model": None if chosen in (None, "-") else chosen, } def _empty_tier(): return { "routed_n": 0, "shadow_n": 0, "policy_n": 0, "completed": 0, "failed": 0, "other_status": 0, "unknown_status": 0, "total_tokens": 0, "usd_micros": 0, "unpriced_tokens": 0, "suggested_models": {}, "suggested_unknown_n": 0, "gate": "not_ready", "sample_shortfall": MIN_SAMPLES, } def evaluate_shadow(audit_rows, tasks_by_id, costs_by_task, explicit_n=0, min_samples=MIN_SAMPLES): """Evaluate shadow evidence from already-fetched rows. ``audit_rows``: dicts with at least ``detail`` (and optionally ``created_at`` for earliest-first dedup; without it, first-seen wins). ``tasks_by_id``: task_id -> {"status": str}. ``costs_by_task``: task_id -> {"total_tokens": int, "usd_micros": int|None, "unpriced_tokens": int}. ``explicit_n``: number of tasks with a creation-time model in the same project (counted separately, never as routed samples). """ tiers = {tier: _empty_tier() for tier in TIERS} seen = set() routed_task_ids = set() for row in audit_rows or []: parsed = parse_worker_routing_detail( (row or {}).get("detail")) if parsed is None: continue task_id = parsed["task_id"] if task_id in seen: continue seen.add(task_id) tier = parsed["tier"] if tier not in tiers: # Unknown tier: do not invent a bucket, do not count it # as a sample for any known tier. continue bucket = tiers[tier] routed_task_ids.add(task_id) bucket["routed_n"] += 1 if parsed["source"] == "shadow": bucket["shadow_n"] += 1 elif parsed["source"] == "policy": bucket["policy_n"] += 1 else: # Other sources (e.g. a future source) are routed # samples, but neither shadow nor applied-policy samples. pass task = tasks_by_id.get(task_id) if tasks_by_id else None status = (task or {}).get("status") if task else None if status == "completed": bucket["completed"] += 1 elif status == "failed": bucket["failed"] += 1 elif status is None: bucket["unknown_status"] += 1 else: bucket["other_status"] += 1 cost = costs_by_task.get(task_id) if costs_by_task else None if cost: bucket["total_tokens"] += int(cost.get("total_tokens") or 0) if cost.get("usd_micros") is not None: bucket["usd_micros"] += int(cost["usd_micros"]) bucket["unpriced_tokens"] += int( cost.get("unpriced_tokens") or 0) if parsed["suggested_model"]: models = bucket["suggested_models"] models[parsed["suggested_model"]] = \ models.get(parsed["suggested_model"], 0) + 1 else: bucket["suggested_unknown_n"] += 1 for tier, bucket in tiers.items(): bucket["gate"] = ( "ready_for_review" if bucket["routed_n"] >= min_samples else "not_ready") bucket["sample_shortfall"] = max( 0, min_samples - bucket["routed_n"]) ready = [t for t in ACTIVATION_ORDER if tiers[t]["gate"] == "ready_for_review"] if ready: recommendation = ( "Ready for manual review (not activation): %s. " "Review plan tier first, then goal, then default; " "activation stays a manual decision and is never " "automatic from this report." % ", ".join(ready)) else: recommendation = ( "Not ready for review: no tier has %d routed samples. " "Keep shadow mode running; do not activate any tier." % min_samples) return { "min_samples": min_samples, "tiers": tiers, "explicit_n": int(explicit_n or 0), "routed_task_n": len(routed_task_ids), "counterfactual": "not_measured", "counterfactual_note": ( "Shadow suggested models never ran, so this report cannot " "compare suggested vs chosen cost or quality; it measures " "sample size, status under the chosen model, and the " "suggested-model distribution only."), "usd_note": ( "usd_micros sums priced cost records only; unpriced " "tokens are kept separate. With the current proxy-price " "overlay, USD is proxy-priced (gpt-6.1-sol), not the " "relay's actual bill."), "activation_order": list(ACTIVATION_ORDER), "ready_for_review": ready, "recommendation": recommendation, }