codex-app-server/cloud/routing_shadow.py

"""Routing v0b: shadow evaluation (read-only, never auto-activates).

This module answers the one question routing v0 left unanswered:
"is there enough shadow evidence to review a tier for activation?"
It does NOT decide activation, and it does NOT claim a counterfactual.

Why the counterfactual is explicitly not measured
--------------------------------------------------
In shadow mode the suggested model never runs. The task's actual
outcome (status, tokens, cost) is the outcome under the chosen
worker-default model, not under the suggested model. So this report
can measure, per tier:

* how many distinct tasks were routed (one task = one sample,
  retries do not double-count),
* their status breakdown under the chosen model,
* the distribution of suggested models,
* total tokens and priced USD, with unpriced tokens kept separate,

but it cannot say the suggested model "would have been cheaper or
better". Any report that compared shadow suggested cost to actual
cost as if both had run would be fabrication. The field
``counterfactual`` in every report says ``not_measured`` for exactly
this reason, and the top-level recommendation never activates a tier:
activation stays a manual decision, plan tier first, per the v0 design.

Data sources, and their limits
-------------------------------
* ``audit_log`` rows with action ``worker_routing``. The v0 format
  was ``<task_id> tier=<tier> source=<source> applied=<bool>``;
  v0b extends it with ``suggested=<model|-> chosen=<model|->``
  (see worker.py). Rows in the old format parse fine, with
  ``suggested_model`` reported as unknown and counted separately
  (``suggested_unknown_n``), never guessed.
* ``tasks`` rows for status and explicit-model separation. A task
  with a creation-time ``model`` is explicit: routing never audits
  it (decide_routing returns source ``explicit`` and the worker
  does not audit), so explicit tasks are counted separately and
  never inflate a tier's sample count.
* ``cost_records`` aggregated per task. USD here is whatever the
  cost store recorded: with the current proxy-price overlay that is
  the gpt-6.1-sol proxy price, NOT the relay's actual bill, and
  unpriced rows contribute tokens but no USD. The report keeps
  ``usd_micros`` (priced only) and ``unpriced_tokens`` apart and
  labels USD as proxy-priced when any unpriced tokens exist.

Pure functions only: this module never touches a DB, a provider,
or an environment variable, so it is unit-testable with plain dicts.
The server endpoint (GET /routing-shadow) is the only DB reader,
and it passes already-aggregated rows in.
"""

import re

MIN_SAMPLES = 20
TIERS = ("plan", "goal", "default")
ACTIVATION_ORDER = ("plan", "goal", "default")

_DETAIL_RE = re.compile(
    r"^(?P<task_id>\S+)\s+tier=(?P<tier>\S+)\s+source=(?P<source>\S+)"
    r"\s+applied=(?P<applied>\S+)"
    r"(?:\s+suggested=(?P<suggested>\S+))?"
    r"(?:\s+chosen=(?P<chosen>\S+))?\s*$")


def parse_worker_routing_detail(detail):
    """Parse one worker_routing audit detail, or None if not parseable.

    Old format (no suggested/chosen) parses with both models None.
    A ``-`` placeholder also means None, never a model named "-".
    """
    if not isinstance(detail, str):
        return None
    match = _DETAIL_RE.match(detail.strip())
    if not match:
        return None
    applied_raw = match.group("applied").lower()
    applied = applied_raw in ("true", "1", "yes")
    suggested = match.group("suggested")
    chosen = match.group("chosen")
    return {
        "task_id": match.group("task_id"),
        "tier": match.group("tier"),
        "source": match.group("source"),
        "applied": applied,
        "suggested_model": None if suggested in (None, "-") else suggested,
        "chosen_model": None if chosen in (None, "-") else chosen,
    }


def _empty_tier():
    return {
        "routed_n": 0,
        "shadow_n": 0,
        "policy_n": 0,
        # Audit fix 3（I8）：显式模型任务（source=explicit）
        # 按档单独计数，不进 routed_n、不占 seen。
        "explicit_n": 0,
        "completed": 0,
        "failed": 0,
        "other_status": 0,
        "unknown_status": 0,
        "total_tokens": 0,
        "usd_micros": 0,
        "unpriced_tokens": 0,
        "suggested_models": {},
        "suggested_unknown_n": 0,
        "gate": "not_ready",
        "sample_shortfall": MIN_SAMPLES,
    }


def evaluate_shadow(audit_rows, tasks_by_id, costs_by_task,
                    explicit_n=0, min_samples=MIN_SAMPLES):
    """Evaluate shadow evidence from already-fetched rows.

    ``audit_rows``: dicts with at least ``detail``. Rows with
    ``source=explicit`` are counted per tier as ``explicit_n``
    only (never ``routed_n``, never dedup slots). The review
    gate is ``completed >= min_samples`` (Audit fix 3, I2):
    failed/unknown/other statuses do not count toward readiness.
    When rows carry ``created_at`` (and optionally ``id`` as a
    tiebreak),
    they are sorted earliest-first inside this function before
    dedup, so the first-seen sample for a task is its earliest
    routing audit regardless of the caller's row order. Rows
    without ``created_at`` sort last (Audit fix 3, M2: the old
    docstring said they kept their given order, but the sort
    key already put them last); among themselves they keep
    stable order.
    The server endpoint passes rows already ordered, and it
    strips ``created_at`` before calling -- that path is
    unaffected, but a future caller passing raw rows no longer
    silently changes dedup semantics. ``tasks_by_id``: task_id
    -> {"status": str}. ``costs_by_task``: task_id ->
    {"total_tokens": int, "usd_micros": int|None,
    "unpriced_tokens": int}. ``explicit_n``: number of tasks
    with a creation-time model in the same project (counted
    separately, never as routed samples).
    """
    tiers = {tier: _empty_tier() for tier in TIERS}
    seen = set()
    routed_task_ids = set()
    rows = list(audit_rows or [])
    if any(isinstance(r, dict) and r.get("created_at") is not None
           for r in rows):
        def _sort_key(row):
            created = row.get("created_at") if isinstance(row, dict) \
                else None
            row_id = row.get("id") if isinstance(row, dict) else None
            # Rows without a timestamp sort first only if they
            # were first already: use a sentinel that keeps
            # stable order among themselves, but never jumps
            # ahead of a timestamped row -- so put them last.
            return (created is None,
                    created if isinstance(created, (int, float))
                    else 0,
                    row_id if isinstance(row_id, (int, float))
                    else 0)
        rows = sorted(rows, key=_sort_key)
    for row in rows:
        parsed = parse_worker_routing_detail(
            (row or {}).get("detail"))
        if parsed is None:
            continue
        task_id = parsed["task_id"]
        tier = parsed["tier"]
        if tier not in tiers:
            # Unknown tier: do not invent a bucket, do not count it
            # as a sample for any known tier, and do not let it
            # occupy the dedup slot (Audit fix 3, M1): a task
            # whose first row is an unknown tier must not lose
            # its later known-tier row to dedup.
            continue
        if parsed["source"] == "explicit":
            # Audit fix 3（I8）：显式行只计 per-tier
            # explicit_n：不计 routed_n、不占 seen（同一任务
            # 日后若出现路由行仍可计为样本），不破坏去重。
            tiers[tier]["explicit_n"] += 1
            continue
        if task_id in seen:
            continue
        seen.add(task_id)
        bucket = tiers[tier]
        routed_task_ids.add(task_id)
        bucket["routed_n"] += 1
        if parsed["source"] == "shadow":
            bucket["shadow_n"] += 1
        elif parsed["source"] == "policy":
            bucket["policy_n"] += 1
        else:
            # Other sources (e.g. a future source) are routed
            # samples, but neither shadow nor applied-policy samples.
            pass
        task = tasks_by_id.get(task_id) if tasks_by_id else None
        status = (task or {}).get("status") if task else None
        if status == "completed":
            bucket["completed"] += 1
        elif status == "failed":
            bucket["failed"] += 1
        elif status is None:
            bucket["unknown_status"] += 1
        else:
            bucket["other_status"] += 1
        cost = costs_by_task.get(task_id) if costs_by_task else None
        if cost:
            bucket["total_tokens"] += int(cost.get("total_tokens") or 0)
            if cost.get("usd_micros") is not None:
                bucket["usd_micros"] += int(cost["usd_micros"])
            bucket["unpriced_tokens"] += int(
                cost.get("unpriced_tokens") or 0)
        if parsed["suggested_model"]:
            models = bucket["suggested_models"]
            models[parsed["suggested_model"]] = \
                models.get(parsed["suggested_model"], 0) + 1
        else:
            bucket["suggested_unknown_n"] += 1
    # Audit fix 3（I2）：就绪门改按已完成任务数计。
    # 旧版按 routed_n（已路由数）计，20 个全失败的档
    # 同样算就绪，与设计稿“已完成任务”的口径不一致；
    # sample_shortfall 同口径，失败/未知/其它状态不算。
    for tier, bucket in tiers.items():
        bucket["gate"] = (
            "ready_for_review" if bucket["completed"] >= min_samples
            else "not_ready")
        bucket["sample_shortfall"] = max(
            0, min_samples - bucket["completed"])
    ready = [t for t in ACTIVATION_ORDER
             if tiers[t]["gate"] == "ready_for_review"]
    if ready:
        recommendation = (
            "Ready for manual review (not activation): %s. "
            "Review plan tier first, then goal, then default; "
            "activation stays a manual decision and is never "
            "automatic from this report." % ", ".join(ready))
    else:
        recommendation = (
            "Not ready for review: no tier has %d completed "
            "samples. Keep shadow mode running; do not activate "
            "any tier." % min_samples)
    return {
        "min_samples": min_samples,
        "tiers": tiers,
        "explicit_n": int(explicit_n or 0),
        "routed_task_n": len(routed_task_ids),
        "counterfactual": "not_measured",
        "counterfactual_note": (
            "Shadow suggested models never ran, so this report cannot "
            "compare suggested vs chosen cost or quality; it measures "
            "sample size, status under the chosen model, and the "
            "suggested-model distribution only."),
        "usd_note": (
            "usd_micros sums priced cost records only; unpriced "
            "tokens are kept separate. With the current proxy-price "
            "overlay, USD is proxy-priced (gpt-6.1-sol), not the "
            "relay's actual bill."),
        "activation_order": list(ACTIVATION_ORDER),
        "ready_for_review": ready,
        "recommendation": recommendation,
    }