306 lines
14 KiB
Python
306 lines
14 KiB
Python
"""Routing v0b: shadow evaluation (read-only, never auto-activates).
|
||
|
||
This module answers the one question routing v0 left unanswered:
|
||
"is there enough shadow evidence to review a tier for activation?"
|
||
It does NOT decide activation, and it does NOT claim a counterfactual.
|
||
|
||
Why the counterfactual is explicitly not measured
|
||
--------------------------------------------------
|
||
In shadow mode the suggested model never runs. The task's actual
|
||
outcome (status, tokens, cost) is the outcome under the chosen
|
||
worker-default model, not under the suggested model. So this report
|
||
can measure, per tier:
|
||
|
||
* how many distinct tasks were routed (one task = one sample,
|
||
retries do not double-count),
|
||
* their status breakdown under the chosen model,
|
||
* the distribution of suggested models,
|
||
* total tokens and priced USD, with unpriced tokens kept separate,
|
||
|
||
but it cannot say the suggested model "would have been cheaper or
|
||
better". Any report that compared shadow suggested cost to actual
|
||
cost as if both had run would be fabrication. The field
|
||
``counterfactual`` in every report says ``not_measured`` for exactly
|
||
this reason, and the top-level recommendation never activates a tier:
|
||
activation stays a manual decision, plan tier first, per the v0 design.
|
||
|
||
Data sources, and their limits
|
||
-------------------------------
|
||
* ``audit_log`` rows with action ``worker_routing``. The v0 format
|
||
was ``<task_id> tier=<tier> source=<source> applied=<bool>``;
|
||
v0b extends it with ``suggested=<model|-> chosen=<model|->``
|
||
(see worker.py). Rows in the old format parse fine, with
|
||
``suggested_model`` reported as unknown and counted separately
|
||
(``suggested_unknown_n``), never guessed.
|
||
* ``tasks`` rows for status and explicit-model separation. A task
|
||
with a creation-time ``model`` is explicit: routing never audits
|
||
it (decide_routing returns source ``explicit`` and the worker
|
||
does not audit), so explicit tasks are counted separately and
|
||
never inflate a tier's sample count.
|
||
* ``cost_records`` aggregated per task. USD here is whatever the
|
||
cost store recorded: with the current proxy-price overlay that is
|
||
the gpt-6.1-sol proxy price, NOT the relay's actual bill, and
|
||
unpriced rows contribute tokens but no USD. The report keeps
|
||
``usd_micros`` (priced only) and ``unpriced_tokens`` apart and
|
||
labels USD as proxy-priced when any unpriced tokens exist.
|
||
|
||
Pure functions only: this module never touches a DB, a provider,
|
||
or an environment variable, so it is unit-testable with plain dicts.
|
||
The server endpoint (GET /routing-shadow) is the only DB reader,
|
||
and it passes already-aggregated rows in.
|
||
"""
|
||
|
||
import re
|
||
|
||
MIN_SAMPLES = 20
|
||
TIERS = ("plan", "goal", "default")
|
||
ACTIVATION_ORDER = ("plan", "goal", "default")
|
||
|
||
_DETAIL_RE = re.compile(
|
||
r"^(?P<task_id>\S+)\s+tier=(?P<tier>\S+)\s+source=(?P<source>\S+)"
|
||
r"\s+applied=(?P<applied>\S+)"
|
||
r"(?:\s+suggested=(?P<suggested>\S+))?"
|
||
# Audit batch 4 (I-6): chosen is the LAST field, so it may
|
||
# greedily run to end of line — a legacy chosen model name
|
||
# containing whitespace ("my model") now parses instead of
|
||
# dropping the whole row. suggested stays \S+ (it is a middle
|
||
# field); new policy models cannot contain whitespace (the
|
||
# batch-4 policy whitelist drops them), so new rows always
|
||
# parse, and legacy suggested-with-whitespace rows still land
|
||
# in the unparseable bucket (counted, batch 2) rather than
|
||
# being silently lost.
|
||
r"(?:\s+chosen=(?P<chosen>.+?))?\s*$")
|
||
|
||
|
||
def parse_worker_routing_detail(detail):
|
||
"""Parse one worker_routing audit detail, or None if not parseable.
|
||
|
||
Old format (no suggested/chosen) parses with both models None.
|
||
A ``-`` placeholder also means None, never a model named "-".
|
||
Audit batch 2 (I-1): 解析语义本身不变（失败仍返回 None），
|
||
但 evaluate_shadow 不再静默跳过：不可解析的行计入
|
||
dropped_rows["unparseable"]，报告里可见。
|
||
Audit batch 4 (I-6): chosen 是行尾字段，含空白的
|
||
legacy chosen 模型名现在能解析（见 _DETAIL_RE 注释）；
|
||
suggested 仍要求无空白，legacy suggested 含空白的行
|
||
仍进 unparseable 桶计数，不静默消失。
|
||
"""
|
||
if not isinstance(detail, str):
|
||
return None
|
||
match = _DETAIL_RE.match(detail.strip())
|
||
if not match:
|
||
return None
|
||
applied_raw = match.group("applied").lower()
|
||
applied = applied_raw in ("true", "1", "yes")
|
||
suggested = match.group("suggested")
|
||
chosen = match.group("chosen")
|
||
return {
|
||
"task_id": match.group("task_id"),
|
||
"tier": match.group("tier"),
|
||
"source": match.group("source"),
|
||
"applied": applied,
|
||
"suggested_model": None if suggested in (None, "-") else suggested,
|
||
"chosen_model": None if chosen in (None, "-") else chosen.strip(),
|
||
}
|
||
|
||
|
||
def _empty_tier():
|
||
return {
|
||
"routed_n": 0,
|
||
"shadow_n": 0,
|
||
"policy_n": 0,
|
||
# Audit fix 3（I8）：显式模型任务（source=explicit）
|
||
# 按档单独计数，不进 routed_n、不占 seen。
|
||
"explicit_n": 0,
|
||
"completed": 0,
|
||
"failed": 0,
|
||
"other_status": 0,
|
||
"unknown_status": 0,
|
||
"total_tokens": 0,
|
||
"usd_micros": 0,
|
||
"unpriced_tokens": 0,
|
||
"suggested_models": {},
|
||
"suggested_unknown_n": 0,
|
||
"gate": "not_ready",
|
||
"sample_shortfall": MIN_SAMPLES,
|
||
}
|
||
|
||
|
||
def evaluate_shadow(audit_rows, tasks_by_id, costs_by_task,
|
||
explicit_n=0, min_samples=MIN_SAMPLES):
|
||
"""Evaluate shadow evidence from already-fetched rows.
|
||
|
||
``audit_rows``: dicts with at least ``detail``. Rows with
|
||
``source=explicit`` are counted per tier as ``explicit_n``
|
||
only (never ``routed_n``, never dedup slots). The review
|
||
gate is ``completed >= min_samples`` (Audit fix 3, I2):
|
||
failed/unknown/other statuses do not count toward readiness.
|
||
When rows carry ``created_at`` (and optionally ``id`` as a
|
||
tiebreak),
|
||
they are sorted earliest-first inside this function before
|
||
dedup, so the first-seen sample for a task is its earliest
|
||
routing audit regardless of the caller's row order. Rows
|
||
without ``created_at`` sort last (Audit fix 3, M2: the old
|
||
docstring said they kept their given order, but the sort
|
||
key already put them last); among themselves they keep
|
||
stable order.
|
||
The server endpoint passes rows already ordered, and it
|
||
strips ``created_at`` before calling -- that path is
|
||
unaffected, but a future caller passing raw rows no longer
|
||
silently changes dedup semantics. ``tasks_by_id``: task_id
|
||
-> {"status": str}. ``costs_by_task``: task_id ->
|
||
{"total_tokens": int, "usd_micros": int|None,
|
||
"unpriced_tokens": int}. ``explicit_n``: number of tasks
|
||
with a creation-time model in the same project (counted
|
||
separately, never as routed samples).
|
||
Audit batch 2 (I-1): every input row lands in exactly one
|
||
bucket — a routed sample, or ``dropped_rows``. Dropped
|
||
buckets are the real skip reasons in the loop below:
|
||
``unparseable`` (detail does not match the format),
|
||
``unknown_tier`` (tier is not a known tier), ``explicit``
|
||
(source=explicit rows, also counted per-tier as explicit_n,
|
||
never as samples), ``duplicate`` (a later row for an
|
||
already-counted task). Identity: ``audit_rows_total ==
|
||
routed_task_n + dropped_total`` — the caller can reconcile
|
||
the report against the audit rows it fetched instead of
|
||
guessing whether samples were lost.
|
||
"""
|
||
tiers = {tier: _empty_tier() for tier in TIERS}
|
||
seen = set()
|
||
routed_task_ids = set()
|
||
dropped_rows = {"unparseable": 0, "unknown_tier": 0,
|
||
"explicit": 0, "duplicate": 0}
|
||
rows = list(audit_rows or [])
|
||
if any(isinstance(r, dict) and r.get("created_at") is not None
|
||
for r in rows):
|
||
def _sort_key(row):
|
||
created = row.get("created_at") if isinstance(row, dict) \
|
||
else None
|
||
row_id = row.get("id") if isinstance(row, dict) else None
|
||
# Rows without a timestamp sort first only if they
|
||
# were first already: use a sentinel that keeps
|
||
# stable order among themselves, but never jumps
|
||
# ahead of a timestamped row -- so put them last.
|
||
return (created is None,
|
||
created if isinstance(created, (int, float))
|
||
else 0,
|
||
row_id if isinstance(row_id, (int, float))
|
||
else 0)
|
||
rows = sorted(rows, key=_sort_key)
|
||
for row in rows:
|
||
parsed = parse_worker_routing_detail(
|
||
(row or {}).get("detail"))
|
||
if parsed is None:
|
||
dropped_rows["unparseable"] += 1
|
||
continue
|
||
task_id = parsed["task_id"]
|
||
tier = parsed["tier"]
|
||
if tier not in tiers:
|
||
# Unknown tier: do not invent a bucket, do not count it
|
||
# as a sample for any known tier, and do not let it
|
||
# occupy the dedup slot (Audit fix 3, M1): a task
|
||
# whose first row is an unknown tier must not lose
|
||
# its later known-tier row to dedup.
|
||
# Audit batch 2 (I-1): the row is counted as dropped,
|
||
# not silently skipped.
|
||
dropped_rows["unknown_tier"] += 1
|
||
continue
|
||
if parsed["source"] == "explicit":
|
||
# Audit fix 3（I8）：显式行只计 per-tier
|
||
# explicit_n：不计 routed_n、不占 seen（同一任务
|
||
# 日后若出现路由行仍可计为样本），不破坏去重。
|
||
# Audit batch 2 (I-1): explicit rows are also
|
||
# counted in dropped_rows so the row identity
|
||
# reconciles; they are not "lost" — they are
|
||
# counted separately by design.
|
||
tiers[tier]["explicit_n"] += 1
|
||
dropped_rows["explicit"] += 1
|
||
continue
|
||
if task_id in seen:
|
||
dropped_rows["duplicate"] += 1
|
||
continue
|
||
seen.add(task_id)
|
||
bucket = tiers[tier]
|
||
routed_task_ids.add(task_id)
|
||
bucket["routed_n"] += 1
|
||
if parsed["source"] == "shadow":
|
||
bucket["shadow_n"] += 1
|
||
elif parsed["source"] == "policy":
|
||
bucket["policy_n"] += 1
|
||
else:
|
||
# Other sources (e.g. a future source) are routed
|
||
# samples, but neither shadow nor applied-policy samples.
|
||
pass
|
||
task = tasks_by_id.get(task_id) if tasks_by_id else None
|
||
status = (task or {}).get("status") if task else None
|
||
if status == "completed":
|
||
bucket["completed"] += 1
|
||
elif status == "failed":
|
||
bucket["failed"] += 1
|
||
elif status is None:
|
||
bucket["unknown_status"] += 1
|
||
else:
|
||
bucket["other_status"] += 1
|
||
cost = costs_by_task.get(task_id) if costs_by_task else None
|
||
if cost:
|
||
bucket["total_tokens"] += int(cost.get("total_tokens") or 0)
|
||
if cost.get("usd_micros") is not None:
|
||
bucket["usd_micros"] += int(cost["usd_micros"])
|
||
bucket["unpriced_tokens"] += int(
|
||
cost.get("unpriced_tokens") or 0)
|
||
if parsed["suggested_model"]:
|
||
models = bucket["suggested_models"]
|
||
models[parsed["suggested_model"]] = \
|
||
models.get(parsed["suggested_model"], 0) + 1
|
||
else:
|
||
bucket["suggested_unknown_n"] += 1
|
||
# Audit fix 3（I2）：就绪门改按已完成任务数计。
|
||
# 旧版按 routed_n（已路由数）计，20 个全失败的档
|
||
# 同样算就绪，与设计稿“已完成任务”的口径不一致；
|
||
# sample_shortfall 同口径，失败/未知/其它状态不算。
|
||
for tier, bucket in tiers.items():
|
||
bucket["gate"] = (
|
||
"ready_for_review" if bucket["completed"] >= min_samples
|
||
else "not_ready")
|
||
bucket["sample_shortfall"] = max(
|
||
0, min_samples - bucket["completed"])
|
||
ready = [t for t in ACTIVATION_ORDER
|
||
if tiers[t]["gate"] == "ready_for_review"]
|
||
if ready:
|
||
recommendation = (
|
||
"Ready for manual review (not activation): %s. "
|
||
"Review plan tier first, then goal, then default; "
|
||
"activation stays a manual decision and is never "
|
||
"automatic from this report." % ", ".join(ready))
|
||
else:
|
||
recommendation = (
|
||
"Not ready for review: no tier has %d completed "
|
||
"samples. Keep shadow mode running; do not activate "
|
||
"any tier." % min_samples)
|
||
return {
|
||
"min_samples": min_samples,
|
||
"tiers": tiers,
|
||
"explicit_n": int(explicit_n or 0),
|
||
"routed_task_n": len(routed_task_ids),
|
||
# Audit batch 2 (I-1): row reconciliation. Every input
|
||
# row is either a routed sample or in dropped_rows, so
|
||
# audit_rows_total == routed_task_n + dropped_total.
|
||
"audit_rows_total": len(rows),
|
||
"dropped_rows": dropped_rows,
|
||
"dropped_total": sum(dropped_rows.values()),
|
||
"counterfactual": "not_measured",
|
||
"counterfactual_note": (
|
||
"Shadow suggested models never ran, so this report cannot "
|
||
"compare suggested vs chosen cost or quality; it measures "
|
||
"sample size, status under the chosen model, and the "
|
||
"suggested-model distribution only."),
|
||
"usd_note": (
|
||
"usd_micros sums priced cost records only; unpriced "
|
||
"tokens are kept separate. With the current proxy-price "
|
||
"overlay, USD is proxy-priced (gpt-6.1-sol), not the "
|
||
"relay's actual bill."),
|
||
"activation_order": list(ACTIVATION_ORDER),
|
||
"ready_for_review": ready,
|
||
"recommendation": recommendation,
|
||
}
|