host
f9a77577c5
- RoutingPolicy.fallback_models: move the cleaning loop into __init__ (it was unreachable after tier_mode's return, so the attribute never existed); still parsed but not applied. - Capability fallback: RoutingDecision gains capability_fallback; the worker audits worker_capability_fallback when it fires, so a tier-model rejection followed by a worker-default fallback is no longer invisible (source stays worker_default, no model_routing event, so shadow samples are unaffected). - Registry: resolve_registry_agent checks skills directly against the loaded definition instead of re-parsing a hand-built dict, so future fields cannot be silently dropped the way tool_approvals was in v9. - Shadow: evaluate_shadow sorts rows earliest-first by created_at (id tiebreak) when timestamps are present, so dedup no longer depends on caller row order; the server path (which strips created_at) is unchanged. - Gateway: when sub2api fills the default model, the worker now checks it against the tier's capability requirements with the same mode semantics as v8 -- shadow only audits, active fails fast with worker_capability_gate_rejected and never calls the runner. No worker-default fallback here: the gateway model is the task's gateway model, falling back to a worker default would run against the wrong gateway.
237 lines
9.7 KiB
Python
237 lines
9.7 KiB
Python
"""Routing v0b: shadow evaluation (read-only, never auto-activates).
|
|
|
|
This module answers the one question routing v0 left unanswered:
|
|
"is there enough shadow evidence to review a tier for activation?"
|
|
It does NOT decide activation, and it does NOT claim a counterfactual.
|
|
|
|
Why the counterfactual is explicitly not measured
|
|
--------------------------------------------------
|
|
In shadow mode the suggested model never runs. The task's actual
|
|
outcome (status, tokens, cost) is the outcome under the chosen
|
|
worker-default model, not under the suggested model. So this report
|
|
can measure, per tier:
|
|
|
|
* how many distinct tasks were routed (one task = one sample,
|
|
retries do not double-count),
|
|
* their status breakdown under the chosen model,
|
|
* the distribution of suggested models,
|
|
* total tokens and priced USD, with unpriced tokens kept separate,
|
|
|
|
but it cannot say the suggested model "would have been cheaper or
|
|
better". Any report that compared shadow suggested cost to actual
|
|
cost as if both had run would be fabrication. The field
|
|
``counterfactual`` in every report says ``not_measured`` for exactly
|
|
this reason, and the top-level recommendation never activates a tier:
|
|
activation stays a manual decision, plan tier first, per the v0 design.
|
|
|
|
Data sources, and their limits
|
|
-------------------------------
|
|
* ``audit_log`` rows with action ``worker_routing``. The v0 format
|
|
was ``<task_id> tier=<tier> source=<source> applied=<bool>``;
|
|
v0b extends it with ``suggested=<model|-> chosen=<model|->``
|
|
(see worker.py). Rows in the old format parse fine, with
|
|
``suggested_model`` reported as unknown and counted separately
|
|
(``suggested_unknown_n``), never guessed.
|
|
* ``tasks`` rows for status and explicit-model separation. A task
|
|
with a creation-time ``model`` is explicit: routing never audits
|
|
it (decide_routing returns source ``explicit`` and the worker
|
|
does not audit), so explicit tasks are counted separately and
|
|
never inflate a tier's sample count.
|
|
* ``cost_records`` aggregated per task. USD here is whatever the
|
|
cost store recorded: with the current proxy-price overlay that is
|
|
the gpt-6.1-sol proxy price, NOT the relay's actual bill, and
|
|
unpriced rows contribute tokens but no USD. The report keeps
|
|
``usd_micros`` (priced only) and ``unpriced_tokens`` apart and
|
|
labels USD as proxy-priced when any unpriced tokens exist.
|
|
|
|
Pure functions only: this module never touches a DB, a provider,
|
|
or an environment variable, so it is unit-testable with plain dicts.
|
|
The server endpoint (GET /routing-shadow) is the only DB reader,
|
|
and it passes already-aggregated rows in.
|
|
"""
|
|
|
|
import re
|
|
|
|
MIN_SAMPLES = 20
|
|
TIERS = ("plan", "goal", "default")
|
|
ACTIVATION_ORDER = ("plan", "goal", "default")
|
|
|
|
_DETAIL_RE = re.compile(
|
|
r"^(?P<task_id>\S+)\s+tier=(?P<tier>\S+)\s+source=(?P<source>\S+)"
|
|
r"\s+applied=(?P<applied>\S+)"
|
|
r"(?:\s+suggested=(?P<suggested>\S+))?"
|
|
r"(?:\s+chosen=(?P<chosen>\S+))?\s*$")
|
|
|
|
|
|
def parse_worker_routing_detail(detail):
|
|
"""Parse one worker_routing audit detail, or None if not parseable.
|
|
|
|
Old format (no suggested/chosen) parses with both models None.
|
|
A ``-`` placeholder also means None, never a model named "-".
|
|
"""
|
|
if not isinstance(detail, str):
|
|
return None
|
|
match = _DETAIL_RE.match(detail.strip())
|
|
if not match:
|
|
return None
|
|
applied_raw = match.group("applied").lower()
|
|
applied = applied_raw in ("true", "1", "yes")
|
|
suggested = match.group("suggested")
|
|
chosen = match.group("chosen")
|
|
return {
|
|
"task_id": match.group("task_id"),
|
|
"tier": match.group("tier"),
|
|
"source": match.group("source"),
|
|
"applied": applied,
|
|
"suggested_model": None if suggested in (None, "-") else suggested,
|
|
"chosen_model": None if chosen in (None, "-") else chosen,
|
|
}
|
|
|
|
|
|
def _empty_tier():
|
|
return {
|
|
"routed_n": 0,
|
|
"shadow_n": 0,
|
|
"policy_n": 0,
|
|
"completed": 0,
|
|
"failed": 0,
|
|
"other_status": 0,
|
|
"unknown_status": 0,
|
|
"total_tokens": 0,
|
|
"usd_micros": 0,
|
|
"unpriced_tokens": 0,
|
|
"suggested_models": {},
|
|
"suggested_unknown_n": 0,
|
|
"gate": "not_ready",
|
|
"sample_shortfall": MIN_SAMPLES,
|
|
}
|
|
|
|
|
|
def evaluate_shadow(audit_rows, tasks_by_id, costs_by_task,
|
|
explicit_n=0, min_samples=MIN_SAMPLES):
|
|
"""Evaluate shadow evidence from already-fetched rows.
|
|
|
|
``audit_rows``: dicts with at least ``detail``. When rows
|
|
carry ``created_at`` (and optionally ``id`` as a tiebreak),
|
|
they are sorted earliest-first inside this function before
|
|
dedup, so the first-seen sample for a task is its earliest
|
|
routing audit regardless of the caller's row order. Rows
|
|
without ``created_at`` keep their given order (stable sort).
|
|
The server endpoint passes rows already ordered, and it
|
|
strips ``created_at`` before calling -- that path is
|
|
unaffected, but a future caller passing raw rows no longer
|
|
silently changes dedup semantics. ``tasks_by_id``: task_id
|
|
-> {"status": str}. ``costs_by_task``: task_id ->
|
|
{"total_tokens": int, "usd_micros": int|None,
|
|
"unpriced_tokens": int}. ``explicit_n``: number of tasks
|
|
with a creation-time model in the same project (counted
|
|
separately, never as routed samples).
|
|
"""
|
|
tiers = {tier: _empty_tier() for tier in TIERS}
|
|
seen = set()
|
|
routed_task_ids = set()
|
|
rows = list(audit_rows or [])
|
|
if any(isinstance(r, dict) and r.get("created_at") is not None
|
|
for r in rows):
|
|
def _sort_key(row):
|
|
created = row.get("created_at") if isinstance(row, dict) \
|
|
else None
|
|
row_id = row.get("id") if isinstance(row, dict) else None
|
|
# Rows without a timestamp sort first only if they
|
|
# were first already: use a sentinel that keeps
|
|
# stable order among themselves, but never jumps
|
|
# ahead of a timestamped row -- so put them last.
|
|
return (created is None,
|
|
created if isinstance(created, (int, float))
|
|
else 0,
|
|
row_id if isinstance(row_id, (int, float))
|
|
else 0)
|
|
rows = sorted(rows, key=_sort_key)
|
|
for row in rows:
|
|
parsed = parse_worker_routing_detail(
|
|
(row or {}).get("detail"))
|
|
if parsed is None:
|
|
continue
|
|
task_id = parsed["task_id"]
|
|
if task_id in seen:
|
|
continue
|
|
seen.add(task_id)
|
|
tier = parsed["tier"]
|
|
if tier not in tiers:
|
|
# Unknown tier: do not invent a bucket, do not count it
|
|
# as a sample for any known tier.
|
|
continue
|
|
bucket = tiers[tier]
|
|
routed_task_ids.add(task_id)
|
|
bucket["routed_n"] += 1
|
|
if parsed["source"] == "shadow":
|
|
bucket["shadow_n"] += 1
|
|
elif parsed["source"] == "policy":
|
|
bucket["policy_n"] += 1
|
|
else:
|
|
# Other sources (e.g. a future source) are routed
|
|
# samples, but neither shadow nor applied-policy samples.
|
|
pass
|
|
task = tasks_by_id.get(task_id) if tasks_by_id else None
|
|
status = (task or {}).get("status") if task else None
|
|
if status == "completed":
|
|
bucket["completed"] += 1
|
|
elif status == "failed":
|
|
bucket["failed"] += 1
|
|
elif status is None:
|
|
bucket["unknown_status"] += 1
|
|
else:
|
|
bucket["other_status"] += 1
|
|
cost = costs_by_task.get(task_id) if costs_by_task else None
|
|
if cost:
|
|
bucket["total_tokens"] += int(cost.get("total_tokens") or 0)
|
|
if cost.get("usd_micros") is not None:
|
|
bucket["usd_micros"] += int(cost["usd_micros"])
|
|
bucket["unpriced_tokens"] += int(
|
|
cost.get("unpriced_tokens") or 0)
|
|
if parsed["suggested_model"]:
|
|
models = bucket["suggested_models"]
|
|
models[parsed["suggested_model"]] = \
|
|
models.get(parsed["suggested_model"], 0) + 1
|
|
else:
|
|
bucket["suggested_unknown_n"] += 1
|
|
for tier, bucket in tiers.items():
|
|
bucket["gate"] = (
|
|
"ready_for_review" if bucket["routed_n"] >= min_samples
|
|
else "not_ready")
|
|
bucket["sample_shortfall"] = max(
|
|
0, min_samples - bucket["routed_n"])
|
|
ready = [t for t in ACTIVATION_ORDER
|
|
if tiers[t]["gate"] == "ready_for_review"]
|
|
if ready:
|
|
recommendation = (
|
|
"Ready for manual review (not activation): %s. "
|
|
"Review plan tier first, then goal, then default; "
|
|
"activation stays a manual decision and is never "
|
|
"automatic from this report." % ", ".join(ready))
|
|
else:
|
|
recommendation = (
|
|
"Not ready for review: no tier has %d routed samples. "
|
|
"Keep shadow mode running; do not activate any tier."
|
|
% min_samples)
|
|
return {
|
|
"min_samples": min_samples,
|
|
"tiers": tiers,
|
|
"explicit_n": int(explicit_n or 0),
|
|
"routed_task_n": len(routed_task_ids),
|
|
"counterfactual": "not_measured",
|
|
"counterfactual_note": (
|
|
"Shadow suggested models never ran, so this report cannot "
|
|
"compare suggested vs chosen cost or quality; it measures "
|
|
"sample size, status under the chosen model, and the "
|
|
"suggested-model distribution only."),
|
|
"usd_note": (
|
|
"usd_micros sums priced cost records only; unpriced "
|
|
"tokens are kept separate. With the current proxy-price "
|
|
"overlay, USD is proxy-priced (gpt-6.1-sol), not the "
|
|
"relay's actual bill."),
|
|
"activation_order": list(ACTIVATION_ORDER),
|
|
"ready_for_review": ready,
|
|
"recommendation": recommendation,
|
|
}
|