refactor: audit batch 3 (I-3 capability verdict, I-4 policy merge)

- I-3: new models.evaluate_capability returning a CapabilityVerdict
  (ok/missing/unknown from check, known/unknown_missing from split,
  status/verb derivations); decide_routing, worker gateway check,
  worker persisted recheck, dry-run tier+gateway checks and server
  shadow endpoint all call it. Blocking rules stay at the call
  sites (gateway has no worker-default fallback, recheck blocks
  only on known, dry-run/shadow never block); all user-visible
  audit details, dry-run notes and shadow notes are byte-identical.
- I-4: new models.merge_policy_documents shared by
  load_routing_policy and dry_run_from_env; the old tiers-update
  block is kept verbatim (it was a no-op after update(): JSON
  tiers replace the whole map, file tiers absent from JSON are
  dropped -- preserved, not fixed, as that would be a behaviour
  change). dropped_tiers/partial_failures remain load-time
  records, not merge products.
- Tests: test_audit_batch3.py, 13 tests (verdict matrix + parity
  with check/split, decide/dry-run wording, merge boundaries +
  300 seeded random pairs + loader/dry-run agreement).
  Full suite: 1694 OK (skipped=11), baseline 1681 + 13, 0 failures.
This commit is contained in:
host committed 2026-11-04 18:02:23 +00:00
1 parent a7f77c8a98
commit 06e4e16bbb
5 files changed
+485 -68

No files matched your search

+117 -17
View File
@@ -289,6 +289,115 @@ def split_model_capability_missing(capabilities, model, requires):
return sorted(known), sorted(unknown)
class CapabilityVerdict:
"""Structured result of one capability check (audit batch 3, I-3).
One call replaces the previous check+split pair at the five
capability sites (decide_routing, worker gateway check, worker
persisted-model recheck, dry-run tier/gateway checks, server
shadow endpoint). The fields are exactly what those sites used
before, so no site changes behaviour by adopting it:
- ``ok`` / ``missing`` / ``unknown`` are the total-check fields
from check_model_capabilities (decide/dry-run/shadow wording);
- ``known_missing`` / ``unknown_missing`` are the split fields
from split_model_capability_missing (gateway explicit split,
persisted recheck);
- ``status`` is "ok", or "unknown" when not ok and any part is
unverifiable, else "fail" (the dry-run/shadow status rule);
- ``verb`` is "cannot verify" when unknown else "lacks" (the
dry-run/gateway note wording); it is only meaningful when
not ok.
Edge preserved, not fixed: for a model absent from the
directory, ``missing`` lists every requires key (the total
check's rule) while ``unknown_missing`` lists only recognised
requirement keys (the split's rule). Callers that previously
used one function keep getting that function's values; the
blocking rules themselves stay at the call sites (gateway has
no worker-default fallback, recheck blocks only on known,
dry-run/shadow never block), they are not folded into this
verdict.
"""
def __init__(self, ok, missing, unknown, known_missing,
unknown_missing):
self.ok = bool(ok)
self.missing = list(missing)
self.unknown = bool(unknown)
self.known_missing = list(known_missing)
self.unknown_missing = list(unknown_missing)
@property
def status(self):
if self.ok:
return "ok"
return "unknown" if self.unknown else "fail"
@property
def verb(self):
return "cannot verify" if self.unknown else "lacks"
def evaluate_capability(capabilities, model, requires):
"""Check one model against requires, returning a CapabilityVerdict.
Pure: it only calls check_model_capabilities and
split_model_capability_missing, so its fields are definitionally
identical to what the five call sites computed before with two
calls. ``requires`` is expected to be already cleaned by
parse_tier_requires (all five sites pass tier_requirements);
uncleaned keys behave exactly as the two underlying functions
behave (see CapabilityVerdict).
"""
ok, missing, unknown = check_model_capabilities(
capabilities, model, requires)
known, unknown_missing = split_model_capability_missing(
capabilities, model, requires)
return CapabilityVerdict(ok, missing, unknown, known,
unknown_missing)
def merge_policy_documents(base, override):
"""Merge a file policy document with a JSON override document.
This is the merge load_routing_policy and dry_run_from_env
always shared line-for-line, now in one place (audit batch 3,
I-4). ``base`` is the file document (or None when unread/absent),
``override`` is the JSON document (or None when absent/unparsed).
Rules, exactly as before:
- override not a dict: base unchanged;
- base not a dict: override;
- otherwise: a shallow copy of base updated with override, so
JSON replaces whole keys — including the whole "tiers" map.
Per-tier merging does NOT happen: the old tiers-update block
after update() copied override's tiers back onto themselves
(merged["tiers"] already WAS override's tiers), i.e. it was
a no-op, and it is kept verbatim below so this function stays
a line-for-line move, not a semantic "fix". File tiers absent
from the JSON document are therefore dropped, not preserved;
changing that would be a behaviour change for another batch.
Source-level failure records (dropped_tiers/partial_failures
from batch 2) are NOT a product of this merge: they are filled
in by parse/load afterwards, and this function neither reads
nor writes them.
"""
if not isinstance(override, dict):
return base
if not isinstance(base, dict):
return override
merged = dict(base)
merged.update(override)
if isinstance(merged.get("tiers"), dict) and \
isinstance(override.get("tiers"), dict):
tiers = dict(merged["tiers"])
tiers.update(override["tiers"])
merged["tiers"] = tiers
return merged
def prompt_len_bucket(prompt) -> str:
"""Bucket a prompt by length for logging only, never for routing.
@@ -578,16 +687,7 @@ def load_routing_policy(env=None):
failures.append("CLOUD_ROUTING_POLICY_JSON: %s" % exc)
override = None
if isinstance(override, dict):
if not isinstance(merged, dict):
merged = override
else:
merged = dict(merged)
merged.update(override)
if isinstance(merged.get("tiers"), dict) and \
isinstance(override.get("tiers"), dict):
tiers = dict(merged["tiers"])
tiers.update(override["tiers"])
merged["tiers"] = tiers
merged = merge_policy_documents(merged, override)
policy = parse_routing_policy(merged)
if policy is None:
if configured:
@@ -741,13 +841,13 @@ def decide_routing(task, worker_model, policy=None,
policy_version=policy.version, applied=True,
reason="%s tier mode is off" % tier, signals=signals)
requires = policy.tier_requirements.get(tier) or {}
cap_ok, cap_missing, cap_unknown = check_model_capabilities(
cap = evaluate_capability(
policy.model_capabilities, tier_model, requires)
cap_note = ""
if requires and not cap_ok:
if requires and not cap.ok:
cap_note = " (capability %s: %s)" % (
"unknown" if cap_unknown else "lacks",
", ".join(cap_missing))
"unknown" if cap.unknown else "lacks",
", ".join(cap.missing))
if effective_mode == "shadow":
return RoutingDecision(
task_id=task.get("id"), tier=tier, chosen_model=worker_model,
@@ -756,7 +856,7 @@ def decide_routing(task, worker_model, policy=None,
reason="%s tier suggests %s (shadow, not applied)%s"
% (tier, tier_model, cap_note), signals=signals)
# effective_mode == "active"
if cap_ok:
if cap.ok:
return RoutingDecision(
task_id=task.get("id"), tier=tier, chosen_model=tier_model,
suggested_model=None, source="policy",
@@ -767,9 +867,9 @@ def decide_routing(task, worker_model, policy=None,
# default against the same requirements before blocking:
# a capable worker default keeps the task moving without
# violating the requirement.
worker_ok, _w_missing, _w_unknown = check_model_capabilities(
worker_cap = evaluate_capability(
policy.model_capabilities, worker_model, requires)
if worker_ok:
if worker_cap.ok:
return RoutingDecision(
task_id=task.get("id"), tier=tier, chosen_model=worker_model,
suggested_model=None, source="worker_default",